From 51470287c63cc22dda527340cb58bfbf3d8f6b95 Mon Sep 17 00:00:00 2001 From: Praneeth Date: Wed, 26 Aug 2026 14:06:58 +0530 Subject: [PATCH 1/6] chore: cleanup logging package --- echopype/calibrate/api.py | 11 +- echopype/calibrate/calibrate_azfp.py | 10 +- echopype/calibrate/calibrate_base.py | 14 +- echopype/calibrate/calibrate_ek.py | 10 +- echopype/calibrate/ecs.py | 5 - echopype/clean/api.py | 9 +- echopype/commongrid/api.py | 3 - echopype/commongrid/utils.py | 11 +- echopype/consolidate/api.py | 14 +- echopype/consolidate/ek_depth_utils.py | 22 +-- echopype/consolidate/loc_utils.py | 5 +- echopype/consolidate/split_beam_angle.py | 9 +- echopype/convert/api.py | 10 +- echopype/convert/parse_azfp.py | 3 - echopype/convert/parse_base.py | 14 +- echopype/convert/parse_uls5.py | 10 +- echopype/convert/parse_uls6.py | 23 ++- echopype/convert/set_groups_base.py | 6 +- echopype/convert/set_groups_ek60.py | 3 - echopype/convert/set_groups_ek80.py | 11 +- echopype/convert/utils/ek_duplicates.py | 11 +- echopype/convert/utils/ek_raw_io.py | 79 +++++----- echopype/convert/utils/ek_raw_parsers.py | 66 ++++---- echopype/echodata/combine.py | 14 +- echopype/echodata/echodata.py | 11 +- .../sensor_ep_version_mapping/v05x_to_v06x.py | 8 +- echopype/echodata/utils_platform.py | 23 +-- echopype/qc/api.py | 9 +- echopype/tests/calibrate/test_calibrate.py | 6 - echopype/tests/clean/test_noise.py | 6 - .../tests/commongrid/test_commongrid_api.py | 101 ++++++------ echopype/tests/consolidate/test_add_depth.py | 24 --- .../tests/consolidate/test_add_location.py | 5 - echopype/tests/convert/test_convert_ek80.py | 18 +-- echopype/tests/echodata/test_echodata.py | 6 - echopype/tests/utils/test_utils_log.py | 144 ------------------ echopype/utils/io.py | 17 ++- echopype/utils/log.py | 137 ----------------- echopype/utils/prov.py | 21 ++- 39 files changed, 269 insertions(+), 640 deletions(-) delete mode 100644 echopype/tests/utils/test_utils_log.py delete mode 100644 echopype/utils/log.py diff --git a/echopype/calibrate/api.py b/echopype/calibrate/api.py index 05574d669..24b6a40fb 100644 --- a/echopype/calibrate/api.py +++ b/echopype/calibrate/api.py @@ -1,10 +1,11 @@ +import warnings + import numpy as np import xarray as xr from ..core import SONAR_MODELS from ..echodata import EchoData from ..echodata.simrad import check_input_args_combination, retrieve_correct_beam_group -from ..utils.log import _init_logger from ..utils.prov import echopype_prov_attrs, source_files_vars from .calibrate_azfp import CalibrateAZFP from .calibrate_ek import CalibrateEK60, CalibrateEK80 @@ -19,8 +20,6 @@ "EA640": CalibrateEK80, } -logger = _init_logger(__name__) - def _compute_cal( cal_type, @@ -44,14 +43,16 @@ def _compute_cal( check_input_args_combination(waveform_mode=waveform_mode, encode_mode=encode_mode) elif echodata.sonar_model in ("EK60", "AZFP", "AZFP6"): if waveform_mode is not None and waveform_mode != "CW": - logger.warning( + warnings.warn( "This sonar model transmits only narrowband signals (waveform_mode='CW'). " "Calibration will be in CW mode", + category=UserWarning, ) if encode_mode is not None and encode_mode != "power": - logger.warning( + warnings.warn( "This sonar model only record data as power or power/angle samples " "(encode_mode='power'). Calibration will be done on the power samples.", + category=UserWarning, ) # Check that assume_single_filter_time is correctly passed in. diff --git a/echopype/calibrate/calibrate_azfp.py b/echopype/calibrate/calibrate_azfp.py index fdebf2449..cb5c522a6 100644 --- a/echopype/calibrate/calibrate_azfp.py +++ b/echopype/calibrate/calibrate_azfp.py @@ -1,16 +1,15 @@ +import warnings + import numpy as np import xarray from scipy.interpolate import LinearNDInterpolator from ..echodata import EchoData -from ..utils.log import _init_logger from .cal_params import get_cal_params_AZFP from .calibrate_ek import CalibrateBase from .env_params import get_env_params_AZFP from .range import compute_range_AZFP -logger = _init_logger(__name__) - # Common Sv_offset values for frequency > 38 kHz SV_OFFSET_HF = { 150: 1.4, @@ -153,9 +152,10 @@ def compute_Sv_offset(self): try: Sv_offset.append(_calc_azfp_Sv_offset(freq, pulse_len * 1e6)) except ValueError: - logger.warning( + warnings.warn( f"The Sv for {freq}Hz and pulse length {pulse_len}us " - "is uncalibrated (Sv_offset=0.0)" + "is uncalibrated (Sv_offset=0.0)", + category=UserWarning, ) Sv_offset.append(0.0) diff --git a/echopype/calibrate/calibrate_base.py b/echopype/calibrate/calibrate_base.py index ccdd89593..070e84399 100644 --- a/echopype/calibrate/calibrate_base.py +++ b/echopype/calibrate/calibrate_base.py @@ -1,11 +1,9 @@ import abc +import warnings from ..echodata import EchoData -from ..utils.log import _init_logger from .ecs import ECSParser -logger = _init_logger(__name__) - class CalibrateBase(abc.ABC): """Class to handle calibration for all sonar models.""" @@ -19,9 +17,10 @@ def __init__(self, echodata: EchoData, env_params=None, cal_params=None, ecs_fil # Set ECS to overwrite user-provided dict if self.ecs_file is not None: if env_params is not None or cal_params is not None: - logger.warning( + warnings.warn( "The ECS file takes precedence when it is provided. " - "Parameter values provided in 'env_params' and 'cal_params' will not be used!" + "Parameter values provided in 'env_params' and 'cal_params' will not be used!", + category=UserWarning, ) # Parse ECS file to a dict @@ -118,11 +117,12 @@ def _check_echodata_backscatter_size(self): # Raise Warning if above 2.0 if total_gb > 2.0: - logger.warning( + warnings.warn( "The Echodata backscatter variables are large and can cause memory issues. " "Consider modifying the workflow that uses compute_Sv as below: " "Prior to `compute_Sv` run `echodata.chunk(CHUNK_DICTIONARY) " "and after `compute_Sv` run `ds_Sv.to_zarr(ZARR_STORE, compute=True)`. " "This will ensure that the computation is lazily evaluated, " - "with the results stored directly in a Zarr store on disk, rather then in memory." + "with the results stored directly in a Zarr store on disk, rather then in memory.", + category=ResourceWarning, ) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 30c974dac..ff98aa8fe 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -1,3 +1,4 @@ +import warnings from typing import Dict import numpy as np @@ -5,7 +6,6 @@ from ..echodata import EchoData from ..echodata.simrad import retrieve_correct_beam_group -from ..utils.log import _init_logger from .cal_params import _get_interp_da, get_cal_params_EK from .calibrate_base import CalibrateBase from .ecs import conform_channel_order, ecs_ds2dict, ecs_ev2ep @@ -19,8 +19,6 @@ from .env_params import get_env_params_EK from .range import compute_range_EK, range_mod_TVG_EK -logger = _init_logger(__name__) - def _slice_beam_vend(beam, vend, slice_dict): beam = beam.sel( @@ -126,10 +124,11 @@ def _cal_power_samples(self, cal_type: str) -> xr.Dataset: ping_time=beam["ping_time"], ) except Exception as e: - logger.warning( + warnings.warn( "Could not compute tau_effective from transmit signal in power encoding mode; " "falling back to transmit_duration_nominal. Error: %s", repr(e), + category=RuntimeWarning, ) tau_effective = beam["transmit_duration_nominal"].isel(ping_time=0) @@ -593,11 +592,12 @@ def _cal_complex_samples(self, cal_type: str) -> xr.Dataset: ping_time=self.beam["ping_time"], ) except Exception as e: - logger.warning( + warnings.warn( "Could not compute tau_effective " "from transmit signal in complex encoding mode; " "falling back to transmit_duration_nominal. Error: %s", repr(e), + category=RuntimeWarning, ) tau_effective = self.beam["transmit_duration_nominal"].isel(ping_time=0) # Use pulse_duration in place of tau_effective for GPT channels diff --git a/echopype/calibrate/ecs.py b/echopype/calibrate/ecs.py index c0a8cd919..5d42f6fc2 100644 --- a/echopype/calibrate/ecs.py +++ b/echopype/calibrate/ecs.py @@ -6,11 +6,6 @@ import numpy as np import xarray as xr -from ..utils.log import _init_logger - -logger = _init_logger(__name__) - - # String matcher for parser SEPARATOR = re.compile(r"#=+#\n") STATUS_CRUDE = re.compile(r"#\s*(?P(.+))\s*#\n") # noqa diff --git a/echopype/clean/api.py b/echopype/clean/api.py index 5dc7b054d..105ad53af 100644 --- a/echopype/clean/api.py +++ b/echopype/clean/api.py @@ -2,6 +2,7 @@ Functions for reducing variabilities in backscatter data. """ +import warnings from functools import partial import numpy as np @@ -9,7 +10,6 @@ from ..commongrid.utils import _parse_x_bin from ..utils.compute import _lin2log, _log2lin -from ..utils.log import _init_logger from ..utils.prov import add_processing_level, echopype_prov_attrs, insert_input_processing_level from .transient_noise.transient_fielding import transient_noise_fielding from .transient_noise.transient_matecho import transient_noise_matecho @@ -24,8 +24,6 @@ pool_Sv, ) -logger = _init_logger(__name__) - def mask_transient_noise( ds_Sv: xr.Dataset, @@ -137,10 +135,11 @@ def mask_transient_noise( elif func == "nanmedian": # Warn when `func=nanmedian` since the sorting overhead makes it incredibly slow compared to # other non-sorting aggregations like `nanmean`. - logger.warning( + warnings.warn( "`func=nanmedian` is an incredibly slow operation due to the overhead sorting. " "We plan to add the Fielding Transient Noise Filter in the future" - "described here: https://github.com/OSOceanAcoustics/echopype/issues/1352" + "described here: https://github.com/OSOceanAcoustics/echopype/issues/1352", + category=ResourceWarning, ) func = np.nanmedian diff --git a/echopype/commongrid/api.py b/echopype/commongrid/api.py index 510dd6dc8..14cb07990 100644 --- a/echopype/commongrid/api.py +++ b/echopype/commongrid/api.py @@ -2,7 +2,6 @@ Functions for enhancing the spatial and temporal coherence of data. """ -import logging import warnings from typing import Literal @@ -29,8 +28,6 @@ ping_time_bin_parsing_and_conversion, ) -logger = logging.getLogger(__name__) - @add_processing_level("L3*") def compute_MVBS( diff --git a/echopype/commongrid/utils.py b/echopype/commongrid/utils.py index 1d451c7f4..9eac9119b 100644 --- a/echopype/commongrid/utils.py +++ b/echopype/commongrid/utils.py @@ -1,5 +1,5 @@ -import logging import re +import warnings from typing import Literal, Optional, Tuple, Union import numpy as np @@ -11,8 +11,6 @@ from ..consolidate.api import POSITION_VARIABLES from ..utils.compute import _lin2log, _log2lin -logger = logging.getLogger(__name__) - def compute_raw_MVBS( ds_Sv: xr.Dataset, @@ -585,7 +583,7 @@ def _groupby_x_along_channels( # Set correct range_var just in case if x_var == "distance_nmi" and range_var != "depth": - logger.warning("x_var is 'distance_nmi', setting range_var to 'depth'") + warnings.warn("x_var is 'distance_nmi', setting range_var to 'depth'", category=UserWarning) range_var = "depth" # average should be done in linear domain @@ -603,8 +601,9 @@ def _groupby_x_along_channels( ) for array_name, array in named_arrays.items(): if np.isnan(array).any(): - logging.warning( - f"The ```{array_name}``` coordinate array contain NaNs. {aggregation_msg}" + warnings.warn( + f"The ```{array_name}``` coordinate array contain NaNs. {aggregation_msg}", + category=UserWarning, ) # Use the first dimension as the grouping dimension for generality diff --git a/echopype/consolidate/api.py b/echopype/consolidate/api.py index c145a57f5..feaa8d0bf 100644 --- a/echopype/consolidate/api.py +++ b/echopype/consolidate/api.py @@ -1,6 +1,7 @@ import datetime import pathlib import sys +import warnings from numbers import Number from pathlib import Path from typing import Optional, Union @@ -14,7 +15,6 @@ from ..echodata.simrad import retrieve_correct_beam_group from ..utils.align import align_to_ping_time from ..utils.io import get_file_format, open_source -from ..utils.log import _init_logger from ..utils.prov import add_processing_level from .ek_depth_utils import ( ek_use_beam_angles, @@ -24,8 +24,6 @@ from .loc_utils import check_and_drop_loc_time_dim_duplicates, check_loc_vars_validity, sel_nmea from .split_beam_angle import get_angle_complex_samples, get_angle_power_samples -logger = _init_logger(__name__) - POSITION_VARIABLES = ["latitude", "longitude"] @@ -137,13 +135,15 @@ def add_depth( # Log warnings when group variables are not used if depth_offset is not None and use_platform_vertical_offsets: - logger.warning( + warnings.warn( "When `depth_offset` is specified, platform vertical offset " - "variables will not be used." + "variables will not be used.", + category=UserWarning, ) if tilt is not None and (use_beam_angles or use_platform_angles): - logger.warning( - "When `tilt` is specified, beam/platform angle variables will " "not be used." + warnings.warn( + "When `tilt` is specified, beam/platform angle variables will " "not be used.", + category=UserWarning, ) if echodata: diff --git a/echopype/consolidate/ek_depth_utils.py b/echopype/consolidate/ek_depth_utils.py index 8a767cfe8..a67cf030d 100644 --- a/echopype/consolidate/ek_depth_utils.py +++ b/echopype/consolidate/ek_depth_utils.py @@ -1,18 +1,17 @@ +import warnings + import numpy as np import xarray as xr from scipy.spatial.transform import Rotation as R from ..utils.align import align_to_ping_time -from ..utils.log import _init_logger - -logger = _init_logger(__name__) def _check_and_log_nans( echodata_group: xr.Dataset, group_name: str, variable_names: list[str] ) -> None: """ - Checks for NaNs in Echodata group variables and raises logger warning. + Checks for NaNs in Echodata group variables and raises UserWarning. """ # Iterate through group variable names for variable_name in variable_names: @@ -20,10 +19,11 @@ def _check_and_log_nans( group_var = echodata_group[variable_name] # Log warning if the group variable contains any NaNs if np.any(np.isnan(group_var.values)): - logger.warning( + warnings.warn( f"The Echodata `{group_name}` group `{variable_name}` variable array contains " "NaNs. This will result in NaNs in the final `depth` array. Consider filling the " - "NaNs and calling `.add_depth(...)` again." + "NaNs and calling `.add_depth(...)` again.", + category=UserWarning, ) @@ -98,14 +98,18 @@ def ek_use_beam_angles(beam_ds: xr.Dataset) -> xr.DataArray: # Warn if any nonzero vector is not normalized tolerance = 1e-8 if ((norm > tolerance) & (np.abs(norm - 1) > tolerance)).any(): - logger.warning( + warnings.warn( "Beam direction vector was not normalized; applying normalization. " - "By definition, it should have been normalized." + "By definition, it should have been normalized.", + category=UserWarning, ) # Warn if any channel has a (nearly) zero vector if (norm < tolerance).any(): - logger.warning("Some beam direction vectors are zero. Outputting NaN for those channels.") + warnings.warn( + "Some beam direction vectors are zero. Outputting NaN for those channels.", + category=UserWarning, + ) # For channels with near-zero norm, we return NaN. Otherwise, we return the normalized # z component. diff --git a/echopype/consolidate/loc_utils.py b/echopype/consolidate/loc_utils.py index fb4ce6bb4..a8ebd1cc4 100644 --- a/echopype/consolidate/loc_utils.py +++ b/echopype/consolidate/loc_utils.py @@ -5,9 +5,6 @@ import xarray as xr from ..echodata import EchoData -from ..utils.log import _init_logger - -logger = _init_logger(__name__) def compute_invalid_check(lat_var: xr.DataArray, lon_var: xr.DataArray, validity_check: str): @@ -105,7 +102,7 @@ def check_loc_vars_validity( if validity_check in ["missing", "all_nan"]: raise ValueError(output_message) elif validity_check in ["some_nan", "some_zero"]: - logger.warning(output_message) + warnings.warn(output_message, category=UserWarning) def check_and_drop_loc_time_dim_duplicates( diff --git a/echopype/consolidate/split_beam_angle.py b/echopype/consolidate/split_beam_angle.py index 48ad03623..b51829436 100644 --- a/echopype/consolidate/split_beam_angle.py +++ b/echopype/consolidate/split_beam_angle.py @@ -3,6 +3,7 @@ angles and add them to a Dataset. """ +import warnings from typing import List, Tuple import dask.array as da @@ -10,9 +11,6 @@ import xarray as xr from ..calibrate.ek80_complex import compress_pulse, get_norm_fac, get_transmit_signal -from ..utils.log import _init_logger - -logger = _init_logger(__name__) # Beam type identifiers BEAM_TYPE_SPLIT_4_SECTOR = 1 # 4-sector split-beam (common Simrad type) @@ -238,7 +236,10 @@ def get_angle_complex_samples( beam_type = ds_beam["beam_type"].sel(channel=ch_id) beam_type = int(beam_type) if beam_type not in SUPPORTED_BEAM_TYPES: - logger.warning(f"Skipping channel {ch_id}: unsupported beam_type {beam_type}") + warnings.warn( + f"Skipping channel {ch_id}: unsupported beam_type {beam_type}", + category=UserWarning, + ) continue theta_ch, phi_ch = _compute_angle_from_complex( diff --git a/echopype/convert/api.py b/echopype/convert/api.py index bb94e726b..e6413a30e 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -14,14 +14,10 @@ from ..echodata.echodata import XARRAY_ENGINE_MAP, EchoData from ..utils import io from ..utils.coding import COMPRESSION_SETTINGS -from ..utils.log import _init_logger from ..utils.prov import add_processing_level BEAM_SUBGROUP_DEFAULT = "Beam_group1" -# Logging setup -logger = _init_logger(__name__) - def to_file( echodata: EchoData, @@ -76,15 +72,15 @@ def to_file( # Sequential or parallel conversion if exists and not overwrite: - logger.info( + print( f"{echodata.source_file} has already been converted to {engine}. " # noqa f"File saving not executed." ) else: if exists: - logger.info(f"overwriting {output_file}") + print(f"overwriting {output_file}") else: - logger.info(f"saving {output_file}") + print(f"saving {output_file}") _save_groups_to_file( echodata, output_path=io.sanitize_file_path( diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index d43fcfaff..e75719fa3 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -4,11 +4,8 @@ import numpy as np -from ..utils.log import _init_logger from .parse_base import ParseBase -logger = _init_logger(__name__) - # Common Sv_offset values for frequency > 38 kHz SV_OFFSET_HF = { 300: 1.1, diff --git a/echopype/convert/parse_base.py b/echopype/convert/parse_base.py index 85a9b6371..7bfc1ea34 100644 --- a/echopype/convert/parse_base.py +++ b/echopype/convert/parse_base.py @@ -12,7 +12,6 @@ from dask.array.core import auto_chunks from ..utils.io import create_temp_zarr_store -from ..utils.log import _init_logger from .utils.ek_raw_io import RawSimradFile, SimradEOF from .utils.ek_swap import calc_final_shapes @@ -23,9 +22,6 @@ # Manufacturer-specific power conversion factor INDEX2POWER = 10.0 * np.log10(2.0) / 256.0 -logger = _init_logger(__name__) - - # --- Windows-safe path component sanitizer (also harmless on POSIX) --- _INVALID_FS_CHARS = r'[<>:"/\\|?*]' @@ -128,9 +124,7 @@ def _print_status(self): self.config_datagram["timestamp"].tolist() / 1e9, datetime.UTC ).strftime("%Y-%b-%d %H:%M:%S") - logger.info( - f"parsing file {os.path.basename(self.source_file)}, " f"time of first ping: {time}" - ) + print(f"parsing file {os.path.basename(self.source_file)}, " f"time of first ping: {time}") @property def num_transducer_sectors(self) -> Dict[Any, int]: @@ -698,7 +692,7 @@ def _read_datagrams(self, fid): # TAG datagrams contain time-stamped annotations inserted via the recording software elif new_datagram["type"].startswith("TAG"): - logger.info("TAG datagram encountered.") + print("TAG datagram encountered.") # BOT datagrams contain sounder detected bottom depths from .bot files elif new_datagram["type"].startswith("BOT"): @@ -717,9 +711,9 @@ def _read_datagrams(self, fid): # DEP datagrams contain sounder detected bottom depths from .out files # as well as reflectivity data elif new_datagram["type"].startswith("DEP"): - logger.info("DEP datagram encountered.") + print("DEP datagram encountered.") else: - logger.info("Unknown datagram type: " + str(new_datagram["type"])) + print("Unknown datagram type: " + str(new_datagram["type"])) def _append_channel_ping_data( self, datagram, raw_type: Literal["transmit", "receive"] = "receive" diff --git a/echopype/convert/parse_uls5.py b/echopype/convert/parse_uls5.py index f949df0f0..5c5df58ae 100644 --- a/echopype/convert/parse_uls5.py +++ b/echopype/convert/parse_uls5.py @@ -1,4 +1,5 @@ import os +import warnings import xml.etree.ElementTree as ET from datetime import datetime as dt from struct import unpack @@ -6,7 +7,6 @@ import fsspec import numpy as np -from ..utils.log import _init_logger from ..utils.misc import camelcase2snakecase from .parse_azfp import ParseAZFP @@ -59,8 +59,6 @@ ("ad", "u2", 2), # AD channel 6 and 7 ) -logger = _init_logger(__name__) - class ParseULS5(ParseAZFP): """Class for converting data from ASL Environmental Sciences AZFP echosounder.""" @@ -322,8 +320,8 @@ def _print_status(self): int(self.unpacked_data["second"][0] + self.unpacked_data["hundredths"][0] / 100), ) timestr = timestamp.strftime("%Y-%b-%d %H:%M:%S") - pathstr, xml_name = os.path.split(self.xml_path) - logger.info(f"parsing file {filename} with {xml_name}, " f"time of first ping: {timestr}") + _, xml_name = os.path.split(self.xml_path) + print(f"parsing file {filename} with {xml_name}, " f"time of first ping: {timestr}") def _split_header(self, raw, ping_num, header_unpacked): """Splits the header information into a dictionary. @@ -345,7 +343,7 @@ def _split_header(self, raw, ping_num, header_unpacked): ): # first field should match hard-coded FILE_TYPE from manufacturer check_eof = raw.read(1) if check_eof: - logger.error("Unknown file type") + warnings.warn("Unknown file type", category=UserWarning) return False header_byte_cnt = 0 diff --git a/echopype/convert/parse_uls6.py b/echopype/convert/parse_uls6.py index 7bc2058b5..24e6e3ae7 100644 --- a/echopype/convert/parse_uls6.py +++ b/echopype/convert/parse_uls6.py @@ -1,4 +1,6 @@ +import logging import os +import warnings import xml.etree.ElementTree as ET from datetime import datetime as dt from io import BytesIO @@ -7,7 +9,6 @@ import fsspec import numpy as np -from ..utils.log import _init_logger from ..utils.misc import camelcase2snakecase from .parse_azfp import ParseAZFP @@ -82,9 +83,6 @@ HEADER_LOOKUP = {**HEADER_LOOKUP, **OLDER_HEADER_LOOKUP} -logger = _init_logger(__name__) - - class ParseULS6(ParseAZFP): """Class for converting data from ASL Environmental Sciences AZFP echosounder.""" @@ -159,7 +157,7 @@ def load_AZFP_xml(self, raw): self.unpacked_data["num_prev_xml_bytes"] = xml_byte_size if int.from_bytes(raw.read(4), "little") != self.XML_END_FLAG: - logger.error("Error reading xml string") + logging.error("Error reading xml string") raise ValueError("Error reading xml string") xml_prev_byte_size = unpack(" xr.Dataset: water_level = self.parser_obj.environment["water_level_draft"] else: water_level = np.nan - logger.info("WARNING: The water_level_draft was not in the file. Value set to NaN.") + warnings.warn( + "WARNING: The water_level_draft was not in the file. Value set to NaN.", + category=UserWarning, + ) time1, msg_type, lat_nmea, lon_nmea = self._extract_NMEA_latlon() time2 = self.parser_obj.mru0.get("timestamp", None) @@ -1162,7 +1163,7 @@ def set_beam(self) -> List[xr.Dataset]: def _remove_duplicates(ds): ping_times = ds["ping_time"].values if len(ping_times) > len(np.unique(ping_times)): - check_unique_ping_time_duplicates(ds, logger) + check_unique_ping_time_duplicates(ds) ds = ds.drop_duplicates(dim="ping_time") return ds diff --git a/echopype/convert/utils/ek_duplicates.py b/echopype/convert/utils/ek_duplicates.py index 8a44988f6..c79f6e68b 100644 --- a/echopype/convert/utils/ek_duplicates.py +++ b/echopype/convert/utils/ek_duplicates.py @@ -1,9 +1,9 @@ -import logging +import warnings import xarray as xr -def check_unique_ping_time_duplicates(ds_data: xr.Dataset, logger: logging.Logger) -> None: +def check_unique_ping_time_duplicates(ds_data: xr.Dataset) -> None: """ Raises a warning if the data stored in duplicate pings is not unique. @@ -11,8 +11,6 @@ def check_unique_ping_time_duplicates(ds_data: xr.Dataset, logger: logging.Logge ---------- ds_data : xr.Dataset Single freq beam dataset being processed in the `SetGroupsEK80.set_beams` class function. - logger : logging.Logger - Warning logger initialized in `SetGroupsEK80` file. """ # Group the dataset by the "ping_time" coordinate groups = ds_data.groupby("ping_time") @@ -36,9 +34,10 @@ def check_unique_ping_time_duplicates(ds_data: xr.Dataset, logger: logging.Logge # Iterate over the remaining entries for i in range(1, data_array.sizes["ping_time"]): if not ref_slice.equals(data_array.isel({"ping_time": i})): - logger.warning( + warnings.warn( f"Duplicate slices in variable '{var}' corresponding to 'ping_time' " f"{ping_time_val} differ in data. All duplicate 'ping_time' entries " - "will be removed, which will result in data loss." + "will be removed, which will result in data loss.", + category=UserWarning, ) break diff --git a/echopype/convert/utils/ek_raw_io.py b/echopype/convert/utils/ek_raw_io.py index 6a1341b51..5d12a269c 100644 --- a/echopype/convert/utils/ek_raw_io.py +++ b/echopype/convert/utils/ek_raw_io.py @@ -7,18 +7,16 @@ """ import struct +import warnings from io import SEEK_CUR, SEEK_END, SEEK_SET, BufferedReader, FileIO import fsspec from fsspec.implementations.local import LocalFileSystem -from ...utils.log import _init_logger from . import ek_raw_parsers as parsers __all__ = ["RawSimradFile"] -logger = _init_logger(__name__) - class SimradEOF(Exception): def __init__(self, message="EOF Reached!"): @@ -265,11 +263,10 @@ def _read_next_dgram(self): # check for invalid time data if (header["low_date"], header["high_date"]) == (0, 0): - logger.warning( - "Skipping %s datagram w/ timestamp of (0, 0) at %sL:%d", - header["type"], - str(self._tell_bytes()), - self.tell(), + warnings.warn( + f"Skipping {header['type']} datagram w/ timestamp of (0, 0) at " + "{str(self._tell_bytes())}L:{self.tell()}", + category=BytesWarning, ) self.skip() return self._read_next_dgram() @@ -277,11 +274,10 @@ def _read_next_dgram(self): # basic sanity check on size if header["size"] < 16: # size can't be smaller than the header size - logger.warning( - "Invalid datagram header: size: %d, type: %s, nt_date: %s. dgram_size < 16", - header["size"], - header["type"], - str((header["low_date"], header["high_date"])), + warnings.warn( + f"Invalid datagram header: size: {header['size']}, type: {header['type']}, " + "nt_date: {str((header['low_date'], header['high_date']))}. dgram_size < 16", + category=BytesWarning, ) # see if we can find the next datagram @@ -303,12 +299,10 @@ def _read_next_dgram(self): # and make sure it checks out if bytes_read < header["size"]: - logger.warning( - "Datagram %d (@%d) shorter than expected length: %d < %d", - self.tell(), - old_file_pos, - bytes_read, - header["size"], + warnings.warn( + f"Datagram {self.tell()} (@{old_file_pos})" + " shorter than expected length: {bytes_read} < {header['size']}", + category=BytesWarning, ) self._find_next_datagram() return self._read_next_dgram() @@ -324,14 +318,12 @@ def _read_next_dgram(self): # make sure they match if header["size"] != dgram_size_check: # self._seek_bytes(old_file_pos, SEEK_SET) - logger.warning( - "Datagram failed size check: %d != %d @ (%d, %d)", - header["size"], - dgram_size_check, - self._tell_bytes(), - self.tell(), + warnings.warn( + f"Datagram failed size check: {header['size']} != {dgram_size_check} @ " + "({self._tell_bytes()}, {self.tell()})", + category=BytesWarning, ) - logger.warning("Skipping to next datagram...") + warnings.warn("Skipping to next datagram...", category=BytesWarning) self._find_next_datagram() return self._read_next_dgram() @@ -472,17 +464,19 @@ def readall(self): def _find_next_datagram(self): old_file_pos = self._tell_bytes() - logger.warning("Attempting to find next valid datagram...") + warnings.warn("Attempting to find next valid datagram...", category=BytesWarning) try: while self.peek()["type"][:3] not in list(self.DGRAM_TYPE_KEY.keys()): self._seek_bytes(1, 1) except DatagramReadError: - logger.warning("No next datagram found. Ending reading of file.") + warnings.warn("No next datagram found. Ending reading of file.", category=BytesWarning) raise SimradEOF() else: - logger.warning("Found next datagram: %s", self.peek()) - logger.warning("Skipped ahead %d bytes", self._tell_bytes() - old_file_pos) + warnings.warn(f"Found next datagram: {self.peek()}", category=BytesWarning) + warnings.warn( + f"Skipped ahead {self._tell_bytes() - old_file_pos} bytes", category=BytesWarning + ) def tell(self): """ @@ -538,11 +532,10 @@ def skip(self): header = self.peek() if header["size"] < 16: - logger.warning( - "Invalid datagram header: size: %d, type: %s, nt_date: %s. dgram_size < 16", - header["size"], - header["type"], - str((header["low_date"], header["high_date"])), + warnings.warn( + f"Invalid datagram header: size: {header['size']}, type: {header['type']}, " + "nt_date: {str((header['low_date'], header['high_date']))}. dgram_size < 16", + category=BytesWarning, ) self._find_next_datagram() @@ -552,14 +545,12 @@ def skip(self): dgram_size_check = self._read_dgram_size() if header["size"] != dgram_size_check: - logger.warning( - "Datagram failed size check: %d != %d @ (%d, %d)", - header["size"], - dgram_size_check, - self._tell_bytes(), - self.tell(), + warnings.warn( + f"Datagram failed size check: {header['size']} != {dgram_size_check} @ " + "({self._tell_bytes()}, {self.tell()})", + category=BytesWarning, ) - logger.warning("Skipping to next datagram... (in skip)") + warnings.warn("Skipping to next datagram... (in skip)", category=UserWarning) self._find_next_datagram() @@ -589,7 +580,7 @@ def skip_back(self): dgram_size = self._read_dgram_size() except DatagramSizeError: - logger.info("Error reading the datagram") + print("Error reading the datagram") self._seek_bytes(old_file_pos, SEEK_SET) raise @@ -614,7 +605,7 @@ def iter_dgrams(self): try: new_dgram = next(self) except Exception: - logger.debug("Caught EOF?") + print("Caught EOF?") raise StopIteration yield new_dgram diff --git a/echopype/convert/utils/ek_raw_parsers.py b/echopype/convert/utils/ek_raw_parsers.py index 4ee9cd68c..cb820409a 100644 --- a/echopype/convert/utils/ek_raw_parsers.py +++ b/echopype/convert/utils/ek_raw_parsers.py @@ -10,12 +10,12 @@ import re import struct import sys +import warnings import xml.etree.ElementTree as ET from collections import Counter import numpy as np -from ...utils.log import _init_logger from ...utils.misc import camelcase2snakecase from .ek_date_conversion import nt_to_unix @@ -30,8 +30,6 @@ "SimradRawParser", ] -logger = _init_logger(__name__) - class _SimradDatagramParser(object): """""" @@ -185,8 +183,12 @@ def _pack_contents(self, data, version): if len(set(lengths)) != 1: min_indx = min(lengths) - logger.warning("Data lengths mismatched: d:%d, r:%d, u:%d, t:%d", *lengths) - logger.warning(" Using minimum value: %d", min_indx) + warnings.warn( + "Data lengths mismatched: " + f"d:{lengths[0]}, r:{lengths[1]}, u:{lengths[2]}, t:{lengths[3]} " + f"Using minimum value: {min_indx}", + category=BytesWarning, + ) data["transceiver_count"] = min_indx else: @@ -274,10 +276,10 @@ def _pack_contents(self, data, version): if version == 0: if len(data["depth"]) != data["transceiver_count"]: - logger.warning( - "# of depth values %d does not match transceiver count %d", - len(data["depth"]), - data["transceiver_count"], + warnings.warn( + f"# of depth values {len(data['depth'])} does not match transceiver " + "count {data['transceiver_count']}", + category=BytesWarning, ) data["transceiver_count"] = len(data["depth"]) @@ -1431,12 +1433,12 @@ def _unpack_contents(self, raw_string, bytes_read, version): transducer_header = self._transducer_headers[sounder_name] _sounder_name_used = sounder_name except KeyError: - logger.warning( - "Unknown sounder_name: %s, (no one of %s)", - sounder_name, - list(self._transducer_headers.keys()), + warnings.warn( + f"Unknown sounder_name: {sounder_name}, " + "(no one of {list(self._transducer_headers.keys())}) " + "will use ER60 transducer config fields as default", + category=UserWarning, ) - logger.warning("Will use ER60 transducer config fields as default") transducer_header = self._transducer_headers["ER60"] _sounder_name_used = "ER60" @@ -1506,7 +1508,11 @@ def _pack_contents(self, data, version): if version == 0: if data["transceiver_count"] != len(data["transceivers"]): - logger.warning("Mismatch between 'transceiver_count' and actual # of transceivers") + warnings.warn( + "Mismatch between 'transceiver_count' and actual # of transceivers " + f"{data['transceiver_count']} != {len(data['transceivers'])}", + category=UserWarning, + ) data["transceiver_count"] = len(data["transceivers"]) sounder_name = data["sounder_name"] @@ -1527,12 +1533,12 @@ def _pack_contents(self, data, version): transducer_header = self._transducer_headers[sounder_name] _sounder_name_used = sounder_name except KeyError: - logger.warning( - "Unknown sounder_name: %s, (no one of %s)", - sounder_name, - list(self._transducer_headers.keys()), + warnings.warn( + f"Unknown sounder_name: {sounder_name}, " + f"(no one of {list(self._transducer_headers.keys())}) " + "will use ER60 transducer config fields as default", + category=UserWarning, ) - logger.warning("Will use ER60 transducer config fields as default") transducer_header = self._transducer_headers["ER60"] _sounder_name_used = "ER60" @@ -1784,21 +1790,27 @@ def _pack_contents(self, data, version): if version == 0: if data["count"] > 0: if (int(data["mode"]) & 0x1) and (len(data.get("power", [])) != data["count"]): - logger.warning( - "Data 'count' = %d, but contains %d power samples. Ignoring power." + warnings.warn( + f"Data 'count' = {data['count']}, " + f"but contains {len(data.get('power', []))} " + "power samples. Ignoring power.", + category=BytesWarning, ) data["mode"] &= ~(1 << 0) if (int(data["mode"]) & 0x2) and (len(data.get("angle", [])) != data["count"]): - logger.warning( - "Data 'count' = %d, but contains %d angle samples. Ignoring angle." + warnings.warn( + f"Data 'count' = {data['count']}, " + f"but contains {len(data.get('angle', []))} " + "angle samples. Ignoring angle.", + category=BytesWarning, ) data["mode"] &= ~(1 << 1) if data["mode"] == 0: - logger.warning( - "Data 'count' = %d, but mode == 0. Setting count to 0", - data["count"], + warnings.warn( + f"Data 'count' = {data['count']}, but mode == 0. " "Setting count to 0", + category=BytesWarning, ) data["count"] = 0 diff --git a/echopype/echodata/combine.py b/echopype/echodata/combine.py index 302d61e86..3a5d69966 100644 --- a/echopype/echodata/combine.py +++ b/echopype/echodata/combine.py @@ -1,9 +1,9 @@ import itertools import re +import warnings from collections import ChainMap from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Tuple, Union -from warnings import warn import fsspec import numpy as np @@ -12,12 +12,9 @@ from xarray import DataTree from ..utils.io import validate_output_path -from ..utils.log import _init_logger from ..utils.prov import echopype_prov_attrs from .echodata import EchoData -logger = _init_logger(__name__) - POSSIBLE_TIME_DIMS = {"time1", "time2", "time3", "time4", "nmea_time", "ping_time", "filter_time"} APPEND_DIMS = {"filenames"}.union(POSSIBLE_TIME_DIMS) DATE_CREATED_ATTR = "date_created" @@ -93,7 +90,7 @@ def check_zarr_path( "different path or set overwrite=True." ) elif exists and overwrite: - logger.info(f"overwriting {validated_path}") + print(f"overwriting {validated_path}") # remove zarr file fs.rm(validated_path, recursive=True) @@ -947,14 +944,17 @@ def combine_echodata( >>> ed2 = echopype.open_raw(raw_file="EK60_file2.raw", sonar_model="EK60") >>> combined = echopype.combine_echodata(echodata_list=[ed1, ed2]) """ - warn( + warnings.warn( "Echopype will stop supporting the `combine_echodata` function in the v0.12.1 release.", category=DeprecationWarning, ) # return empty EchoData object, if no EchoData objects are provided if echodata_list is None: - warn("No EchoData objects were provided, returning an empty EchoData object.") + warnings.warn( + "No EchoData objects were provided, returning an empty EchoData object.", + category=UserWarning, + ) return EchoData() # Ensure the list of all EchoData objects to be combined are valid diff --git a/echopype/echodata/echodata.py b/echopype/echodata/echodata.py index 24ac2473f..4563b5858 100644 --- a/echopype/echodata/echodata.py +++ b/echopype/echodata/echodata.py @@ -16,9 +16,8 @@ if TYPE_CHECKING: from ..core import EngineHint, FileFormatHint, PathHint, SonarModelsHint -from ..echodata.utils_platform import _clip_by_time_dim, get_mappings_expanded +from ..echodata.utils_platform import clip_by_time_dim, get_mappings_expanded from ..utils.coding import sanitize_dtypes, set_time_encodings -from ..utils.log import _init_logger from ..utils.prov import add_processing_level from .convention import sonarnetcdf_1 from .widgets.utils import tree_repr @@ -37,8 +36,6 @@ "EA640": 0, } -logger = _init_logger(__name__) - class EchoData: """Echo data model class for handling raw converted data, @@ -427,9 +424,7 @@ def update_platform( # Retain only variable_mappings items where # either the Platform group or extra_platform_data # contain the corresponding variables or contain valid (not all nan) data - mappings_expanded = get_mappings_expanded( - logger, extra_platform_data, variable_mappings, platform - ) + mappings_expanded = get_mappings_expanded(extra_platform_data, variable_mappings, platform) # Create names for required new time dimensions ext_time_dims = list( @@ -451,7 +446,7 @@ def update_platform( k: v for k, v in mappings_expanded.items() if v["ext_time_dim_name"] == ext_time_dim } ext_vars = [v["external_var"] for v in mappings_selected.values()] - ext_ds = _clip_by_time_dim( + ext_ds = clip_by_time_dim( extra_platform_data[ext_vars], ext_time_dim, self["Sonar/Beam_group1"]["ping_time"] ) diff --git a/echopype/echodata/sensor_ep_version_mapping/v05x_to_v06x.py b/echopype/echodata/sensor_ep_version_mapping/v05x_to_v06x.py index ed5f8135d..bf9d19c71 100644 --- a/echopype/echodata/sensor_ep_version_mapping/v05x_to_v06x.py +++ b/echopype/echodata/sensor_ep_version_mapping/v05x_to_v06x.py @@ -1,3 +1,4 @@ +import warnings import xml.etree.ElementTree as ET import numpy as np @@ -5,11 +6,9 @@ # TODO: turn this into an absolute import! from ...core import SONAR_MODELS -from ...utils.log import _init_logger from ..convention import sonarnetcdf_1 _varattrs = sonarnetcdf_1.yaml_dict["variable_and_varattributes"] -logger = _init_logger(__name__) def _get_sensor(sensor_model): @@ -1108,12 +1107,13 @@ def convert_v05x_to_v06x(echodata_obj): """ # TODO: put in an appropriate link to the v5 to v6 conversion outline - logger.warning( + warnings.warn( "Converting echopype version 0.5.x file to 0.6.0." " For specific details on how items have been changed," " please see the echopype documentation. It is recommended " "that one creates the file using echopype.open_raw again, " - "rather than relying on this conversion." + "rather than relying on this conversion.", + category=RuntimeWarning, ) # get the sensor used to create the v0.5.x file. diff --git a/echopype/echodata/utils_platform.py b/echopype/echodata/utils_platform.py index 4696c6871..971aba6fb 100644 --- a/echopype/echodata/utils_platform.py +++ b/echopype/echodata/utils_platform.py @@ -1,3 +1,5 @@ +import warnings + import numpy as np @@ -17,7 +19,7 @@ def _extvar_properties(ds, name): return False, False, None -def _clip_by_time_dim(external_ds, ext_time_dim_name, ping_time): +def clip_by_time_dim(external_ds, ext_time_dim_name, ping_time): """ Clip incoming time to 1 less than min of EchoData["Sonar/Beam_group1"]["ping_time"] and 1 greater than max of EchoData["Sonar/Beam_group1"]["ping_time"]. @@ -52,14 +54,12 @@ def _clip_by_time_dim(external_ds, ext_time_dim_name, ping_time): ) -def get_mappings_expanded(logger, extra_platform_data, variable_mappings, platform): +def get_mappings_expanded(extra_platform_data, variable_mappings, platform): """ Generate a dictionary of mappings between Platform group variables and external variables. Parameters ---------- - logger : logging.Logger - A logger object to log warnings and errors. extra_platform_data : xr.Dataset An `xr.Dataset` containing the additional platform data to be added to the `EchoData["Platform"]` group. @@ -99,9 +99,10 @@ def get_mappings_expanded(logger, extra_platform_data, variable_mappings, platfo # Generate warning if mappings_expanded is empty if not mappings_expanded: - logger.warning( + warnings.warn( "No variables will be updated, " - "check variable_mappings to ensure variable names are correctly specified!" + "check variable_mappings to ensure variable names are correctly specified!", + category=UserWarning, ) # If longitude or latitude are requested, verify that both are present @@ -128,16 +129,18 @@ def get_mappings_expanded(logger, extra_platform_data, variable_mappings, platfo # Generate warnings regarding variables that will be updated vars_not_handled = set(variable_mappings.keys()).difference(mappings_expanded.keys()) if len(vars_not_handled) > 0: - logger.warning( - f"The following requested variables will not be updated: {', '.join(vars_not_handled)}" # noqa + warnings.warn( + f"The following requested variables will not be updated: {', '.join(vars_not_handled)}", # noqa + category=UserWarning, ) vars_notnan_replaced = [ platform_var for platform_var, v in mappings_expanded.items() if v["platform_validvalues"] ] if len(vars_notnan_replaced) > 0: - logger.warning( - f"Some variables with valid data in the original Platform group will be overwritten: {', '.join(vars_notnan_replaced)}" # noqa + warnings.warn( + f"Some variables with valid data in the original Platform group will be overwritten: {', '.join(vars_notnan_replaced)}", # noqa + category=RuntimeWarning, ) return mappings_expanded diff --git a/echopype/qc/api.py b/echopype/qc/api.py index ee7c0e022..5ce5695a5 100644 --- a/echopype/qc/api.py +++ b/echopype/qc/api.py @@ -1,12 +1,10 @@ +import warnings from typing import List, Optional import numpy as np import xarray as xr from ..echodata import EchoData -from ..utils.log import _init_logger - -logger = _init_logger(__name__) def _clean_reversed(time_old: np.ndarray, win_len: int): @@ -124,9 +122,10 @@ def check_and_correct_reversed_time( """ if time_str in combined_group and exist_reversed_time(combined_group, time_str): - logger.warning( + warnings.warn( f"{ed_group} {time_str} reversal detected; {time_str} will be corrected" # noqa - " (see https://github.com/OSOceanAcoustics/echopype/pull/297)" + " (see https://github.com/OSOceanAcoustics/echopype/pull/297)", + category=UserWarning, ) old_time = combined_group[time_str].copy() coerce_increasing_time(combined_group, time_name=time_str) diff --git a/echopype/tests/calibrate/test_calibrate.py b/echopype/tests/calibrate/test_calibrate.py index 482077092..aec86e47a 100644 --- a/echopype/tests/calibrate/test_calibrate.py +++ b/echopype/tests/calibrate/test_calibrate.py @@ -497,9 +497,6 @@ def test_check_echodata_backscatter_size( } ) - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - # Run Backscatter Size check cal_obj._check_echodata_backscatter_size() @@ -514,9 +511,6 @@ def test_check_echodata_backscatter_size( ) assert warning_message == caplog.records[0].message - # Turn off logger verbosity - ep.utils.log.verbose(override=False) - @pytest.mark.integration def test_fm_equals_bb(ek80_path): diff --git a/echopype/tests/clean/test_noise.py b/echopype/tests/clean/test_noise.py index c1a3c1548..b70dbf6b7 100644 --- a/echopype/tests/clean/test_noise.py +++ b/echopype/tests/clean/test_noise.py @@ -114,9 +114,6 @@ def test_transient_mask_noise_func_error_and_warnings(caplog, ek60_path): ### Check for `nanmedian` warning: - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - # Compute transient noise mask ep.clean.mask_transient_noise( ds_Sv, @@ -134,9 +131,6 @@ def test_transient_mask_noise_func_error_and_warnings(caplog, ek60_path): ) assert any(expected_warning in record.message for record in caplog.records) - # Turn off logger verbosity - ep.utils.log.verbose(override=False) - # Check for func value error: with pytest.raises(ValueError, match="Input `func` is `nanmode`. `func` must be `nanmean` or `nanmedian`."): # noqa: E501 ep.clean.mask_transient_noise( diff --git a/echopype/tests/commongrid/test_commongrid_api.py b/echopype/tests/commongrid/test_commongrid_api.py index d7b12d821..b3c4e95df 100644 --- a/echopype/tests/commongrid/test_commongrid_api.py +++ b/echopype/tests/commongrid/test_commongrid_api.py @@ -22,7 +22,7 @@ def ek80_path(test_path): @pytest.fixture def calculate_total_energy(): """ - Returns a function that calculates the total integrated energy + Returns a function that calculates the total integrated energy (Linear Sv * Thickness) per ping for a specific channel. """ def _calc(ds, channel): @@ -30,55 +30,55 @@ def _calc(ds, channel): ds_channel = ds.sel(channel=channel) sv = ds_channel['Sv'].values echo_range = ds_channel['echo_range'].values - + n_pings = sv.shape[0] total_energy = np.zeros(n_pings) - + for i in range(n_pings): # Extract row range_row = echo_range[i, :] sv_row = sv[i, :] - + # Mask valid_range = ~np.isnan(range_row) - + if not np.any(valid_range): total_energy[i] = 0.0 continue - + # Get valid geometry range_valid = range_row[valid_range] sv_valid = sv_row[valid_range] - - # Calculate Thickness using Midpoints + + # Calculate Thickness using Midpoints if len(range_valid) > 1: midpoints = 0.5 * (range_valid[:-1] + range_valid[1:]) d_start = range_valid[1] - range_valid[0] d_end = range_valid[-1] - range_valid[-2] - + edges = np.concatenate([ - [range_valid[0] - d_start/2], - midpoints, + [range_valid[0] - d_start/2], + midpoints, [range_valid[-1] + d_end/2] ]) - + thickness = np.diff(edges) else: thickness = np.array([1.0]) linear_sv = 10 ** (sv_valid / 10.0) - + linear_sv = np.nan_to_num(linear_sv, nan=0.0) - + total_energy[i] = np.sum(linear_sv * thickness) - + return total_energy - + return _calc - - + + # Utilities Tests @pytest.mark.unit @@ -154,7 +154,7 @@ def test__groupby_x_along_channels(request, range_var, lat_lon): # Check that the range_var is in the dimension assert f"{range_var}_bins" in sv_mean.dims - + # NASC Tests @pytest.mark.integration @pytest.mark.parametrize("compute_mvbs", [True, False]) @@ -583,9 +583,6 @@ def test_compute_MVBS_NASC_skipna_nan_and_non_nan_values( # Compute MVBS / Compute NASC if operation == "MVBS": if range_var == "echo_range": - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - da = ep.commongrid.compute_MVBS( subset_ds_Sv, range_var=range_var, @@ -603,8 +600,6 @@ def test_compute_MVBS_NASC_skipna_nan_and_non_nan_values( expected_warning = f"The ```echo_range``` coordinate array contain NaNs. {aggregation_msg}" # noqa: E501 assert any(expected_warning in record.message for record in caplog.records) - # Turn off logger verbosity - ep.utils.log.verbose(override=False) else: da = ep.commongrid.compute_NASC(subset_ds_Sv, range_bin="2m", skipna=skipna)["NASC"] @@ -698,15 +693,15 @@ def test_compute_reindex_non_NaN_not_map_reduce(request): ["scenario", "source_params", "target_params", "expected_value"], [ # Downsampling) - ("downsample_const", {"start": 0, "stop": 1000, "step": 0.1, "val": 5.0}, + ("downsample_const", {"start": 0, "stop": 1000, "step": 0.1, "val": 5.0}, {"start": 0, "stop": 1000, "step": 2.0}, 5.0), - + # Upsampling - ("upsample_const", {"start": 0, "stop": 1000, "step": 0.2, "val": 10.0}, + ("upsample_const", {"start": 0, "stop": 1000, "step": 0.2, "val": 10.0}, {"start": 0, "stop": 1000, "step": 0.1}, 10.0), - + # No Change - ("identity", {"start": 0, "stop": 1000, "step": 1.0, "val": 42.0}, + ("identity", {"start": 0, "stop": 1000, "step": 1.0, "val": 42.0}, {"start": 0, "stop": 1000, "step": 1.0}, 42.0), ], ) @@ -714,24 +709,24 @@ def test__weighted_mean_kernel(scenario, source_params, target_params, expected_ """ Tests the Numba/Numpy regridding kernel for energy/magnitude conservation. """ - + source_ranges = np.arange(source_params["start"], source_params["stop"], source_params["step"]) source_values = np.full_like(source_ranges, source_params["val"]) - + target_ranges = np.arange(target_params["start"], target_params["stop"], target_params["step"]) output = _weighted_mean_kernel(target_ranges, source_ranges, source_values) - - assert output.shape == target_ranges.shape + + assert output.shape == target_ranges.shape # Energy Conservation Check source_width = source_params["step"] target_width = target_params["step"] - - + + source_energy = np.sum(source_values) * source_width target_energy = np.nansum(output) * target_width - + assert np.isclose(source_energy, target_energy, rtol=0.05), \ f"Scenario '{scenario}' failed energy conservation." @@ -891,29 +886,29 @@ def test_range_spacing(ek80_path): ds_Sv = ep.calibrate.compute_Sv( echodata, waveform_mode='CW', encode_mode='complex' ) - + channel = ds_Sv["channel"].values[0] ds_regridded = ep.commongrid.resample_to_geometry(ds_Sv, target_variable="Sv", target_channel=channel) - c = float(ds_Sv["sound_speed"].values) + c = float(ds_Sv["sound_speed"].values) dt = float(echodata["Sonar/Beam_group1"]["sample_interval"].sel(channel=channel).median("ping_time").values) delta_expected = c * dt / 2.0 for ch in ds_regridded.channel.values: r = ds_regridded["echo_range"].sel(channel=ch).isel(ping_time=0).values - + idx = np.where(np.isfinite(r))[0][:2] - + assert len(idx) == 2, f"Not enough finite echo_range values to compute delta for channel {ch}" - + delta_actual = float(r[idx[1]] - r[idx[0]]) np.testing.assert_allclose( - delta_actual, - delta_expected, - rtol=1e-4, + delta_actual, + delta_expected, + rtol=1e-4, err_msg=f"Resolution mismatch on channel {ch}. Expected {delta_expected}, got {delta_actual}." ) @@ -949,7 +944,7 @@ def test_resample_log_variable_sp(ds_Sv_echo_range_regular): rtol=1e-12, atol=1e-12, ) - + @pytest.mark.unit def test_resample_log_variable_ts(ds_Sv_echo_range_regular): """Test explicitly resampling TS in the logarithmic domain.""" @@ -982,7 +977,7 @@ def test_resample_log_variable_ts(ds_Sv_echo_range_regular): rtol=1e-12, atol=1e-12, ) - + @pytest.mark.unit def test_resample_requires_exactly_one_target( ds_Sv_echo_range_regular, @@ -1024,7 +1019,7 @@ def test_resample_requires_exactly_one_target( target_channel=target_channel, target_grid=target_grid, ) - + @pytest.mark.unit def test_resample_linear_angle_variable( ds_Sv_echo_range_regular, @@ -1264,7 +1259,7 @@ def test_resample_matches_echoview_match_geometry(test_path): # transducer are represented after regridding. Exclude this region # and compare only the overlapping valid portion of the water column. min_range_m = 2.0 - + diff = ( ds_regridded["Sv"] .where(ds_echoview["echo_range"] > min_range_m) @@ -1282,7 +1277,7 @@ def test_resample_matches_echoview_match_geometry(test_path): atol=0.003, rtol=0, ) - + @pytest.mark.integration def test_resample_shared_depth_and_range_geometry(test_path): """ @@ -1338,8 +1333,8 @@ def test_resample_shared_depth_and_range_geometry(test_path): atol=0, rtol=0, ) - - + + @pytest.mark.integration def test_resample_with_one_dimensional_longer_target_grid( ds_Sv_echo_range_regular, @@ -1411,7 +1406,7 @@ def test_resample_with_one_dimensional_longer_target_grid( ds_regridded["echo_range"], expected_grid, ) - + @pytest.mark.unit def test_resample_warns_and_corrects_reversed_ping_time( ds_Sv_echo_range_regular, @@ -1441,4 +1436,4 @@ def test_resample_warns_and_corrects_reversed_ping_time( assert np.all( np.diff(ds_regridded["ping_time"].values) >= np.timedelta64(0, "ns") - ) \ No newline at end of file + ) diff --git a/echopype/tests/consolidate/test_add_depth.py b/echopype/tests/consolidate/test_add_depth.py index d9645e7ef..deaeff2c6 100644 --- a/echopype/tests/consolidate/test_add_depth.py +++ b/echopype/tests/consolidate/test_add_depth.py @@ -139,15 +139,9 @@ def test_ek_use_beam_angles_output(caplog): coords={"channel": channel_da} ) - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - # Compute beam angle echo range scaling echo_range_scaling = ep.consolidate.ek_depth_utils.ek_use_beam_angles(beam_ds) - # Turn off logger verbosity - ep.utils.log.verbose(override=False) - # Verify the correct warning assert "Beam direction vector was not normalized" in caplog.text @@ -170,9 +164,6 @@ def test_warning_zero_vector(caplog): } ) - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - # Compute beam angle echo range scaling echo_range_scaling = ep.consolidate.ek_depth_utils.ek_use_beam_angles(beam_ds) @@ -183,9 +174,6 @@ def test_warning_zero_vector(caplog): assert np.isnan(echo_range_scaling.values[0]) assert np.isclose(echo_range_scaling.values[1], 0.0) - # Turn off logger verbosity - ep.utils.log.verbose(override=False) - @pytest.mark.integration @pytest.mark.parametrize( @@ -257,9 +245,6 @@ def test_ek_depth_utils_group_variable_NaNs_logger_warnings(caplog, ek80_path): ed["Sonar/Beam_group1"]["beam_direction_y"].values[0] = np.nan ed["Sonar/Beam_group1"]["beam_direction_z"].values[0] = np.nan - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - # Run EK depth util functions: ek_use_platform_vertical_offsets(platform_ds=ed["Platform"], ping_time_da=ds_Sv["ping_time"]) ek_use_platform_angles(platform_ds=ed["Platform"], ping_time_da=ds_Sv["ping_time"]) @@ -286,9 +271,6 @@ def test_ek_depth_utils_group_variable_NaNs_logger_warnings(caplog, ek80_path): ) assert any(expected_warning in record.message for record in caplog.records) - # Turn off logger verbosity - ep.utils.log.verbose(override=False) - @pytest.mark.integration def test_add_depth_tilt_depth_use_arg_logger_warnings(caplog, ek80_path): @@ -304,9 +286,6 @@ def test_add_depth_tilt_depth_use_arg_logger_warnings(caplog, ek80_path): ed = ep.open_raw(raw_file, sonar_model="EK80") ds_Sv = ep.calibrate.compute_Sv(ed, waveform_mode="CW", encode_mode="power") - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - # Run `add_depth` with `tilt`, `depth_offset` as Non-NaN, using beam group angles, # and platform vertical offset values ep.consolidate.add_depth( @@ -329,9 +308,6 @@ def test_add_depth_tilt_depth_use_arg_logger_warnings(caplog, ek80_path): for warning in [depth_offset_warning, tilt_warning]: assert any(warning in record.message for record in caplog.records) - # Turn off logger verbosity - ep.utils.log.verbose(override=False) - @pytest.mark.integration def test_add_depth_without_echodata(): diff --git a/echopype/tests/consolidate/test_add_location.py b/echopype/tests/consolidate/test_add_location.py index 90930ed3b..2b256452d 100644 --- a/echopype/tests/consolidate/test_add_location.py +++ b/echopype/tests/consolidate/test_add_location.py @@ -443,9 +443,6 @@ def test_add_location_lat_lon_0_NaN_warnings( ed["Platform"]["latitude"][0] = np.nan ed["Platform"]["longitude"][0] = 0 - # Turn on logger verbosity - ep.utils.log.verbose(override=True) - # Run add location with 0 and NaN lat/lon values ep.consolidate.add_location(ds=ds, echodata=ed, datagram_type=datagram_type) @@ -453,5 +450,3 @@ def test_add_location_lat_lon_0_NaN_warnings( for warning in expected_warnings: assert any(warning in record.message for record in caplog.records) - # Turn off logger verbosity - ep.utils.log.verbose(override=False) diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index b508a68d7..4455d52cd 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -10,7 +10,6 @@ from echopype.calibrate import compute_Sv from echopype.convert.parse_ek80 import ParseEK80 from echopype.convert.set_groups_ek80 import SetGroupsEK80, WIDE_BAND_TRANS, PULSE_COMPRESS, FILTER_IMAG, FILTER_REAL, DECIMATION # noqa: E501 -from echopype.utils import log from echopype.convert.utils.ek_duplicates import check_unique_ping_time_duplicates @@ -552,9 +551,6 @@ def test_duplicate_ping_times(caplog, ek80_dupe_ping_path): """ Tests that RAW file with duplicate ping times can be parsed and that the correct warning has been raised. """ # noqa: E501 - # Turn on logger verbosity - log.verbose(override=True) - # Open RAW ed = open_raw(ek80_dupe_ping_path / "Hake-D20210913-T130612.raw", sonar_model="EK80") @@ -567,21 +563,12 @@ def test_duplicate_ping_times(caplog, ek80_dupe_ping_path): not_expected_warning = ("All duplicate ping_time entries' will be removed, resulting in potential data loss.") # noqa: E501 assert not any(not_expected_warning in record.message for record in caplog.records) - # Turn off logger verbosity - log.verbose(override=False) - @pytest.mark.unit def test_check_unique_ping_time_duplicates(caplog, ek80_dupe_ping_path): """ Checks that `check_unique_ping_time_duplicates` raises a warning when the data for duplicate ping times is not unique. """ # noqa: E501 - # Initialize logger - logger = log._init_logger(__name__) - - # Turn on logger verbosity - log.verbose(override=True) - # Open duplicate ping time beam dataset ds_data = xr.open_zarr(ek80_dupe_ping_path / "duplicate_beam_ds.zarr") @@ -589,10 +576,7 @@ def test_check_unique_ping_time_duplicates(caplog, ek80_dupe_ping_path): ds_data["backscatter_r"][0,0,0] = 0 # Check for ping time duplicates - check_unique_ping_time_duplicates(ds_data, logger) - - # Turn off logger verbosity - log.verbose(override=False) + check_unique_ping_time_duplicates(ds_data) # Check if the expected warning is logged expected_warning = ( diff --git a/echopype/tests/echodata/test_echodata.py b/echopype/tests/echodata/test_echodata.py index 0bc2a9e87..fa8a165b3 100644 --- a/echopype/tests/echodata/test_echodata.py +++ b/echopype/tests/echodata/test_echodata.py @@ -836,15 +836,9 @@ def test_echodata_delete(caplog, ek60_path): # Check that temp zarr path exists assert os.path.exists(temp_zarr_path) - # Turn on logger verbosity - echopype.utils.log.verbose(override=True) - # Delete temp zarr in temp zarr path ed.__del__() - # Turn off logger verbosity - echopype.utils.log.verbose(override=False) - # Check that no exceptions were wrapped by warnings assert not any("Warning: Exception ignored in:" in record.message for record in caplog.records) diff --git a/echopype/tests/utils/test_utils_log.py b/echopype/tests/utils/test_utils_log.py deleted file mode 100644 index 4f3664a30..000000000 --- a/echopype/tests/utils/test_utils_log.py +++ /dev/null @@ -1,144 +0,0 @@ -import pytest -import os.path -import platform - -pytestmark = pytest.mark.unit - -EXPECTED_MESSAGE = "Testing log function" - - -def logging_func(logger): - logger.info("Testing log function") - - -@pytest.fixture(params=[False, True]) -def verbose(request): - return request.param - - -def test_init_logger(): - import logging - from echopype.utils import log - logger = log._init_logger('echopype.testing0') - handlers = [h.name for h in logger.handlers] - - assert isinstance(logger, logging.Logger) is True - assert logger.name == 'echopype.testing0' - assert len(logger.handlers) == 2 - assert log.STDERR_NAME in handlers - assert log.STDOUT_NAME in handlers - - -def test_set_log_file(): - from echopype.utils import log - logger = log._init_logger('echopype.testing1') - from tempfile import TemporaryDirectory - tmpdir = TemporaryDirectory() - tmpfile = os.path.join(tmpdir.name, "testfile.log") - log._set_logfile(logger, tmpfile) - handlers = [h.name for h in logger.handlers] - - assert log.LOGFILE_HANDLE_NAME in handlers - - # when done with temporary directory - # see: https://www.scivision.dev/python-tempfile-permission-error-windows/ - try: - tmpdir.cleanup() - except Exception as e: - if platform.system() == "Windows": - pass - else: - raise e - - -def test_set_verbose(verbose, capsys): - from echopype.utils import log - logger = log._init_logger(f'echopype.testing_{str(verbose).lower()}') - - # To pass through in caplog need to propagate - # logger.propagate = True - - log._set_verbose(logger, verbose) - - logging_func(logger) - - captured = capsys.readouterr() - - if verbose: - assert EXPECTED_MESSAGE in captured.out - else: - assert "" in captured.out - - -def test_get_all_loggers(): - import logging - from echopype.utils import log - all_loggers = log._get_all_loggers() - loggers = [logging.getLogger()] # get the root logger - loggers = loggers + [logging.getLogger(name) for name in logging.root.manager.loggerDict] - assert all_loggers == loggers - - -def run_verbose_test(logger, override, logfile, capsys): - import echopype as ep - import os - - ep.verbose(logfile=logfile, override=override) - - logging_func(logger) - - captured = capsys.readouterr() - - if override is False: - assert captured.out == "" - else: - assert EXPECTED_MESSAGE in captured.out - - if logfile is not None: - assert os.path.exists(logfile) - with open(logfile) as f: - assert EXPECTED_MESSAGE in f.read() - - -@pytest.mark.parametrize(["id", "override", "logfile"], [ - ("fn", True, None), - ("tn", False, None), - ("tf", True, 'test.log') -]) -def test_verbose(id, override, logfile, capsys): - from echopype.utils import log - logger = log._init_logger(f'echopype.testing_{id}') - - if logfile is not None: - from tempfile import TemporaryDirectory - tmpdir = TemporaryDirectory() - tmpfile = os.path.join(tmpdir.name, logfile) - run_verbose_test(logger, override, tmpfile, capsys) - - # when done with temporary directory - # see: https://www.scivision.dev/python-tempfile-permission-error-windows/ - try: - tmpdir.cleanup() - except Exception as e: - if platform.system() == "Windows": - pass - else: - raise e - else: - run_verbose_test(logger, override, logfile, capsys) - - -def test_verbose_per_package(capsys): - from echopype.utils import log - - convert_logger = log._init_logger("echopype.convert.testing") - calibrate_logger = log._init_logger("echopype.calibrate.testing") - - log.verbose(override=False, package_verbosity={"echopype.convert.testing": True}) - - convert_logger.info("Testing convert function") - calibrate_logger.info("Testing calibrate function") - - captured = capsys.readouterr() - assert captured.out.count("Testing convert function") == 1 - assert captured.out.count("Testing calibrate function") == 0 diff --git a/echopype/utils/io.py b/echopype/utils/io.py index 10d14ad2c..ca8673599 100644 --- a/echopype/utils/io.py +++ b/echopype/utils/io.py @@ -9,6 +9,7 @@ import sys import tempfile import uuid +import warnings from pathlib import Path, WindowsPath from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union @@ -22,7 +23,6 @@ from ..echodata import EchoData from ..echodata.api import open_converted from ..utils.coding import set_storage_encodings -from ..utils.log import _init_logger if TYPE_CHECKING: from ..core import PathHint @@ -35,7 +35,6 @@ }, } -logger = _init_logger(__name__) # Get root echopype package name ECHOPYPE = __name__.split(".")[0] @@ -238,13 +237,13 @@ def validate_output_path( file_ext = SUPPORTED_ENGINES[engine]["ext"] if save_path is None: - logger.warning("A directory or file path is not provided!") + warnings.warn("A directory or file path is not provided!", category=UserWarning) out_dir = ECHOPYPE_DIR / "temp_output" if not out_dir.exists(): out_dir.mkdir(parents=True) - logger.warning(f"Resulting converted file(s) will be available at {str(out_dir)}") + print(f"Resulting converted file(s) will be available at {str(out_dir)}") out_path = str(out_dir / (Path(source_file).stem + file_ext)) elif not isinstance(save_path, Path) and not isinstance(save_path, str): raise TypeError("save_path must be a string or Path") @@ -285,8 +284,9 @@ def validate_output_path( final_path = Path(save_path) out_path = save_path if final_path.suffix != file_ext: - logger.warning( - "Mismatch between specified engine and save_path found; forcing output format to engine." # noqa + warnings.warn( + "Mismatch between specified engine and save_path found; forcing output format to engine.", # noqa + category=UserWarning, ) return out_path @@ -344,7 +344,10 @@ def check_file_permissions(FILE_DIR): FILE_DIR = Path(FILE_DIR) if not FILE_DIR.exists(): - logger.warning(f"{str(FILE_DIR)} does not exist. Attempting to create it.") + warnings.warn( + f"{str(FILE_DIR)} does not exist. Attempting to create it.", + category=UserWarning, + ) FILE_DIR.mkdir(exist_ok=True, parents=True) TEST_FILE = FILE_DIR.joinpath(Path(fname)) TEST_FILE.write_text("testing\n") diff --git a/echopype/utils/log.py b/echopype/utils/log.py deleted file mode 100644 index e7cd074d0..000000000 --- a/echopype/utils/log.py +++ /dev/null @@ -1,137 +0,0 @@ -import logging -import sys -from typing import Dict, List, Optional - -LOG_FORMAT = "{asctime}:{name}:{levelname}: {message}" -LOG_FORMATTER = logging.Formatter(LOG_FORMAT, style="{") -STDOUT_NAME = "stdout_stream_handler" -STDERR_NAME = "stderr_stream_handler" -LOGFILE_HANDLE_NAME = "logfile_file_handler" - - -class _ExcludeWarningsFilter(logging.Filter): - def filter(self, record): # noqa - """Only lets through log messages with log level below ERROR.""" - return record.levelno < logging.WARNING - - -def verbose( - logfile: Optional[str] = None, - override: bool = True, - package_verbosity: Optional[Dict[str, bool]] = None, -) -> None: - """Set the verbosity for echopype print outs. - If called it will output logs to terminal by default. - - Parameters - ---------- - logfile : str, optional - Optional string path to the desired log file. - override: bool - Boolean flag to override verbosity, - which turns off verbosity if the value is `False`. - Default is `True`. - package_verbosity: dict, optional - Dictionary of package names and their verbosity levels. - Default is `None` which will use the `override` value for all packages. - Example: - { - "echopype.convert": True, - "echopype.calibrate": False, - } - - Returns - ------- - None - """ - if not isinstance(override, bool): - raise ValueError("override argument must be a boolean") - - if package_verbosity is not None: - if not isinstance(package_verbosity, dict): - raise ValueError("package_verbosity argument must be a dictionary") - for logger_name, verbose in package_verbosity.items(): - if not isinstance(logger_name, str): - raise ValueError( - f"package_verbosity keys must be strings, got {logger_name} for {verbose}" - ) - if not isinstance(verbose, bool): - raise ValueError( - f"package_verbosity values must be booleans, got {verbose} for {logger_name}" - ) - else: - package_verbosity = {} - - package_name = __name__.split(".")[0] # Get the package name - for logger in _get_all_loggers(): - if package_name not in logger.name: - continue - _set_verbose(logger, package_verbosity.get(logger.name, override)) - handlers = [h.name for h in logger.handlers] - if logfile is None: - if LOGFILE_HANDLE_NAME in handlers: - # Remove log file handler if it exists - handler = next(filter(lambda h: h.name == LOGFILE_HANDLE_NAME, logger.handlers)) - logger.removeHandler(handler) - elif LOGFILE_HANDLE_NAME not in handlers: - # Only add the logfile handler if it doesn't exist - _set_logfile(logger, logfile) - - logger.propagate = logfile is None - - -def _get_all_loggers() -> List[logging.Logger]: - """Get all loggers""" - loggers = [logging.getLogger()] # get the root logger - return loggers + [logging.getLogger(name) for name in logging.root.manager.loggerDict] - - -def _init_logger(name) -> logging.Logger: - """Initialize logger with the default stdout stream handler - - Parameters - ---------- - name : str - Logger name - - Returns - ------- - logging.Logger - """ - # Logging setup - logger = logging.getLogger(name) - logger.setLevel(logging.DEBUG) - - # Setup stream handler - STREAM_HANDLER = logging.StreamHandler(sys.stdout) - STREAM_HANDLER.setLevel(logging.DEBUG) - STREAM_HANDLER.set_name(STDOUT_NAME) - STREAM_HANDLER.setFormatter(LOG_FORMATTER) - STREAM_HANDLER.addFilter(_ExcludeWarningsFilter()) - logger.addHandler(STREAM_HANDLER) - - # Setup err stream handler - ERR_STREAM_HANDLER = logging.StreamHandler(sys.stderr) - ERR_STREAM_HANDLER.setLevel(logging.WARNING) - ERR_STREAM_HANDLER.set_name(STDERR_NAME) - ERR_STREAM_HANDLER.setFormatter(LOG_FORMATTER) - logger.addHandler(ERR_STREAM_HANDLER) - return logger - - -def _set_verbose(logger: logging.Logger, verbose: bool) -> None: - """Set the verbosity for echopype logs.""" - if verbose: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(logging.WARNING) - - -def _set_logfile(logger: logging.Logger, logfile: Optional[str] = None) -> logging.Logger: - """Adds log file handler to logger""" - if not logfile: - raise ValueError("Please provide logfile path") - file_handler = logging.FileHandler(logfile) - file_handler.set_name(LOGFILE_HANDLE_NAME) - file_handler.setFormatter(LOG_FORMATTER) - logger.addHandler(file_handler) diff --git a/echopype/utils/prov.py b/echopype/utils/prov.py index 60e233f7e..adadfa42d 100644 --- a/echopype/utils/prov.py +++ b/echopype/utils/prov.py @@ -2,6 +2,7 @@ import functools import re import sys +import warnings from pathlib import Path from typing import Any, Dict, List, Tuple, Union @@ -11,15 +12,11 @@ from numpy.typing import NDArray from typing_extensions import Literal -from .log import _init_logger - ProcessType = Literal["conversion", "combination", "processing", "mask"] # Note that this PathHint is defined differently from the one in ..core PathHint = Union[str, Path] PathSequenceHint = Union[List[PathHint], Tuple[PathHint], NDArray[PathHint]] -logger = _init_logger(__name__) - def echopype_prov_attrs(process_type: ProcessType) -> Dict[str, str]: """ @@ -70,15 +67,17 @@ def _sanitize_source_files(paths: Union[PathHint, PathSequenceHint]): elif isinstance(p, sequence_types): paths_list += [str(pp) for pp in p if isinstance(pp, (str, Path))] else: - logger.warning( + warnings.warn( "Unrecognized file path element type, path element will not be" - f" written to (meta)source_file provenance attribute. {p}" + f" written to (meta)source_file provenance attribute. {p}", + category=UserWarning, ) return paths_list else: - logger.warning( + warnings.warn( "Unrecognized file path element type, path element will not be" - f" written to (meta)source_file provenance attribute. {paths}" + f" written to (meta)source_file provenance attribute. {paths}", + category=UserWarning, ) return [] @@ -227,7 +226,7 @@ def inner(self, *args, **kwargs): _attrs_dict(processing_level) ) else: - logger.info( + print( "EchoData object (converted raw file) does not contain " "valid Platform location data. Processing level attributes " "will not be added." @@ -248,7 +247,7 @@ def inner(*args, **kwargs): _attrs_dict(processing_level) ) else: - logger.info( + print( "EchoData object (converted raw file) does not contain " "valid Platform location data. Processing level attributes " "will not be added." @@ -289,7 +288,7 @@ def inner(*args, **kwargs): ds = ds.assign_attrs(_attrs_dict(processing_level)) else: - logger.info( + print( "xarray Dataset does not contain valid location data. " "Processing level attributes will not be added." ) From b8e9e0b7019e462145c247bf55613ad85adbc249 Mon Sep 17 00:00:00 2001 From: Praneeth Date: Wed, 26 Aug 2026 14:21:06 +0530 Subject: [PATCH 2/6] fix build --- echopype/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/echopype/__init__.py b/echopype/__init__.py index f3dedafa3..b0c1f2aea 100644 --- a/echopype/__init__.py +++ b/echopype/__init__.py @@ -7,10 +7,6 @@ from .echodata.api import open_converted from .echodata.combine import combine_echodata from .utils.io import init_ep_dir -from .utils.log import verbose - -# Turn off verbosity for echopype -verbose(override=False) init_ep_dir() @@ -25,5 +21,4 @@ "open_converted", "open_raw", "utils", - "verbose", ] From 6b62e4837b6b13e2a5abbd6b548c6180290380d6 Mon Sep 17 00:00:00 2001 From: Praneeth Date: Wed, 26 Aug 2026 14:31:29 +0530 Subject: [PATCH 3/6] minor fix --- echopype/tests/commongrid/test_commongrid_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/echopype/tests/commongrid/test_commongrid_api.py b/echopype/tests/commongrid/test_commongrid_api.py index b3c4e95df..a2080c4cc 100644 --- a/echopype/tests/commongrid/test_commongrid_api.py +++ b/echopype/tests/commongrid/test_commongrid_api.py @@ -582,7 +582,6 @@ def test_compute_MVBS_NASC_skipna_nan_and_non_nan_values( # Compute MVBS / Compute NASC if operation == "MVBS": - if range_var == "echo_range": da = ep.commongrid.compute_MVBS( subset_ds_Sv, range_var=range_var, From 081ea66d2beb759173ee4d2f6967fbc17767acab Mon Sep 17 00:00:00 2001 From: Praneeth Date: Wed, 26 Aug 2026 14:49:13 +0530 Subject: [PATCH 4/6] fix unit tests --- echopype/calibrate/calibrate_ek.py | 6 ++---- echopype/convert/utils/ek_raw_io.py | 12 +++++------ echopype/convert/utils/ek_raw_parsers.py | 4 ++-- echopype/qc/api.py | 2 +- echopype/tests/calibrate/test_calibrate.py | 4 ++-- echopype/tests/clean/test_noise.py | 4 ++-- .../tests/commongrid/test_commongrid_api.py | 4 ++-- echopype/tests/consolidate/test_add_depth.py | 21 ++++++++++++------- .../tests/consolidate/test_add_location.py | 4 ++-- echopype/tests/convert/test_convert_ek80.py | 8 +++---- echopype/tests/echodata/test_echodata.py | 4 ++-- 11 files changed, 38 insertions(+), 35 deletions(-) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index ff98aa8fe..0d7f4f121 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -126,8 +126,7 @@ def _cal_power_samples(self, cal_type: str) -> xr.Dataset: except Exception as e: warnings.warn( "Could not compute tau_effective from transmit signal in power encoding mode; " - "falling back to transmit_duration_nominal. Error: %s", - repr(e), + f"falling back to transmit_duration_nominal. Error: {e!r}", category=RuntimeWarning, ) tau_effective = beam["transmit_duration_nominal"].isel(ping_time=0) @@ -595,8 +594,7 @@ def _cal_complex_samples(self, cal_type: str) -> xr.Dataset: warnings.warn( "Could not compute tau_effective " "from transmit signal in complex encoding mode; " - "falling back to transmit_duration_nominal. Error: %s", - repr(e), + f"falling back to transmit_duration_nominal. Error: {e!r}", category=RuntimeWarning, ) tau_effective = self.beam["transmit_duration_nominal"].isel(ping_time=0) diff --git a/echopype/convert/utils/ek_raw_io.py b/echopype/convert/utils/ek_raw_io.py index 5d12a269c..9a05d33e2 100644 --- a/echopype/convert/utils/ek_raw_io.py +++ b/echopype/convert/utils/ek_raw_io.py @@ -265,7 +265,7 @@ def _read_next_dgram(self): if (header["low_date"], header["high_date"]) == (0, 0): warnings.warn( f"Skipping {header['type']} datagram w/ timestamp of (0, 0) at " - "{str(self._tell_bytes())}L:{self.tell()}", + f"{str(self._tell_bytes())}L:{self.tell()}", category=BytesWarning, ) self.skip() @@ -276,7 +276,7 @@ def _read_next_dgram(self): # size can't be smaller than the header size warnings.warn( f"Invalid datagram header: size: {header['size']}, type: {header['type']}, " - "nt_date: {str((header['low_date'], header['high_date']))}. dgram_size < 16", + f"nt_date: {str((header['low_date'], header['high_date']))}. dgram_size < 16", category=BytesWarning, ) @@ -301,7 +301,7 @@ def _read_next_dgram(self): if bytes_read < header["size"]: warnings.warn( f"Datagram {self.tell()} (@{old_file_pos})" - " shorter than expected length: {bytes_read} < {header['size']}", + f" shorter than expected length: {bytes_read} < {header['size']}", category=BytesWarning, ) self._find_next_datagram() @@ -320,7 +320,7 @@ def _read_next_dgram(self): # self._seek_bytes(old_file_pos, SEEK_SET) warnings.warn( f"Datagram failed size check: {header['size']} != {dgram_size_check} @ " - "({self._tell_bytes()}, {self.tell()})", + f"({self._tell_bytes()}, {self.tell()})", category=BytesWarning, ) warnings.warn("Skipping to next datagram...", category=BytesWarning) @@ -534,7 +534,7 @@ def skip(self): if header["size"] < 16: warnings.warn( f"Invalid datagram header: size: {header['size']}, type: {header['type']}, " - "nt_date: {str((header['low_date'], header['high_date']))}. dgram_size < 16", + f"nt_date: {str((header['low_date'], header['high_date']))}. dgram_size < 16", category=BytesWarning, ) @@ -547,7 +547,7 @@ def skip(self): if header["size"] != dgram_size_check: warnings.warn( f"Datagram failed size check: {header['size']} != {dgram_size_check} @ " - "({self._tell_bytes()}, {self.tell()})", + f"({self._tell_bytes()}, {self.tell()})", category=BytesWarning, ) warnings.warn("Skipping to next datagram... (in skip)", category=UserWarning) diff --git a/echopype/convert/utils/ek_raw_parsers.py b/echopype/convert/utils/ek_raw_parsers.py index cb820409a..a7e6fe8c8 100644 --- a/echopype/convert/utils/ek_raw_parsers.py +++ b/echopype/convert/utils/ek_raw_parsers.py @@ -278,7 +278,7 @@ def _pack_contents(self, data, version): if len(data["depth"]) != data["transceiver_count"]: warnings.warn( f"# of depth values {len(data['depth'])} does not match transceiver " - "count {data['transceiver_count']}", + f"count {data['transceiver_count']}", category=BytesWarning, ) @@ -1435,7 +1435,7 @@ def _unpack_contents(self, raw_string, bytes_read, version): except KeyError: warnings.warn( f"Unknown sounder_name: {sounder_name}, " - "(no one of {list(self._transducer_headers.keys())}) " + f"(no one of {list(self._transducer_headers.keys())}) " "will use ER60 transducer config fields as default", category=UserWarning, ) diff --git a/echopype/qc/api.py b/echopype/qc/api.py index 5ce5695a5..fe17d6cd5 100644 --- a/echopype/qc/api.py +++ b/echopype/qc/api.py @@ -123,7 +123,7 @@ def check_and_correct_reversed_time( if time_str in combined_group and exist_reversed_time(combined_group, time_str): warnings.warn( - f"{ed_group} {time_str} reversal detected; {time_str} will be corrected" # noqa + f"{ed_group} {time_str} reversal detected; {time_str} will be corrected" " (see https://github.com/OSOceanAcoustics/echopype/pull/297)", category=UserWarning, ) diff --git a/echopype/tests/calibrate/test_calibrate.py b/echopype/tests/calibrate/test_calibrate.py index aec86e47a..25772df0d 100644 --- a/echopype/tests/calibrate/test_calibrate.py +++ b/echopype/tests/calibrate/test_calibrate.py @@ -419,7 +419,7 @@ def test_check_echodata_backscatter_size( xml_path, waveform_mode, encode_mode, - caplog, + recwarn, azfp_path, ek60_path, ek80_path @@ -509,7 +509,7 @@ def test_check_echodata_backscatter_size( "This will ensure that the computation is lazily evaluated, " "with the results stored directly in a Zarr store on disk, rather then in memory." ) - assert warning_message == caplog.records[0].message + assert any(warning_message == str(record.message) for record in recwarn) @pytest.mark.integration diff --git a/echopype/tests/clean/test_noise.py b/echopype/tests/clean/test_noise.py index b70dbf6b7..e9ae11582 100644 --- a/echopype/tests/clean/test_noise.py +++ b/echopype/tests/clean/test_noise.py @@ -94,7 +94,7 @@ def test_mask_functions_dimensions(ek60_path): @pytest.mark.integration -def test_transient_mask_noise_func_error_and_warnings(caplog, ek60_path): +def test_transient_mask_noise_func_error_and_warnings(recwarn, ek60_path): """Check if appropriate warnings and errors are raised for transient noise mask func input.""" # Open raw, calibrate, and add depth ed = ep.open_raw( @@ -129,7 +129,7 @@ def test_transient_mask_noise_func_error_and_warnings(caplog, ek60_path): "We plan to add the Fielding Transient Noise Filter in the future" "described here: https://github.com/OSOceanAcoustics/echopype/issues/1352" ) - assert any(expected_warning in record.message for record in caplog.records) + assert any(expected_warning in str(record.message) for record in recwarn) # Check for func value error: with pytest.raises(ValueError, match="Input `func` is `nanmode`. `func` must be `nanmean` or `nanmedian`."): # noqa: E501 diff --git a/echopype/tests/commongrid/test_commongrid_api.py b/echopype/tests/commongrid/test_commongrid_api.py index a2080c4cc..4806a5e41 100644 --- a/echopype/tests/commongrid/test_commongrid_api.py +++ b/echopype/tests/commongrid/test_commongrid_api.py @@ -571,7 +571,7 @@ def test_compute_MVBS_NASC_skipna_nan_and_non_nan_values( operation, skipna, range_var, - caplog, + recwarn, ): # Create subset dataset with 2 channels, 2 ping times, and 20 range samples: @@ -597,7 +597,7 @@ def test_compute_MVBS_NASC_skipna_nan_and_non_nan_values( "these values before calling your intended commongrid function." ) expected_warning = f"The ```echo_range``` coordinate array contain NaNs. {aggregation_msg}" # noqa: E501 - assert any(expected_warning in record.message for record in caplog.records) + assert any(expected_warning in str(record.message) for record in recwarn) else: da = ep.commongrid.compute_NASC(subset_ds_Sv, range_bin="2m", skipna=skipna)["NASC"] diff --git a/echopype/tests/consolidate/test_add_depth.py b/echopype/tests/consolidate/test_add_depth.py index deaeff2c6..280757a6f 100644 --- a/echopype/tests/consolidate/test_add_depth.py +++ b/echopype/tests/consolidate/test_add_depth.py @@ -117,7 +117,7 @@ def test_ek_use_platform_angles_output(): @pytest.mark.unit -def test_ek_use_beam_angles_output(caplog): +def test_ek_use_beam_angles_output(recwarn): """ Test `use_beam_angle` outputs for 2 sideways looking beams, 1 vertical looking beam, and 1 beam that does not have normalized directions. @@ -143,7 +143,10 @@ def test_ek_use_beam_angles_output(caplog): echo_range_scaling = ep.consolidate.ek_depth_utils.ek_use_beam_angles(beam_ds) # Verify the correct warning - assert "Beam direction vector was not normalized" in caplog.text + assert any( + "Beam direction vector was not normalized" in str(record.message) + for record in recwarn + ) # Compute and compare manual test values with function values fourth_value = (np.sqrt(3)/2) / np.sqrt(((np.sqrt(3)/2) ** 2) + 1) @@ -151,7 +154,7 @@ def test_ek_use_beam_angles_output(caplog): @pytest.mark.unit -def test_warning_zero_vector(caplog): +def test_warning_zero_vector(recwarn): """ Test that a warning is logged and NaN is returned for channels with zero beam direction vector. """ @@ -168,7 +171,9 @@ def test_warning_zero_vector(caplog): echo_range_scaling = ep.consolidate.ek_depth_utils.ek_use_beam_angles(beam_ds) # Verify the correct warning - assert "Some beam direction vectors are zero" in caplog.text + assert any( + "Some beam direction vectors are zero" in str(record.message) for record in recwarn + ) # Check that channel 0 output is NaN and channel 1 output is 0 assert np.isnan(echo_range_scaling.values[0]) @@ -221,7 +226,7 @@ def test_ek_depth_utils_dims(relpath, sonar_model, compute_Sv_kwargs, ek60_path, @pytest.mark.integration -def test_ek_depth_utils_group_variable_NaNs_logger_warnings(caplog, ek80_path): +def test_ek_depth_utils_group_variable_NaNs_logger_warnings(recwarn, ek80_path): """ Tests `ek_use_platform_vertical_offsets`, `ek_use_platform_angles`, and `ek_use_beam_angles` for correct logger warnings when NaNs exist in group @@ -269,11 +274,11 @@ def test_ek_depth_utils_group_variable_NaNs_logger_warnings(caplog, ek80_path): "NaNs. This will result in NaNs in the final `depth` array. Consider filling the " "NaNs and calling `.add_depth(...)` again." ) - assert any(expected_warning in record.message for record in caplog.records) + assert any(expected_warning in str(record.message) for record in recwarn) @pytest.mark.integration -def test_add_depth_tilt_depth_use_arg_logger_warnings(caplog, ek80_path): +def test_add_depth_tilt_depth_use_arg_logger_warnings(recwarn, ek80_path): """ Tests warnings when `tilt` and `depth_offset` are being passed in when other `use_*` arguments are passed in as `True`. @@ -306,7 +311,7 @@ def test_add_depth_tilt_depth_use_arg_logger_warnings(caplog, ek80_path): "When `tilt` is specified, beam/platform angle variables will not be used." ) for warning in [depth_offset_warning, tilt_warning]: - assert any(warning in record.message for record in caplog.records) + assert any(warning in str(record.message) for record in recwarn) @pytest.mark.integration diff --git a/echopype/tests/consolidate/test_add_location.py b/echopype/tests/consolidate/test_add_location.py index 2b256452d..ccf2616da 100644 --- a/echopype/tests/consolidate/test_add_location.py +++ b/echopype/tests/consolidate/test_add_location.py @@ -422,7 +422,7 @@ def test_add_location_lat_lon_missing_all_NaN_errors( ], ) def test_add_location_lat_lon_0_NaN_warnings( - ek80_path, raw_path, sonar_model, datagram_type, parse_idx, compute_Sv_kwargs, expected_warnings, caplog # noqa: E501 + ek80_path, raw_path, sonar_model, datagram_type, parse_idx, compute_Sv_kwargs, expected_warnings, recwarn # noqa: E501 ): """Tests for lat lon 0 and NaN value warnings.""" # Open raw and compute the Sv dataset @@ -448,5 +448,5 @@ def test_add_location_lat_lon_0_NaN_warnings( # Check if the expected warnings are logged for warning in expected_warnings: - assert any(warning in record.message for record in caplog.records) + assert any(warning in str(record.message) for record in recwarn) diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index 4455d52cd..ff082d655 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -547,7 +547,7 @@ def test_parse_missing_sound_velocity_profile(ek80_missing_sound_path): @pytest.mark.unit -def test_duplicate_ping_times(caplog, ek80_dupe_ping_path): +def test_duplicate_ping_times(recwarn, ek80_dupe_ping_path): """ Tests that RAW file with duplicate ping times can be parsed and that the correct warning has been raised. """ # noqa: E501 @@ -561,11 +561,11 @@ def test_duplicate_ping_times(caplog, ek80_dupe_ping_path): # Check that no warning is logged since the data for all duplicate pings is unique not_expected_warning = ("All duplicate ping_time entries' will be removed, resulting in potential data loss.") # noqa: E501 - assert not any(not_expected_warning in record.message for record in caplog.records) + assert not any(not_expected_warning in str(record.message) for record in recwarn) @pytest.mark.unit -def test_check_unique_ping_time_duplicates(caplog, ek80_dupe_ping_path): +def test_check_unique_ping_time_duplicates(recwarn, ek80_dupe_ping_path): """ Checks that `check_unique_ping_time_duplicates` raises a warning when the data for duplicate ping times is not unique. """ # noqa: E501 @@ -584,7 +584,7 @@ def test_check_unique_ping_time_duplicates(caplog, ek80_dupe_ping_path): f"{str(ds_data['ping_time'].values[0])} differ in data. All duplicate " "'ping_time' entries will be removed, which will result in data loss." ) - assert any(expected_warning in record.message for record in caplog.records) + assert any(expected_warning in str(record.message) for record in recwarn) @pytest.mark.unit diff --git a/echopype/tests/echodata/test_echodata.py b/echopype/tests/echodata/test_echodata.py index fa8a165b3..69a2b9f6f 100644 --- a/echopype/tests/echodata/test_echodata.py +++ b/echopype/tests/echodata/test_echodata.py @@ -798,7 +798,7 @@ def test_convert_legacy_versions_ek80(legacy_datatree, legacy_datatree_filename) @pytest.mark.unit -def test_echodata_delete(caplog, ek60_path): +def test_echodata_delete(recwarn, ek60_path): """ Check for correct removal behavior and no warnings captured in echodata delete. """ @@ -840,7 +840,7 @@ def test_echodata_delete(caplog, ek60_path): ed.__del__() # Check that no exceptions were wrapped by warnings - assert not any("Warning: Exception ignored in:" in record.message for record in caplog.records) + assert not any("Warning: Exception ignored in:" in str(record.message) for record in recwarn) # Check that it doesn't exist assert not os.path.exists(temp_zarr_path) From 2b94df28bb2d41fcfa25604a671342709d09e326 Mon Sep 17 00:00:00 2001 From: Lloyd Izard <76954858+LOCEANlloydizard@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:05:25 -0700 Subject: [PATCH 5/6] fix warning handling after logger removal --- echopype/convert/parse_base.py | 6 +++- echopype/convert/parse_uls6.py | 30 +++++++++++------- echopype/convert/utils/ek_raw_io.py | 32 +++++++++++--------- echopype/convert/utils/ek_raw_parsers.py | 10 +++--- echopype/qc/api.py | 4 +-- echopype/tests/consolidate/test_add_depth.py | 2 +- 6 files changed, 50 insertions(+), 34 deletions(-) diff --git a/echopype/convert/parse_base.py b/echopype/convert/parse_base.py index 7bfc1ea34..df514c0cb 100644 --- a/echopype/convert/parse_base.py +++ b/echopype/convert/parse_base.py @@ -2,6 +2,7 @@ import os import re import sys +import warnings from collections import defaultdict from typing import Any, Dict, Literal, Optional, Tuple, Union @@ -713,7 +714,10 @@ def _read_datagrams(self, fid): elif new_datagram["type"].startswith("DEP"): print("DEP datagram encountered.") else: - print("Unknown datagram type: " + str(new_datagram["type"])) + warnings.warn( + f"Unknown datagram type: {new_datagram['type']}", + category=UserWarning, + ) def _append_channel_ping_data( self, datagram, raw_type: Literal["transmit", "receive"] = "receive" diff --git a/echopype/convert/parse_uls6.py b/echopype/convert/parse_uls6.py index 24e6e3ae7..37aed5b5a 100644 --- a/echopype/convert/parse_uls6.py +++ b/echopype/convert/parse_uls6.py @@ -1,4 +1,3 @@ -import logging import os import warnings import xml.etree.ElementTree as ET @@ -157,7 +156,6 @@ def load_AZFP_xml(self, raw): self.unpacked_data["num_prev_xml_bytes"] = xml_byte_size if int.from_bytes(raw.read(4), "little") != self.XML_END_FLAG: - logging.error("Error reading xml string") raise ValueError("Error reading xml string") xml_prev_byte_size = unpack(" Date: Tue, 1 Sep 2026 09:47:52 -0700 Subject: [PATCH 6/6] fix ULS6 indentation --- echopype/convert/parse_uls6.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/echopype/convert/parse_uls6.py b/echopype/convert/parse_uls6.py index 37aed5b5a..3cde869ce 100644 --- a/echopype/convert/parse_uls6.py +++ b/echopype/convert/parse_uls6.py @@ -482,27 +482,28 @@ def _split_header(self, raw, ping_num): _, byte_code, byte_size, array_size = self._get_masked_data(field_code) val = unpack("<" + byte_code * array_size, raw.read(byte_size * array_size)) header_byte_cnt += 2 + byte_size * array_size + try: field = HEADER_LOOKUP[field_code].lower() - except: # Unknown field + except: field = f"code_{hex(field_code)}" warnings.warn( f"Unknown code found in file: {hex(field_code)}, field stored as {field}", category=UserWarning, ) - self.unpacked_data[field].append(*val if len(val) == 1 else [val]) # list(val) + self.unpacked_data[field].append(*val if len(val) == 1 else [val]) if field_code == HEADER_CODES["LAST_HEADER_RECORD"]: break - if header_byte_cnt != self.unpacked_data["header_bytes"][0]: - warnings.warn( - "Error reading header: {} != {}".format( - header_byte_cnt, self.unpacked_data["header_bytes"][0] - ), - category=UserWarning, - ) + if header_byte_cnt != self.unpacked_data["header_bytes"][0]: + warnings.warn( + "Error reading header: {} != {}".format( + header_byte_cnt, self.unpacked_data["header_bytes"][0] + ), + category=UserWarning, + ) return False # TODO: this is a bit hacky, convert the parameters to a numpy array and make a extra dim?