diff --git a/echopype/convert/set_groups_base.py b/echopype/convert/set_groups_base.py index f0985cd58..5cfdf3a70 100644 --- a/echopype/convert/set_groups_base.py +++ b/echopype/convert/set_groups_base.py @@ -119,7 +119,13 @@ def _nan_timestamp_handler(self, time_val) -> List: model_family = SONAR_MODELS[self.sonar_model]["family"] # set time_val to earliest ping_time among all channels if model_family in ["Ex60", "Ex80"]: - return [np.array([v[0] for v in self.parser_obj.ping_time.values()]).min()] + ping_times = [v[0] for v in self.parser_obj.ping_time.values()] + # Partial/truncated raw files can yield no channels or empty ping arrays + if len(ping_times) == 0 or all( + hasattr(t, "__len__") and len(t) == 0 for t in ping_times + ): + return [np.nan] + return [np.array(ping_times).min()] elif model_family == "AZFP": return [self.parser_obj.ping_time[0]] else: diff --git a/echopype/tests/convert/test_set_groups_common.py b/echopype/tests/convert/test_set_groups_common.py index 84fc54c55..0b8362b56 100644 --- a/echopype/tests/convert/test_set_groups_common.py +++ b/echopype/tests/convert/test_set_groups_common.py @@ -1,9 +1,12 @@ from typing import Dict, List, Any +from types import SimpleNamespace import xarray as xr import numpy as np import pytest +from echopype.convert.set_groups_base import SetGroupsBase + pytestmark = pytest.mark.unit @@ -71,3 +74,62 @@ def test_backscatter_concat_jitter_ping_time(mock_ping_data_dict_power_angle_jit # Check equivalent ping times assert np.array_equal(da["ping_time"].to_numpy(), np.array(ping_times[ch])) + + +# Regression tests for _nan_timestamp_handler on partial/truncated raw files. +# See PR #1624. + + +def _call_nan_timestamp_handler(sonar_model, parser_obj, time_val): + """Invoke the unbound method to avoid instantiating the abstract SetGroupsBase.""" + fake_self = SimpleNamespace(sonar_model=sonar_model, parser_obj=parser_obj) + return SetGroupsBase._nan_timestamp_handler(fake_self, time_val) + + +@pytest.mark.parametrize("sonar_model", ["EK60", "ES70", "EK80", "ES80", "EA640"]) +def test_nan_timestamp_handler_ek_empty_ping_time_dict(sonar_model): + """No channels at all -> [nan] instead of crashing on np.array([]).min().""" + parser_obj = SimpleNamespace(ping_time={}) + + result = _call_nan_timestamp_handler(sonar_model, parser_obj, [np.nan]) + + assert len(result) == 1 + assert np.isnan(result[0]) + + +@pytest.mark.parametrize("sonar_model", ["EK60", "EK80"]) +def test_nan_timestamp_handler_ek_all_channels_empty(sonar_model): + """All channels present but each has an empty ping_time array -> [nan].""" + parser_obj = SimpleNamespace( + ping_time={ + "ch1": [np.array([], dtype="datetime64[ns]")], + "ch2": [np.array([], dtype="datetime64[ns]")], + } + ) + + result = _call_nan_timestamp_handler(sonar_model, parser_obj, [np.nan]) + + assert len(result) == 1 + assert np.isnan(result[0]) + + +def test_nan_timestamp_handler_ek_returns_earliest_when_populated(): + """Sanity check: non-empty ping times still return the earliest one.""" + t_ch1 = np.array(["2024-01-01T00:00:05"], dtype="datetime64[ns]") + t_ch2 = np.array(["2024-01-01T00:00:02"], dtype="datetime64[ns]") + parser_obj = SimpleNamespace(ping_time={"ch1": [t_ch1], "ch2": [t_ch2]}) + + result = _call_nan_timestamp_handler("EK60", parser_obj, [np.nan]) + + assert len(result) == 1 + assert result[0] == np.datetime64("2024-01-01T00:00:02", "ns") + + +def test_nan_timestamp_handler_passthrough_when_time_val_valid(): + """Non-NaN input should be returned unchanged without touching parser_obj.""" + time_val = [np.datetime64("2024-01-01T00:00:00", "ns")] + parser_obj = SimpleNamespace(ping_time={}) + + result = _call_nan_timestamp_handler("EK60", parser_obj, time_val) + + assert result is time_val diff --git a/echopype/tests/utils/test_coding.py b/echopype/tests/utils/test_coding.py index b2ee20f43..337c196ad 100644 --- a/echopype/tests/utils/test_coding.py +++ b/echopype/tests/utils/test_coding.py @@ -5,7 +5,15 @@ import dask import warnings -from echopype.utils.coding import _get_dask_auto_chunk, set_netcdf_encodings, _encode_time_dataarray, DEFAULT_TIME_ENCODING # noqa: E501 +from echopype.utils.coding import ( + _get_dask_auto_chunk, + set_netcdf_encodings, + set_time_encodings, + set_zarr_encodings, + _encode_time_dataarray, + COMPRESSION_SETTINGS, + DEFAULT_TIME_ENCODING, +) @pytest.mark.unit @pytest.mark.parametrize( @@ -136,3 +144,88 @@ def test_encode_time_dataarray_on_encoded_time_data(): # Check to see if value error is raised when we pass in an encoded float datetime array with pytest.raises(ValueError, match="Encoded time data array must be of type ```np.int64```."): _encode_time_dataarray(encoded_datetime_array.astype(np.float64)) + + +# Regression tests for zero-length / all-NaN handling on partial/truncated raw files. +# See PR #1624. + +@pytest.mark.unit +def test_set_time_encodings_skips_all_nan_time_var(): + """All-NaN time variables must be skipped so xarray's encoder isn't called on them.""" + ds = xr.Dataset( + { + "ping_time": xr.DataArray(np.array([np.nan, np.nan]), dims="ping_time"), + "backscatter_r": xr.DataArray(np.zeros((2, 3)), dims=("ping_time", "range_sample")), + } + ) + + new_ds = set_time_encodings(ds) + + # Values are preserved and no encoding was forced onto the all-NaN time var. + assert np.all(np.isnan(new_ds["ping_time"].values)) + assert new_ds["ping_time"].encoding == {} + + +@pytest.mark.unit +def test_set_time_encodings_all_nan_mixed_with_valid_time(): + """A valid time var should still be encoded when another time var is all-NaN.""" + valid_times = np.array( + ["2024-01-01T00:00:00", "2024-01-01T00:00:01"], dtype="datetime64[ns]" + ) + ds = xr.Dataset( + { + "ping_time": xr.DataArray(np.array([np.nan, np.nan]), dims="ping_time"), + "time1": xr.DataArray(valid_times, dims="time1"), + } + ) + + new_ds = set_time_encodings(ds) + + assert np.all(np.isnan(new_ds["ping_time"].values)) + assert new_ds["ping_time"].encoding == {} + assert np.issubdtype(new_ds["time1"].dtype, np.datetime64) + assert new_ds["time1"].encoding == DEFAULT_TIME_ENCODING + + +@pytest.mark.unit +def test_set_zarr_encodings_zero_length_dim_sets_chunks_to_none(): + """A variable with a zero-length dim must not trigger division by zero in chunk calc.""" + ds = xr.Dataset( + { + "backscatter_r": xr.DataArray( + np.zeros((0, 3), dtype=np.float32), dims=("ping_time", "range_sample") + ), + } + ) + + encoding = set_zarr_encodings(ds, COMPRESSION_SETTINGS["zarr"]) + + assert encoding["backscatter_r"]["chunks"] is None + + +@pytest.mark.unit +def test_set_zarr_encodings_scalar_variable_sets_chunks_to_none(): + """A scalar (zero-dim) variable must be given chunks=None rather than tripping the chunker.""" + ds = xr.Dataset({"scalar_var": xr.DataArray(np.float32(1.5))}) + + encoding = set_zarr_encodings(ds, COMPRESSION_SETTINGS["zarr"]) + + assert encoding["scalar_var"]["chunks"] is None + + +@pytest.mark.unit +def test_set_zarr_encodings_normal_variable_still_chunked(): + """Sanity check: non-empty, non-scalar variables still receive a chunk list.""" + ds = xr.Dataset( + { + "backscatter_r": xr.DataArray( + np.zeros((10, 20), dtype=np.float32), dims=("ping_time", "range_sample") + ), + } + ) + + encoding = set_zarr_encodings(ds, COMPRESSION_SETTINGS["zarr"]) + + chunks = encoding["backscatter_r"]["chunks"] + assert chunks is not None + assert len(chunks) == 2 diff --git a/echopype/utils/coding.py b/echopype/utils/coding.py index 0295d9046..bed4845aa 100644 --- a/echopype/utils/coding.py +++ b/echopype/utils/coding.py @@ -141,14 +141,21 @@ def set_time_encodings(ds: xr.Dataset) -> xr.Dataset: for var, encoding in DEFAULT_ENCODINGS.items(): if var in new_ds: da = new_ds[var].copy() + # Process all variable names matching the patterns *_time* or time # Examples: ping_time, ping_time_2, time1, time2 if bool(search(r"_time|^time[\d]+$", var)): - new_ds[var] = xr.apply_ufunc( - _encode_time_dataarray, - da, - keep_attrs=True, - ) + if np.isnan(da).all(): + continue + + try: + new_ds[var] = xr.apply_ufunc( + _encode_time_dataarray, + da, + keep_attrs=True, + ) + except ValueError as e: + raise e new_ds[var].encoding = encoding @@ -206,28 +213,36 @@ def set_zarr_encodings( chunk_size_tolerance = parse_bytes(ctol) if len(val.shape) > 0: - rechunk = True - if existing_chunks is not None: - # Perform chunk optimization - # 1. Get the chunk total from existing chunks - chunk_total = np.prod(existing_chunks) * val.dtype.itemsize - # 2. Get chunk size difference from the optimal chunk size - chunk_diff = optimal_chunk_size - chunk_total - # 3. Check difference from tolerance, if diff is less than - # tolerance then no need to rechunk - if chunk_diff < chunk_size_tolerance: - rechunk = False - chunks = existing_chunks - - if rechunk: - # Use dask auto chunk to determine the optimal chunk - # spread for optimal chunk size - chunks = _get_dask_auto_chunk(val, chunk_size=chunk_size) - # Dask expects chunk dictionary but Zarr expects list-like iterable of - # values in encoding - chunks = [*chunks.values()] - - encoding[name]["chunks"] = chunks + if any(dim_size == 0 for dim_size in val.shape): + # If any dimension is zero, set chunks to None to avoid division by zero + encoding[name]["chunks"] = None + else: + rechunk = True + if existing_chunks is not None: + # Perform chunk optimization + # 1. Get the chunk total from existing chunks + chunk_total = np.prod(existing_chunks) * val.dtype.itemsize + # 2. Get chunk size difference from the optimal chunk size + chunk_diff = optimal_chunk_size - chunk_total + # 3. Check difference from tolerance, if diff is less than + # tolerance then no need to rechunk + if chunk_diff < chunk_size_tolerance: + rechunk = False + chunks = existing_chunks + + if rechunk: + # Use dask auto chunk to determine the optimal chunk + # spread for optimal chunk size + chunks = _get_dask_auto_chunk(val, chunk_size=chunk_size) + # Dask expects chunk dictionary but Zarr expects list-like iterable of + # values in encoding + chunks = [*chunks.values()] + + encoding[name]["chunks"] = chunks + else: + # Variable has no shape (scalar), set chunks to None + encoding[name]["chunks"] = None + if PREFERRED_CHUNKS in encoding[name]: # Remove 'preferred_chunks', use chunks only instead encoding[name].pop(PREFERRED_CHUNKS)