Skip to content

Commit e93682c

Browse files
authored
Merge pull request #763 from sentinel-hub/develop
Release 1.5.1
2 parents 596fdc1 + f311aa7 commit e93682c

21 files changed

+262
-99
lines changed

.pre-commit-config.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
repos:
22
- repo: https://github.com/pre-commit/pre-commit-hooks
3-
rev: v4.4.0
3+
rev: v4.5.0
44
hooks:
55
- id: end-of-file-fixer
66
- id: requirements-txt-fixer
@@ -13,13 +13,13 @@ repos:
1313
- id: debug-statements
1414

1515
- repo: https://github.com/psf/black
16-
rev: 23.7.0
16+
rev: 23.9.1
1717
hooks:
1818
- id: black
1919
language_version: python3
2020

2121
- repo: https://github.com/charliermarsh/ruff-pre-commit
22-
rev: "v0.0.282"
22+
rev: "v0.0.292"
2323
hooks:
2424
- id: ruff
2525

CHANGELOG.md

+7
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
## [Version 1.5.1] - 2023-10-17
2+
3+
- `MorphologicalFilterTask` adapted to work on boolean values.
4+
- Added `temporal_subset` method to `EOPatch`, which can be used to extract a subset of an `EOPatch` by filtering out temporal slices. Also added a corresponding `TemporalSubsetTask`.
5+
- `EOExecutor` now has an option to treat `TemporalDimensionWarning` as an exception.
6+
- String representation of `EOPatch` objects was revisited to avoid edge cases where the output would print enormous objects.
7+
18
## [Version 1.5.0] - 2023-09-06
29

310
The release focuses on making `eo-learn` much simpler to install, reducing the number of dependencies, and improving validation of soundness of `EOPatch` data.

docs/source/conf.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@
134134
# When Sphinx documents class signature it prioritizes __new__ method over __init__ method. The following hack puts
135135
# EOTask.__new__ method to the blacklist so that __init__ method signature will be taken instead. This seems the
136136
# cleanest way even though a private object is accessed.
137-
sphinx.ext.autodoc._CLASS_NEW_BLACKLIST.append("{0.__module__}.{0.__qualname__}".format(EOTask.__new__)) # noqa: SLF001
137+
sphinx.ext.autodoc._CLASS_NEW_BLACKLIST.append(f"{EOTask.__module__}.{EOTask.__new__.__qualname__}") # noqa: SLF001
138138

139139

140140
EXAMPLES_FOLDER = "./examples"

eolearn/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""Main module of the `eolearn` package."""
2-
__version__ = "1.5.0"
2+
__version__ = "1.5.1"
33

44
import importlib.util
55
import warnings

eolearn/core/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
RemoveFeatureTask,
2020
RenameFeatureTask,
2121
SaveTask,
22+
TemporalSubsetTask,
2223
ZipFeatureTask,
2324
)
2425
from .eodata import EOPatch

eolearn/core/core_tasks.py

+24
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,30 @@ def execute(self, src_eopatch: EOPatch, dst_eopatch: EOPatch) -> EOPatch:
413413
return dst_eopatch
414414

415415

416+
class TemporalSubsetTask(EOTask):
417+
"""Extracts a temporal subset of the EOPatch."""
418+
419+
def __init__(
420+
self, timestamps: None | list[dt.datetime] | list[int] | Callable[[list[dt.datetime]], Iterable[bool]] = None
421+
):
422+
"""
423+
:param timestamps: Input for the `temporal_subset` method of EOPatch. Can also be provided in execution
424+
arguments. Value in execution arguments takes precedence.
425+
"""
426+
self.timestamps = timestamps
427+
428+
def execute(
429+
self,
430+
eopatch: EOPatch,
431+
*,
432+
timestamps: None | list[dt.datetime] | list[int] | Callable[[list[dt.datetime]], Iterable[bool]] = None,
433+
) -> EOPatch:
434+
timestamps = timestamps if timestamps is not None else self.timestamps
435+
if timestamps is None:
436+
raise ValueError("Value for `timestamps` must be provided on initialization or as an execution argument.")
437+
return eopatch.temporal_subset(timestamps)
438+
439+
416440
class MapFeatureTask(EOTask):
417441
"""Applies a function to each feature in input_features of a patch and stores the results in a set of
418442
output_features.

eolearn/core/eodata.py

+55-15
Original file line numberDiff line numberDiff line change
@@ -455,36 +455,36 @@ def __repr__(self) -> str:
455455

456456
@staticmethod
457457
def _repr_value(value: object) -> str:
458-
"""Creates a representation string for different types of data.
459-
460-
:param value: data in any type
461-
:return: representation string
462-
"""
458+
"""Creates a representation string for different types of data."""
463459
if isinstance(value, np.ndarray):
464460
return f"{EOPatch._repr_value_class(value)}(shape={value.shape}, dtype={value.dtype})"
465461

466462
if isinstance(value, gpd.GeoDataFrame):
467463
crs = CRS(value.crs).ogc_string() if value.crs else value.crs
468464
return f"{EOPatch._repr_value_class(value)}(columns={list(value)}, length={len(value)}, crs={crs})"
469465

466+
repr_str = str(value)
467+
if len(repr_str) <= MAX_DATA_REPR_LEN:
468+
return repr_str
469+
470470
if isinstance(value, (list, tuple, dict)) and value:
471-
repr_str = str(value)
472-
if len(repr_str) <= MAX_DATA_REPR_LEN:
473-
return repr_str
471+
lb, rb = ("[", "]") if isinstance(value, list) else ("(", ")") if isinstance(value, tuple) else ("{", "}")
474472

475-
l_bracket, r_bracket = ("[", "]") if isinstance(value, list) else ("(", ")")
476-
if isinstance(value, (list, tuple)) and len(value) > 2:
477-
repr_str = f"{l_bracket}{value[0]!r}, ..., {value[-1]!r}{r_bracket}"
473+
if isinstance(value, dict): # generate representation of first element or (key, value) pair
474+
some_key = next(iter(value))
475+
repr_of_el = f"{EOPatch._repr_value(some_key)}: {EOPatch._repr_value(value[some_key])}"
476+
else:
477+
repr_of_el = EOPatch._repr_value(value[0])
478478

479-
if len(repr_str) > MAX_DATA_REPR_LEN and isinstance(value, (list, tuple)) and len(value) > 1:
480-
repr_str = f"{l_bracket}{value[0]!r}, ...{r_bracket}"
479+
many_elements_visual = ", ..." if len(value) > 1 else "" # add ellipsis if there are multiple elements
480+
repr_str = f"{lb}{repr_of_el}{many_elements_visual}{rb}"
481481

482482
if len(repr_str) > MAX_DATA_REPR_LEN:
483483
repr_str = str(type(value))
484484

485-
return f"{repr_str}, length={len(value)}"
485+
return f"{repr_str}<length={len(value)}>"
486486

487-
return repr(value)
487+
return str(type(value))
488488

489489
@staticmethod
490490
def _repr_value_class(value: object) -> str:
@@ -726,6 +726,7 @@ def merge(
726726
self, *eopatches, features=features, time_dependent_op=time_dependent_op, timeless_op=timeless_op
727727
)
728728

729+
@deprecated_function(EODeprecationWarning, "Please use the method `temporal_subset` instead.")
729730
def consolidate_timestamps(self, timestamps: list[dt.datetime]) -> set[dt.datetime]:
730731
"""Removes all frames from the EOPatch with a date not found in the provided timestamps list.
731732
@@ -750,6 +751,45 @@ def consolidate_timestamps(self, timestamps: list[dt.datetime]) -> set[dt.dateti
750751

751752
return remove_from_patch
752753

754+
def temporal_subset(
755+
self, timestamps: Iterable[dt.datetime] | Iterable[int] | Callable[[list[dt.datetime]], Iterable[bool]]
756+
) -> EOPatch:
757+
"""Returns an EOPatch that only contains data for the temporal subset corresponding to `timestamps`.
758+
759+
For array-based data appropriate temporal slices are extracted. For vector data a filtration is performed.
760+
761+
:param timestamps: Parameter that defines the temporal subset. Can be a collection of timestamps, a
762+
collection of timestamp indices. It is possible to also provide a callable that maps a list of timestamps
763+
to a sequence of booleans, which determine if a given timestamp is included in the subset or not.
764+
"""
765+
timestamp_indices = self._parse_temporal_subset_input(timestamps)
766+
new_timestamps = [ts for i, ts in enumerate(self.get_timestamps()) if i in timestamp_indices]
767+
new_patch = EOPatch(bbox=self.bbox, timestamps=new_timestamps)
768+
769+
for ftype, fname in self.get_features():
770+
if ftype.is_timeless() or ftype.is_meta():
771+
new_patch[ftype, fname] = self[ftype, fname]
772+
elif ftype.is_vector():
773+
gdf: gpd.GeoDataFrame = self[ftype, fname]
774+
new_patch[ftype, fname] = gdf[gdf[TIMESTAMP_COLUMN].isin(new_timestamps)]
775+
else:
776+
new_patch[ftype, fname] = self[ftype, fname][timestamp_indices]
777+
778+
return new_patch
779+
780+
def _parse_temporal_subset_input(
781+
self, timestamps: Iterable[dt.datetime] | Iterable[int] | Callable[[list[dt.datetime]], Iterable[bool]]
782+
) -> list[int]:
783+
"""Parses input into a list of timestamp indices. Also adds implicit support for strings via `parse_time`."""
784+
if callable(timestamps):
785+
accepted_timestamps = timestamps(self.get_timestamps())
786+
return [i for i, accepted in enumerate(accepted_timestamps) if accepted]
787+
ts_or_idx = list(timestamps)
788+
if all(isinstance(ts, int) for ts in ts_or_idx):
789+
return ts_or_idx # type: ignore[return-value]
790+
parsed_timestamps = {parse_time(ts, force_datetime=True) for ts in ts_or_idx} # type: ignore[call-overload]
791+
return [i for i, ts in enumerate(self.get_timestamps()) if ts in parsed_timestamps]
792+
753793
def plot(
754794
self,
755795
feature: Feature,

eolearn/core/eoexecution.py

+11-4
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
from .eonode import EONode
2929
from .eoworkflow import EOWorkflow, WorkflowResults
30-
from .exceptions import EORuntimeWarning
30+
from .exceptions import EORuntimeWarning, TemporalDimensionWarning
3131
from .utils.fs import get_base_filesystem_and_path, get_full_path, pickle_fs, unpickle_fs
3232
from .utils.logging import LogFileFilter
3333
from .utils.parallelize import _decide_processing_type, _ProcessingType, parallelize
@@ -36,8 +36,7 @@
3636
class _HandlerWithFsFactoryType(Protocol):
3737
"""Type definition for a callable that accepts a path and a filesystem object"""
3838

39-
def __call__(self, path: str, filesystem: FS, **kwargs: Any) -> Handler:
40-
...
39+
def __call__(self, path: str, filesystem: FS, **kwargs: Any) -> Handler: ...
4140

4241

4342
# pylint: disable=invalid-name
@@ -56,6 +55,7 @@ class _ProcessingData:
5655
filter_logs_by_thread: bool
5756
logs_filter: Filter | None
5857
logs_handler_factory: _HandlerFactoryType
58+
raise_on_temporal_mismatch: bool
5959

6060

6161
@dataclass(frozen=True)
@@ -87,6 +87,7 @@ def __init__(
8787
filesystem: FS | None = None,
8888
logs_filter: Filter | None = None,
8989
logs_handler_factory: _HandlerFactoryType = FileHandler,
90+
raise_on_temporal_mismatch: bool = False,
9091
):
9192
"""
9293
:param workflow: A prepared instance of EOWorkflow class
@@ -109,6 +110,7 @@ def __init__(
109110
object.
110111
111112
The 2nd option is chosen only if `filesystem` parameter exists in the signature.
113+
:param raise_on_temporal_mismatch: Whether to treat `TemporalDimensionWarning` as an exception.
112114
"""
113115
self.workflow = workflow
114116
self.execution_kwargs = self._parse_and_validate_execution_kwargs(execution_kwargs)
@@ -117,6 +119,7 @@ def __init__(
117119
self.filesystem, self.logs_folder = self._parse_logs_filesystem(filesystem, logs_folder)
118120
self.logs_filter = logs_filter
119121
self.logs_handler_factory = logs_handler_factory
122+
self.raise_on_temporal_mismatch = raise_on_temporal_mismatch
120123

121124
self.start_time: dt.datetime | None = None
122125
self.report_folder: str | None = None
@@ -194,6 +197,7 @@ def run(self, workers: int | None = 1, multiprocess: bool = True, **tqdm_kwargs:
194197
filter_logs_by_thread=filter_logs_by_thread,
195198
logs_filter=self.logs_filter,
196199
logs_handler_factory=self.logs_handler_factory,
200+
raise_on_temporal_mismatch=self.raise_on_temporal_mismatch,
197201
)
198202
for workflow_kwargs, log_path in zip(self.execution_kwargs, log_paths)
199203
]
@@ -264,7 +268,10 @@ def _execute_workflow(cls, data: _ProcessingData) -> WorkflowResults:
264268
data.logs_handler_factory,
265269
)
266270

267-
results = data.workflow.execute(data.workflow_kwargs, raise_errors=False)
271+
with warnings.catch_warnings():
272+
if data.raise_on_temporal_mismatch:
273+
warnings.simplefilter("error", TemporalDimensionWarning)
274+
results = data.workflow.execute(data.workflow_kwargs, raise_errors=False)
268275

269276
cls._try_remove_logging(data.log_path, logger, handler)
270277
return results

eolearn/core/eoworkflow.py

+2-4
Original file line numberDiff line numberDiff line change
@@ -307,12 +307,10 @@ def get_nodes(self) -> list[EONode]:
307307
return self._nodes[:]
308308

309309
@overload
310-
def get_node_with_uid(self, uid: str, fail_if_missing: Literal[True] = ...) -> EONode:
311-
...
310+
def get_node_with_uid(self, uid: str, fail_if_missing: Literal[True] = ...) -> EONode: ...
312311

313312
@overload
314-
def get_node_with_uid(self, uid: str, fail_if_missing: Literal[False] = ...) -> EONode | None:
315-
...
313+
def get_node_with_uid(self, uid: str, fail_if_missing: Literal[False] = ...) -> EONode | None: ...
316314

317315
def get_node_with_uid(self, uid: str, fail_if_missing: bool = False) -> EONode | None:
318316
"""Returns node with give uid, if it exists in the workflow."""

eolearn/core/extra/ray.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def _get_processing_type(*_: Any, **__: Any) -> _ProcessingType:
6161
def _ray_workflow_executor(workflow_args: _ProcessingData) -> WorkflowResults:
6262
"""Called to execute a workflow on a ray worker"""
6363
# pylint: disable=protected-access
64-
return RayExecutor._execute_workflow(workflow_args)
64+
return RayExecutor._execute_workflow(workflow_args) # noqa: SLF001
6565

6666

6767
def parallelize_with_ray(

eolearn/core/utils/parallelize.py

+4-16
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,7 @@ def _decide_processing_type(workers: int | None, multiprocess: bool) -> _Process
4949
"""
5050
if workers == 1:
5151
return _ProcessingType.SINGLE_PROCESS
52-
if multiprocess:
53-
return _ProcessingType.MULTIPROCESSING
54-
return _ProcessingType.MULTITHREADING
52+
return _ProcessingType.MULTIPROCESSING if multiprocess else _ProcessingType.MULTITHREADING
5553

5654

5755
def parallelize(
@@ -74,10 +72,7 @@ def parallelize(
7472
:return: A list of function results.
7573
"""
7674
if not params:
77-
raise ValueError(
78-
"At least 1 list of parameters should be given. Otherwise it is not clear how many times the"
79-
"function has to be executed."
80-
)
75+
return []
8176
processing_type = _decide_processing_type(workers=workers, multiprocess=multiprocess)
8277

8378
if processing_type is _ProcessingType.SINGLE_PROCESS:
@@ -105,7 +100,6 @@ def execute_with_mp_lock(function: Callable[..., OutputType], *args: Any, **kwar
105100
:param function: A function
106101
:param args: Function's positional arguments
107102
:param kwargs: Function's keyword arguments
108-
:return: Function's results
109103
"""
110104
if multiprocessing.current_process().name == "MainProcess" or MULTIPROCESSING_LOCK is None:
111105
return function(*args, **kwargs)
@@ -165,10 +159,7 @@ def join_futures_iter(
165159
"""
166160

167161
def _wait_function(remaining_futures: Collection[Future]) -> tuple[Collection[Future], Collection[Future]]:
168-
done, not_done = concurrent.futures.wait(
169-
remaining_futures, timeout=float(update_interval), return_when=FIRST_COMPLETED
170-
)
171-
return done, not_done
162+
return concurrent.futures.wait(remaining_futures, timeout=float(update_interval), return_when=FIRST_COMPLETED)
172163

173164
def _get_result(future: Future) -> Any:
174165
return future.result()
@@ -184,8 +175,6 @@ def _base_join_futures_iter(
184175
) -> Generator[tuple[int, OutputType], None, None]:
185176
"""A generalized utility function that resolves futures, monitors progress, and serves as an iterator over
186177
results."""
187-
if not isinstance(futures, list):
188-
raise ValueError(f"Parameters 'futures' should be a list but {type(futures)} was given")
189178
remaining_futures: Collection[FutureType] = _make_copy_and_empty_given(futures)
190179

191180
id_to_position_map = {id(future): index for index, future in enumerate(remaining_futures)}
@@ -195,9 +184,8 @@ def _base_join_futures_iter(
195184
done, remaining_futures = wait_function(remaining_futures)
196185
for future in done:
197186
result = get_result_function(future)
198-
result_position = id_to_position_map[id(future)]
199187
pbar.update(1)
200-
yield result_position, result
188+
yield id_to_position_map[id(future)], result
201189

202190

203191
def _make_copy_and_empty_given(items: list[T]) -> list[T]:

eolearn/features/extra/clustering.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def execute(self, eopatch: EOPatch) -> EOPatch:
8888

8989
# All connections to masked pixels are removed
9090
if self.mask_name is not None:
91-
mask = eopatch.mask_timeless[self.mask_name].squeeze()
91+
mask = eopatch.mask_timeless[self.mask_name].squeeze(axis=-1)
9292
graph_args["mask"] = mask
9393
data = data[np.ravel(mask) != 0]
9494

eolearn/geometry/morphology.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ def __init__(
4848
self.no_data_label = no_data_label
4949

5050
def execute(self, eopatch: EOPatch) -> EOPatch:
51-
feature_array = eopatch[(self.mask_type, self.mask_name)].squeeze().copy()
51+
feature_array = eopatch[(self.mask_type, self.mask_name)].squeeze(axis=-1).copy()
5252

5353
all_labels = np.unique(feature_array)
5454
erode_labels = self.erode_labels if self.erode_labels else all_labels
@@ -148,6 +148,10 @@ def __init__(
148148
def map_method(self, feature: np.ndarray) -> np.ndarray:
149149
"""Applies the morphological operation to a raster feature."""
150150
feature = feature.copy()
151+
is_bool = feature.dtype == bool
152+
if is_bool:
153+
feature = feature.astype(np.uint8)
154+
151155
morph_func = partial(cv2.morphologyEx, kernel=self.struct_elem, op=self.morph_operation)
152156
if feature.ndim == 3:
153157
for channel in range(feature.shape[2]):
@@ -158,4 +162,4 @@ def map_method(self, feature: np.ndarray) -> np.ndarray:
158162
else:
159163
raise ValueError(f"Invalid number of dimensions: {feature.ndim}")
160164

161-
return feature
165+
return feature.astype(bool) if is_bool else feature

0 commit comments

Comments
 (0)