|
1 | 1 | # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2 | 2 |
|
3 |
| -from typing import Any, Hashable, Iterable, Mapping, Optional |
| 3 | +from typing import Any, Iterable, Mapping, Optional |
4 | 4 |
|
5 | 5 | from airbyte_cdk.sources.declarative.retrievers import Retriever
|
6 | 6 | from airbyte_cdk.sources.message import MessageRepository
|
@@ -89,21 +89,5 @@ def __init__(
|
89 | 89 | self._stream_slicer = stream_slicer
|
90 | 90 |
|
91 | 91 | def generate(self) -> Iterable[Partition]:
|
92 |
| - # Yield partitions for unique stream slices, avoiding duplicates |
93 |
| - seen_slices: set[Hashable] = set() |
94 | 92 | for stream_slice in self._stream_slicer.stream_slices():
|
95 |
| - slice_key = self._make_hashable(stream_slice) |
96 |
| - if slice_key in seen_slices: |
97 |
| - continue |
98 |
| - seen_slices.add(slice_key) |
99 | 93 | yield self._partition_factory.create(stream_slice)
|
100 |
| - |
101 |
| - @staticmethod |
102 |
| - def _make_hashable(obj: Any) -> Any: |
103 |
| - if isinstance(obj, dict): |
104 |
| - return frozenset( |
105 |
| - (k, StreamSlicerPartitionGenerator._make_hashable(v)) for k, v in obj.items() |
106 |
| - ) |
107 |
| - if isinstance(obj, list): |
108 |
| - return tuple(StreamSlicerPartitionGenerator._make_hashable(i) for i in obj) |
109 |
| - return obj |
0 commit comments