memory and file system

Author: vdusek

src/crawlee/_service_locator.py

@@ -77,9 +77,9 @@ def set_event_manager(self, event_manager: EventManager) -> None:
     def get_storage_client(self) -> StorageClient:
         """Get the storage client."""
         if self._storage_client is None:
-            from crawlee.storage_clients import MemoryStorageClient
+            from crawlee.storage_clients import file_system_storage_client
 
-            self._storage_client = MemoryStorageClient()
+            self._storage_client = file_system_storage_client
 
         self._storage_client_was_retrieved = True
         return self._storage_client

src/crawlee/_types.py

@@ -274,10 +274,6 @@ async def push_data(
         **kwargs: Unpack[PushDataKwargs],
     ) -> None:
         """Track a call to the `push_data` context helper."""
-        from crawlee.storages._dataset import Dataset
-
-        await Dataset._check_and_serialize(data)
-
         self.push_data_calls.append(
             PushDataFunctionCall(
                 data=data,

src/crawlee/storage_clients/__init__.py

@@ -1,4 +1,9 @@
 from ._base import StorageClient
-from ._memory import MemoryStorageClient
+from ._file_system import file_system_storage_client
+from ._memory import memory_storage_client
 
-__all__ = ['MemoryStorageClient', 'StorageClient']
+__all__ = [
+    'StorageClient',
+    'file_system_storage_client',
+    'memory_storage_client'
+]

src/crawlee/storage_clients/_base/__init__.py

@@ -2,15 +2,10 @@
 from ._key_value_store_client import KeyValueStoreClient
 from ._request_queue_client import RequestQueueClient
 from ._storage_client import StorageClient
-from ._types import ResourceClient
 
 __all__ = [
     'DatasetClient',
-    'DatasetCollectionClient',
     'KeyValueStoreClient',
-    'KeyValueStoreCollectionClient',
     'RequestQueueClient',
-    'RequestQueueCollectionClient',
-    'ResourceClient',
     'StorageClient',
 ]

src/crawlee/storage_clients/_base/_dataset_client.py

@@ -76,7 +76,7 @@ async def open(
 
     @abstractmethod
     async def drop(self) -> None:
-        """Drop the dataset and all its data.
+        """Drop the whole dataset and remove all its items.
 
         The backend method for the `Dataset.drop` call.
         """

src/crawlee/storage_clients/_base/_storage_client.py

@@ -1,45 +1,16 @@
 from __future__ import annotations
 
-from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-from crawlee._utils.docs import docs_group
-
 if TYPE_CHECKING:
     from ._dataset_client import DatasetClient
     from ._key_value_store_client import KeyValueStoreClient
     from ._request_queue_client import RequestQueueClient
 
 
-@docs_group('Abstract classes')
-class StorageClient(ABC):
-    """Defines an abstract base for storage clients.
-
-    It offers interfaces to get subclients for interacting with storage resources like datasets, key-value stores,
-    and request queues.
-    """
-
-    @abstractmethod
-    def dataset(self) -> type[DatasetClient]:
-        """Get a dataset client class."""
-
-    @abstractmethod
-    def key_value_store(self) -> type[KeyValueStoreClient]:
-        """Get a key-value store client class."""
-
-    @abstractmethod
-    def request_queue(self) -> type[RequestQueueClient]:
-        """Get a request queue client class."""
-
-    def get_rate_limit_errors(self) -> dict[int, int]:
-        """Return statistics about rate limit errors encountered by the HTTP client in storage client."""
-        return {}
-
-    # @abstractmethod
-    # async def purge_on_start(self) -> None:
-    #     """Performs a purge of the default storages.
-
-    #     This method ensures that the purge is executed only once during the lifetime of the instance.
-    #     It is primarily used to clean up residual data from previous runs to maintain a clean state.
-    #     If the storage client does not support purging, leave it empty.
-    #     """
+@dataclass
+class StorageClient:
+    dataset_client_class: type[DatasetClient]
+    key_value_store_client_class: type[KeyValueStoreClient]
+    request_queue_client_class: type[RequestQueueClient]

src/crawlee/storage_clients/_base/_types.py

@@ -0,0 +1,3 @@
+from ._storage_client import file_system_storage_client
+
+__all__ = ['file_system_storage_client']