Init of request queue and its clients

Author: vdusek

src/crawlee/storage_clients/_base/_dataset_client.py

@@ -7,11 +7,10 @@
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
-    from datetime import datetime
     from pathlib import Path
     from typing import Any
 
-    from crawlee.storage_clients.models import DatasetItemsListPage
+    from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
 
 @docs_group('Abstract classes')
@@ -30,33 +29,8 @@ class DatasetClient(ABC):
 
     @property
     @abstractmethod
-    def id(self) -> str:
-        """The ID of the dataet, a unique identifier, typically a UUID or similar value."""
-
-    @property
-    @abstractmethod
-    def name(self) -> str | None:
-        """The optional human-readable name of the dataset."""
-
-    @property
-    @abstractmethod
-    def created_at(self) -> datetime:
-        """Timestamp when the dataset was first created, remains unchanged."""
-
-    @property
-    @abstractmethod
-    def accessed_at(self) -> datetime:
-        """Timestamp of last access to the dataset, updated on read or write operations."""
-
-    @property
-    @abstractmethod
-    def modified_at(self) -> datetime:
-        """Timestamp of last modification of the dataset, updated when new data are added."""
-
-    @property
-    @abstractmethod
-    def item_count(self) -> int:
-        """Total count of data items stored in the dataset."""
+    def metadata(self) -> DatasetMetadata:
+        """The metadata of the dataset."""
 
     @classmethod
     @abstractmethod

src/crawlee/storage_clients/_base/_key_value_store_client.py

@@ -7,10 +7,9 @@
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
-    from datetime import datetime
     from pathlib import Path
 
-    from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata
+    from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
 
 @docs_group('Abstract classes')
@@ -29,28 +28,8 @@ class KeyValueStoreClient(ABC):
 
     @property
     @abstractmethod
-    def id(self) -> str:
-        """The unique identifier of the key-value store (typically a UUID)."""
-
-    @property
-    @abstractmethod
-    def name(self) -> str | None:
-        """The optional human-readable name for the KVS."""
-
-    @property
-    @abstractmethod
-    def created_at(self) -> datetime:
-        """Timestamp when the KVS was first created, remains unchanged."""
-
-    @property
-    @abstractmethod
-    def accessed_at(self) -> datetime:
-        """Timestamp of last access to the KVS, updated on read or write operations."""
-
-    @property
-    @abstractmethod
-    def modified_at(self) -> datetime:
-        """Timestamp of last modification of the KVS, updated when new data are added, updated or deleted."""
+    def metadata(self) -> KeyValueStoreMetadata:
+        """The metadata of the key-value store."""
 
     @classmethod
     @abstractmethod

src/crawlee/storage_clients/_base/_request_queue_client.py

@@ -1,22 +1,21 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from datetime import datetime
 from typing import TYPE_CHECKING
 
 from crawlee._utils.docs import docs_group
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
-    from datetime import datetime
+    from pathlib import Path
 
     from crawlee.storage_clients.models import (
         BatchRequestsOperationResponse,
         ProcessedRequest,
         ProlongRequestLockResponse,
         Request,
-        RequestQueueHead,
         RequestQueueHeadWithLocks,
+        RequestQueueMetadata,
     )
 
 
@@ -30,58 +29,29 @@ class RequestQueueClient(ABC):
 
     @property
     @abstractmethod
-    def id(self) -> str:
-        """The ID of the dataset."""
+    def metadata(self) -> RequestQueueMetadata:
+        """The metadata of the request queue."""
 
-    @property
-    @abstractmethod
-    def name(self) -> str | None:
-        """The name of the dataset."""
-
-    @property
-    @abstractmethod
-    def created_at(self) -> datetime:
-        """The time at which the dataset was created."""
-
-    @property
-    @abstractmethod
-    def accessed_at(self) -> datetime:
-        """The time at which the dataset was last accessed."""
-
-    @property
-    @abstractmethod
-    def modified_at(self) -> datetime:
-        """The time at which the dataset was last modified."""
-
-    @property
+    @classmethod
     @abstractmethod
-    def had_multiple_clients(self) -> bool:
-        """TODO."""
-
-    @property
-    @abstractmethod
-    def handled_request_count(self) -> int:
-        """TODO."""
-
-    @property
-    @abstractmethod
-    def pending_request_count(self) -> int:
-        """TODO."""
-
-    @property
-    @abstractmethod
-    def stats(self) -> dict:
-        """TODO."""
+    async def open(
+        cls,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        storage_dir: Path | None = None,
+    ) -> RequestQueueClient:
+        """Open a request queue client.
 
-    @property
-    @abstractmethod
-    def total_request_count(self) -> int:
-        """TODO."""
+        Args:
+            id: ID of the queue to open. If not provided, a new queue will be created with a random ID.
+            name: Name of the queue to open. If not provided, the queue will be unnamed.
+            purge_on_start: If True, the queue will be purged before opening.
+            storage_dir: Directory to store the queue data in. If not provided, uses the default storage directory.
 
-    @property
-    @abstractmethod
-    def resource_directory(self) -> str:
-        """TODO."""
+        Returns:
+            A request queue client.
+        """
 
     @abstractmethod
     async def drop(self) -> None:
@@ -90,17 +60,6 @@ async def drop(self) -> None:
         The backend method for the `RequestQueue.drop` call.
         """
 
-    @abstractmethod
-    async def list_head(self, *, limit: int | None = None) -> RequestQueueHead:
-        """Retrieve a given number of requests from the beginning of the queue.
-
-        Args:
-            limit: How many requests to retrieve.
-
-        Returns:
-            The desired number of requests from the beginning of the queue.
-        """
-
     @abstractmethod
     async def list_and_lock_head(self, *, lock_secs: int, limit: int | None = None) -> RequestQueueHeadWithLocks:
         """Fetch and lock a specified number of requests from the start of the queue.
@@ -117,33 +76,16 @@ async def list_and_lock_head(self, *, lock_secs: int, limit: int | None = None)
         """
 
     @abstractmethod
-    async def add_request(
-        self,
-        request: Request,
-        *,
-        forefront: bool = False,
-    ) -> ProcessedRequest:
-        """Add a request to the queue.
-
-        Args:
-            request: The request to add to the queue.
-            forefront: Whether to add the request to the head or the end of the queue.
-
-        Returns:
-            Request queue operation information.
-        """
-
-    @abstractmethod
-    async def batch_add_requests(
+    async def add_requests_batch(
         self,
         requests: Sequence[Request],
         *,
         forefront: bool = False,
     ) -> BatchRequestsOperationResponse:
-        """Add a batch of requests to the queue.
+        """Add a requests to the queue in batches.
 
         Args:
-            requests: The requests to add to the queue.
+            requests: The batch of requests to add to the queue.
             forefront: Whether to add the requests to the head or the end of the queue.
 
         Returns:
@@ -187,7 +129,7 @@ async def delete_request(self, request_id: str) -> None:
         """
 
     @abstractmethod
-    async def batch_delete_requests(self, requests: list[Request]) -> BatchRequestsOperationResponse:
+    async def delete_requests_batch(self, requests: list[Request]) -> BatchRequestsOperationResponse:
         """Delete given requests from the queue.
 
         Args:

src/crawlee/storage_clients/_base/_storage_client.py

@@ -23,7 +23,7 @@ async def open_dataset_client(
         purge_on_start: bool = True,
         storage_dir: Path | None = None,
     ) -> DatasetClient:
-        """Open the dataset client."""
+        """Open a dataset client."""
 
     @abstractmethod
     async def open_key_value_store_client(
@@ -34,7 +34,7 @@ async def open_key_value_store_client(
         purge_on_start: bool = True,
         storage_dir: Path | None = None,
     ) -> KeyValueStoreClient:
-        """Open the key-value store client."""
+        """Open a key-value store client."""
 
     @abstractmethod
     async def open_request_queue_client(
@@ -45,4 +45,4 @@ async def open_request_queue_client(
         purge_on_start: bool = True,
         storage_dir: Path | None = None,
     ) -> RequestQueueClient:
-        """Open the request queue client."""
+        """Open a request queue client."""

src/crawlee/storage_clients/_file_system/__init__.py

@@ -1,3 +1,11 @@
+from ._dataset_client import FileSystemDatasetClient
+from ._key_value_store_client import FileSystemKeyValueStoreClient
+from ._request_queue_client import FileSystemRequestQueueClient
 from ._storage_client import FileSystemStorageClient
 
-__all__ = ['FileSystemStorageClient']
+__all__ = [
+    'FileSystemDatasetClient',
+    'FileSystemKeyValueStoreClient',
+    'FileSystemRequestQueueClient',
+    'FileSystemStorageClient',
+]

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -76,38 +76,13 @@ def __init__(
 
     @override
     @property
-    def id(self) -> str:
-        return self._metadata.id
-
-    @override
-    @property
-    def name(self) -> str:
-        return self._metadata.name
-
-    @override
-    @property
-    def created_at(self) -> datetime:
-        return self._metadata.created_at
-
-    @override
-    @property
-    def accessed_at(self) -> datetime:
-        return self._metadata.accessed_at
-
-    @override
-    @property
-    def modified_at(self) -> datetime:
-        return self._metadata.modified_at
-
-    @override
-    @property
-    def item_count(self) -> int:
-        return self._metadata.item_count
+    def metadata(self) -> DatasetMetadata:
+        return self._metadata
 
     @property
     def path_to_dataset(self) -> Path:
         """The full path to the dataset directory."""
-        return self._storage_dir / self._STORAGE_SUBDIR / self.name
+        return self._storage_dir / self._STORAGE_SUBDIR / self.metadata.name
 
     @property
     def path_to_metadata(self) -> Path:
@@ -170,12 +145,13 @@ async def open(
 
         # Otherwise, create a new dataset client.
         else:
+            now = datetime.now(timezone.utc)
             client = cls(
                 id=crypto_random_object_id(),
                 name=name,
-                created_at=datetime.now(timezone.utc),
-                accessed_at=datetime.now(timezone.utc),
-                modified_at=datetime.now(timezone.utc),
+                created_at=now,
+                accessed_at=now,
+                modified_at=now,
                 item_count=0,
                 storage_dir=storage_dir,
             )
@@ -194,12 +170,12 @@ async def drop(self) -> None:
                 await asyncio.to_thread(shutil.rmtree, self.path_to_dataset)
 
         # Remove the client from the cache.
-        if self.name in self.__class__._cache_by_name:  # noqa: SLF001
-            del self.__class__._cache_by_name[self.name]  # noqa: SLF001
+        if self.metadata.name in self.__class__._cache_by_name:  # noqa: SLF001
+            del self.__class__._cache_by_name[self.metadata.name]  # noqa: SLF001
 
     @override
     async def push_data(self, data: list[Any] | dict[str, Any]) -> None:
-        new_item_count = self.item_count
+        new_item_count = self.metadata.item_count
 
         # If data is a list, push each item individually.
         if isinstance(data, list):