Utilize pathlib and use Config in constructors

Author: vdusek

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -660,14 +660,12 @@ async def _use_state(
         self,
         default_value: dict[str, JsonSerializable] | None = None,
     ) -> dict[str, JsonSerializable]:
-        kvs = await self.get_key_value_store()
-        # TODO:
-        # return some kvs value
+        # TODO: implement
+        return {}
 
     async def _save_crawler_state(self) -> None:
-        kvs = await self.get_key_value_store()
-        # TODO:
-        # some kvs call
+        pass
+        # TODO: implement
 
     async def get_data(
         self,
@@ -697,16 +695,16 @@ async def export_data(
         dataset_id: str | None = None,
         dataset_name: str | None = None,
     ) -> None:
-        """Export data from a `Dataset`.
+        """Export all items from a Dataset to a JSON or CSV file.
 
-        This helper method simplifies the process of exporting data from a `Dataset`. It opens the specified
-        one and then exports the data based on the provided parameters. If you need to pass options
-        specific to the output format, use the `export_data_csv` or `export_data_json` method instead.
+        This method simplifies the process of exporting data collected during crawling. It automatically
+        determines the export format based on the file extension (`.json` or `.csv`) and handles
+        the conversion of `Dataset` items to the appropriate format.
 
         Args:
-            path: The destination path.
-            dataset_id: The ID of the `Dataset`.
-            dataset_name: The name of the `Dataset`.
+            path: The destination file path. Must end with '.json' or '.csv'.
+            dataset_id: The ID of the Dataset to export from. If None, uses `name` parameter instead.
+            dataset_name: The name of the Dataset to export from. If None, uses `id` parameter instead.
         """
         dataset = await self.get_dataset(id=dataset_id, name=dataset_name)
 

src/crawlee/project_template/hooks/post_gen_project.py

@@ -2,7 +2,6 @@
 import subprocess
 from pathlib import Path
 
-
 # % if cookiecutter.package_manager in ['poetry', 'uv']
 Path('requirements.txt').unlink()
 
@@ -32,8 +31,9 @@
 
 # Install requirements and generate requirements.txt as an impromptu lockfile
 subprocess.check_call([str(path / 'pip'), 'install', '-r', 'requirements.txt'])
-with open('requirements.txt', 'w') as requirements_txt:
-    subprocess.check_call([str(path / 'pip'), 'freeze'], stdout=requirements_txt)
+Path('requirements.txt').write_text(
+    subprocess.check_output([str(path / 'pip'), 'freeze']).decode()
+)
 
 # % if cookiecutter.crawler_type == 'playwright'
 subprocess.check_call([str(path / 'playwright'), 'install'])

src/crawlee/storage_clients/_base/_dataset_client.py

@@ -7,9 +7,9 @@
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
-    from pathlib import Path
     from typing import Any
 
+    from crawlee.configuration import Configuration
     from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
 
@@ -37,9 +37,9 @@ def metadata(self) -> DatasetMetadata:
     async def open(
         cls,
         *,
-        id: str | None = None,
-        name: str | None = None,
-        storage_dir: Path | None = None,
+        id: str | None,
+        name: str | None,
+        configuration: Configuration,
     ) -> DatasetClient:
         """Open existing or create a new dataset client.
 
@@ -51,8 +51,7 @@ async def open(
         Args:
             id: The ID of the dataset. If not provided, an ID may be generated.
             name: The name of the dataset. If not provided a default name may be used.
-            storage_dir: The path to the storage directory. If the client persists data,
-                it should use this directory. May be ignored by non-persistent implementations.
+            configuration: The configuration object.
 
         Returns:
             A dataset client instance.

src/crawlee/storage_clients/_base/_key_value_store_client.py

@@ -7,8 +7,8 @@
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
-    from pathlib import Path
 
+    from crawlee.configuration import Configuration
     from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
 
@@ -36,9 +36,9 @@ def metadata(self) -> KeyValueStoreMetadata:
     async def open(
         cls,
         *,
-        id: str | None = None,
-        name: str | None = None,
-        storage_dir: Path | None = None,
+        id: str | None,
+        name: str | None,
+        configuration: Configuration,
     ) -> KeyValueStoreClient:
         """Open existing or create a new key-value store client.
 
@@ -51,8 +51,7 @@ async def open(
         Args:
             id: The ID of the key-value store. If not provided, an ID may be generated.
             name: The name of the key-value store. If not provided a default name may be used.
-            storage_dir: The path to the storage directory. If the client persists data,
-                it should use this directory. May be ignored by non-persistent implementations.
+            configuration: The configuration object.
 
         Returns:
             A key-value store client instance.

src/crawlee/storage_clients/_base/_request_queue_client.py

@@ -7,8 +7,8 @@
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
-    from pathlib import Path
 
+    from crawlee.configuration import Configuration
     from crawlee.storage_clients.models import (
         BatchRequestsOperationResponse,
         ProcessedRequest,
@@ -37,17 +37,16 @@ def metadata(self) -> RequestQueueMetadata:
     async def open(
         cls,
         *,
-        id: str | None = None,
-        name: str | None = None,
-        storage_dir: Path | None = None,
+        id: str | None,
+        name: str | None,
+        configuration: Configuration,
     ) -> RequestQueueClient:
         """Open a request queue client.
 
         Args:
             id: ID of the queue to open. If not provided, a new queue will be created with a random ID.
             name: Name of the queue to open. If not provided, the queue will be unnamed.
-            purge_on_start: If True, the queue will be purged before opening.
-            storage_dir: Directory to store the queue data in. If not provided, uses the default storage directory.
+            configuration: The configuration object.
 
         Returns:
             A request queue client.

src/crawlee/storage_clients/_base/_storage_client.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from pathlib import Path
+    from crawlee.configuration import Configuration
 
     from ._dataset_client import DatasetClient
     from ._key_value_store_client import KeyValueStoreClient
@@ -20,8 +20,7 @@ async def open_dataset_client(
         *,
         id: str | None = None,
         name: str | None = None,
-        purge_on_start: bool = True,
-        storage_dir: Path | None = None,
+        configuration: Configuration | None = None,
     ) -> DatasetClient:
         """Open a dataset client."""
 
@@ -31,8 +30,7 @@ async def open_key_value_store_client(
         *,
         id: str | None = None,
         name: str | None = None,
-        purge_on_start: bool = True,
-        storage_dir: Path | None = None,
+        configuration: Configuration | None = None,
     ) -> KeyValueStoreClient:
         """Open a key-value store client."""
 
@@ -42,7 +40,6 @@ async def open_request_queue_client(
         *,
         id: str | None = None,
         name: str | None = None,
-        purge_on_start: bool = True,
-        storage_dir: Path | None = None,
+        configuration: Configuration | None = None,
     ) -> RequestQueueClient:
         """Open a request queue client."""

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -21,6 +21,8 @@
     from collections.abc import AsyncIterator
     from typing import Any
 
+    from crawlee.configuration import Configuration
+
 logger = getLogger(__name__)
 
 
@@ -32,14 +34,11 @@ class FileSystemDatasetClient(DatasetClient):
     filename, allowing for easy ordering and pagination.
     """
 
-    _DEFAULT_NAME = 'default'
-    """The default name for the dataset when no name is provided."""
-
     _STORAGE_SUBDIR = 'datasets'
     """The name of the subdirectory where datasets are stored."""
 
-    _LOCAL_ENTRY_NAME_DIGITS = 9
-    """Number of digits used for the file names (e.g., 000000019.json)."""
+    _ITEM_FILENAME_DIGITS = 9
+    """Number of digits used for the dataset item file names (e.g., 000000019.json)."""
 
     _cache_by_name: ClassVar[dict[str, FileSystemDatasetClient]] = {}
     """A dictionary to cache clients by their names."""
@@ -72,7 +71,7 @@ def __init__(
 
         # Internal attributes
         self._lock = asyncio.Lock()
-        """A lock to ensure that only one file operation is performed at a time."""
+        """A lock to ensure that only one operation is performed at a time."""
 
     @override
     @property
@@ -94,24 +93,24 @@ def path_to_metadata(self) -> Path:
     async def open(
         cls,
         *,
-        id: str | None = None,
-        name: str | None = None,
-        storage_dir: Path | None = None,
+        id: str | None,
+        name: str | None,
+        configuration: Configuration,
     ) -> FileSystemDatasetClient:
         if id:
             raise ValueError(
                 'Opening a dataset by "id" is not supported for file system storage client, use "name" instead.'
             )
 
-        name = name or cls._DEFAULT_NAME
+        name = name or configuration.default_dataset_id
 
         # Check if the client is already cached by name.
         if name in cls._cache_by_name:
             client = cls._cache_by_name[name]
             await client._update_metadata(update_accessed_at=True)  # noqa: SLF001
             return client
 
-        storage_dir = storage_dir or Path.cwd()
+        storage_dir = Path(configuration.storage_dir)
         dataset_path = storage_dir / cls._STORAGE_SUBDIR / name
         metadata_path = dataset_path / METADATA_FILENAME
 
@@ -386,7 +385,7 @@ async def _push_item(self, item: dict[str, Any], item_id: int) -> None:
         # Acquire the lock to perform file operations safely.
         async with self._lock:
             # Generate the filename for the new item using zero-padded numbering.
-            filename = f'{str(item_id).zfill(self._LOCAL_ENTRY_NAME_DIGITS)}.json'
+            filename = f'{str(item_id).zfill(self._ITEM_FILENAME_DIGITS)}.json'
             file_path = self.path_to_dataset / filename
 
             # Ensure the dataset directory exists.

src/crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -21,6 +21,8 @@
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
 
+    from crawlee.configuration import Configuration
+
 
 logger = getLogger(__name__)
 
@@ -33,9 +35,6 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
     in an accompanying file.
     """
 
-    _DEFAULT_NAME = 'default'
-    """The default name for the unnamed key-value store."""
-
     _STORAGE_SUBDIR = 'key_value_stores'
     """The name of the subdirectory where key-value stores are stored."""
 
@@ -68,7 +67,7 @@ def __init__(
 
         # Internal attributes
         self._lock = asyncio.Lock()
-        """A lock to ensure that only one file operation is performed at a time."""
+        """A lock to ensure that only one operation is performed at a time."""
 
     @override
     @property
@@ -90,22 +89,24 @@ def path_to_metadata(self) -> Path:
     async def open(
         cls,
         *,
-        id: str | None = None,
-        name: str | None = None,
-        storage_dir: Path | None = None,
+        id: str | None,
+        name: str | None,
+        configuration: Configuration,
     ) -> FileSystemKeyValueStoreClient:
         if id:
             raise ValueError(
                 'Opening a key-value store by "id" is not supported for file system storage client, use "name" instead.'
             )
 
-        name = name or cls._DEFAULT_NAME
+        name = name or configuration.default_dataset_id
 
         # Check if the client is already cached by name.
         if name in cls._cache_by_name:
-            return cls._cache_by_name[name]
+            client = cls._cache_by_name[name]
+            await client._update_metadata(update_accessed_at=True)  # noqa: SLF001
+            return client
 
-        storage_dir = storage_dir or Path.cwd()
+        storage_dir = Path(configuration.storage_dir)
         kvs_path = storage_dir / cls._STORAGE_SUBDIR / name
         metadata_path = kvs_path / METADATA_FILENAME