From 2f75c54aaf709075c03e9d6c16bb0ebec36dfda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jan 2025 11:56:47 +0100 Subject: [PATCH] Add-on (#216) --- README.rst | 36 ++++++---- docs/api_reference.rst | 17 +++-- docs/index.rst | 4 +- docs/intro/advanced-tutorial.rst | 16 ----- docs/intro/install.rst | 51 -------------- docs/intro/setup.rst | 56 +++++++++++++++ example/example/settings.py | 16 +---- scrapy_poet/__init__.py | 1 + scrapy_poet/_addon.py | 103 ++++++++++++++++++++++++++++ scrapy_poet/utils/testing.py | 54 +++++++++++---- tests/conftest.py | 4 +- tests/test_cache.py | 4 +- tests/test_downloader.py | 6 +- tests/test_middleware.py | 30 +++++--- tests/test_providers.py | 2 +- tests/test_request_fingerprinter.py | 12 +++- tests/test_utils.py | 4 +- tests/test_web_poet_rules.py | 4 +- tox.ini | 2 +- 19 files changed, 286 insertions(+), 136 deletions(-) delete mode 100644 docs/intro/install.rst create mode 100644 docs/intro/setup.rst create mode 100644 scrapy_poet/_addon.py diff --git a/README.rst b/README.rst index 31dd67f7..c69fd9ba 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,8 @@ scrapy-poet With ``scrapy-poet`` is possible to make a single spider that supports many sites with different layouts. +Requires **Python 3.9+** and **Scrapy >= 2.6.0**. + Read the `documentation `_ for more information. License is BSD 3-clause. @@ -48,24 +50,32 @@ Installation pip install scrapy-poet -Requires **Python 3.9+** and **Scrapy >= 2.6.0**. - Usage in a Scrapy Project ========================= Add the following inside Scrapy's ``settings.py`` file: -.. code-block:: python - - DOWNLOADER_MIDDLEWARES = { - "scrapy_poet.InjectionMiddleware": 543, - "scrapy.downloadermiddlewares.stats.DownloaderStats": None, - "scrapy_poet.DownloaderStatsMiddleware": 850, - } - SPIDER_MIDDLEWARES = { - "scrapy_poet.RetryMiddleware": 275, - } - REQUEST_FINGERPRINTER_CLASS = "scrapy_poet.ScrapyPoetRequestFingerprinter" +- Scrapy ≥ 2.10: + + .. code-block:: python + + ADDONS = { + "scrapy_poet.Addon": 300, + } + +- Scrapy < 2.10: + + .. code-block:: python + + DOWNLOADER_MIDDLEWARES = { + "scrapy_poet.InjectionMiddleware": 543, + "scrapy.downloadermiddlewares.stats.DownloaderStats": None, + "scrapy_poet.DownloaderStatsMiddleware": 850, + } + REQUEST_FINGERPRINTER_CLASS = "scrapy_poet.ScrapyPoetRequestFingerprinter" + SPIDER_MIDDLEWARES = { + "scrapy_poet.RetryMiddleware": 275, + } Developing ========== diff --git a/docs/api_reference.rst b/docs/api_reference.rst index ce58b44e..4fc48294 100644 --- a/docs/api_reference.rst +++ b/docs/api_reference.rst @@ -11,13 +11,22 @@ API :members: :no-special-members: -Injection Middleware -==================== +Scrapy components +================= + +.. autoclass:: scrapy_poet.DownloaderStatsMiddleware + :members: + +.. autoclass:: scrapy_poet.InjectionMiddleware + :members: + +.. autoclass:: scrapy_poet.RetryMiddleware + :members: -.. automodule:: scrapy_poet.downloadermiddlewares +.. autoclass:: scrapy_poet.ScrapyPoetRequestFingerprinter :members: -Page Input Providers +Page input providers ==================== .. automodule:: scrapy_poet.page_input_providers diff --git a/docs/index.rst b/docs/index.rst index 1b022b83..4edc29b2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,7 @@ testability and reusability. Concrete integrations are not provided by ``web-poet``, but ``scrapy-poet`` makes them possbile. -To get started, see :ref:`intro-install` and :ref:`intro-tutorial`. +To get started, see :ref:`setup` and :ref:`intro-tutorial`. :ref:`license` is BSD 3-clause. @@ -34,7 +34,7 @@ To get started, see :ref:`intro-install` and :ref:`intro-tutorial`. :caption: Getting started :maxdepth: 1 - intro/install + intro/setup intro/basic-tutorial intro/advanced-tutorial intro/pitfalls diff --git a/docs/intro/advanced-tutorial.rst b/docs/intro/advanced-tutorial.rst index f4f83b43..5eff1fa6 100644 --- a/docs/intro/advanced-tutorial.rst +++ b/docs/intro/advanced-tutorial.rst @@ -77,14 +77,6 @@ It can be directly used inside the spider as: class ProductSpider(scrapy.Spider): - custom_settings = { - "DOWNLOADER_MIDDLEWARES": { - "scrapy_poet.InjectionMiddleware": 543, - "scrapy.downloadermiddlewares.stats.DownloaderStats": None, - "scrapy_poet.DownloaderStatsMiddleware": 850, - } - } - def start_requests(self): for url in [ "https://example.com/category/product/item?id=123", @@ -152,14 +144,6 @@ Let's see it in action: class ProductSpider(scrapy.Spider): - custom_settings = { - "DOWNLOADER_MIDDLEWARES": { - "scrapy_poet.InjectionMiddleware": 543, - "scrapy.downloadermiddlewares.stats.DownloaderStats": None, - "scrapy_poet.DownloaderStatsMiddleware": 850, - } - } - start_urls = [ "https://example.com/category/product/item?id=123", "https://example.com/category/product/item?id=989", diff --git a/docs/intro/install.rst b/docs/intro/install.rst deleted file mode 100644 index ed3f5b9a..00000000 --- a/docs/intro/install.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. _intro-install: - -============ -Installation -============ - -Installing scrapy-poet -====================== - -``scrapy-poet`` is a Scrapy extension that runs on Python 3.9 and above. - -If you’re already familiar with installation of Python packages, you can install -``scrapy-poet`` and its dependencies from PyPI with: - -:: - - pip install scrapy-poet - -Scrapy 2.6.0 or above is required and it has to be installed separately. - -Configuring the project -======================= - -To use ``scrapy-poet``, enable its middlewares in the ``settings.py`` file -of your Scrapy project: - -.. code-block:: python - - DOWNLOADER_MIDDLEWARES = { - "scrapy_poet.InjectionMiddleware": 543, - "scrapy.downloadermiddlewares.stats.DownloaderStats": None, - "scrapy_poet.DownloaderStatsMiddleware": 850, - } - SPIDER_MIDDLEWARES = { - "scrapy_poet.RetryMiddleware": 275, - } - REQUEST_FINGERPRINTER_CLASS = "scrapy_poet.ScrapyPoetRequestFingerprinter" - -Things that are good to know -============================ - -``scrapy-poet`` is written in pure Python and depends on a few key Python packages -(among others): - -- web-poet_, core library used for Page Object pattern -- andi_, provides annotation-based dependency injection -- parsel_, responsible for css and xpath selectors - -.. _web-poet: https://github.com/scrapinghub/web-poet -.. _andi: https://github.com/scrapinghub/andi -.. _parsel: https://github.com/scrapy/parsel diff --git a/docs/intro/setup.rst b/docs/intro/setup.rst new file mode 100644 index 00000000..24c3aece --- /dev/null +++ b/docs/intro/setup.rst @@ -0,0 +1,56 @@ +.. _setup: + +===== +Setup +===== + +.. _intro-install: + +Install from PyPI:: + + pip install scrapy-poet + +Then configure: + +- For Scrapy ≥ 2.10, install the add-on: + + .. code-block:: python + :caption: settings.py + + ADDONS = { + "scrapy_poet.Addon": 300, + } + + .. _addon-changes: + + This is what the add-on changes: + + - In :setting:`DOWNLOADER_MIDDLEWARES`: + + - Sets :class:`~scrapy_poet.InjectionMiddleware` with value ``543``. + + - Replaces + :class:`scrapy.downloadermiddlewares.stats.DownloaderStats` + with :class:`scrapy_poet.DownloaderStatsMiddleware`. + + - Sets :setting:`REQUEST_FINGERPRINTER_CLASS` to + :class:`~scrapy_poet.ScrapyPoetRequestFingerprinter`. + + - In :setting:`SPIDER_MIDDLEWARES`, sets + :class:`~scrapy_poet.RetryMiddleware` with value ``275``. + +- For Scrapy < 2.10, manually apply :ref:`the add-on changes + `. For example: + + .. code-block:: python + :caption: settings.py + + DOWNLOADER_MIDDLEWARES = { + "scrapy_poet.InjectionMiddleware": 543, + "scrapy.downloadermiddlewares.stats.DownloaderStats": None, + "scrapy_poet.DownloaderStatsMiddleware": 850, + } + REQUEST_FINGERPRINTER_CLASS = "scrapy_poet.ScrapyPoetRequestFingerprinter" + SPIDER_MIDDLEWARES = { + "scrapy_poet.RetryMiddleware": 275, + } diff --git a/example/example/settings.py b/example/example/settings.py index a8e9ef63..117cde8d 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -9,26 +9,16 @@ from example.autoextract import AutoextractProductProvider -from scrapy_poet import ScrapyPoetRequestFingerprinter - BOT_NAME = "example" SPIDER_MODULES = ["example.spiders"] NEWSPIDER_MODULE = "example.spiders" -SCRAPY_POET_PROVIDERS = {AutoextractProductProvider: 500} - # Obey robots.txt rules ROBOTSTXT_OBEY = True -DOWNLOADER_MIDDLEWARES = { - "scrapy_poet.InjectionMiddleware": 543, - "scrapy.downloadermiddlewares.stats.DownloaderStats": None, - "scrapy_poet.DownloaderStatsMiddleware": 850, +ADDONS = { + "scrapy_poet.Addon": 300, } -REQUEST_FINGERPRINTER_CLASS = ScrapyPoetRequestFingerprinter - -SPIDER_MIDDLEWARES = { - "scrapy_poet.RetryMiddleware": 275, -} +SCRAPY_POET_PROVIDERS = {AutoextractProductProvider: 500} diff --git a/scrapy_poet/__init__.py b/scrapy_poet/__init__.py index 393fd303..63bde65a 100644 --- a/scrapy_poet/__init__.py +++ b/scrapy_poet/__init__.py @@ -4,3 +4,4 @@ from .page_input_providers import HttpResponseProvider, PageObjectInputProvider from .spidermiddlewares import RetryMiddleware from ._request_fingerprinter import ScrapyPoetRequestFingerprinter +from ._addon import Addon diff --git a/scrapy_poet/_addon.py b/scrapy_poet/_addon.py new file mode 100644 index 00000000..b75de290 --- /dev/null +++ b/scrapy_poet/_addon.py @@ -0,0 +1,103 @@ +from logging import getLogger + +from scrapy.downloadermiddlewares.stats import DownloaderStats +from scrapy.settings import BaseSettings +from scrapy.utils.misc import load_object + +from ._request_fingerprinter import ScrapyPoetRequestFingerprinter +from .downloadermiddlewares import DownloaderStatsMiddleware, InjectionMiddleware +from .spidermiddlewares import RetryMiddleware + +logger = getLogger(__name__) + + +# https://github.com/zytedata/zyte-spider-templates/blob/1b72aa8912f6009d43bf87a5bd1920537d458744/zyte_spider_templates/_addon.py#L33C1-L88C37 +def _replace_builtin( + settings: BaseSettings, setting: str, builtin_cls: type, new_cls: type +) -> None: + setting_value = settings[setting] + if not setting_value: + logger.warning( + f"Setting {setting!r} is empty. Could not replace the built-in " + f"{builtin_cls} entry with {new_cls}. Add {new_cls} manually to " + f"silence this warning." + ) + return None + + if new_cls in setting_value: + return None + for cls_or_path in setting_value: + if isinstance(cls_or_path, str): + _cls = load_object(cls_or_path) + if _cls == new_cls: + return None + + builtin_entry: object = None + for _setting_value in (setting_value, settings[f"{setting}_BASE"]): + if builtin_cls in _setting_value: + builtin_entry = builtin_cls + pos = _setting_value[builtin_entry] + break + for cls_or_path in _setting_value: + if isinstance(cls_or_path, str): + _cls = load_object(cls_or_path) + if _cls == builtin_cls: + builtin_entry = cls_or_path + pos = _setting_value[builtin_entry] + break + if builtin_entry: + break + + if not builtin_entry: + logger.warning( + f"Settings {setting!r} and {setting + '_BASE'!r} are both " + f"missing built-in entry {builtin_cls}. Cannot replace it with {new_cls}. " + f"Add {new_cls} manually to silence this warning." + ) + return None + + if pos is None: + logger.warning( + f"Built-in entry {builtin_cls} of setting {setting!r} is disabled " + f"(None). Cannot replace it with {new_cls}. Add {new_cls} " + f"manually to silence this warning. If you had replaced " + f"{builtin_cls} with some other entry, you might also need to " + f"disable that other entry for things to work as expected." + ) + return + + settings[setting][builtin_entry] = None + settings[setting][new_cls] = pos + + +# https://github.com/scrapy-plugins/scrapy-zyte-api/blob/a1d81d11854b420248f38e7db49c685a8d46d943/scrapy_zyte_api/addon.py#L12 +def _setdefault(settings, setting, cls, pos): + setting_value = settings[setting] + if not setting_value: + settings[setting] = {cls: pos} + return + if cls in setting_value: + return + for cls_or_path in setting_value: + if isinstance(cls_or_path, str): + _cls = load_object(cls_or_path) + if _cls == cls: + return + settings[setting][cls] = pos + + +class Addon: + def update_settings(self, settings: BaseSettings) -> None: + settings.set( + "REQUEST_FINGERPRINTER_CLASS", + ScrapyPoetRequestFingerprinter, + priority="addon", + ) + _setdefault(settings, "DOWNLOADER_MIDDLEWARES", InjectionMiddleware, 543) + _setdefault(settings, "SPIDER_MIDDLEWARES", RetryMiddleware, 275) + _replace_builtin( + settings, + "DOWNLOADER_MIDDLEWARES", + DownloaderStats, + DownloaderStatsMiddleware, + ) diff --git a/scrapy_poet/utils/testing.py b/scrapy_poet/utils/testing.py index 788128a9..345718f9 100644 --- a/scrapy_poet/utils/testing.py +++ b/scrapy_poet/utils/testing.py @@ -1,6 +1,7 @@ import json from inspect import isasyncgenfunction from typing import Dict +from warnings import warn from scrapy import Spider, signals from scrapy.crawler import Crawler @@ -159,10 +160,10 @@ def get_download_handler(crawler, schema): def make_crawler(spider_cls, settings=None): settings = settings or {} if isinstance(settings, dict): - _settings = create_scrapy_settings() + _settings = _get_test_settings() _settings.update(settings) else: - _settings = create_scrapy_settings() + _settings = _get_test_settings() for k, v in dict(settings).items(): _settings.set(k, v, priority=settings.getpriority(k)) settings = _settings @@ -186,6 +187,11 @@ def setup_crawler_engine(crawler: Crawler): crawler.crawling = True crawler.spider = crawler._create_spider() + crawler.settings.frozen = False + try: + crawler._apply_settings() + except AttributeError: + pass # Scrapy < 2.10 crawler.engine = crawler._create_engine() handler = get_download_handler(crawler, "https") @@ -229,26 +235,46 @@ def process_response(self, request, response, spider): return response -def create_scrapy_settings(): - """Default scrapy-poet settings""" - s = dict( +def _get_test_settings(): + settings = { # collect scraped items to crawler.spider.collected_items - ITEM_PIPELINES={ + "ITEM_PIPELINES": { CollectorPipeline: 100, }, - DOWNLOADER_MIDDLEWARES={ + "DOWNLOADER_MIDDLEWARES": { # collect injected dependencies to crawler.spider.collected_response_deps InjectedDependenciesCollectorMiddleware: 542, - "scrapy_poet.InjectionMiddleware": 543, - "scrapy.downloadermiddlewares.stats.DownloaderStats": None, - "scrapy_poet.DownloaderStatsMiddleware": 850, }, - REQUEST_FINGERPRINTER_CLASS=ScrapyPoetRequestFingerprinter, - SPIDER_MIDDLEWARES={ + } + try: + import scrapy.addons # noqa: F401 + except ImportError: + settings["DOWNLOADER_MIDDLEWARES"]["scrapy_poet.InjectionMiddleware"] = 543 + settings["DOWNLOADER_MIDDLEWARES"][ + "scrapy.downloadermiddlewares.stats.DownloaderStats" + ] = None + settings["DOWNLOADER_MIDDLEWARES"][ + "scrapy_poet.DownloaderStatsMiddleware" + ] = 850 + settings["REQUEST_FINGERPRINTER_CLASS"] = ScrapyPoetRequestFingerprinter + settings["SPIDER_MIDDLEWARES"] = { "scrapy_poet.RetryMiddleware": 275, - }, + } + else: + settings["ADDONS"] = { + "scrapy_poet.Addon": 300, + } + return settings + + +def create_scrapy_settings(): + """Return the default scrapy-poet settings.""" + warn( + "The scrapy_poet.utils.create_scrapy_settings() function is deprecated.", + DeprecationWarning, + stacklevel=2, ) - return Settings(s) + return Settings(_get_test_settings()) def capture_exceptions(callback): diff --git a/tests/conftest.py b/tests/conftest.py index af670a87..b66fd233 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,8 @@ import pytest -from scrapy_poet.utils.testing import create_scrapy_settings +from scrapy_poet.utils.testing import _get_test_settings @pytest.fixture() def settings(): - return create_scrapy_settings() + return _get_test_settings() diff --git a/tests/test_cache.py b/tests/test_cache.py index 93c9011a..5248b7ac 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -5,7 +5,7 @@ from web_poet import WebPage, field from scrapy_poet.utils.mockserver import MockServer -from scrapy_poet.utils.testing import EchoResource, create_scrapy_settings, make_crawler +from scrapy_poet.utils.testing import EchoResource, _get_test_settings, make_crawler @inlineCallbacks @@ -22,7 +22,7 @@ class CacheSpider(Spider): name = "cache" custom_settings = { - **create_scrapy_settings(), + **_get_test_settings(), "SCRAPY_POET_CACHE": cache_dir, } diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 45e86b66..7e2630f1 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -32,7 +32,7 @@ DelayedResource, EchoResource, StatusResource, - create_scrapy_settings, + _get_test_settings, make_crawler, ) @@ -284,7 +284,7 @@ async def parse(self, response, page: ItemPage): item = await page.to_item() items.append(item) - settings = create_scrapy_settings() + settings = _get_test_settings() settings["DOWNLOADER_MIDDLEWARES"][TestDownloaderMiddleware] = 1 crawler = make_crawler(TestSpider, settings) yield crawler.crawl() @@ -338,7 +338,7 @@ async def parse(self, response, page: ItemPage): item = await page.to_item() items.append(item) - settings = create_scrapy_settings() + settings = _get_test_settings() settings["DOWNLOADER_MIDDLEWARES"][TestDownloaderMiddleware] = 1 crawler = make_crawler(TestSpider) yield crawler.crawl() diff --git a/tests/test_middleware.py b/tests/test_middleware.py index aa85536b..9bcd5966 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,5 +1,6 @@ import socket from pathlib import Path +from textwrap import dedent from typing import Optional, Type, Union import attr @@ -445,15 +446,28 @@ def test_skip_download_request_url_page(settings): @inlineCallbacks def test_scrapy_shell(tmp_path): - Path(tmp_path, "settings.py").write_text( + try: + import scrapy.addons # noqa: F401 + except ImportError: + settings = """ + DOWNLOADER_MIDDLEWARES = { + "scrapy_poet.InjectionMiddleware": 543, + "scrapy.downloadermiddlewares.stats.DownloaderStats": None, + "scrapy_poet.DownloaderStatsMiddleware": 850, + } + REQUEST_FINGERPRINTER_CLASS = "scrapy_poet.ScrapyPoetRequestFingerprinter" + SPIDER_MIDDLEWARES = { + "scrapy_poet.RetryMiddleware": 275, + } """ -DOWNLOADER_MIDDLEWARES = { - "scrapy_poet.InjectionMiddleware": 543, - "scrapy.downloadermiddlewares.stats.DownloaderStats": None, - "scrapy_poet.DownloaderStatsMiddleware": 850, -} -""" - ) + else: + settings = """ + ADDONS = { + "scrapy_poet.Addon": 300, + } + """ + settings = dedent(settings) + Path(tmp_path, "settings.py").write_text(settings) pt = ProcessTest() pt.command = "shell" pt.cwd = tmp_path diff --git a/tests/test_providers.py b/tests/test_providers.py index d60e21d0..e8ff7808 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -158,7 +158,7 @@ class NameFirstMultiProviderSpider(PriceFirstMultiProviderSpider): def test_name_first_spider(settings, tmp_path): port = get_ephemeral_port() cache = tmp_path / "cache" - settings.set("SCRAPY_POET_CACHE", str(cache)) + settings["SCRAPY_POET_CACHE"] = str(cache) item, _, _ = yield crawl_single_item( NameFirstMultiProviderSpider, ProductHtml, settings, port=port ) diff --git a/tests/test_request_fingerprinter.py b/tests/test_request_fingerprinter.py index 961a4142..d39ceccc 100644 --- a/tests/test_request_fingerprinter.py +++ b/tests/test_request_fingerprinter.py @@ -37,12 +37,19 @@ from scrapy_poet.downloadermiddlewares import DEFAULT_PROVIDERS from scrapy_poet.injection import Injector, is_class_provided_by_any_provider_fn from scrapy_poet.page_input_providers import PageObjectInputProvider -from scrapy_poet.utils.testing import create_scrapy_settings +from scrapy_poet.utils.testing import _get_test_settings from scrapy_poet.utils.testing import get_crawler as _get_crawler ANDI_VERSION = Version(package_version("andi")) -SETTINGS = create_scrapy_settings() +SETTINGS = _get_test_settings() + +try: + import scrapy.addons # noqa: F401 +except ImportError: + ADDON_SUPPORT = False +else: + ADDON_SUPPORT = True def get_crawler(spider_cls=None, settings=None, ensure_providers_for=None): @@ -452,6 +459,7 @@ def fingerprint(self, request): assert fingerprinter.fingerprint(request) != b"foo" +@pytest.mark.skipif(ADDON_SUPPORT, reason="Using the add-on") def test_missing_middleware(): settings = {**SETTINGS, "DOWNLOADER_MIDDLEWARES": {}} crawler = get_crawler(settings=settings) diff --git a/tests/test_utils.py b/tests/test_utils.py index 27fa4222..e4e5ff54 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -259,7 +259,7 @@ def test_http_response_to_scrapy_response(scrapy_response, http_response): @mock.patch("scrapy_poet.utils.consume_modules") def test_create_registry_instance_SCRAPY_POET_DISCOVER(mock_consume_modules, settings): - settings.set("SCRAPY_POET_RULES", []) + settings["SCRAPY_POET_RULES"] = [] mock_cls = mock.Mock() fake_crawler = get_crawler(Spider, settings) @@ -268,7 +268,7 @@ def test_create_registry_instance_SCRAPY_POET_DISCOVER(mock_consume_modules, set mock_cls.assert_called_once_with(rules=[]) mock_cls = mock.Mock() - settings.set("SCRAPY_POET_DISCOVER", ["a.b.c", "x.y"]) + settings["SCRAPY_POET_DISCOVER"] = ["a.b.c", "x.y"] fake_crawler = get_crawler(Spider, settings) create_registry_instance(mock_cls, fake_crawler) assert mock_consume_modules.call_args_list == [mock.call("a.b.c"), mock.call("x.y")] diff --git a/tests/test_web_poet_rules.py b/tests/test_web_poet_rules.py index 0697c556..608a36cc 100644 --- a/tests/test_web_poet_rules.py +++ b/tests/test_web_poet_rules.py @@ -36,9 +36,9 @@ from scrapy_poet.utils import is_min_scrapy_version from scrapy_poet.utils.mockserver import get_ephemeral_port from scrapy_poet.utils.testing import ( + _get_test_settings, capture_exceptions, crawl_single_item, - create_scrapy_settings, ) from tests.test_middleware import ProductHtml @@ -48,7 +48,7 @@ def rules_settings() -> dict: - settings = create_scrapy_settings() + settings = _get_test_settings() settings["SCRAPY_POET_RULES"] = default_registry.get_rules() return settings diff --git a/tox.ini b/tox.ini index 8487e181..76bf3aac 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = min,py39,py310,py311,py312,py313,asyncio,asyncio-min,mypy,docs +envlist = min,pinned-scrapy-2x7,pinned-scrapy-2x8,py39,py310,py311,py312,py313,asyncio,asyncio-min,mypy,docs [testenv] deps =