diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 35aca3ed..820c29d6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af983264..5d202cbb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,20 +16,20 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.8" + - python-version: "3.9" toxenv: "min" - - python-version: "3.8" + - python-version: "3.9" toxenv: "pinned-scrapy-2x7" - - python-version: "3.8" + - python-version: "3.9" toxenv: "pinned-scrapy-2x8" - - python-version: "3.8" + - python-version: "3.9" toxenv: "asyncio-min" - - python-version: "3.8" - python-version: "3.9" - python-version: "3.10" - python-version: "3.11" - python-version: "3.12" - - python-version: "3.12" + - python-version: "3.13" + - python-version: "3.13" toxenv: "asyncio" steps: @@ -54,7 +54,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.12'] + python-version: ['3.12'] # Keep in sync with .readthedocs.yml tox-job: ["mypy", "docs", "linters", "twinecheck"] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99c37e9a..5aa77fbf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,12 +3,12 @@ repos: - id: black language_version: python3 repo: https://github.com/ambv/black - rev: 22.12.0 + rev: 24.10.0 - hooks: - id: isort language_version: python3 repo: https://github.com/PyCQA/isort - rev: 5.11.5 + rev: 5.13.2 - hooks: - id: flake8 language_version: python3 @@ -19,4 +19,4 @@ repos: - flake8-docstrings - flake8-string-format repo: https://github.com/pycqa/flake8 - rev: 6.1.0 + rev: 7.1.1 diff --git a/.readthedocs.yml b/.readthedocs.yml index 9961cf58..5ff2e5ec 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,7 +6,7 @@ sphinx: build: os: ubuntu-22.04 tools: - python: "3.12" # Keep in sync with .github/workflows/tests.yml + python: "3.12" # Keep in sync with .github/workflows/test.yml python: install: diff --git a/MANIFEST.in b/MANIFEST.in index dab18b23..3a5db1a4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include CHANGES.rst +include CHANGELOG.rst include LICENSE include README.rst diff --git a/README.rst b/README.rst index c5537dc6..31dd67f7 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ Installation pip install scrapy-poet -Requires **Python 3.8+** and **Scrapy >= 2.6.0**. +Requires **Python 3.9+** and **Scrapy >= 2.6.0**. Usage in a Scrapy Project ========================= diff --git a/docs/intro/install.rst b/docs/intro/install.rst index b1601b60..ed3f5b9a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,7 @@ Installation Installing scrapy-poet ====================== -``scrapy-poet`` is a Scrapy extension that runs on Python 3.8 and above. +``scrapy-poet`` is a Scrapy extension that runs on Python 3.9 and above. If you’re already familiar with installation of Python packages, you can install ``scrapy-poet`` and its dependencies from PyPI with: diff --git a/docs/providers.rst b/docs/providers.rst index 53bd6c61..338acbf0 100644 --- a/docs/providers.rst +++ b/docs/providers.rst @@ -327,8 +327,6 @@ you could implement those limits in the library itself. Attaching metadata to dependencies ================================== -.. note:: This feature requires Python 3.9+. - Providers can support dependencies with arbitrary metadata attached and use that metadata when creating them. Attaching the metadata is done by wrapping the dependency class in :data:`typing.Annotated`: diff --git a/example/example/autoextract.py b/example/example/autoextract.py index 49cff130..816056cd 100644 --- a/example/example/autoextract.py +++ b/example/example/autoextract.py @@ -2,6 +2,7 @@ Example of how to create a PageObject with a very different input data, which even requires an API request. """ + from typing import Any, Dict import attr diff --git a/example/example/spiders/books_01.py b/example/example/spiders/books_01.py index a7d65039..474607ff 100644 --- a/example/example/spiders/books_01.py +++ b/example/example/spiders/books_01.py @@ -1,6 +1,7 @@ """ Baseline: regular Scrapy spider, sweet & easy. """ + import scrapy diff --git a/example/example/spiders/books_02.py b/example/example/spiders/books_02.py index 06867076..8966ef88 100644 --- a/example/example/spiders/books_02.py +++ b/example/example/spiders/books_02.py @@ -2,6 +2,7 @@ Scrapy spider which uses Page Objects to make extraction code more reusable. BookPage is now independent of Scrapy. """ + import scrapy from web_poet import WebPage diff --git a/example/example/spiders/books_02_1.py b/example/example/spiders/books_02_1.py index b6835512..6a71cc22 100644 --- a/example/example/spiders/books_02_1.py +++ b/example/example/spiders/books_02_1.py @@ -3,6 +3,7 @@ BookPage is now independent of Scrapy. callback_for is used to reduce boilerplate. """ + import scrapy from web_poet import WebPage diff --git a/example/example/spiders/books_02_2.py b/example/example/spiders/books_02_2.py index a81960c5..2841a050 100644 --- a/example/example/spiders/books_02_2.py +++ b/example/example/spiders/books_02_2.py @@ -10,6 +10,7 @@ has problems now, it is used in the latter examples, because as an API it is better than defining callback explicitly. """ + import scrapy from web_poet import WebPage diff --git a/example/example/spiders/books_02_3.py b/example/example/spiders/books_02_3.py index 66bc7e76..8d3ccac6 100644 --- a/example/example/spiders/books_02_3.py +++ b/example/example/spiders/books_02_3.py @@ -7,6 +7,7 @@ Page object is used instead of callback below. It doesn't work now, but it can be implemented, with Scrapy support. """ + import scrapy from web_poet import WebPage diff --git a/example/example/spiders/books_03.py b/example/example/spiders/books_03.py index 61efb4f7..274eb211 100644 --- a/example/example/spiders/books_03.py +++ b/example/example/spiders/books_03.py @@ -1,6 +1,7 @@ """ Scrapy spider which uses AutoExtract API, to extract books as products. """ + import scrapy from example.autoextract import ProductPage diff --git a/example/example/spiders/books_04.py b/example/example/spiders/books_04.py index 1171aeb8..84c4e42f 100644 --- a/example/example/spiders/books_04.py +++ b/example/example/spiders/books_04.py @@ -1,6 +1,7 @@ """ Scrapy spider which uses Page Objects both for crawling and extraction. """ + import scrapy from web_poet import WebPage diff --git a/example/example/spiders/books_04_overrides_01.py b/example/example/spiders/books_04_overrides_01.py index c59b80d9..991ef005 100644 --- a/example/example/spiders/books_04_overrides_01.py +++ b/example/example/spiders/books_04_overrides_01.py @@ -5,6 +5,7 @@ The default configured PO logic contains the logic for books.toscrape.com """ + import scrapy from web_poet import ApplyRule, WebPage diff --git a/example/example/spiders/books_04_overrides_02.py b/example/example/spiders/books_04_overrides_02.py index 7bd4cc0b..61664854 100644 --- a/example/example/spiders/books_04_overrides_02.py +++ b/example/example/spiders/books_04_overrides_02.py @@ -6,6 +6,7 @@ No configured default logic: if used for an unregistered domain, no logic at all is applied. """ + import scrapy from web_poet import WebPage from web_poet.rules import ApplyRule diff --git a/example/example/spiders/books_04_overrides_03.py b/example/example/spiders/books_04_overrides_03.py index 1b3c671d..99b4892b 100644 --- a/example/example/spiders/books_04_overrides_03.py +++ b/example/example/spiders/books_04_overrides_03.py @@ -10,6 +10,7 @@ difference is that this example is using the ``@handle_urls`` decorator to store the rules in web-poet's registry. """ + import scrapy from web_poet import WebPage, default_registry, handle_urls diff --git a/example/example/spiders/books_05.py b/example/example/spiders/books_05.py index 1c60d2f9..347a98a3 100644 --- a/example/example/spiders/books_05.py +++ b/example/example/spiders/books_05.py @@ -2,6 +2,7 @@ Scrapy spider which uses Page Objects both for crawling and extraction. You can mix various page types freely. """ + import scrapy from example.autoextract import ProductPage from web_poet import WebPage diff --git a/scrapy_poet/_request_fingerprinter.py b/scrapy_poet/_request_fingerprinter.py index 7073754f..4ba6555a 100644 --- a/scrapy_poet/_request_fingerprinter.py +++ b/scrapy_poet/_request_fingerprinter.py @@ -10,7 +10,7 @@ import json from functools import cached_property from logging import getLogger - from typing import Callable, Dict, List, Optional, get_args, get_origin + from typing import Annotated, Callable, Dict, List, Optional, get_args, get_origin from weakref import WeakKeyDictionary from andi import CustomBuilder @@ -37,10 +37,6 @@ def _serialize_dep(cls): if isinstance(cls, CustomBuilder): cls = cls.result_class_or_fn - try: - from typing import Annotated - except ImportError: - pass else: if get_origin(cls) is Annotated: annotated, *annotations = get_args(cls) diff --git a/scrapy_poet/downloadermiddlewares.py b/scrapy_poet/downloadermiddlewares.py index bf36d4ba..86cdceef 100644 --- a/scrapy_poet/downloadermiddlewares.py +++ b/scrapy_poet/downloadermiddlewares.py @@ -2,6 +2,7 @@ responsible for injecting Page Input dependencies before the request callbacks are executed. """ + import inspect import logging import warnings diff --git a/scrapy_poet/page_input_providers.py b/scrapy_poet/page_input_providers.py index d5d70fd0..3746fddf 100644 --- a/scrapy_poet/page_input_providers.py +++ b/scrapy_poet/page_input_providers.py @@ -8,6 +8,7 @@ different providers in order to acquire data from multiple external sources, for example, from scrapy-playwright or from an API for automatic extraction. """ + from typing import Any, Callable, ClassVar, FrozenSet, List, Set, Union from warnings import warn diff --git a/setup.py b/setup.py index b5b14956..f9e2b55c 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ "scrapy.commands": ["savefixture = scrapy_poet.commands:SaveFixtureCommand"] }, package_data={"scrapy_poet": ["VERSION"]}, - python_requires=">=3.8", + python_requires=">=3.9", install_requires=[ "andi >= 0.6.0", "attrs >= 21.3.0", @@ -39,10 +39,10 @@ "Operating System :: OS Independent", "Framework :: Scrapy", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], ) diff --git a/tests/test_commands.py b/tests/test_commands.py index 116fe11f..e82386e0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,7 +6,6 @@ import tempfile from pathlib import Path -import pytest from twisted.web.resource import Resource from web_poet.testing import Fixture @@ -246,9 +245,6 @@ class CustomItemAdapter(ItemAdapter): result.assert_outcomes(passed=3) -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) def test_savefixture_annotated(pytester) -> None: project_name = "foo" cwd = Path(pytester.path) diff --git a/tests/test_injection.py b/tests/test_injection.py index a2a216e5..44ca6eb0 100644 --- a/tests/test_injection.py +++ b/tests/test_injection.py @@ -1,7 +1,6 @@ import re import shutil -import sys -from typing import Any, Callable, Dict, Generator, Optional +from typing import Annotated, Any, Callable, Dict, Generator, Optional import andi import attr @@ -312,21 +311,11 @@ def _assert_instances( kwargs = yield from injector.build_callback_dependencies(request, response) assert kwargs == expected_kwargs - @pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" - ) def test_annotated_provide(self, injector): - from typing import Annotated - assert injector.is_class_provided_by_any_provider(Annotated[Cls1, 42]) - @pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" - ) @inlineCallbacks def test_annotated_build(self, injector): - from typing import Annotated - def callback( a: Cls1, b: Annotated[Cls2, 42], @@ -345,13 +334,8 @@ def callback( injector, callback, expected_instances, expected_kwargs ) - @pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" - ) @inlineCallbacks def test_annotated_build_only(self, injector): - from typing import Annotated - def callback( a: Annotated[Cls1, 42], ): @@ -367,13 +351,8 @@ def callback( injector, callback, expected_instances, expected_kwargs ) - @pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" - ) @inlineCallbacks def test_annotated_build_duplicate(self, injector): - from typing import Annotated - def callback( a: Cls1, b: Cls2, @@ -398,13 +377,8 @@ def callback( injector, callback, expected_instances, expected_kwargs ) - @pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" - ) @inlineCallbacks def test_annotated_build_no_support(self, injector): - from typing import Annotated - # get_provider_requiring_response() returns a provider that doesn't support Annotated def callback( a: Cls1, @@ -423,15 +397,10 @@ def callback( Cls1: Cls1(), } - @pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" - ) @inlineCallbacks def test_annotated_build_duplicate_forbidden( self, ): - from typing import Annotated - class Provider(PageObjectInputProvider): provided_classes = {Cls1} require_response = False @@ -666,13 +635,8 @@ def callback(dd: DynamicDeps): instances = yield from injector.build_instances(request, response, plan) assert set(instances) == {TestItemPage, TestItem, DynamicDeps} - @pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" - ) @inlineCallbacks def test_dynamic_deps_annotated(self): - from typing import Annotated - def callback(dd: DynamicDeps): pass @@ -779,8 +743,9 @@ class TestInjectorStats: def test_stats(self, cb_args, expected, injector): def callback_factory(): args = ", ".join([f"{k}: {v.__name__}" for k, v in cb_args.items()]) - exec(f"def callback(response: DummyResponse, {args}): pass") - return locals().get("callback") + ns = {} + exec(f"def callback(response: DummyResponse, {args}): pass", None, ns) + return ns["callback"] callback = callback_factory() response = get_response_for_testing(callback) @@ -1018,12 +983,7 @@ def test_dynamic_deps_factory(): assert dd == {int: 42, Cls1: c} -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) def test_dynamic_deps_factory_annotated(): - from typing import Annotated - fn = Injector._get_dynamic_deps_factory( [Annotated[Cls1, 42], Annotated[Cls2, "foo"]] ) diff --git a/tests/test_request_fingerprinter.py b/tests/test_request_fingerprinter.py index 74b94396..961a4142 100644 --- a/tests/test_request_fingerprinter.py +++ b/tests/test_request_fingerprinter.py @@ -1,6 +1,5 @@ -import sys from itertools import combinations -from typing import Callable, Set +from typing import Annotated, Callable, Set from unittest.mock import patch import pytest @@ -373,16 +372,11 @@ def test_meta(meta): assert fingerprint1 == fingerprint2 -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @pytest.mark.skipif( ANDI_VERSION <= Version("0.4.1"), reason="https://github.com/scrapinghub/andi/pull/25", ) def test_different_annotations(): - from typing import Annotated - class TestSpider(Spider): name = "test_spider" @@ -405,12 +399,7 @@ def test_serialize_dep(): assert _serialize_dep(HttpResponse) == "web_poet.page_inputs.http.HttpResponse" -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) def test_serialize_dep_annotated(): - from typing import Annotated - assert ( _serialize_dep(Annotated[HttpResponse, "foo"]) == "web_poet.page_inputs.http.HttpResponse['foo']" diff --git a/tox.ini b/tox.ini index 7ac907ed..8487e181 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,12 @@ [tox] -envlist = min,py38,py39,py310,py311,py312,asyncio,asyncio-min,mypy,docs +envlist = min,py39,py310,py311,py312,py313,asyncio,asyncio-min,mypy,docs [testenv] deps = pytest pytest-cov pytest-twisted - Twisted<=22.10.0 + Twisted commands = py.test \ @@ -29,7 +29,7 @@ deps = tldextract<3.6 [testenv:min] -basepython = python3.8 +basepython = python3.9 deps = {[pinned]deps} scrapy==2.6.0 @@ -43,18 +43,20 @@ deps = # Before ``scrapy.http.request.NO_CALLBACK`` was introduced. # See: https://github.com/scrapinghub/scrapy-poet/issues/48 [testenv:pinned-scrapy-2x7] -basepython=python3.8 +basepython=python3.9 deps = {[pinned]deps} scrapy==2.7.0 + Twisted<23.8.0 # After ``scrapy.http.request.NO_CALLBACK`` was introduced. # See: https://github.com/scrapinghub/scrapy-poet/issues/118 [testenv:pinned-scrapy-2x8] -basepython=python3.8 +basepython=python3.9 deps = {[pinned]deps} scrapy==2.8.0 + Twisted<23.8.0 [testenv:asyncio] setenv = @@ -62,10 +64,10 @@ setenv = commands = {[testenv]commands} --reactor=asyncio deps = - {[pinned]deps} + {[testenv]deps} [testenv:asyncio-min] -basepython = python3.8 +basepython = python3.9 setenv = {[testenv:asyncio]setenv} commands = @@ -75,9 +77,10 @@ deps = [testenv:mypy] deps = - mypy==0.991 + mypy==1.11.2 + pytest -commands = mypy --ignore-missing-imports --no-warn-no-return scrapy_poet tests +commands = mypy --ignore-missing-imports scrapy_poet tests [testenv:docs] basepython = python3 @@ -94,8 +97,8 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:twinecheck] basepython = python3 deps = - twine==5.0.0 - build==0.10.0 + twine==5.1.1 + build==1.2.2 commands = python -m build --sdist twine check dist/*