handle_urls decorator using a new PageObjectRegistry

CHANGELOG.rst

@@ -5,10 +5,11 @@ Changelog
 TBR
 ------------------
 
+* ``handle_urls`` decorator and ``find_page_object_overrides`` function added.
+* new CLI tool for displaying all available Page Objects: ``web_poet <path>``
 * removed support for Python 3.6
 * added support for Python 3.10
 
-
 0.1.1 (2021-06-02)
 ------------------
 

docs/api_reference.rst

@@ -45,3 +45,13 @@ Mixins
 .. autoclass:: web_poet.mixins.ResponseShortcutsMixin
    :members:
    :no-special-members:
+
+
+Overrides
+=========
+
+.. autofunction:: web_poet.handle_urls
+
+.. automodule:: web_poet.overrides
+   :members:
+   :exclude-members: handle_urls

docs/conf.py

@@ -192,4 +192,5 @@
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None, ),
     'scrapy': ('https://docs.scrapy.org/en/latest', None, ),
+    'url-matcher': ('https://url-matcher.readthedocs.io/en/stable/', None, ),
 }

docs/requirements.txt

@@ -5,4 +5,4 @@ sphinxcontrib-devhelp==1.0.2
 sphinxcontrib-htmlhelp==2.0.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-serializinghtml==1.1.5

setup.py

@@ -14,6 +14,7 @@
     author='Scrapinghub',
     author_email='[email protected]',
     url='https://github.com/scrapinghub/web-poet',
+    entry_points={'console_scripts': ['web_poet = web_poet.__main__:main']},
     packages=find_packages(
         exclude=(
             'tests',
@@ -22,6 +23,8 @@
     install_requires=(
         'attrs',
         'parsel',
+        'url-matcher',
+        'tabulate',
     ),
     classifiers=(
         'Development Status :: 2 - Pre-Alpha',

tests/po_lib/__init__.py

@@ -0,0 +1,43 @@
+"""
+This package is just for overrides testing purposes.
+"""
+from typing import Dict, Any, Callable
+
+from url_matcher import Patterns
+
+from web_poet import handle_urls, PageObjectRegistry
+
+
+class POBase:
+    expected_overrides: Callable
+    expected_patterns: Patterns
+    expected_meta: Dict[str, Any]
+
+
+class POTopLevelOverriden1:
+    ...
+
+
+class POTopLevelOverriden2:
+    ...
+
+
+secondary_registry = PageObjectRegistry(name="secondary")
+
+
+# This first annotation is ignored. A single annotation per registry is allowed
+@handle_urls("example.com", POTopLevelOverriden1)
+@handle_urls("example.com", POTopLevelOverriden1, exclude="/*.jpg|", priority=300)
+class POTopLevel1(POBase):
+    expected_overrides = POTopLevelOverriden1
+    expected_patterns = Patterns(["example.com"], ["/*.jpg|"], priority=300)
+    expected_meta = {}  # type: ignore
+
+
+# The second annotation is for a different registry
+@handle_urls("example.com", POTopLevelOverriden2)
+@secondary_registry.handle_urls("example.org", POTopLevelOverriden2)
+class POTopLevel2(POBase):
+    expected_overrides = POTopLevelOverriden2
+    expected_patterns = Patterns(["example.com"])
+    expected_meta = {}  # type: ignore

tests/po_lib/a_module.py

@@ -0,0 +1,16 @@
+from url_matcher import Patterns
+
+from tests.po_lib import POBase
+from web_poet import handle_urls
+
+
+class POModuleOverriden:
+    ...
+
+
+@handle_urls("example.com", overrides=POModuleOverriden, extra_arg="foo")
+class POModule(POBase):
+    expected_overrides = POModuleOverriden
+    expected_patterns = Patterns(["example.com"])
+    expected_meta = {"extra_arg": "foo"}  # type: ignore
+

tests/po_lib/nested_package/__init__.py

@@ -0,0 +1,15 @@
+from url_matcher import Patterns
+
+from tests.po_lib import POBase
+from web_poet import handle_urls
+
+
+class PONestedPkgOverriden:
+    ...
+
+
+@handle_urls(include=["example.com", "example.org"], exclude=["/*.jpg|"], overrides=PONestedPkgOverriden)
+class PONestedPkg(POBase):
+    expected_overrides = PONestedPkgOverriden
+    expected_patterns = Patterns(["example.com", "example.org"], ["/*.jpg|"])
+    expected_meta = {}  # type: ignore

tests/po_lib/nested_package/a_nested_module.py

@@ -0,0 +1,21 @@
+from url_matcher import Patterns
+
+from tests.po_lib import POBase, secondary_registry
+from web_poet import handle_urls
+
+
+class PONestedModuleOverriden:
+    ...
+
+
+class PONestedModuleOverridenSecondary:
+    ...
+
+
+@handle_urls(include=["example.com", "example.org"], exclude=["/*.jpg|"], overrides=PONestedModuleOverriden)
+@secondary_registry.handle_urls("example.com", PONestedModuleOverridenSecondary)
+class PONestedModule(POBase):
+    expected_overrides = PONestedModuleOverriden
+    expected_patterns = Patterns(include=["example.com", "example.org"], exclude=["/*.jpg|"])
+    expected_meta = {}  # type: ignore
+

tests/test_overrides.py

@@ -0,0 +1,69 @@
+import pytest
+from url_matcher import Patterns
+
+from tests.po_lib import POTopLevel1, POTopLevel2, POTopLevelOverriden2
+from tests.po_lib.a_module import POModule
+from tests.po_lib.nested_package import PONestedPkg
+from tests.po_lib.nested_package.a_nested_module import PONestedModule, PONestedModuleOverridenSecondary
+from web_poet.overrides import find_page_object_overrides
+
+
+POS = {POTopLevel1, POTopLevel2, POModule, PONestedPkg, PONestedModule}
+
+
+def test_list_page_objects_from_pkg():
+    """Tests that metadata is extracted properly from the po_lib package"""
+    rules = find_page_object_overrides("tests.po_lib")
+    assert {po.use for po in rules} == POS
+
+    for rule in rules:
+        assert rule.instead_of == rule.use.expected_overrides, rule.use
+        assert rule.for_patterns == rule.use.expected_patterns, rule.use
+        assert rule.meta == rule.use.expected_meta, rule.use
+
+
+def test_list_page_objects_from_module():
+    rules = find_page_object_overrides("tests.po_lib.a_module")
+    assert len(rules) == 1
+    rule = rules[0]
+    assert rule.use == POModule
+    assert rule.for_patterns == POModule.expected_patterns
+    assert rule.instead_of == POModule.expected_overrides
+
+
+def test_list_page_objects_from_empty_module():
+    rules = find_page_object_overrides("tests.po_lib.an_empty_module")
+    assert len(rules) == 0
+
+
+def test_list_page_objects_from_empty_pkg():
+    rules = find_page_object_overrides("tests.po_lib.an_empty_package")
+    assert len(rules) == 0
+
+
+def test_list_page_objects_from_unknown_module():
+    with pytest.raises(ImportError):
+        find_page_object_overrides("tests.po_lib.unknown_module")
+
+
+def test_list_page_objects_from_imported_registry():
+    rules = find_page_object_overrides("tests.po_lib", registry_name="secondary")
+    assert len(rules) == 2
+    rule_for = {po.use: po for po in rules}
+
+    potop2 = rule_for[POTopLevel2]
+    assert potop2.for_patterns == Patterns(["example.org"])
+    assert potop2.instead_of == POTopLevelOverriden2
+
+    pones = rule_for[PONestedModule]
+    assert pones.for_patterns == Patterns(["example.com"])
+    assert pones.instead_of == PONestedModuleOverridenSecondary
+
+
+def test_list_page_objects_from_non_existing_registry():
+    assert find_page_object_overrides("tests.po_lib", registry_name="not-exist") == []
+
+
+def test_cmd():
+    from  web_poet.__main__ import main
+    main(["tests.po_lib"])

tox.ini

@@ -15,6 +15,7 @@ commands =
 [testenv:mypy]
 deps =
     mypy
+    types-tabulate
 
 commands = mypy --ignore-missing-imports  web_poet tests
 

web_poet/__init__.py

@@ -1,2 +1,3 @@
 from .pages import WebPage, ItemPage, ItemWebPage, Injectable
-from .page_inputs import ResponseData
+from .page_inputs import ResponseData
+from .overrides import handle_urls, PageObjectRegistry

web_poet/__main__.py

@@ -0,0 +1,57 @@
+import argparse
+from typing import Callable
+
+import tabulate
+
+from web_poet.overrides import find_page_object_overrides
+
+
+def qualified_name(cls: Callable) -> str:
+    return f"{cls.__module__}.{cls.__name__}"
+
+
+def main(args=None):
+    parser = argparse.ArgumentParser(
+        description="Tool that list the Page Object overrides from a package or module recursively"
+    )
+    parser.add_argument(
+        "module",
+        metavar="PKG_OR_MODULE",
+        type=str,
+        help="A package or module to list overrides from",
+    )
+    parser.add_argument(
+        "--registry",
+        "-n",
+        metavar="REGISTRY_NAME",
+        type=str,
+        help="Registry name to list overrides from",
+        default="default",
+    )
+    args = parser.parse_args(args)
+    table = [
+        (
+            "Use this",
+            "instead of",
+            "for the URL patterns",
+            "except for the patterns",
+            "with priority",
+            "meta",
+        )
+    ]
+    table += [
+        (
+            qualified_name(rule.use),
+            qualified_name(rule.instead_of),
+            rule.for_patterns.include,
+            rule.for_patterns.exclude,
+            rule.for_patterns.priority,
+            rule.meta,
+        )
+        for rule in find_page_object_overrides(args.module, registry_name=args.registry)
+    ]
+    print(tabulate.tabulate(table, headers="firstrow"))
+
+
+if __name__ == "__main__":
+    main()