Do not count DummyResponse into downloader stats (#187)

README.rst

@@ -59,6 +59,8 @@ Add the following inside Scrapy's ``settings.py`` file:
 
     DOWNLOADER_MIDDLEWARES = {
         "scrapy_poet.InjectionMiddleware": 543,
+        "scrapy.downloadermiddlewares.stats.DownloaderStats": None,
+        "scrapy_poet.DownloaderStatsMiddleware": 850,
     }
     SPIDER_MIDDLEWARES = {
         "scrapy_poet.RetryMiddleware": 275,

docs/intro/advanced-tutorial.rst

@@ -80,6 +80,8 @@ It can be directly used inside the spider as:
         custom_settings = {
             "DOWNLOADER_MIDDLEWARES": {
                 "scrapy_poet.InjectionMiddleware": 543,
+                "scrapy.downloadermiddlewares.stats.DownloaderStats": None,
+                "scrapy_poet.DownloaderStatsMiddleware": 850,
             }
         }
 
@@ -153,6 +155,8 @@ Let's see it in action:
         custom_settings = {
             "DOWNLOADER_MIDDLEWARES": {
                 "scrapy_poet.InjectionMiddleware": 543,
+                "scrapy.downloadermiddlewares.stats.DownloaderStats": None,
+                "scrapy_poet.DownloaderStatsMiddleware": 850,
             }
         }
 

docs/intro/install.rst

@@ -28,6 +28,8 @@ of your Scrapy project:
 
     DOWNLOADER_MIDDLEWARES = {
         "scrapy_poet.InjectionMiddleware": 543,
+        "scrapy.downloadermiddlewares.stats.DownloaderStats": None,
+        "scrapy_poet.DownloaderStatsMiddleware": 850,
     }
     SPIDER_MIDDLEWARES = {
         "scrapy_poet.RetryMiddleware": 275,

example/example/settings.py

@@ -6,8 +6,11 @@
 #     https://docs.scrapy.org/en/latest/topics/settings.html
 #     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
 #     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
 from example.autoextract import AutoextractProductProvider
 
+from scrapy_poet import ScrapyPoetRequestFingerprinter
+
 BOT_NAME = "example"
 
 SPIDER_MODULES = ["example.spiders"]
@@ -20,4 +23,12 @@
 
 DOWNLOADER_MIDDLEWARES = {
     "scrapy_poet.InjectionMiddleware": 543,
+    "scrapy.downloadermiddlewares.stats.DownloaderStats": None,
+    "scrapy_poet.DownloaderStatsMiddleware": 850,
+}
+
+REQUEST_FINGERPRINTER_CLASS = ScrapyPoetRequestFingerprinter
+
+SPIDER_MIDDLEWARES = {
+    "scrapy_poet.RetryMiddleware": 275,
 }

scrapy_poet/__init__.py

@@ -1,5 +1,5 @@
 from .api import AnnotatedResult, DummyResponse, callback_for
-from .downloadermiddlewares import InjectionMiddleware
+from .downloadermiddlewares import DownloaderStatsMiddleware, InjectionMiddleware
 from .page_input_providers import HttpResponseProvider, PageObjectInputProvider
 from .spidermiddlewares import RetryMiddleware
 from ._request_fingerprinter import ScrapyPoetRequestFingerprinter

scrapy_poet/downloadermiddlewares.py

@@ -5,10 +5,11 @@
 import inspect
 import logging
 import warnings
-from typing import Generator, Optional, Type, TypeVar
+from typing import Generator, Optional, Type, TypeVar, Union
 
 from scrapy import Spider
 from scrapy.crawler import Crawler
+from scrapy.downloadermiddlewares.stats import DownloaderStats
 from scrapy.http import Request, Response
 from twisted.internet.defer import Deferred, inlineCallbacks
 from web_poet import RulesRegistry
@@ -28,6 +29,15 @@
 logger = logging.getLogger(__name__)
 
 
+class DownloaderStatsMiddleware(DownloaderStats):
+    def process_response(
+        self, request: Request, response: Response, spider: Spider
+    ) -> Union[Request, Response]:
+        if isinstance(response, DummyResponse):
+            return response
+        return super().process_response(request, response, spider)
+
+
 DEFAULT_PROVIDERS = {
     HttpResponseProvider: 500,
     HttpClientProvider: 600,

scrapy_poet/utils/testing.py

@@ -157,7 +157,10 @@ def get_download_handler(crawler, schema):
     return crawler.engine.downloader.handlers._get_handler(schema)
 
 
-def make_crawler(spider_cls, settings):
+def make_crawler(spider_cls, settings=None):
+    settings = settings or {}
+    settings = {**create_scrapy_settings(), **settings}
+
     if not getattr(spider_cls, "name", None):
 
         class Spider(spider_cls):
@@ -220,7 +223,7 @@ def process_response(self, request, response, spider):
         return response
 
 
-def create_scrapy_settings(request):
+def create_scrapy_settings():
     """Default scrapy-poet settings"""
     s = dict(
         # collect scraped items to crawler.spider.collected_items
@@ -231,8 +234,13 @@ def create_scrapy_settings(request):
             # collect injected dependencies to crawler.spider.collected_response_deps
             InjectedDependenciesCollectorMiddleware: 542,
             "scrapy_poet.InjectionMiddleware": 543,
+            "scrapy.downloadermiddlewares.stats.DownloaderStats": None,
+            "scrapy_poet.DownloaderStatsMiddleware": 850,
         },
         REQUEST_FINGERPRINTER_CLASS=ScrapyPoetRequestFingerprinter,
+        SPIDER_MIDDLEWARES={
+            "scrapy_poet.RetryMiddleware": 275,
+        },
     )
     return Settings(s)
 

tests/conftest.py

@@ -4,5 +4,5 @@
 
 
 @pytest.fixture()
-def settings(request):
-    return create_scrapy_settings(request)
+def settings():
+    return create_scrapy_settings()

tests/test_cache.py

@@ -5,7 +5,7 @@
 from web_poet import WebPage, field
 
 from scrapy_poet.utils.mockserver import MockServer
-from scrapy_poet.utils.testing import EchoResource, make_crawler
+from scrapy_poet.utils.testing import EchoResource, create_scrapy_settings, make_crawler
 
 
 @inlineCallbacks
@@ -22,10 +22,7 @@ class CacheSpider(Spider):
                 name = "cache"
 
                 custom_settings = {
-                    "DOWNLOADER_MIDDLEWARES": {
-                        "scrapy_poet.InjectionMiddleware": 543,
-                    },
-                    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+                    **create_scrapy_settings(),
                     "SCRAPY_POET_CACHE": cache_dir,
                 }