If Retry has a message, use it as retry reason

Author: Gallaecio

scrapy_poet/spidermiddlewares.py

@@ -24,7 +24,7 @@ def process_spider_exception(
         new_request_or_none = get_retry_request(
             response.request,
             spider=spider,
-            reason="page_object_retry",
+            reason=str(exception) or "page_object_retry",
         )
         if not new_request_or_none:
             return []

tests/test_retries.py

@@ -53,6 +53,40 @@ def parse(self, response, page: SamplePage):
     _assert_all_unique_instances(page_response_instances)
 
 
+@inlineCallbacks
+def test_retry_reason():
+    retries = deque([True, False])
+    items, page_instances, page_response_instances = [], [], []
+
+    with MockServer(EchoResource) as server:
+
+        class SamplePage(WebPage):
+            def to_item(self):
+                page_instances.append(self)
+                page_response_instances.append(self.response)
+                if retries.popleft():
+                    raise Retry("foo")
+                return {"foo": "bar"}
+
+        class TestSpider(BaseSpider):
+            def start_requests(self):
+                yield Request(server.root_url, callback=self.parse)
+
+            def parse(self, response, page: SamplePage):
+                items.append(page.to_item())
+
+        crawler = make_crawler(TestSpider)
+        yield crawler.crawl()
+
+    assert items == [{"foo": "bar"}]
+    assert crawler.stats.get_value("downloader/request_count") == 2
+    assert crawler.stats.get_value("retry/count") == 1
+    assert crawler.stats.get_value("retry/reason_count/foo") == 1
+    assert crawler.stats.get_value("retry/max_reached") is None
+    _assert_all_unique_instances(page_instances)
+    _assert_all_unique_instances(page_response_instances)
+
+
 @inlineCallbacks
 def test_retry_max():
     # The default value of the RETRY_TIMES Scrapy setting is 2.