get more text

shaform · Aug 27, 2016 · 457e0bd · 457e0bd
1 parent 4a2c3fc
commit 457e0bd
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/scrapy/ptt/spiders/ptt.py b/scrapy/ptt/spiders/ptt.py
@@ -2,6 +2,7 @@
 
 from datetime import datetime
 
+import html2text
 import scrapy
 
 from scrapy.http import FormRequest
@@ -61,8 +62,9 @@ def parse_post(self, response):
                 0].extract()
         item['date'] = datetime.strptime(datetime_str, '%a %b %d %H:%M:%S %Y')
 
-        item['content'] = response.xpath('//div[@id="main-content"]/text()')[
-            0].extract()
+        converter = html2text.HTML2Text()
+        converter.ignore_links = True
+        item['content'] = converter.handle(response.xpath('//div[@id="main-content"]')[ 0].extract())
 
         comments = []
         total_score = 0

diff --git a/scrapy/requirements.txt b/scrapy/requirements.txt
@@ -6,3 +6,4 @@ scipy>=0.17.0
 scikit-learn>=0.16.1
 cryptography>=1.2.2
 pyOpenSSL>=0.15.1
+html2text>=2016.5.29