Merge pull request #145 from Datenschule/always-use-school-spider

Make all scrapers extend SchoolSpider
Datenschule · Aug 12, 2024 · fb52281 · fb52281
2 parents a5e157f + 8535e71
commit fb52281
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 7 deletions.
diff --git a/jedeschule/spiders/bayern.py b/jedeschule/spiders/bayern.py
@@ -3,9 +3,10 @@
 from scrapy import Item
 
 from jedeschule.items import School
+from jedeschule.spiders.school_spider import SchoolSpider
 
 
-class BayernSpider(scrapy.Spider):
+class BayernSpider(SchoolSpider):
     name = "bayern"
     start_urls = ['https://gdiserv.bayern.de/srv112940/services/schulstandortebayern-wfs?SERVICE=WFS&VERSION=2.0.0&REQUEST=GetCapabilities']
 

diff --git a/jedeschule/spiders/berlin.py b/jedeschule/spiders/berlin.py
@@ -1,11 +1,12 @@
 import xml.etree.ElementTree as ET
 
-import scrapy
 from jedeschule.items import School
 from scrapy import Item
 
+from jedeschule.spiders.school_spider import SchoolSpider
 
-class BerlinSpider(scrapy.Spider):
+
+class BerlinSpider(SchoolSpider):
     name = "berlin"
     start_urls = ['https://gdi.berlin.de/services/wfs/schulen?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetFeature&srsname=EPSG:4326&typename=fis:schulen']
 

diff --git a/jedeschule/spiders/schleswig_holstein.py b/jedeschule/spiders/schleswig_holstein.py
@@ -1,8 +1,5 @@
 import csv
-import json
 
-import scrapy
-import scrapy.http
 from scrapy import Item
 
 from jedeschule.items import School

diff --git a/test_changes.py b/test_changes.py
@@ -20,7 +20,7 @@ def sort_dict(data):
 
 
 def get_clean_item(data):
-    return set({key: value for key, value in data.items() if value is not None})
+    return set({key: value for key, value in data.items() if value is not None and key != "update_timestamp"})
 
 
 def compare_schools(new_school, old_school):