PoS spider: use OAI-PMH spider

docker-compose.test.yml

@@ -64,7 +64,7 @@ services:
     depends_on:
       scrapyd:
         condition: service_healthy
-      http-server.local:
+      pos-http-server.local:
         condition: service_healthy
 
   unit:
@@ -118,22 +118,6 @@ services:
       - ${PWD}/tests/functional/wsp/fixtures/ftp_server/WSP:/home/ftpusers/bob/WSP
       - ${PWD}/tests/functional/wsp/fixtures/ftp_server/pureftpd.passwd:/etc/pure-ftpd/passwd/pureftpd.passwd
 
-  http-server.local:
-    image: nginx:stable-alpine
-    volumes:
-      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
-      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/ssl:/etc/nginx/ssl
-      - ${PWD}/tests/functional/pos/fixtures/https_server/records:/etc/nginx/html/
-    ports:
-      - 443:443
-    healthcheck:
-      timeout: 5s
-      interval: 5s
-      retries: 5
-      test:
-        - "CMD-SHELL"
-        - "curl https://localhost:443/"
-
   functional_cds:
     <<: *service_base
     command: py.test -vv tests/functional/cds
@@ -173,6 +157,21 @@ services:
         - "CMD-SHELL"
         - "curl http://localhost:80/"
 
+  pos-http-server.local:
+    image: nginx:stable-alpine
+    volumes:
+      - ${PWD}/tests/functional/pos/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
+      - ${PWD}/tests/functional/pos/fixtures/http_server/records:/etc/nginx/html/
+    ports:
+      - 80:80
+    healthcheck:
+      timeout: 5s
+      interval: 5s
+      retries: 5
+      test:
+        - "CMD-SHELL"
+        - "curl http://localhost:80/"
+
   rabbitmq:
     image: rabbitmq
     healthcheck:

hepcrawl/settings.py

@@ -33,9 +33,6 @@
 # user-agent
 USER_AGENT = 'hepcrawl (+http://www.inspirehep.net)'
 
-# Allow duplicate requests
-DUPEFILTER_CLASS = "scrapy.dupefilters.BaseDupeFilter"
-
 # URI base prefix for $schema to be used during record generation
 SCHEMA_BASE_URI = os.environ.get(
     'APP_SCHEMA_BASE_URI',