Add "FRDR" catalog alongside DataCite catalog

.gitignore

@@ -119,4 +119,6 @@ dmypy.json
 .vscode
 
 # Env configs
-*.env
+*.env
+
+.DS_Store

Pipfile

@@ -13,7 +13,9 @@ python-dotenv = "*"
 
 [packages]
 flask = "*"
+ftfy = "*"
 pyoai = "*"
+psycopg2-binary = "*"
 requests = "*"
 python-dateutil = "*"
 lxml = "*"
@@ -23,4 +25,4 @@ sentry-sdk = {extras = ["flask"],version = "*"}
 python-dotenv = "*"
 
 [requires]
-python_version = "3.6"
+python_version = "3.5"

vendor/docker/env.conf

@@ -3,7 +3,13 @@ env OAIPMH_BASE_URL;
 env DATACITE_API_URL;
 env OAIPMH_REPOS_NAME;
 env OAIPMH_ADMIN_EMAIL;
+env OAIPMH_IDENTIFIER;
 env SENTRY_DSN;
 env DATACITE_API_ADMIN_USERNAME;
 env DATACITE_API_ADMIN_PASSWORD;
 env RESULT_SET_SIZE;
+env CATALOG_SET;
+env POSTGRES_SERVER;
+env POSTGRES_DB;
+env POSTGRES_USER;
+env POSTGRES_PASSWORD;

viringo/catalogs.py

@@ -12,6 +12,7 @@
 
 from viringo import config
 from .services import datacite
+from .services import frdr
 
 
 class DataCiteOAIServer():
@@ -285,6 +286,267 @@ def build_metadata_map(self, result):
         return metadata
 
 
+class FRDROAIServer():
+    """Build OAI-PMH responses from the FRDR Postgres server"""
+    def identify(self):
+        """Construct common identification for the OAI service"""
+
+        identify = common.Identify(
+            repositoryName=config.OAIPMH_REPOS_NAME,
+            baseURL=config.OAIPMH_BASE_URL,
+            protocolVersion="2.0",
+            adminEmails=[config.OAIPMH_ADMIN_EMAIL],
+            earliestDatestamp=datetime(2011, 1, 1),
+            deletedRecord='no',
+            granularity='YYYY-MM-DDThh:mm:ssZ',
+            compression=['gzip', 'deflate'],
+            toolkit_description=False)
+
+        # Specify a custom description
+        frdr_desc = """
+        <oai-identifier xmlns="http://www.openarchives.org/OAI/2.0/oai-identifier" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai-identifier http://www.openarchives.org/OAI/2.0/oai-identifier.xsd">
+            <scheme>oai</scheme>
+            <repositoryIdentifier>""" + config.OAIPMH_IDENTIFIER + """</repositoryIdentifier>
+            <delimiter>:</delimiter>
+            <sampleIdentifier>oai""" + config.OAIPMH_IDENTIFIER + """:1</sampleIdentifier>
+        </oai-identifier>
+        """
+
+        identify.add_description(xml_string=frdr_desc)
+
+        return identify
+
+    def listMetadataFormats(self, identifier=None):
+        #pylint: disable=no-self-use,invalid-name
+        """Returns metadata formats available for the repository
+
+        Identifier does nothing as our repository responds in all formats for all dois
+        """
+        # PyOAI Expects result format (metadataPrefix, schema, metadataNamespace)
+
+        format_oai_dc = (
+            'oai_dc',
+            'http://www.openarchives.org/OAI/2.0/oai_dc.xsd',
+            'http://www.openarchives.org/OAI/2.0/oai_dc/'
+        )
+
+        format_oai_datacite = (
+            'oai_datacite',
+            'http://schema.datacite.org/oai/oai-1.1/oai.xsd',
+            'http://schema.datacite.org/oai/oai-1.1/'
+        )
+
+        format_datacite = (
+            'datacite',
+            'http://schema.datacite.org/meta/nonexistant/nonexistant.xsd',
+            'http://datacite.org/schema/nonexistant'
+        )
+
+        return [format_oai_dc, format_oai_datacite, format_datacite]
+
+    def getRecord(self, metadataPrefix, identifier):
+        #pylint: disable=no-self-use,invalid-name
+        """Returns pyoai data tuple for specific record"""
+
+        # Should we implement this based on source_url and local_identifier the way we currently do for the harvester?
+
+        result = frdr.get_metadata(identifier, db=config.POSTGRES_DB, user=config.POSTGRES_USER, password=config.POSTGRES_PASSWORD, server=config.POSTGRES_SERVER, port=config.POSTGRES_PORT)
+        if not result:
+            raise error.IdDoesNotExistError(
+                "\"%s\" is unknown or illegal in this repository" % identifier
+            )
+
+        # Build metadata based on requested format and result
+        metadata = self.build_metadata_map(result)
+
+        header = self.build_header(result)
+        record = self.build_record(metadata)
+        data = (
+            header,
+            record,
+            None # About string - not used
+        )
+
+        return data
+
+    def listRecords(
+            self,
+            metadataPrefix=None,
+            from_=None,
+            until=None,
+            set=None,
+            paging_cursor=None
+        ):
+
+        #pylint: disable=no-self-use,invalid-name
+        """Returns pyoai data tuple for list of records"""
+
+        # If available get the search query from the set param
+        search_query = set_to_search_query(set)
+
+        results, total_records, paging_cursor = frdr.get_metadata_list(
+            server=config.POSTGRES_SERVER,
+            db=config.POSTGRES_DB,
+            user=config.POSTGRES_USER,
+            password=config.POSTGRES_PASSWORD,
+            port=config.POSTGRES_PORT,
+            query=search_query,
+            set=set,
+            from_datetime=from_,
+            until_datetime=until,
+            cursor=paging_cursor
+        )
+
+        if paging_cursor >= total_records:
+            paging_cursor = None
+
+        records = []
+        if results:
+            for result in results:
+                # Build metadata based on requested format and result
+                metadata = self.build_metadata_map(result)
+
+                header = self.build_header(result)
+                record = self.build_record(metadata)
+
+                data = (
+                    header,
+                    record,
+                    None # About string - not used
+                )
+
+                records.append(data)
+
+        # This differs from the pyoai implementation in that we have to return a cursor here
+        # But this is okay as we have a custom server to handle it.
+        return records, total_records, paging_cursor
+
+    def listIdentifiers(
+            self,
+            metadataPrefix=None,
+            from_=None,
+            until=None,
+            set=None,
+            paging_cursor=None
+        ):
+        #pylint: disable=no-self-use,invalid-name
+        """Returns pyoai data tuple for list of identifiers"""
+
+        # If available get the search query from the set param
+        search_query = set_to_search_query(set)
+
+        results, total_records, paging_cursor = frdr.get_metadata_list(
+            server=config.POSTGRES_SERVER,
+            db=config.POSTGRES_DB,
+            user=config.POSTGRES_USER,
+            password=config.POSTGRES_PASSWORD,
+            port=config.POSTGRES_PORT,
+            query=search_query,
+            set=set,
+            from_datetime=from_,
+            until_datetime=until,
+            cursor=paging_cursor
+        )
+
+        if paging_cursor >= total_records:
+            paging_cursor = None
+
+        records = []
+        if results:
+            for result in results:
+                header = self.build_header(result)
+
+                records.append(header)
+
+        # This differs from the pyoai implementation in that we have to return a cursor here
+        # But this is okay as we have a custom server to handle it.
+        return records, total_records, paging_cursor
+
+    def listSets(
+            self,
+            paging_cursor=0
+        ):
+        #pylint: disable=no-self-use,invalid-name
+        """Returns pyoai data tuple for list of sets"""
+
+        # Note this implementation is not super efficient as we request
+        # the full set everytime regardles of actual paging
+        # The paging is handled just by offsetting the records returned.
+        # This is however acceptable given sets are a small subset of data.
+
+        # We know we're always dealing with a integer value here
+        paging_cursor = int(paging_cursor)
+
+        batch_size = 50
+        next_batch = paging_cursor + batch_size
+        results, total_results = frdr.get_sets(db=config.POSTGRES_DB, user=config.POSTGRES_USER, password=config.POSTGRES_PASSWORD, server=config.POSTGRES_SERVER, port=config.POSTGRES_PORT)
+        results = results[paging_cursor: next_batch]
+
+        if len(results) < batch_size:
+            paging_cursor = None
+        else:
+            paging_cursor = next_batch
+
+        records = []
+        if results:
+            for identifier, name in results:
+                # Format of a set is setSpec, setName, setDescription
+                records.append((identifier, name, None))
+
+        # This differs from the pyoai implementation in that we have to return a cursor here
+        # But this is okay as we have a custom server to handle it.
+        return records, total_results, paging_cursor
+
+    def build_header(self, result):
+        """Construct a OAI-PMH record header"""
+
+        return common.Header(
+            None,
+            str(result.identifier),
+            result.updated_datetime,
+            setspec=[result.client],
+            deleted=not result.active
+        )
+
+    def build_record(self, metadata):
+        """Construct a OAI-PMH payload for a record"""
+
+        return common.Metadata(
+            None,
+            metadata
+        )
+
+    def build_metadata_map(self, result):
+        """Construct a metadata map object for oai metadata writing"""
+        identifiers = result.identifiers
+
+        relations = [
+            identifier_to_string(relation)
+            for relation in result.relations
+        ]
+
+        metadata = {
+            'title': result.titles,
+            'creator': result.creators,
+            'subject': result.subjects,
+            'description': result.descriptions,
+            'publisher': [result.publisher] if result.publisher else [],
+            'contributor': result.contributors,
+            'date': result.dates,
+            'type': result.resource_types,
+            'format': result.formats,
+            'identifier': identifiers,
+            'relation': relations,
+            'language': [result.language] if result.language else [],
+            'rights': result.rights,
+            'xml': result.xml,
+            'set': result.client,
+            'metadata_version': result.metadata_version
+        }
+
+        return metadata
+
+
 def set_to_search_query(unparsed_set):
     """Take a oai set and extract any base64url encoded search query"""
 
@@ -298,7 +560,6 @@ def set_to_search_query(unparsed_set):
 
     return ""
 
-
 def set_to_provider_client(unparsed_set):
     """Take a oai set and convert into provider_id and client_id"""
 

viringo/config.py

@@ -16,5 +16,19 @@
 OAIPMH_BASE_URL = os.getenv('OAIPMH_BASE_URL', 'https://oai.datacite.org/oai')
 # Admin e-mail for the OAI-PMH service
 OAIPMH_ADMIN_EMAIL = os.getenv('OAIPMH_ADMIN_EMAIL', 'support@datacite.org')
+# OAI repository identifier
+OAIPMH_IDENTIFIER = os.getenv('OAIPMH_IDENTIFIER', 'oai.datacite.org')
 # Page size of results shown for result listings
 RESULT_SET_SIZE = int(os.getenv('RESULT_SET_SIZE', '50'))
+# Source metadata catalog (DataCite or FRDR)
+CATALOG_SET = os.getenv('OAIPMH_CATALOG', 'DataCite')
+# FRDR Postgres server
+POSTGRES_SERVER = os.getenv('OAIPMH_POSTGRES_SERVER', '')
+# FRDR Postgres db
+POSTGRES_DB = os.getenv('OAIPMH_POSTGRES_DB', '')
+# FRDR Postgres user
+POSTGRES_USER = os.getenv('OAIPMH_POSTGRES_USER', '')
+# FRDR Postgres password
+POSTGRES_PASSWORD = os.getenv('OAIPMH_POSTGRES_PASSWORD', '')
+# FRDR Postgres port
+POSTGRES_PORT = os.getenv('OAIPMH_POSTGRES_PORT', '5432')

viringo/metadata.py

@@ -1,6 +1,7 @@
 """This module deals with handling the representation of metadata formats for OAI"""
 
 import re
+import ftfy
 from lxml import etree
 
 NS_OAIPMH = 'http://www.openarchives.org/OAI/2.0/'
@@ -43,10 +44,20 @@ def nsdc(name):
         ]:
         for value in _map.get(name, []):
             if value:
+                if isinstance(value, list):
+                    if len(value) == 1:
+                        value = value[0]
+                    else:
+                        value = str(value)
                 new_element = etree.SubElement(e_dc, nsdc(name))
-                # The regular expression here is to filter only valid XML chars
-                # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
-                new_element.text = re.sub(u'[^\u0020-\uD7FF\u0009\u000A\u000D\uE000-\uFFFD\U00010000-\U0010FFFF]+', '', value)
+                if isinstance(value, str):
+                    try:
+                        value = value.replace('\x0c', " ")
+                        new_element.text = ftfy.fix_text(value)
+                    except:
+                        new_element.text = ''
+                else:
+                    new_element.text = ''
 
 def datacite_writer(element: etree.Element, metadata):
     """Writer for writing data in a metadata object out into raw datacite format"""
@@ -62,7 +73,10 @@ def oai_datacite_writer(element: etree.Element, metadata):
     _map = metadata.getMap()
     raw_xml = _map.get('xml', '')
 
-    xml_resource_element = etree.fromstring(raw_xml)
+    try:
+        xml_resource_element = etree.fromstring(raw_xml)
+    except:
+        print(raw_xml)
 
     e_oai_datacite = etree.SubElement(
         element, "oai_datacite", {'xmlns': 'http://schema.datacite.org/oai/oai-1.1/'},