Merge pull request #142 from unt-libraries/py3

Py3

.travis.yml

@@ -1,7 +1,7 @@
-dist: trusty
+dist: xenial
 language: python
 sudo: false
-python: 2.7
+python: 3.7
 services:
     - mysql
 env:

Dockerfile

@@ -1,10 +1,11 @@
 # vim: set ft=conf
-FROM python:2.7-stretch
+FROM python:3.7-stretch
 
 RUN echo "US/Central" > /etc/timezone
 RUN dpkg-reconfigure -f noninteractive tzdata
 
 ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONPATH /app:/app/coda
 
 RUN mkdir /app

coda/coda_mdstore/mdstore_oai.py

@@ -17,7 +17,7 @@ def __init__(self, identifyDict={}, resultSize=10, domain="", debug=False):
             'baseURL': 'http://not.a.good.base.url.com/fixit/kurt/',
             'protocolVersion': "2.0",
             'adminEmails': ['[email protected]'],
-            'earliestDatestamp': datetime(2004, 05, 19),
+            'earliestDatestamp': datetime(2004, 5, 19),
             'deletedRecord': 'transient',
             'granularity': 'YYYY-MM-DDThh:mm:ssZ',
             # 'granularity':'YYYY-MM-DD',
@@ -42,7 +42,7 @@ def getRecord(self, metadataPrefix, identifier):
         try:
             bagObject = Bag.objects.get(name=id)
         except Bag.DoesNotExist:
-            raise error.IdDoesNotExistError, "Id doesnt exist: %s" % identifier
+            raise error.IdDoesNotExistError("Id doesn't exist: %s" % identifier)
         record = makeDataRecord(
             bagObject, domain=self.domain,
             metadataPrefix=metadataPrefix
@@ -186,18 +186,19 @@ def makeDataRecord(
         raise Exception("dcDict is empty")
     setList = []
     header = common.Header(
+        None,
         id,
         date,
         setList,
         False,
     )
     dublinStruct = dcDict
     if metadataPrefix == "oai_dc":
-        metadata = common.Metadata(dublinStruct)
+        metadata = common.Metadata(None, dublinStruct)
     elif metadataPrefix == "coda_bag":
         bagMap = {}
         bagMap["bag"] = bagObject
-        metadata = common.Metadata(bagMap)
+        metadata = common.Metadata(None, bagMap)
     return (header, metadata, None)
 
 

coda/coda_mdstore/migrations/0003_auto_20191022_0935.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.24 on 2019-10-22 09:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('coda_mdstore', '0002_add_fulltext_index'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bag',
+            name='bagging_date',
+            field=models.DateField(help_text='Date of Bag Creation'),
+        ),
+        migrations.AlterField(
+            model_name='bag',
+            name='bagit_version',
+            field=models.CharField(help_text='BagIt version number', max_length=10),
+        ),
+        migrations.AlterField(
+            model_name='bag',
+            name='files',
+            field=models.IntegerField(help_text='Number of files in Bag'),
+        ),
+        migrations.AlterField(
+            model_name='bag',
+            name='last_verified_date',
+            field=models.DateField(help_text='Date of last Bag Verification'),
+        ),
+        migrations.AlterField(
+            model_name='bag',
+            name='last_verified_status',
+            field=models.CharField(help_text='Status of last bag Verification', max_length=25),
+        ),
+        migrations.AlterField(
+            model_name='bag',
+            name='name',
+            field=models.CharField(help_text='Name of Bag', max_length=255, primary_key=True,
+                                   serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='bag',
+            name='size',
+            field=models.BigIntegerField(help_text="Size of Bag's Payload (in bytes)"),
+        ),
+        migrations.AlterField(
+            model_name='bag_info',
+            name='field_body',
+            field=models.TextField(help_text='Field Body'),
+        ),
+        migrations.AlterField(
+            model_name='bag_info',
+            name='field_name',
+            field=models.CharField(db_index=True, help_text='Field Name', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='node',
+            name='last_checked',
+            field=models.DateTimeField(blank=True, help_text='Date node size last checked'),
+        ),
+        migrations.AlterField(
+            model_name='node',
+            name='node_capacity',
+            field=models.BigIntegerField(blank=True,
+                                         help_text='The total amount of storage (in bytes)'),
+        ),
+        migrations.AlterField(
+            model_name='node',
+            name='node_name',
+            field=models.CharField(db_index=True, help_text='The name of the node',
+                                   max_length=255, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='node',
+            name='node_path',
+            field=models.CharField(help_text="The path on disk to the node's root",
+                                   max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='node',
+            name='node_size',
+            field=models.BigIntegerField(blank=True,
+                                         help_text='The current size of files on disk (in bytes)'),
+        ),
+        migrations.AlterField(
+            model_name='node',
+            name='node_url',
+            field=models.TextField(help_text='The external url for the root of the node'),
+        ),
+    ]

coda/coda_mdstore/models.py

@@ -31,7 +31,7 @@ class Bag(models.Model):
     def oxum(self):
         return '{size}.{files}'.format(**vars(self))
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 
@@ -46,7 +46,7 @@ class Bag_Info(models.Model):
         help_text="Field Body"
     )
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s:%s" % (self.bag_name, self.field_name)
 
     class Meta:
@@ -83,5 +83,5 @@ class External_Identifier(models.Model):
     class Meta:
         ordering = ['value']
 
-    def __unicode__(self):
+    def __str__(self):
         return self.value

coda/coda_mdstore/presentation.py

@@ -1,10 +1,9 @@
 import os
 import re
-import urllib
-import urllib2
-import urlparse
+import urllib.request
+import urllib.parse
 
-from BeautifulSoup import BeautifulSoup as BSoup
+from bs4 import BeautifulSoup as BSoup
 from codalib import APP_AUTHOR
 from codalib.bagatom import (
     wrapAtom, ATOM, ATOM_NSMAP, BAG, BAG_NSMAP, TIME_FORMAT_STRING
@@ -27,21 +26,17 @@ def getFileList(url):
     """
 
     fileList = []
-    handle = urllib2.urlopen(url)
+    handle = urllib.request.urlopen(url)
     soup = BSoup(handle)
-    trList = soup.findAll('tr')
+    trList = soup.find_all('tr')
     for tr in trList:
-        tds = tr.findAll('td')
+        tds = tr.find_all('td')
         for td in tds:
-            anchors = td.findAll('a')
+            anchors = td.find_all('a')
             for anchor in anchors:
-                try:
-                    # if anchor.contents and "Parent Directory" in contents:
-                    if anchor['href'][-1] == "/":
-                        continue
-                    fileList.append(anchor['href'])
-                except Exception, e:
-                    raise e
+                if anchor['href'][-1] == "/":
+                    continue
+                fileList.append(anchor['href'])
     return fileList
 
 
@@ -57,13 +52,13 @@ def getFileHandle(codaId, codaPath):
     codaPathParts = codaPath.split("/")
     for i in range(len(codaPathParts)):
         part = codaPathParts[i]
-        codaPathParts[i] = urllib.quote(part)
+        codaPathParts[i] = urllib.parse.quote(part)
     escapedCodaPath = "/".join(codaPathParts)
     nodeList = Node.objects.all()
     exceptionList = []
     for node in nodeList:
-        url_parts = urlparse.urlparse(node.node_url)
-        url = urlparse.urljoin(
+        url_parts = urllib.parse.urlparse(node.node_url)
+        url = urllib.parse.urljoin(
             "http://%s" % url_parts.hostname,
             os.path.join(
                 url_parts.path,
@@ -75,9 +70,9 @@ def getFileHandle(codaId, codaPath):
         )
         # urlList.append(url)
         try:
-            fileHandle = urllib2.urlopen(url)
+            fileHandle = urllib.request.urlopen(url)
             return fileHandle
-        except Exception, e:
+        except Exception as e:
             exceptionList.append(str(e))
             pass
     raise Exception(
@@ -190,7 +185,7 @@ def xmlToBagObject(codaXML):
         name = codaXML.xpath("*[local-name() = 'name']")[0].text.strip()
         bagObject = Bag.objects.get(Bag, name=name)
     # if we can't get the object, then we're just making a new one.
-    except Exception, e:
+    except Exception:
         bagObject = Bag()
     dateFormatString = "%Y-%m-%d"
     try:
@@ -206,7 +201,7 @@ def xmlToBagObject(codaXML):
     try:
         bagObject.size = int(codaXML.xpath(
             "*[local-name() = 'payloadSize']")[0].text.strip())
-    except Exception, e:
+    except Exception as e:
         return (None, None, "Unable to set 'size' attribute: %s" % (e,))
     try:
         bagObject.bagit_version = codaXML.xpath(
@@ -345,8 +340,8 @@ def updateNode(request):
     node_size = nodeXML.xpath("*[local-name() = 'size']")[0].text.strip()
     node_path = nodeXML.xpath("*[local-name() = 'path']")[0].text.strip()
     node_url = nodeXML.xpath("*[local-name() = 'url']")[0].text.strip()
-    node.node_capacity = long(node_capacity)
-    node.node_size = long(node_size)
+    node.node_capacity = int(node_capacity)
+    node.node_size = int(node_size)
     node.node_path = node_path
     node.node_name = node_name
     node.node_url = node_url
@@ -373,8 +368,8 @@ def createNode(request):
     node_path = nodeXML.xpath("*[local-name() = 'path']")[0].text.strip()
     node_url = nodeXML.xpath("*[local-name() = 'url']")[0].text.strip()
     node = Node()
-    node.node_capacity = long(node_capacity)
-    node.node_size = long(node_size)
+    node.node_capacity = int(node_capacity)
+    node.node_size = int(node_size)
     node.node_path = node_path
     node.node_name = node_name
     node.node_url = node_url

coda/coda_mdstore/resourcesync.py

@@ -1,4 +1,4 @@
-from urlparse import urlparse
+from urllib.parse import urlparse
 import warnings
 
 from django.contrib.sitemaps import Sitemap, views
@@ -16,7 +16,7 @@
     MOST_RECENT_BAGGING_DATE = Bag.objects.latest(
         'bagging_date'
     ).bagging_date.strftime(TIME_FORMAT_STRING)
-except Exception, e:
+except Exception:
     MOST_RECENT_BAGGING_DATE = '2012-12-12T00:00:00Z'
 
 
@@ -45,7 +45,7 @@ def index(
     req_site = get_current_site(request)
 
     sites = []
-    for section, site in sitemaps.iteritems():
+    for section, site in sitemaps.items():
         if callable(site):
             site = site()
         protocol = req_protocol if site.protocol is None else site.protocol

coda/coda_mdstore/tests/test_mdstore_oai.py

@@ -270,7 +270,7 @@ def test_coda_bag_writer():
     element = etree.Element('root')
 
     oai.coda_bag_writer(element, metadata)
-    assert 'bag:codaXML' in etree.tostring(element)
+    assert b'bag:codaXML' in etree.tostring(element)
 
 
 @pytest.mark.django_db

coda/coda_mdstore/tests/test_models.py

@@ -8,6 +8,6 @@ def test_oxum(self):
         expected = '{0}.{1}'.format(bag.size, bag.files)
         assert bag.oxum == expected
 
-    def test_unicode(self):
+    def test_str(self):
         bag = factories.BagFactory.build()
-        assert unicode(bag) == bag.name
+        assert str(bag) == bag.name

coda/coda_mdstore/tests/test_utils.py

@@ -4,7 +4,7 @@
 from django.conf import settings
 from lxml import etree, objectify
 from codalib import bagatom
-import mock
+from unittest import mock
 import pytest
 
 from coda_mdstore import factories, models, presentation, views, exceptions
@@ -591,3 +591,22 @@ def test_existing_bag_info_objects_are_update(self, bag_xml, rf):
         update_bag_infos = updated_bag.bag_info_set.all()
         assert old_bag_info1.field_name not in [b.field_name for b in update_bag_infos]
         assert old_bag_info2.field_name not in [b.field_name for b in update_bag_infos]
+
+
+@mock.patch('coda_mdstore.presentation.urllib.request.urlopen')
+def test_getFileList(mock_urlopen):
+    """Test all attribute values are extracted as files."""
+    text = b"""<html>
+                 <body>
+                 <tr> <td>test</td> <td>data</td> </tr>
+                 <tr> <td>of </td> </tr>
+                 <tr> <td>
+                    <a href='bag-info.txt'>url</a>
+                    <a href='manifest-md5.txt'>here</a>
+                    <a href='bagit.txt'>here</a>
+                 </td> </tr>
+                 </body>
+              </html>"""
+    mock_urlopen.return_value = text
+    filelist = presentation.getFileList('https://coda/testurl')
+    assert ['bag-info.txt', 'manifest-md5.txt', 'bagit.txt'] == filelist