Migrate to Python3

CHANGES.rst

@@ -142,6 +142,8 @@ Development
 - [mw] Improve settings for having per-vendor OPS credentials
 - [ui] More flexbox for header layout
 - [ui] Improve comment editing usability
+- [mw] No need to manually encode form fields with "mechanize" anymore.
+  Thanks, `Kovid <https://github.com/kovidgoyal>`_!
 
 
 2019-05-08 0.165.0

Makefile

@@ -1,18 +1,16 @@
 #VERSION := $(shell cat patzilla/version.py | awk '{ print $$3 }' | tr -d "'")
 #$(error VERSION=$(VERSION))
 
-$(eval venvpath     := .venv2)
+$(eval venvpath     := .venv3)
 $(eval pip          := $(venvpath)/bin/pip)
 $(eval twine        := $(venvpath)/bin/twine)
 $(eval python       := $(venvpath)/bin/python)
 $(eval pserve       := $(venvpath)/bin/pserve)
 $(eval pytest       := $(venvpath)/bin/pytest)
 $(eval bumpversion  := $(venvpath)/bin/bumpversion)
 $(eval fab          := $(venvpath)/bin/fab)
-
-$(eval venv3path     := .venv)
-$(eval yarn          := $(venv3path)/bin/yarn)
-$(eval npx           := $(venv3path)/bin/npx)
+$(eval yarn         := $(venvpath)/bin/yarn)
+$(eval npx          := $(venvpath)/bin/npx)
 
 
 setup: setup-py
@@ -65,7 +63,7 @@ upload-pypi:
 
 # Setup Python virtualenv.
 setup-virtualenv:
-	@test -e $(python) || virtualenv --python=python2 $(venvpath)
+	@test -e $(python) || virtualenv --python=python3 $(venvpath)
 
 setup-py: setup-virtualenv
 	$(pip) install --editable=.[test]

docs/conf.py

@@ -49,18 +49,18 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'PatZilla'
-copyright = u'2013-2022, The PatZilla authors'
-author = u'The PatZilla authors'
+project = 'PatZilla'
+copyright = '2013-2012, The PatZilla authors'
+author = 'The PatZilla authors'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = u'0.169.3'
+version = '0.169.3'
 # The full version, including alpha/beta/rc tags.
-release = u'0.169.3'
+release = '0.169.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -145,8 +145,8 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'PatZilla.tex', u'PatZilla Documentation',
-     u'The PatZilla authors', 'manual'),
+    (master_doc, 'PatZilla.tex', 'PatZilla Documentation',
+     'The PatZilla authors', 'manual'),
 ]
 
 
@@ -155,7 +155,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'patzilla', u'PatZilla Documentation',
+    (master_doc, 'patzilla', 'PatZilla Documentation',
      [author], 1)
 ]
 
@@ -166,7 +166,7 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'PatZilla', u'PatZilla Documentation',
+    (master_doc, 'PatZilla', 'PatZilla Documentation',
      author, 'PatZilla', 'One line description of project.',
      'Miscellaneous'),
 ]

fabfile.py

@@ -34,7 +34,7 @@ def install(version, target):
     if not version:
         version = pkg_version
 
-    print 'Installing package {0}, version {1} to target {2}.'.format(*map(yellow, [pkg_name, version, target]))
+    print('Installing package {0}, version {1} to target {2}.'.format(*list(map(yellow, [pkg_name, version, target]))))
     if env.confirm:
         response = ask('Proceed (y/n)? ', ('y', 'n'))
     else:
@@ -72,7 +72,7 @@ def install(version, target):
         restart_service(target)
 
     else:
-        print yellow('Skipped package install due to user request.')
+        print(yellow('Skipped package install due to user request.'))
 
 def setup_package(package, virtualenv, options=''):
     #--index-url=http://c.pypi.python.org/simple
@@ -100,7 +100,7 @@ def restart_service(target):
     if uwsgi_name:
         run('service uwsgi reload %s' % uwsgi_name)
     else:
-        print(red('WARNING: Could not restart service "%s"' % target))
+        print((red('WARNING: Could not restart service "%s"' % target)))
 
 @task
 @hosts(INSTALLATION_HOST)

patzilla/access/cipo/drawing.py

@@ -3,7 +3,7 @@
 import re
 import logging
 import requests
-from BeautifulSoup import BeautifulSoup
+from bs4 import BeautifulSoup
 from patzilla.util.numbers.common import split_patent_number
 
 log = logging.getLogger(__name__)
@@ -70,7 +70,7 @@ def get_first_drawing_url(patent):
     images_index_html = fetch_images_index(images_index_url)
     soup = BeautifulSoup(images_index_html)
     # <img src="/opic-cipo/cpd/page/141597_20130713_drawings_page1_scale25_rotate0.gif?page=3&amp;section=drawings&amp;scale=25&amp;rotation=0&amp;type=" alt="Canadian Patent Document 141597. Drawings page. Image 1 of 3" />
-    first_drawing_url = cipo_baseurl + soup.find('img', src=re.compile(ur'/opic-cipo/cpd/page'))['src']
+    first_drawing_url = cipo_baseurl + soup.find('img', src=re.compile(r'/opic-cipo/cpd/page'))['src']
 
     return first_drawing_url
 
@@ -83,6 +83,6 @@ def get_first_drawing_url(patent):
         payload = fetch_first_drawing(split_patent_number(number))
         if payload:
             #print "payload length:", len(payload)
-            print payload
+            print(payload)
         else:
-            print "not found"
+            print("not found")

patzilla/access/depatech/client.py

@@ -13,7 +13,7 @@
 from patzilla.access.depatech import get_depatech_client
 from patzilla.access.generic.exceptions import NoResultsException, GenericAdapterException, SearchException
 from patzilla.access.generic.search import GenericSearchResponse, GenericSearchClient
-from patzilla.util.data.container import SmartBunch
+from patzilla.util.data.container import SmartMunch
 from patzilla.util.numbers.normalize import normalize_patent
 
 log = logging.getLogger(__name__)
@@ -55,7 +55,7 @@ def search(self, query, options=None):
         return self.search_real(query, options=options)
 
     def search_real(self, query, options=None):
-        options = options or SmartBunch()
+        options = options or SmartMunch()
 
         options.setdefault('offset', 0)
         options.setdefault('limit', self.pagesize)
@@ -73,7 +73,7 @@ def search_real(self, query, options=None):
             transport = 'json'
             query.expression = self.translate_deparom_query(query.expression)
 
-        log.info(u"{backend_name}: searching documents, expression='{0}', offset={1}, limit={2}; user={username}".format(
+        log.info("{backend_name}: searching documents, expression='{0}', offset={1}, limit={2}; user={username}".format(
             query.expression, offset, limit, **self.__dict__))
 
         starttime = timeit.default_timer()
@@ -92,7 +92,7 @@ def search_real(self, query, options=None):
             'from': offset, 'size': limit,
         }
 
-        log.info(u'{backend_name}: query={query}, uri={uri}, params={params}, options={options}'.format(
+        log.info('{backend_name}: query={query}, uri={uri}, params={params}, options={options}'.format(
             query=query, uri=uri, params=params, options=options.dump(), backend_name=self.backend_name))
 
         # Perform search request
@@ -164,10 +164,10 @@ def search_real(self, query, options=None):
                 if 'reason' not in upstream_error:
                     upstream_error['reason'] = 'Reason unknown'
 
-                message = u'Response status code: {code}\n\n{reason}'.format(**upstream_error)
+                message = 'Response status code: {code}\n\n{reason}'.format(**upstream_error)
 
                 raise self.search_failed(
-                    user_info=u'Error searching depa.tech.',
+                    user_info='Error searching depa.tech.',
                     message=message,
                     response=response)
 
@@ -180,7 +180,7 @@ def translate_deparom_query(self, expression):
 
         expression = expression.replace(upstream_prefix, '').replace('deparom:', '')
 
-        log.info(u'{backend_name}: Translate DEPAROM query expression={expression}, uri={uri}'.format(
+        log.info('{backend_name}: Translate DEPAROM query expression={expression}, uri={uri}'.format(
             expression=expression, uri=uri, backend_name=self.backend_name))
 
         expression = upstream_prefix + expression
@@ -212,7 +212,7 @@ def translate_deparom_query(self, expression):
 
         elif response.status_code >= 400:
 
-            message = u'Reason unknown'
+            message = 'Reason unknown'
 
             if response.headers.get('Content-Type', '').startswith('application/json'):
 
@@ -224,15 +224,15 @@ def translate_deparom_query(self, expression):
                     upstream_error['code'] = response_data['status']
 
                     if 'reason' not in upstream_error:
-                        upstream_error['reason'] = u'Reason unknown'
+                        upstream_error['reason'] = 'Reason unknown'
 
-                    message = u'Response status code: {code}\n\n{reason}'.format(**upstream_error)
+                    message = 'Response status code: {code}\n\n{reason}'.format(**upstream_error)
 
             else:
                 message = response.content
 
             raise self.search_failed(
-                user_info=u'Translating DEPAROM query expression failed',
+                user_info='Translating DEPAROM query expression failed',
                 message=message,
                 response=response)
 
@@ -298,23 +298,23 @@ def read(self):
             'name': 'depatech',
             'time': self.input['took'],
             'status': 'success',
-            #'params': SmartBunch.bunchify(self.input['content']['responseHeader']['params']),
-            #'pager': SmartBunch.bunchify(self.input['content']['responseHeader'].get('pager', {})),
+            #'params': SmartMunch.munchify(self.input['content']['responseHeader']['params']),
+            #'pager': SmartMunch.munchify(self.input['content']['responseHeader'].get('pager', {})),
         })
 
         self.meta.navigator.count_total = int(self.input['hits']['total'])
         #self.meta.navigator.count_page  = int(self.meta.upstream.pager.entriesOnThisPage)
         self.meta.navigator.offset      = int(self.options.offset)
         self.meta.navigator.limit       = int(self.options.limit)
         self.meta.navigator.max_hits   = int(self.options.max_hits)
-        self.meta.navigator.postprocess = SmartBunch()
+        self.meta.navigator.postprocess = SmartMunch()
 
         # Read content
         self.documents = self.input['hits']['hits']
         self.read_documents()
 
     def document_to_number(self, document):
-        _id = document[u'_id']
+        _id = document['_id']
         cc, docno, kindcode = _id.split('.')
         publication_number = cc + docno + kindcode
         number = normalize_patent(publication_number)
@@ -326,7 +326,7 @@ def document_to_family_id(self, document):
 
 def depatech_search(query, options=None):
 
-    options = options or SmartBunch()
+    options = options or SmartMunch()
 
     client = get_depatech_client()
     try:

patzilla/access/depatech/clientpool.py

@@ -3,8 +3,8 @@
 import logging
 import os
 from pyramid.httpexceptions import HTTPUnauthorized
-from zope.interface.declarations import implements
 from zope.interface.interface import Interface
+from zope.interface import implementer
 from patzilla.access.depatech.client import DepaTechClient
 from patzilla.access.generic.credentials import AbstractCredentialsGetter, DatasourceCredentialsManager
 
@@ -78,13 +78,12 @@ class IDepaTechClientPool(Interface):
     pass
 
 
+@implementer(IDepaTechClientPool)
 class DepaTechClientPool(object):
     """
     depa.tech client pool as Pyramid utility implementation.
     """
 
-    implements(IDepaTechClientPool)
-
     def __init__(self, api_uri):
         logger.info("Creating upstream client pool for depa.tech")
         self.api_uri = api_uri