Merge branch 'remote-fonts'

Author: chrissimpkins

.travis.yml

@@ -17,25 +17,25 @@ matrix:
       dist: xenial
       env: TOX_ENV=coverage
       install:
-        - pip install --upgrade coverage pytest
+        - pip install --upgrade coverage pytest pytest-asyncio
         - pip install .
       script:
         - make test-coverage
       after_success:
         - bash <(curl -s https://codecov.io/bash)
     - python: 3.6
       env: TOX_ENV=py36
-      install: pip install --upgrade tox pytest
+      install: pip install --upgrade tox pytest pytest-asyncio
       script: tox -e $TOX_ENV
     - python: 3.7
       env: TOX_ENV=py37
-      install: pip install --upgrade tox pytest
+      install: pip install --upgrade tox pytest pytest-asyncio
       dist: xenial
       script: tox -e $TOX_ENV
     - os: osx
       language: generic
       osx_image: xcode11    # Python 3.7.4 running on macOS 10.14.4
-      install: pip3 install --upgrade tox pytest
+      install: pip3 install --upgrade tox pytest pytest-asyncio
       env: TOX_ENV=py37
       script: tox -e $TOX_ENV
 

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## v0.4.0
+
+- Added support for remote font files with asynchronous I/O GET requests.  This feature supports combinations of local and remote font file comparisons.
+    - `fdiff` executable: added support for remote font files with command line URL arguments
+    - `fdiff` executable: refactored unified diff error message formatting
+    - Library: add new `fdiff.remote` module
+    - Library: add new `fdiff.aio` module
+    - Library: add new `fdiff.exceptions` module
+    - Library: refactored `fdiff.diff.unified_diff()` function to support remote files through URL
+    - Library: refactored local file path checks to support remote files via URL
+    - added new aiohttp, aiodns, aiofiles dependencies to requirements.txt
+    - added new aiohttp, aiodns, aiofiles dependencies to setup.py
+    - added pytest-asyncio dependency to setup.py [dev] install target
+    - added pytest-asyncio dependency instatllation to tox.ini, .travis.yml, .appveyor.yml configuration files
+- Py3.6+ updates: removed `# -*- coding: utf-8 -*-` header definitions (Thanks Niko!)
+- updated fontTools dependency to v4.0.1 (from v4.0.0)
+- Updated README.md documentation
+
 ## v0.3.0
 
 - Added support for head and tail diff output filter functionality

README.md

@@ -13,15 +13,15 @@
 
 ## About
 
-`fdiff` is a Python command line comparison tool for differences in the OpenType table data between font files.  The tool provides cross-platform support on macOS, Windows, and Linux systems with a Python v3.6+ interpreter.
+`fdiff` is a Python command line comparison tool for assessment of differences in the OpenType table data between font files.  The tool provides cross-platform support on macOS, Windows, and Linux systems with a Python v3.6+ interpreter.
 
 <p align="center">
 <img src="https://raw.githubusercontent.com/source-foundry/fdiff/img/img/diff-example-crunch.png" width="500"/>
 </p>
 
 ## What it does
 
-- Takes two font file path arguments for comparison
+- Takes two font file path (or URL for remote fonts) arguments for the font comparison
 - Dumps OpenType table data in the fontTools library TTX format (XML)
 - Compares the OpenType table data across the two files using the unified diff format with 3 lines of surrounding context
 
@@ -79,16 +79,34 @@ $ pip3 install --ignore-installed -r requirements.txt -e ".[dev]"
 
 ## Usage
 
+#### Local font files
+
 ```
 $ fdiff [OPTIONS] [PRE-FONT FILE PATH] [POST-FONT FILE PATH]
 ```
 
-By default, an uncolored unified diff is performed on the two files defined with the local file paths in the above command.
+#### Remote font files
+
+`fdiff` supports GET requests for publicly accessible remote font files.  Replace the file path arguments with URL:
+
+```
+$ fdiff [OPTIONS] [PRE-FONT FILE URL] [POST-FONT FILE URL]
+```
+
+`fdiff` works with any combination of local and remote font files. For example, to compare a local post font file with a remote pre font file to assess local changes against a font file that was previously pushed to a remote, use the following syntax:
+
+```
+$ fdiff [OPTIONS] [PRE-FONT FILE URL] [POST-FONT FILE FILE PATH]
+```
+
+⭐ **Tip**: Remote git repository hosting services (like Github) support access to files on different git branches by URL.  Use these repository branch URL to compare fonts across git branches in your repository.
 
 ### Options
 
 #### Color diffs
 
+Uncolored diffs are performed by default.
+
 To view a colored diff in your terminal, include either the `-c` or `--color` option in your command:
 
 ```

appveyor.yml

@@ -20,7 +20,7 @@ install:
   - "python -m pip install --disable-pip-version-check --user --upgrade pip setuptools virtualenv"
 
   # install the dependencies to run the tests
-  - "python -m pip install tox"
+  - "python -m pip install --upgrade tox pytest pytest-asyncio"
 
 build: false
 

codecov.yml

@@ -1,7 +1,16 @@
 codecov:
   max_report_age: off
 
-comment: off
+coverage:
+  status:
+    project:
+      default:
+        # basic
+        target: auto
+        threshold: 2%
+        base: auto
 
 ignore:
   - "lib/fdiff/thirdparty"
+
+comment: off

lib/fdiff/__main__.py

@@ -85,12 +85,12 @@ def run(argv):
     #  File path argument validations
     # -------------------------------
 
-    if not file_exists(args.PREFILE):
+    if not args.PREFILE.startswith("http") and not file_exists(args.PREFILE):
         sys.stderr.write(
             f"[*] ERROR: The file path '{args.PREFILE}' can not be found.{os.linesep}"
         )
         sys.exit(1)
-    if not file_exists(args.POSTFILE):
+    if not args.PREFILE.startswith("http") and not file_exists(args.POSTFILE):
         sys.stderr.write(
             f"[*] ERROR: The file path '{args.POSTFILE}' can not be found.{os.linesep}"
         )
@@ -123,10 +123,7 @@ def run(argv):
             exclude_tables=exclude_list,
         )
     except Exception as e:
-        sys.stderr.write(
-            f"[*] ERROR: During the attempt to diff the requested files the following error was encountered: "
-            f"{e}{os.linesep}"
-        )
+        sys.stderr.write(f"[*] ERROR: {e}{os.linesep}")
         sys.exit(1)
 
     # re-define the line contents of the diff iterable

lib/fdiff/aio.py

@@ -0,0 +1,7 @@
+import aiofiles
+
+
+async def async_write_bin(path, binary):
+    """Asynchronous IO writes of binary data `binary` to disk on the file path `path`"""
+    async with aiofiles.open(path, "wb") as f:
+        await f.write(binary)

lib/fdiff/diff.py

@@ -1,10 +1,16 @@
+import asyncio
 import os
 import tempfile
 
 from fontTools.ttLib import TTFont
 
-from fdiff.utils import get_file_modtime
+from fdiff.exceptions import AIOError
+from fdiff.remote import (
+    _get_filepath_from_url,
+    create_async_get_request_session_and_run,
+)
 from fdiff.thirdparty.fdifflib import unified_diff
+from fdiff.utils import get_file_modtime
 
 
 def u_diff(
@@ -13,8 +19,8 @@ def u_diff(
     """Performs a unified diff on a TTX serialized data format dump of font binary data using
     a modified version of the Python standard libary difflib module.
 
-    filepath_a: (string) pre-file path
-    filepath_b: (string) post-file path
+    filepath_a: (string) pre-file local file path or URL path
+    filepath_b: (string) post-file local file path or URL path
     context_lines: (int) number of context lines to include in the diff (default=3)
     include_tables: (list of str) Python list of OpenType tables to include in the diff
     exclude_tables: (list of str) Python list of OpentType tables to exclude from the diff
@@ -24,39 +30,81 @@ def u_diff(
 
     :returns: Generator of ordered diff line strings that include newline line endings
     :raises: KeyError if include_tables or exclude_tables includes a mis-specified table
-    that is not included in filepath_a OR filepath_b"""
-    tt_left = TTFont(filepath_a)
-    tt_right = TTFont(filepath_b)
-
-    # Validation: include_tables request should be for tables that are in one of
-    # the two fonts. This otherwise silently passes with exit status code 0 which
-    # could lead to the interpretation of no diff between two files when the table
-    # entry is incorrectly defined or is a typo.  Let's be conservative and consider
-    # this an error, force user to use explicit definitions that include tables in
-    # one of the two files, and understand that the diff request was for one or more
-    # tables that are not present.
-    if include_tables is not None:
-        for table in include_tables:
-            if table not in tt_left and table not in tt_right:
-                raise KeyError(
-                    f"'{table}' table was not identified for inclusion in either font"
-                )
-
-    # Validation: exclude_tables request should be for tables that are in one of
-    # the two fonts.  Mis-specified OT table definitions could otherwise result
-    # in the presence of a table in the diff when the request was to exclude it.
-    # For example, when an "OS/2" table request is entered as "OS2".
-    if exclude_tables is not None:
-        for table in exclude_tables:
-            if table not in tt_left and table not in tt_right:
-                raise KeyError(
-                    f"'{table}' table was not identified for exclusion in either font"
-                )
-
-    fromdate = get_file_modtime(filepath_a)
-    todate = get_file_modtime(filepath_b)
-
+    that is not included in filepath_a OR filepath_b
+    :raises: fdiff.exceptions.AIOError if exception raised during execution of async I/O
+             GET request for URL or file write
+    :raises: fdiff.exceptions.AIOError if GET request to URL returned non-200 response status code"""
     with tempfile.TemporaryDirectory() as tmpdirname:
+        # define the file paths with either local file requests
+        # or pulls of remote files based on the command line request
+        urls = []
+        if filepath_a.startswith("http"):
+            urls.append(filepath_a)
+            prepath = _get_filepath_from_url(filepath_a, tmpdirname)
+            # keep URL as path name for remote file requests
+            pre_pathname = filepath_a
+        else:
+            prepath = filepath_a
+            pre_pathname = filepath_a
+
+        if filepath_b.startswith("http"):
+            urls.append(filepath_b)
+            postpath = _get_filepath_from_url(filepath_b, tmpdirname)
+            # keep URL as path name for remote file requests
+            post_pathname = filepath_b
+        else:
+            postpath = filepath_b
+            post_pathname = filepath_b
+
+        # Async IO fetch and write of any remote file requests
+        if len(urls) > 0:
+            loop = asyncio.get_event_loop()
+            tasks = loop.run_until_complete(
+                create_async_get_request_session_and_run(urls, tmpdirname)
+            )
+            for task in tasks:
+                if task.exception():
+                    # raise exception here to notify calling code that something
+                    # did not work
+                    raise AIOError(f"{task.exception()}")
+                elif task.result().http_status != 200:
+                    # handle non-200 HTTP response status codes + file write fails
+                    raise AIOError(
+                        f"failed to pull '{task.result().url}' with HTTP status code {task.result().http_status}"
+                    )
+
+        # instantiate left and right fontTools.ttLib.TTFont objects
+        tt_left = TTFont(prepath)
+        tt_right = TTFont(postpath)
+
+        # Validation: include_tables request should be for tables that are in one of
+        # the two fonts. This otherwise silently passes with exit status code 0 which
+        # could lead to the interpretation of no diff between two files when the table
+        # entry is incorrectly defined or is a typo.  Let's be conservative and consider
+        # this an error, force user to use explicit definitions that include tables in
+        # one of the two files, and understand that the diff request was for one or more
+        # tables that are not present.
+        if include_tables is not None:
+            for table in include_tables:
+                if table not in tt_left and table not in tt_right:
+                    raise KeyError(
+                        f"'{table}' table was not identified for inclusion in either font"
+                    )
+
+        # Validation: exclude_tables request should be for tables that are in one of
+        # the two fonts.  Mis-specified OT table definitions could otherwise result
+        # in the presence of a table in the diff when the request was to exclude it.
+        # For example, when an "OS/2" table request is entered as "OS2".
+        if exclude_tables is not None:
+            for table in exclude_tables:
+                if table not in tt_left and table not in tt_right:
+                    raise KeyError(
+                        f"'{table}' table was not identified for exclusion in either font"
+                    )
+
+        fromdate = get_file_modtime(prepath)
+        todate = get_file_modtime(postpath)
+
         tt_left.saveXML(
             os.path.join(tmpdirname, "left.ttx"),
             tables=include_tables,
@@ -76,8 +124,8 @@ def u_diff(
         return unified_diff(
             fromlines,
             tolines,
-            filepath_a,
-            filepath_b,
+            pre_pathname,
+            post_pathname,
             fromdate,
             todate,
             n=context_lines,

lib/fdiff/exceptions.py

@@ -0,0 +1,2 @@
+class AIOError(Exception):
+    pass

lib/fdiff/remote.py

@@ -0,0 +1,66 @@
+import os.path
+import urllib.parse
+
+from collections import namedtuple
+
+import aiohttp
+import asyncio
+
+from fdiff.aio import async_write_bin
+
+
+def _get_filepath_from_url(url, dirpath):
+    """Returns filepath from base file name in URL and directory path."""
+    url_path_list = urllib.parse.urlsplit(url)
+    abs_filepath = url_path_list.path
+    basepath = os.path.split(abs_filepath)[-1]
+    return os.path.join(dirpath, basepath)
+
+
+async def async_fetch(session, url):
+    """Asynchronous I/O HTTP GET request with a ClientSession instantiated from the aiohttp library."""
+    async with session.get(url) as response:
+        status = response.status
+        if status != 200:
+            binary = None
+        else:
+            binary = await response.read()
+        return url, status, binary
+
+
+async def async_fetch_and_write(session, url, dirpath):
+    """Asynchronous I/O HTTP GET request with a ClientSession instantiated from the aiohttp library, followed
+    by an asynchronous I/O file write of the binary to disk with the aiofiles library.
+
+    :returns `FWRes` namedtuple with url, filepath, http_status, write_success fields"""
+    FWResponse = namedtuple(
+        "FWRes", ["url", "filepath", "http_status", "write_success"]
+    )
+    url, status, binary = await async_fetch(session, url)
+    if status != 200:
+        filepath = None
+        write_success = False
+    else:
+        filepath = _get_filepath_from_url(url, dirpath)
+        await async_write_bin(filepath, binary)
+        write_success = True
+
+    return FWResponse(
+        url=url, filepath=filepath, http_status=status, write_success=write_success
+    )
+
+
+async def create_async_get_request_session_and_run(urls, dirpath):
+    """Creates an aiohttp library ClientSession and performs asynchronous GET requests +
+    binary file writes with the binary response from the GET request.
+
+    :returns list of asyncio Tasks that include `FWRes` namedtuple instances (defined in async_fetch_and_write)"""
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for url in urls:
+            # use asyncio.ensure_future instead of .run() here to maintain
+            # Py3.6 compatibility
+            task = asyncio.ensure_future(async_fetch_and_write(session, url, dirpath))
+            tasks.append(task)
+        await asyncio.gather(*tasks, return_exceptions=True)
+        return tasks