Update setup (#434)

* Update cythonize in setup

* Set cython language_level=2

* Exclude generated .cpp files

* Relax constraints for blis and numpy in pyproject.toml

Remove python version constraints from blis and numpy pins in
`pyproject.toml`:

* `numpy` build pins for released wheels are now only maintained in
`build-constraints.txt` because it is too difficult to maintain
reasonable/workable constraints across architectures for sdist installs

* `blis` is easier to build from source as of v0.7.4 so there's less
reason to add an unnecessary restriction, which was only intended to
make binary installs for python<3.6 easier

* Update blis pins in requirements and setup

* Update CI to prefer binary for python 2.7/3.5

Author: adrianeboyd

MANIFEST.in

@@ -3,3 +3,4 @@ include buildbot.json
 include LICENSE
 include README.rst
 prune tmp/
+recursive-exclude *.cpp

azure-pipelines.yml

@@ -72,12 +72,19 @@ jobs:
 
   - script: |
       python -m pip install --upgrade pip setuptools
-      pip install -r requirements.txt --prefer-binary
     displayName: 'Install dependencies'
 
   - script: |
-      pip install -e . --only-binary :all:
+      pip install --prefer-binary -e .
+    displayName: 'Install (python 2.7/3.5: prefer binary)'
+    condition: or(eq(variables['python.version'], '2.7'), eq(variables['python.version'], '3.5'))
+
+  - script: |
+      pip install -e .
     displayName: 'Install'
+    condition: not(or(eq(variables['python.version'], '2.7'), eq(variables['python.version'], '3.5')))
 
-  - script: python -m pytest thinc
+  - script: |
+      pip install -r requirements.txt --prefer-binary
+      python -m pytest thinc
     displayName: 'Run tests'

bin/cythonize.py

@@ -5,8 +5,7 @@ requires = [
     "murmurhash>=0.28.0,<1.1.0",
     "cymem>=2.0.2,<2.1.0",
     "preshed>=1.0.1,<3.1.0",
-    "blis>=0.4.0,<0.5.0; python_version<'3.6'",
-    "blis>=0.4.0,<0.8.0; python_version>='3.6'",
+    "blis>=0.4.0,<0.8.0",
     "numpy>=1.15.0",
 ]
 build-backend = "setuptools.build_meta"

pyproject.toml

@@ -2,8 +2,7 @@
 murmurhash>=0.28.0,<1.1.0
 cymem>=2.0.2,<2.1.0
 preshed>=1.0.1,<3.1.0
-blis>=0.4.0,<0.5.0; python_version<'3.6'
-blis>=0.4.0,<0.8.0; python_version>='3.6'
+blis>=0.4.0,<0.8.0
 srsly>=0.0.6,<1.1.0
 wasabi>=0.0.9,<1.1.0
 catalogue>=0.0.7,<1.1.0

requirements.txt

@@ -3,19 +3,16 @@
 import io
 import os
 import os.path
-import subprocess
 import sys
 import contextlib
+from setuptools import Extension, setup
 import distutils.util
 from distutils.command.build_ext import build_ext
 from distutils.sysconfig import get_python_inc
 from distutils import ccompiler, msvccompiler
-from distutils.ccompiler import new_compiler
-import platform
+from Cython.Build import cythonize
 import numpy
 
-from setuptools import Extension, setup
-
 
 def is_new_osx():
     """Check whether we're on OSX >= 10.10"""
@@ -83,12 +80,12 @@ def is_new_osx():
     # See: https://stackoverflow.com/questions/1653047/avoid-linking-to-libstdc
     LINK_OPTIONS["other"].append("-nodefaultlibs")
 
+
 # By subclassing build_extensions we have the actual compiler that will be used
 # which is really known only after finalize_options
 # http://stackoverflow.com/questions/724664/python-distutils-how-to-get-a-compiler-that-is-going-to-be-used
 class build_ext_options:
     def build_options(self):
-        src_dir = os.path.join(os.path.dirname(__file__), "thinc", "_files")
         if hasattr(self.compiler, "initialize"):
             self.compiler.initialize()
         self.compiler.platform = sys.platform[:6]
@@ -107,16 +104,6 @@ def build_extensions(self):
         build_ext.build_extensions(self)
 
 
-def generate_cython(root, source):
-    print("Cythonizing sources")
-    p = subprocess.call(
-        [sys.executable, os.path.join(root, "bin", "cythonize.py"), source],
-        env=os.environ,
-    )
-    if p != 0:
-        raise RuntimeError("Running cythonize failed")
-
-
 def find_in_path(name, path):
     "Find a file in a search path"
     # adapted fom http://code.activestate.com/recipes/52224-find-a-file-given-a-search-path/
@@ -127,10 +114,6 @@ def find_in_path(name, path):
     return None
 
 
-def is_source_release(path):
-    return os.path.exists(os.path.join(path, "PKG-INFO"))
-
-
 def clean(path):
     for name in MOD_NAMES:
         name = name.replace(".", "/")
@@ -180,10 +163,9 @@ def setup_package():
 
         ext_modules = []
         for mod_name in MOD_NAMES:
-            mod_path = mod_name.replace(".", "/") + ".cpp"
             if mod_name.endswith("gpu_ops"):
                 continue
-            mod_path = mod_name.replace(".", "/") + ".cpp"
+            mod_path = mod_name.replace(".", "/") + ".pyx"
             ext_modules.append(
                 Extension(
                     mod_name, [mod_path], language="c++", include_dirs=include_dirs
@@ -197,9 +179,6 @@ def setup_package():
             )
         )
 
-        if not is_source_release(root):
-            generate_cython(root, "thinc")
-
         setup(
             name="thinc",
             zip_safe=False,
@@ -213,7 +192,7 @@ def setup_package():
             version=about["__version__"],
             url=about["__uri__"],
             license=about["__license__"],
-            ext_modules=ext_modules,
+            ext_modules=cythonize(ext_modules, language_level=2),
             setup_requires=[
                 "numpy>=1.15.0",
                 "cython>=0.25",