Bundle the symlink_import() function with the package

Regular copy import is too slow for this package.

Author: dean0x7d

conanfile.py

@@ -14,21 +14,22 @@ class EmbeddedPython(ConanFile):
     settings = {"os": ["Windows"]}
     options = {"pip_packages": "ANY"}
     default_options = "pip_packages=None"
+    exports = "embedded_python_tools.py"
     short_paths = True  # some of the pip packages go over the 260 char path limit on Windows
 
     def _get_binaries(self):
         """Get the binaries from the special embeddable Python package"""
         url = "https://www.python.org/ftp/python/{0}/python-{0}-embed-amd64.zip"
-        tools.get(url.format(self.pyversion), md5=self.md5)
+        tools.get(url.format(self.pyversion), md5=self.md5, destination="embedded_python")
 
     def _get_headers_and_lib(self):
         """We also need headers and the `python3.lib` file to link against"""
         url = "https://www.python.org/ftp/python/{0}/python-{0}-amd64-webinstall.exe"
         tools.download(url.format(self.pyversion), filename="tmp\\installer.exe")
         self.run("tmp\\installer.exe /quiet /layout")
-        self.run("msiexec.exe /a {0}\\tmp\\dev.msi targetdir={0}".format(self.build_folder))
+        dst = os.path.join(self.build_folder, "embedded_python")
+        self.run(f"msiexec.exe /a {self.build_folder}\\tmp\\dev.msi targetdir={dst}")
         tools.rmdir("tmp")
-        os.remove(self.build_folder + "\\dev.msi")
 
     def build(self):
         self._get_binaries()
@@ -39,12 +40,15 @@ def build(self):
 
         # Enable site-packages, i.e. additional non-system packages
         pyver = "".join(self.pyversion.split(".")[:2]) # e.g. 3.7.3 -> 37
-        tools.replace_in_file("python{}._pth".format(pyver), "#import site", "import site")
+        tools.replace_in_file("embedded_python/python{}._pth".format(pyver), "#import site", "import site")
 
-        target = self.build_folder + "/Lib/site-packages"
         packages = self.options.pip_packages.value
         packages += " setuptools==41.0.1"  # some modules always assume it's installed (e.g. pytest)
+        target = self.build_folder + "/embedded_python/Lib/site-packages"
         self.run(f'{sys.executable} -m pip install --no-deps --python-version {pyver} --target "{target}" {packages}')
 
     def package(self):
-        self.copy("*", dst="embedded_python", keep_path=True)
+        self.copy("*", keep_path=True)
+
+    def package_info(self):
+        self.env_info.PYTHONPATH.append(self.package_folder)

embedded_python_tools.py

@@ -0,0 +1,50 @@
+import os
+import shutil
+import pathlib
+
+
+def symlink_import(self, dst="bin/python/interpreter", bin="bin"):
+    """Copying the entire embedded Python environment is extremely slow, so we just symlink it
+
+    Usage:
+    ```python
+    def imports(self):
+        import embedded_python_tools
+        embedded_python_tools.symlink_import(self, dst="bin/python/interpreter")
+    ```
+
+    On Windows, symlinks require admin privileges, so we use a directory junction instead.
+    It points to the Conan package location. We still want to copy in `python*.dll` and
+    `python*.zip` right next to the executable so that they can be found, but the rest of
+    the Python environment is in a subfolder:
+
+    bin
+    |- python/interpreter
+    |  |- Lib
+    |  \- ...
+    |- <main>.exe
+    |- python*.dll
+    |- python*.zip
+    \- ...
+    """
+    if self.settings.os != "Windows":
+        return
+
+    import _winapi
+
+    dst = pathlib.Path(dst).absolute()
+    if not dst.parent.exists():
+        dst.parent.mkdir(parents=True)
+
+    if dst.exists():
+        try:  # to remove any existing junction
+            os.remove(dst)
+        except:  # this seems to be the only way to find out this is not a junction 
+            shutil.rmtree(dst)
+
+    src = pathlib.Path(self.deps_cpp_info["embedded_python"].rootpath) / "embedded_python"
+    _winapi.CreateJunction(str(src), str(dst))
+
+    bin = pathlib.Path(bin).absolute()
+    self.copy("python*.dll", dst=bin, src="embedded_python", keep_path=False)
+    self.copy("python*.zip", dst=bin, src="embedded_python", keep_path=False)

test_package/conanfile.py

@@ -1,11 +1,13 @@
+import sys
 from conans import ConanFile
 
 
 class TestEmbeddedPython(ConanFile):
-    settings = None
+    settings = "os"
 
     def imports(self):
-        self.copy("*", dst="python", src="embedded_python")
+        import embedded_python_tools
+        embedded_python_tools.symlink_import(self, dst="bin/python")
 
     def test(self):
-        self.run(".\\python\\python.exe --version")
+        self.run(".\\bin\\python\\python.exe --version")