Macos embedding python and Qt5.11.1 bugfix

macbuild/ReadMe.md

@@ -97,6 +97,9 @@ Homebrew's installation of python3 (`brew install python3`) places a `Python.fra
 ```
 # Build step
 ./build4mac.py -p B36 -q Qt5Brew
+# build with log
+./build4mac.py -p B36 -q Qt5Brew 2>&1 | tee qt5.build.macos-HighSierra-release-version.log
+
 
 # Deploy step
 ./build4mac.py -p B36 -q Qt5Brew -y  # normal deploy

macbuild/Resources/klayout_console

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+if [ "$1" != "" ]; then
+/Applications/klayout.app/Contents/MacOS/klayout $1 -r /Applications/klayout.app/Contents/MacOS/start-console.py
+else
+/Applications/klayout.app/Contents/MacOS/klayout -z -r /Applications/klayout.app/Contents/MacOS/start-console.py
+fi

macbuild/Resources/start-console.py

@@ -0,0 +1,16 @@
+#!/Applications/klayout.app/Contents/MacOS/klayout -b -r
+import readline
+import code
+import sys
+import os
+pwd = os.getcwd()
+sys.path.append(pwd)
+
+variables = globals().copy()
+variables.update(locals())
+shell = code.InteractiveConsole(variables)
+cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
+banner = "Python %s on %s\n%s\n(%s)" % (sys.version, sys.platform,
+    cprt, "KLayout Python Console")
+exit_msg = 'now exiting %s...' % "KLayout Python Console"
+shell.interact(banner, exit_msg)

macbuild/build4mac.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 #===============================================================================
@@ -538,8 +538,8 @@ def DeployBinariesForBundle():
   if not DeploymentF and not DeploymentP:
     return 1
   if DeploymentF and NonOSStdLang:
-    print( "WARNING!!! You chose <-y|--deploy> while using non-OS-standard script language.", file=sys.stderr )
-    print( "    Use <-Y|--DEPLOY> instead", file=sys.stderr )
+    print( "     WARNING!!! You chose <-y|--deploy> while using non-OS-standard script language.", file=sys.stderr )
+    print( "         Consider using <-Y|--DEPLOY> instead", file=sys.stderr )
     #return 1
   if not os.path.isfile(MacBuildLog):
     print( "!!! Build log file <%s> does not present !!!" % MacBuildLog, file=sys.stderr )
@@ -756,51 +756,119 @@ def DeployBinariesForBundle():
     os.chdir(ProjectDir)
     os.chdir(MacPkgDir)
     command = "%s %s %s" % ( deploytool, app_bundle, options )
-    print(command)
     if subprocess.call( command, shell=True ) != 0:
       msg = "!!! Failed to deploy applications on OSX !!!"
       print( msg, file=sys.stderr )
       print("")
       os.chdir(ProjectDir)
       return 1
 
-    deploymentPython = False
-    if deploymentPython:
-      # TODO Code incomplete! To be finished.
-      from macbuild.build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
-
-      bundlePath = 'qt5.pkg.macos-HighSierra-release/klayout.app'
-      bundleExecPathAbs = os.getcwd() + '/%s/Contents/MacOS/' % bundlePath
-
-      # rsync -a --safe-links /usr/local/opt/python/Frameworks/Python.framework/ qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework
-      # cp -RL /usr/local/opt/python/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework/Versions/3.6/lib/python3.6/
-      # rm -rf qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/test
+    deploymentPython = True
+    if deploymentPython and NonOSStdLang:
+      from build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
+      from pathlib import Path
 
+      bundlePath = AbsMacPkgDir + '/klayout.app'
+      # bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
+      bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
+      pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
       pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
-      depdict = WalkFrameworkPaths(pythonFrameworkPath)
 
-      pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
+      print(f" [8.1] Deploying Python from {pythonOriginalFrameworkPath} ..." )
+      print("  [1] Copying Python Framework")
+      shell_commands = list()
+      shell_commands.append(f"rm -rf {pythonFrameworkPath}")
+      shell_commands.append(f"rsync -a --safe-links {pythonOriginalFrameworkPath}/ {pythonFrameworkPath}")
+      shell_commands.append(f"mkdir {pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
+      shell_commands.append(f"cp -RL {pythonOriginalFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/{{pip*,pkg_resources,setuptools*,wheel*}} " +
+                            f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
+      shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/lib/python3.6/test")
+      shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/Resources")
+      shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/bin")
+
+      for command in shell_commands:
+        if subprocess.call( command, shell=True ) != 0:
+          msg = "command failed: %s"
+          print( msg % command, file=sys.stderr )
+          exit(1)
+
+      shutil.copy2( sourceDir2 + "/start-console.py", targetDirM )
+      shutil.copy2( sourceDir2 + "/klayout_console", targetDirM )
+      os.chmod( targetDirM + "/klayout_console",      0o0755 )
+
+      print("  [2] Relinking dylib dependencies inside Python.framework")
+      print("   [2.1] Patching Python Framework")
+      depdict = WalkFrameworkPaths(pythonFrameworkPath)
       appPythonFrameworkPath = '@executable_path/../Frameworks/Python.framework/'
-      PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
-
-      klayoutPath = bundleExecPathAbs
-      depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
-      PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
-
-      klayoutPath = bundleExecPathAbs + '../Frameworks'
-      depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'libklayout')
-      PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
-
+      PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs)
 
+      print("   [2.2] Patching /usr/local/opt/ libs")
       usrLocalPath = '/usr/local/opt/'
       appUsrLocalPath = '@executable_path/../Frameworks/'
-      depdict = WalkFrameworkPaths(pythonFrameworkPath)
-      PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)
+      replacePairs = [(usrLocalPath, appUsrLocalPath, True)]
+      depdict = WalkFrameworkPaths(pythonFrameworkPath, search_path_filter=r'\t+/usr/local/(opt|Cellar)')
+      PerformChanges(depdict, replacePairs, bundleExecPathAbs)
 
-      # usrLocalPath = '/usr/local/lib/'
-      # appUsrLocalPath = '@executable_path/../Frameworks/'
-      # depdict = WalkFrameworkPaths(pythonFrameworkPath)
-      # PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)
+      print("   [2.3] Patching openssl, gdbm, readline, sqlite, tcl-tk, xz")
+      usrLocalPath = '/usr/local/opt'
+      appUsrLocalPath = '@executable_path/../Frameworks/'
+      replacePairs = [(usrLocalPath, appUsrLocalPath, True)]
+      replacePairs.extend([(openssl_version, '@executable_path/../Frameworks/openssl', True)
+        for openssl_version in list(Path('/usr/local/Cellar/openssl').glob('*'))])
+      depdict = WalkFrameworkPaths([pythonFrameworkPath + '/../openssl',
+                                    pythonFrameworkPath + '/../gdbm',
+                                    pythonFrameworkPath + '/../readline',
+                                    pythonFrameworkPath + '/../sqlite',
+                                    pythonFrameworkPath + '/../tcl-tk',
+                                    pythonFrameworkPath + '/../xz'], search_path_filter=r'\t+/usr/local/(opt|Cellar)')
+
+      PerformChanges(depdict, replacePairs, bundleExecPathAbs)
+
+      print("  [3] Relinking dylib dependencies for klayout")
+      klayoutPath = bundleExecPathAbs
+      depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
+      PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs)
+
+      libKlayoutPath = bundleExecPathAbs + '../Frameworks'
+      depdict = WalkFrameworkPaths(libKlayoutPath, filter_regex=r'libklayout')
+      PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs)
+
+      print("  [4] Patching site.py, pip/, and distutils/")
+      site_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site.py"
+      with open(site_module, 'r') as site:
+        buf = site.readlines()
+      with open(site_module, 'w') as site:
+        import re
+        for line in buf:
+          # This will fool pip into thinking it's inside a virtual environment
+          # and install new packates to the correct site-packages
+          if re.match("^PREFIXES", line) is not None:
+            line = line + "sys.real_prefix = sys.prefix\n"
+          # do not allow installation in the user folder.
+          if re.match("^ENABLE_USER_SITE", line) is not None:
+            line = "ENABLE_USER_SITE = False\n"
+          site.write(line)
+
+      pip_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/pip/__init__.py"
+      with open(pip_module, 'r') as pip:
+        buf = pip.readlines()
+      with open(pip_module, 'w') as pip:
+        import re
+        for line in buf:
+          # this will reject user's configuration of pip, forcing the isolated mode
+          line = re.sub("return isolated$", "return isolated or True", line)
+          pip.write(line)
+
+      distutilsconfig = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/distutils/distutils.cfg"
+      with open(distutilsconfig, 'r') as file:
+        buf = file.readlines()
+      with open(distutilsconfig, 'w') as file:
+        import re
+        for line in buf:
+          # This will cause all packages to be installed to sys.prefix
+          if re.match('prefix=', line) is not None:
+            continue
+          file.write(line)
 
   else:
     print( " [8] Skipped deploying Qt's Frameworks ..." )

macbuild/build4mac_util.py

@@ -200,32 +200,32 @@ def WalkLibDependencyTree( dylibPath, depth=0, filter_regex=r'\t+/usr/local/opt'
           lib = str(Path(lib))
           if lib != list(keys)[0]:
             deplibs[idx] = WalkLibDependencyTree(lib, depth+1, filter_regex)
-      else:
-        return NOTHINGTODO
       if depth == 0:
         return deplibs
       return exedepdic
     else:
       raise RuntimeError("Exceeded maximum recursion depth.")
 
-  def WalkFrameworkPaths(frameworkPaths, filter_regex=r'\.(so|dylib)$'):
-    try:
-      frameworkPathsIter = iter(frameworkPaths)
-    except TypeError:
+  def WalkFrameworkPaths(frameworkPaths, filter_regex=r'\.(so|dylib)$', search_path_filter=r'\t+/usr/local/opt'):
+
+    if isinstance(frameworkPaths, str):
       frameworkPathsIter = [frameworkPaths]
+    else:
+      frameworkPathsIter = frameworkPaths
 
     dependency_dict = dict()
 
-    for frameworkPath in frameworkPaths:
+    for frameworkPath in frameworkPathsIter:
       frameworkPath = str(Path(frameworkPath))
+      # print("Calling:", 'find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex))
       find_grep_results = os.popen('find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex)).read().split('\n')
       framework_files = filter(lambda x: x != '',
                             map(lambda x: x.strip(),
                                   find_grep_results))
 
       dependency_dict[frameworkPath] = list()
       for idx, file in enumerate(framework_files):
-        dict_file = {file: WalkLibDependencyTree(file)}
+        dict_file = {file: WalkLibDependencyTree(file, filter_regex=search_path_filter)}
         dependency_dict[frameworkPath].append(dict_file)
     return dependency_dict
 
@@ -263,15 +263,12 @@ def FindFramework(path, root_path):
     relPath = path.relative_to(root_path)
     return str(root_path / relPath.parts[0])
 
-  def ReplaceExecutablePath(path, executable_path):
+  def ResolveExecutablePath(path, executable_path):
+    """ Transforms @executable_path into executable_path"""
     executable_path = str(executable_path)
     p = Path(str(path).replace("@executable_path", "/%s/" % executable_path))
     return p
 
-  def FileExists(file_path, executable_path):
-    p = ReplaceExecutablePath(file_path, executable_path)
-    return p.exists()
-
   def DetectChanges(frameworkDependencyDict):
     visited_files = list()
     libNameChanges = list()
@@ -282,7 +279,7 @@ def DetectChanges(frameworkDependencyDict):
 
     return libNameChanges
 
-  def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout", libdir=False):
+  def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout"):
     libNameChanges = DetectChanges(frameworkDependencyDict)
     cmdNameId = XcodeToolChain['nameID']
     cmdNameChg = XcodeToolChain['nameCH']
@@ -295,45 +292,49 @@ def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_
           dependencies = next(libNameChangeIterator)
         except StopIteration:
           dependencies = list()
-        for replaceFrom, replaceTo in replaceFromToPairs:
+        for replaceFrom, replaceTo, libdir in replaceFromToPairs:
           replaceFrom = str(Path(replaceFrom))
           replaceTo = str(Path(replaceTo))
 
+          fileName = ResolveExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
+          if str(fileName).startswith('/usr'):
+            # print(f'skipping fileName: {fileName}')
+            continue
+
           if lib.find(replaceFrom) >= 0:
             if libdir:
               frameworkPath = FindFramework(lib, replaceFrom)
             else:
               frameworkPath = lib
             destFrameworkPath = frameworkPath.replace(replaceFrom, replaceTo)
-            destFrameworkPath = ReplaceExecutablePath(destFrameworkPath, executable_path)
-            if not FileExists(lib.replace(replaceFrom, replaceTo), executable_path):
+            destFrameworkPath = ResolveExecutablePath(destFrameworkPath, executable_path)
+
+            if not fileName.exists():
               print (lib.replace(replaceFrom, replaceTo), "DOES NOT EXIST")
               print ("COPY", frameworkPath, " -> ", destFrameworkPath)
               shutil.copytree(frameworkPath, destFrameworkPath)
 
-            fileName = ReplaceExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
             nameId = lib.replace(replaceFrom, replaceTo)
             command = "%s %s %s" % ( cmdNameId, nameId, fileName )
             if not os.access(fileName, os.W_OK):
               command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
-            print("\t%s" % command)
+            # print("\t%s" % command)
             if subprocess.call( command, shell=True ) != 0:
               msg = "!!! Failed to set the new identification name to <%s> !!!"
               print( msg % fileName, file=sys.stderr )
               return 1
 
-          fileName = ReplaceExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
           for dependency in dependencies:
             if dependency.find(replaceFrom) >= 0:
-              print("In:", fileName)
+              print("\tIn:", fileName)
               print("\tRENAME", dependency, " -> ", dependency.replace(replaceFrom, replaceTo))
 
               # Try changing id first
               nameId = dependency.replace(replaceFrom, replaceTo)
               command = "%s %s %s" % ( cmdNameId, nameId, fileName )
               if not os.access(fileName, os.W_OK):
                 command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
-              print("\t%s" % command)
+              # print("\t%s" % command)
               if subprocess.call( command, shell=True ) != 0:
                 msg = "!!! Failed to set the new identification name to <%s> !!!"
                 print( msg % fileName, file=sys.stderr )
@@ -346,7 +347,7 @@ def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_
               if not os.access(fileName, os.W_OK):
                 command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
 
-              print("\t%s" % command)
+              # print("\t%s" % command)
               if subprocess.call( command, shell=True ) != 0:
                 msg = "!!! Failed to set the new identification name to <%s> !!!"
                 print( msg % fileName, file=sys.stderr )

src/lay/lay/laySaltDownloadManager.cc

@@ -68,7 +68,7 @@ ConfirmationDialog::add_info (const std::string &name, bool update, const std::s
   item->setText (3, tl::to_qstring (url));
 
   for (int column = 0; column < list->colorCount (); ++column) {
-    item->setData (column, Qt::ForegroundRole, QVariant (QBrush (update ? Qt::blue : Qt::black)));
+    item->setData (column, Qt::ForegroundRole, QVariant (QBrush (update ? QColor (Qt::blue) : QColor (Qt::black))));
   }
 }
 
@@ -86,7 +86,7 @@ ConfirmationDialog::mark_fetching (const std::string &name)
     list->scrollToItem (i->second);
     for (int c = 0; c < list->columnCount (); ++c) {
       i->second->setData (c, Qt::BackgroundColorRole, QColor (224, 244, 244));
-      i->second->setData (c, Qt::TextColorRole, Qt::blue);
+      i->second->setData (c, Qt::TextColorRole, QColor (Qt::blue));
     }
     i->second->setData (1, Qt::DisplayRole, tr ("FETCHING"));
   }
@@ -102,7 +102,7 @@ ConfirmationDialog::mark_error (const std::string &name)
     list->scrollToItem (i->second);
     for (int c = 0; c < list->columnCount (); ++c) {
       i->second->setData (c, Qt::BackgroundColorRole, QColor (255, 224, 244));
-      i->second->setData (c, Qt::TextColorRole, Qt::black);
+      i->second->setData (c, Qt::TextColorRole, QColor (Qt::black));
     }
     i->second->setData (1, Qt::DisplayRole, tr ("ERROR"));
   }
@@ -118,7 +118,7 @@ ConfirmationDialog::mark_success (const std::string &name)
     list->scrollToItem (i->second);
     for (int c = 0; c < list->columnCount (); ++c) {
       i->second->setData (c, Qt::BackgroundColorRole, QColor (160, 255, 160));
-      i->second->setData (c, Qt::TextColorRole, Qt::black);
+      i->second->setData (c, Qt::TextColorRole, QColor (Qt::black));
     }
     i->second->setData (1, Qt::DisplayRole, tr ("INSTALLED"));
   }