windows: link _ctypes to libffi in static builds

CPython 3.8 moved to an out-of-tree libffi. As far as I can tell, our
first official Python 3.8 static builds did not handle libffi correctly.

In 3.8 and 3.9 builds, we hacked up the Python project to link against
libffi.lib. This ensured the symbols were present at link time. However,
we didn't ship libffi.lib and the dependency of _ctypes on libffi.lib
wasn't captured in PYTHON.json. If we attempted to produce a new link
library from the object files + static libraries (such as what
PyOxidizer does), we'd get a linker error for missing libffi symbols.

This commit attempts to fix matters.

We now explicitly link the _ctypes extension to libffi.lib via
project files hackery. This dependency is captured in the metadata
so the link dependency is present in PYTHON.json. We also need to
manually copy libffi.lib into the build directory so it is
available.

Finally, we undo the dependency of pythoncore on libffi because I now
believe it is redundant with _ctypes. Although I'm not 100% confident
about this.

This should hopefully fix
indygreg/PyOxidizer#360.

Author: indygreg

cpython-windows/build.py

@@ -47,7 +47,10 @@
         "allow_missing_preprocessor": True
     },
     "_bz2": {},
-    "_ctypes": {"shared_depends": ["libffi-7"]},
+    "_ctypes": {
+        "shared_depends": ["libffi-7"],
+        "static_depends_no_project": ["libffi"],
+    },
     "_decimal": {},
     "_elementtree": {},
     "_hashlib": {
@@ -775,6 +778,7 @@ def hack_props(
         )
 
         # We need to copy linking settings for dynamic libraries to static libraries.
+        copy_link_to_lib(pcbuild_path / "libffi.props")
         copy_link_to_lib(pcbuild_path / "openssl.props")
 
         # We should look against the static library variants.
@@ -892,32 +896,6 @@ def hack_project_files(
 
     pythoncore_proj = pcbuild_path / "pythoncore.vcxproj"
 
-    # normally the _ctypes extension/project pulls in libffi via
-    # <Link><AdditionalDependencies>. However, as part of converting this
-    # extension to static, we lose the transitive dependency. Here, we
-    # hack pythoncore as a one-off to add the dependency. Ideally we would
-    # handle this when hacking the extension's project. But it is easier to
-    # do here.
-    if static:
-        libffi_path = td / "libffi" / "libffi.lib"
-        try:
-            # Python 3.9 version
-            static_replace_in_file(
-                pythoncore_proj,
-                b"<AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;pathcch.lib;%(AdditionalDependencies)</AdditionalDependencies>",
-                b"<AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;pathcch.lib;"
-                + bytes(libffi_path)
-                + b";%(AdditionalDependencies)</AdditionalDependencies>",
-            )
-        except NoSearchStringError:
-            static_replace_in_file(
-                pythoncore_proj,
-                b"<AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>",
-                b"<AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;"
-                + bytes(libffi_path)
-                + b";%(AdditionalDependencies)</AdditionalDependencies>",
-            )
-
     if static:
         for extension, entry in sorted(CONVERT_TO_BUILTIN_EXTENSIONS.items()):
             if entry.get("ignore_static"):
@@ -2170,6 +2148,13 @@ def build_cpython(
             for record in entries:
                 record["required"] = extension in REQUIRED_EXTENSIONS
 
+        # Copy libffi static library as a one-off.
+        if static:
+            source = td / "libffi" / "libffi.lib"
+            dest = out_dir / "python" / "build" / "lib" / "libffi.lib"
+            log("copying %s to %s" % (source, dest))
+            shutil.copyfile(source, dest)
+
         # Copy OpenSSL libraries as a one-off.
         for lib in ("crypto", "ssl"):
             if static:

src/main.rs

@@ -359,6 +359,19 @@ const MACHO_BANNED_SYMBOLS_NON_AARCH64: &[&str] = &[
     "_preadv", "_pwritev",
 ];
 
+static WANTED_WINDOWS_STATIC_PATHS: Lazy<BTreeSet<PathBuf>> = Lazy::new(|| {
+    [
+        PathBuf::from("python/build/lib/libffi.lib"),
+        PathBuf::from("python/build/lib/libcrypto_static.lib"),
+        PathBuf::from("python/build/lib/liblzma.lib"),
+        PathBuf::from("python/build/lib/libssl_static.lib"),
+        PathBuf::from("python/build/lib/sqlite3.lib"),
+    ]
+    .iter()
+    .cloned()
+    .collect()
+});
+
 fn allowed_dylibs_for_triple(triple: &str) -> Vec<MachOAllowedDylib> {
     match triple {
         "aarch64-apple-darwin" => DARWIN_ALLOWED_DYLIBS.clone(),
@@ -563,6 +576,7 @@ fn validate_json(json: &PythonJsonMain, triple: &str) -> Result<Vec<String>> {
 fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
     let mut errors = vec![];
     let mut seen_dylibs = BTreeSet::new();
+    let mut seen_paths = BTreeSet::new();
 
     let fh = std::fs::File::open(&dist_path)
         .with_context(|| format!("unable to open {}", dist_path.display()))?;
@@ -589,6 +603,8 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
         let mut entry = entry.map_err(|e| anyhow!("failed to iterate over archive: {}", e))?;
         let path = entry.path()?.to_path_buf();
 
+        seen_paths.insert(path.clone());
+
         let mut data = Vec::new();
         entry.read_to_end(&mut data)?;
 
@@ -642,6 +658,12 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
         errors.push(format!("required library dependency {} not seen", lib));
     }
 
+    if triple.contains("-windows-") && dist_path.to_string_lossy().contains("-static-") {
+        for path in WANTED_WINDOWS_STATIC_PATHS.difference(&seen_paths) {
+            errors.push(format!("required path {} not seen", path.display()));
+        }
+    }
+
     Ok(errors)
 }