ci: Pin transitive dependencies for tests on Python>=3.8

scripts/populate_tox/populate_tox.py

@@ -10,7 +10,8 @@
 import os
 import re
 import subprocess
 import sys
+import tempfile
 import time
 from bisect import bisect_left
 from collections import defaultdict
@@ -20,6 +21,7 @@
 from pathlib import Path
 from typing import Optional, Union
 
+from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 
@@ -77,6 +79,19 @@
 MIN_FREE_THREADING_SUPPORT = Version("3.14")
 
 
+class DryRunFailed(Exception):
+    def __init__(self, result: subprocess.CompletedProcess[str]) -> None:
+        self.result = result
+        message = f"Command failed with exit code {result.returncode}.\n"
+        stderr = result.stderr.strip()
+        if stderr:
+            message += f"\nStderr:\n{stderr}"
+        stdout = result.stdout.strip()
+        if stdout:
+            message += f"\nStdout:\n{stdout}"
+        super().__init__(message)
+
+
 class PackageVersion(Version):
     # Convenience wrapper around Version. It's convenient to be able to set
     # attributes on a Version in toxgen, but we can't because the class now
@@ -94,8 +109,11 @@
         self.version = Version(version) if isinstance(version, str) else version
         self.no_gil = no_gil
 
+    def __hash__(self):
+        return hash((self.version, self.no_gil))
+
     def __str__(self):
-        version = f"py{self.version.major}.{self.version.minor}"
+        version = f"{self.version.major}.{self.version.minor}"
         if self.no_gil:
             version += "t"
 
@@ -146,34 +164,94 @@
     return release
 
 
+def _get_dependency_probe_constraints(
+    integration: str,
+    release: Version,
+    python_version: ThreadedVersion,
+) -> tuple[str, ...]:
+    constraints = []
+    for rule, dependencies in TEST_SUITE_CONFIG[integration].get("deps", {}).items():
+        # Skip if rule does not apply to current package or Python version
+        if (
+            rule == "*"
+            or (rule.startswith("py3") and f"py{python_version}" not in rule.split(","))
+            or (
+                not rule.startswith("py3")
+                and release not in SpecifierSet(rule, prereleases=True)
+            )
+        ):
+            continue
+
+        for dependency in dependencies:
+            requirement = Requirement(dependency)
+
+            # Constraints are useful only when they actually constrain versions.
+            if (
+                requirement.specifier
+                and not requirement.extras
+                and requirement.url is None
+            ):
+                constraints.append(dependency)
+    return tuple(constraints)
+
+
 @functools.cache
-def fetch_package_dependencies(package: str, version: Version) -> dict:
+def fetch_package_dependencies(
+    package: str,
+    version: Version,
+    python_version: ThreadedVersion,
+    constraints: tuple[str, ...] = (),
+) -> dict:
     """Fetch package dependencies metadata from cache or, failing that, PyPI."""
-    package_dependencies = _fetch_package_dependencies_from_cache(package, version)
+    package_dependencies = _fetch_package_dependencies_from_cache(
+        package, version, python_version
+    )
     if package_dependencies is not None:
         return package_dependencies
 
     # Removing non-report output with -qqq may be brittle, but avoids file I/O.
     # Currently -qqq supresses all non-report output that would break json.loads().
-    pip_report = subprocess.run(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            f"{package}=={str(version)}",
-            "--dry-run",
-            "--ignore-installed",
-            "--report",
-            "-",
-            "-qqq",
-        ],
-        capture_output=True,
-        text=True,
-    ).stdout.strip()
+    cmd = [
+        "uv",
+        "run",
+        "--no-project",
+        "--python",
+        str(python_version),
+        "--with",
+        "pip",
+        "python",
+        "-m",
+        "pip",
+        "install",
+        f"{package}=={version}",
+        "--dry-run",
+        "--ignore-installed",
+        "--report",
+        "-",
+        "-qqq",
+    ]
 
+    if constraints:
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f:
+            f.write("\n".join(constraints))
+            f.flush()
+            result = subprocess.run(
+                [*cmd, "--constraint", f.name],
+                capture_output=True,
+                text=True,
+            )
+    else:
+        result = subprocess.run(cmd, capture_output=True, text=True)
+
+    if result.returncode != 0:
+        # Some failures are expected because uv installs packages which pip rejects for having bad metadata.
+        raise DryRunFailed(result)
+
+    pip_report = result.stdout.strip()
     dependencies_info = json.loads(pip_report)["install"]
-    _save_to_package_dependencies_cache(package, version, dependencies_info)
+    _save_to_package_dependencies_cache(
+        package, version, python_version, dependencies_info
+    )
 
     return dependencies_info
 
@@ -185,15 +263,20 @@
         return CACHE[package][str(version)]
 
     return None
 
 
 def _fetch_package_dependencies_from_cache(
-    package: str, version: Version
+    package: str,
+    version: Version,
+    python_version: ThreadedVersion,
 ) -> Optional[dict]:
     package = _normalize_name(package)
-    if package in DEPENDENCIES_CACHE and str(version) in DEPENDENCIES_CACHE[package]:
-        DEPENDENCIES_CACHE[package][str(version)]["_accessed"] = True
-        return DEPENDENCIES_CACHE[package][str(version)]["dependencies"]
+    cache_entry = (
+        DEPENDENCIES_CACHE[package].get(str(version), {}).get(str(python_version), None)
+    )
+    if cache_entry is not None:
+        cache_entry["_accessed"] = True
+        return cache_entry["dependencies"]
 
     return None
 
@@ -207,18 +290,30 @@
 
 
 def _save_to_package_dependencies_cache(
-    package: str, version: Version, release: Optional[dict]
+    package: str,
+    version: Version,
+    python_version: ThreadedVersion,
+    release: Optional[dict],
 ) -> None:
+    normalized_dependencies = _normalize_package_dependencies(release)
+
     with open(DEPENDENCIES_CACHE_FILE, "a") as releases_cache:
-        line = {
-            "name": package,
-            "version": str(version),
-            "dependencies": _normalize_package_dependencies(release),
-        }
-        releases_cache.write(json.dumps(line) + "\n")
+        releases_cache.write(
+            json.dumps(
+                {
+                    "name": package,
+                    "version": str(version),
+                    "python_version": str(python_version),
+                    "dependencies": normalized_dependencies,
+                }
+            )
+            + "\n"
+        )
 
-    DEPENDENCIES_CACHE[_normalize_name(package)][str(version)] = {
-        "info": release,
+    DEPENDENCIES_CACHE[_normalize_name(package)].setdefault(str(version), {})[
+        str(python_version)
+    ] = {
+        "dependencies": normalized_dependencies,
         "_accessed": True,
     }
 
@@ -623,7 +718,9 @@
     - no wheel targets the platform on which the script is run, but PyPI distributes a wheel
       satisfying one of the above conditions.
     """
-    dependencies_info = fetch_package_dependencies(package_name, release)
+    dependencies_info = fetch_package_dependencies(
+        package_name, release, ThreadedVersion(python_version, no_gil=True)
+    )
 
     for dependency_info in dependencies_info:
         wheel_filename = dependency_info["download_info"]["url"].split("/")[-1]
@@ -698,7 +795,7 @@
 
 
 def _render_python_versions(python_versions: list[ThreadedVersion]) -> str:
-    return "{" + ",".join(str(version) for version in python_versions) + "}"
+    return "{" + ",".join(f"py{str(version)}" for version in python_versions) + "}"
 
 
 def _render_dependencies(integration: str, releases: list[Version]) -> list[str]:
@@ -724,9 +821,7 @@
     return rendered
 
 
-def _render_latest_dependencies(
-    integration: str, latest_release: Version
-) -> list[str]:
+def _render_latest_dependencies(integration: str, latest_release: Version) -> list[str]:
     """Render version-specific dependencies for the 'latest' alias.
 
     Dependencies with "*" or "py3.*" constraints already match the latest
@@ -829,6 +924,27 @@
             )
 
 
+def _render_transitive_dependencies(
+    integration: str,
+    package: str,
+    release: Version,
+    python_version: ThreadedVersion,
+) -> list[str]:
+    deps = []
+    for dependency in fetch_package_dependencies(
+        package,
+        release,
+        python_version,
+        _get_dependency_probe_constraints(integration, release, python_version),
+    ):
+        name = dependency["metadata"]["name"]
+        version = dependency["metadata"]["version"]
+        if _normalize_name(name) == _normalize_name(package):
+            continue
+        deps.append(f"py{python_version}-{integration}-v{release}: {name}=={version}")
+    return deps
+
+
 def _add_python_versions_to_release(
     integration: str, package: str, release: PackageVersion
 ) -> None:
@@ -945,9 +1061,13 @@
     """Filter out unneeded parts of the package dependencies JSON."""
     normalized = [
         {
-            "download_info": {"url": depedency["download_info"]["url"]},
+            "metadata": {
+                "name": dependency["metadata"]["name"],
+                "version": dependency["metadata"]["version"],
+            },
+            "download_info": {"url": dependency["download_info"]["url"]},
         }
-        for depedency in package_dependencies
+        for dependency in package_dependencies
     ]
 
     return normalized
@@ -1036,7 +1156,8 @@
             release = json.loads(line)
             name = _normalize_name(release["name"])
             version = release["version"]
-            DEPENDENCIES_CACHE[name][version] = {
+            python_version = release["python_version"]
+            DEPENDENCIES_CACHE[name].setdefault(version, {})[python_version] = {
                 "dependencies": release["dependencies"],
                 "_accessed": False,
             }
@@ -1081,6 +1202,23 @@
                 _add_python_versions_to_release(integration, package, release)
                 if not release.python_versions:
                     print(f"  Release {release} has no Python versions, skipping.")
+                    continue
+
+                release.transitive_dependencies = []
+                for python_version in release.python_versions:
+                    if python_version < ThreadedVersion("3.8"):
+                        continue
+                    try:
+                        dependencies = _render_transitive_dependencies(
+                            integration, package, release, python_version
+                        )
+                    except DryRunFailed as error:
+                        print(
+                            f"\npip dry run failed for version {release} of {package} on Python {python_version}:\n{error}"
+                        )
+                        continue
+
+                    release.transitive_dependencies.append(dependencies)
 
             test_releases = [
                 release for release in test_releases if release.python_versions
@@ -1126,7 +1264,7 @@
             if (
                 DEPENDENCIES_CACHE[_normalize_name(release["name"])][
                     release["version"]
-                ]["_accessed"]
+                ][release["python_version"]]["_accessed"]
                 is True
             ):
                 releases_cache.write(json.dumps(release) + "\n")