feat: allow populating binary's venv site-packages with symlinks

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
 
 test --test_output=errors
 

CHANGELOG.md

@@ -60,6 +60,14 @@ Unreleased changes template.
 
 {#v0-0-0-added}
 ### Added
+* (providers) {obj}`PyInfo.site_packages_symlinks` field added to allow
+  specifying links to create within the venv site packages
+  (only applicable with {obj}`--bootstrap_impl=script`)
+  ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)).
+* (rules) {obj}`py_library.site_packages_root` attribute added to allow
+  specifying a library's sources follow a site-packages file layout.
+  (only applicable with {obj}`--bootstrap_impl=script`)
+  ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)).
 * {obj}`//python/bin:python`: convenience target for directly running an
   interpreter. {obj}`--//python/bin:python_src` can be used to specify a
   binary whose interpreter to use.

MODULE.bazel

@@ -85,6 +85,7 @@ bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
 bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
 bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
 bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
+bazel_dep(name = "other", version = "0", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.
@@ -106,6 +107,11 @@ local_path_override(
     path = "gazelle",
 )
 
+local_path_override(
+    module_name = "other",
+    path = "tests/modules/other",
+)
+
 dev_python = use_extension(
     "//python/extensions:python.bzl",
     "python",

docs/BUILD.bazel

@@ -81,6 +81,7 @@ sphinx_stardocs(
     name = "bzl_api_docs",
     srcs = [
         "//python:defs_bzl",
+        "//python:features_bzl",
         "//python:packaging_bzl",
         "//python:pip_bzl",
         "//python:py_binary_bzl",

python/BUILD.bazel

@@ -79,6 +79,9 @@ bzl_library(
 bzl_library(
     name = "features_bzl",
     srcs = ["features.bzl"],
+    deps = [
+        "@rules_python_internal//:rules_python_config_bzl",
+    ],
 )
 
 bzl_library(

python/features.bzl

@@ -19,8 +19,49 @@ load("@rules_python_internal//:rules_python_config.bzl", "config")
 # See https://git-scm.com/docs/git-archive/2.29.0#Documentation/git-archive.txt-export-subst
 _VERSION_PRIVATE = "$Format:%(describe:tags=true)$"
 
+def _features_typedef():
+    """Information about features rules_python has implemented.
+
+    ::::{field} precompile
+    :type: bool
+
+    True if the precompile attributes are available.
+
+    :::{versionadded} 0.33.0
+    :::
+    ::::
+
+    ::::{field} site_packages_root_attr
+    :type: bool
+
+    True if the {obj}`site_packages_root` attribute is available.
+
+    :::{versionadded} VERSION_NEXT_FEATURE
+    :::
+    ::::
+
+    ::::{field} uses_builtin_rules
+    :type: bool
+
+    True if the rules are using the Bazel-builtin implementation.
+
+    :::{versionadded} 1.1.0
+    :::
+    ::::
+
+    ::::{field} version
+    :type: str
+
+    The rules_python version. This is a semver format, e.g. `X.Y.Z` with
+    optional trailing `-rcN`. For unreleased versions, it is an empty string.
+    :::{versionadded} 0.38.0
+    ::::
+    """
+
 features = struct(
-    version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "",
+    TYPEDEF = _features_typedef,
     precompile = True,
+    site_packages_root_attr = True,
     uses_builtin_rules = not config.enable_pystar,
+    version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "",
 )

python/private/attributes.bzl

@@ -282,6 +282,8 @@ that depend on this rule. The strings are repo-runfiles-root relative,
 
 Absolute paths (paths that start with `/`) and paths that references a path
 above the execution root are not allowed and will result in an error.
+
+This attribute is mutually exclusive with the {attr}`site_packages_root` attribute.
 """,
     ),
 }
@@ -308,6 +310,17 @@ These are typically `py_library` rules.
 
 Targets that only provide data files used at runtime belong in the `data`
 attribute.
+
+:::{note}
+The order of this list can matter because it affects the order that information
+from dependencies is merged in, which can be relevant depending on the ordering
+mode of depsets that are merged.
+
+* {obj}`PyInfo.site_packages_symlinks` uses topological ordering.
+
+See {obj}`PyInfo` for more information about the ordering of its depsets and
+how its fields are merged.
+:::
 """,
         ),
         "precompile": attr.string(

python/private/builders.bzl

@@ -15,12 +15,19 @@
 
 load("@bazel_skylib//lib:types.bzl", "types")
 
-def _DepsetBuilder():
-    """Create a builder for a depset."""
+def _DepsetBuilder(order = None):
+    """Create a builder for a depset.
+
+    Args:
+        order: {type}`str | None` The order to initialize the depset to, if any.
+
+    Returns:
+        {type}`DepsetBuilder`
+    """
 
     # buildifier: disable=uninitialized
     self = struct(
-        _order = [None],
+        _order = [order],
         add = lambda *a, **k: _DepsetBuilder_add(self, *a, **k),
         build = lambda *a, **k: _DepsetBuilder_build(self, *a, **k),
         direct = [],

python/private/common.bzl

@@ -30,6 +30,16 @@ PackageSpecificationInfo = getattr(py_internal, "PackageSpecificationInfo", None
 # Extensions without the dot
 _PYTHON_SOURCE_EXTENSIONS = ["py"]
 
+# Extensions that mean a file is relevant to Python
+PYTHON_FILE_EXTENSIONS = [
+    "dll",  # Python C modules, Windows specific
+    "dylib",  # Python C modules, Mac specific
+    "py",
+    "pyc",
+    "pyi",
+    "so",  # Python C modules, usually Linux
+]
+
 def create_binary_semantics_struct(
         *,
         create_executable,
@@ -413,7 +423,8 @@ def create_py_info(
         required_pyc_files,
         implicit_pyc_files,
         implicit_pyc_source_files,
-        imports):
+        imports,
+        site_packages_symlinks = []):
     """Create PyInfo provider.
 
     Args:
@@ -431,13 +442,17 @@ def create_py_info(
         implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files
             that a binary can choose to include.
         imports: depset of strings; the import path values to propagate.
+        site_packages_symlinks: {type}`list[tuple[str, str]]` tuples of
+            `(runfiles_path, site_packages_path)` for symlinks to create
+            in the consuming binary's venv site packages.
 
     Returns:
         A tuple of the PyInfo instance and a depset of the
         transitive sources collected from dependencies (the latter is only
         necessary for deprecated extra actions support).
     """
     py_info = PyInfoBuilder()
+    py_info.site_packages_symlinks.add(site_packages_symlinks)
     py_info.direct_original_sources.add(original_sources)
     py_info.direct_pyc_files.add(required_pyc_files)
     py_info.direct_pyi_files.add(ctx.files.pyi_srcs)

python/private/py_executable.bzl

@@ -591,15 +591,89 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
         },
         computed_substitutions = computed_subs,
     )
+    site_packages_symlinks = _create_site_packages_symlinks(ctx, site_packages)
 
     return struct(
         interpreter = interpreter,
         recreate_venv_at_runtime = not venvs_use_declare_symlink_enabled,
         # Runfiles root relative path or absolute path
         interpreter_actual_path = interpreter_actual_path,
-        files_without_interpreter = [pyvenv_cfg, pth, site_init],
+        files_without_interpreter = [pyvenv_cfg, pth, site_init] + site_packages_symlinks,
     )
 
+def _create_site_packages_symlinks(ctx, site_packages):
+    """Creates symlinks within site-packages.
+
+    Args:
+        ctx: current rule ctx
+        site_packages: runfiles-root-relative path to the site-packages directory
+
+    Returns:
+        {type}`list[File]` list of the File symlink objects created.
+    """
+
+    # maps site-package symlink to the runfiles path it should point to
+    entries = depset(
+        # NOTE: Topological ordering is used so that dependencies closer to the
+        # binary have precedence in creating their symlinks. This allows the
+        # binary a modicum of control over the result.
+        order = "topological",
+        transitive = [
+            dep[PyInfo].site_packages_symlinks
+            for dep in ctx.attr.deps
+            if PyInfo in dep
+        ],
+    ).to_list()
+    link_map = _build_link_map(entries)
+
+    sp_files = []
+    for sp_dir_path, link_to in link_map.items():
+        sp_link = ctx.actions.declare_symlink(paths.join(site_packages, sp_dir_path))
+        sp_link_rf_path = runfiles_root_path(ctx, sp_link.short_path)
+        rel_path = relative_path(
+            # dirname is necessary because a relative symlink is relative to
+            # the directory the symlink resides within.
+            from_ = paths.dirname(sp_link_rf_path),
+            to = link_to,
+        )
+        ctx.actions.symlink(output = sp_link, target_path = rel_path)
+        sp_files.append(sp_link)
+    return sp_files
+
+def _build_link_map(entries):
+    link_map = {}
+    for link_to_runfiles_path, site_packages_path in entries:
+        if site_packages_path in link_map:
+            # We ignore duplicates by design. The dependency closer to the
+            # binary gets precedence due to the topological ordering.
+            continue
+        else:
+            link_map[site_packages_path] = link_to_runfiles_path
+
+    # An empty link_to value means to not create the site package symlink.
+    # Because of the topological ordering, this allows binaries to remove
+    # entries by having an earlier dependency produce empty link_to values.
+    for sp_dir_path, link_to in link_map.items():
+        if not link_to:
+            link_map.pop(sp_dir_path)
+
+    # Remove entries that would be a child path of a created symlink.
+    # Earlier entries have precedence to match how exact matches are handled.
+    keep_link_map = {}
+    for _ in range(len(link_map)):
+        if not link_map:
+            break
+        dirname, value = link_map.popitem()
+        keep_link_map[dirname] = value
+
+        prefix = dirname + "/"  # Add slash to prevent /X matching /XY
+        for maybe_suffix in link_map.keys():
+            maybe_suffix += "/"  # Add slash to prevent /X matching /XY
+            if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix):
+                link_map.pop(maybe_suffix)
+
+    return keep_link_map
+
 def _map_each_identity(v):
     return v