diff --git a/.bazelrc b/.bazelrc index ada5c5a0a7..4e6f2fa187 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6287da0b..1a042f9254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,14 @@ Unreleased changes template. {#v0-0-0-added} ### Added +* (providers) {obj}`PyInfo.site_packages_symlinks` field added to allow + specifying links to create within the venv site packages + (only applicable with {obj}`--bootstrap_impl=script`) + ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)). +* (rules) {obj}`py_library.site_packages_root` attribute added to allow + specifying a library's sources follow a site-packages file layout. + (only applicable with {obj}`--bootstrap_impl=script`) + ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)). * {obj}`//python/bin:python`: convenience target for directly running an interpreter. {obj}`--//python/bin:python_src` can be used to specify a binary whose interpreter to use. diff --git a/MODULE.bazel b/MODULE.bazel index dc2193cec2..de1164a5fa 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -85,6 +85,7 @@ bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True) bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True) bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True) +bazel_dep(name = "other", version = "0", dev_dependency = True) # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests. # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides. @@ -106,6 +107,11 @@ local_path_override( path = "gazelle", ) +local_path_override( + module_name = "other", + path = "tests/modules/other", +) + dev_python = use_extension( "//python/extensions:python.bzl", "python", diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 09de21b86a..76e58fc668 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -87,6 +87,7 @@ sphinx_stardocs( name = "bzl_api_docs", srcs = [ "//python:defs_bzl", + "//python:features_bzl", "//python:packaging_bzl", "//python:pip_bzl", "//python:py_binary_bzl", diff --git a/internal_dev_deps.bzl b/internal_dev_deps.bzl index cd33475f43..87690be1ad 100644 --- a/internal_dev_deps.bzl +++ b/internal_dev_deps.bzl @@ -15,6 +15,7 @@ """Dependencies that are needed for development and testing of rules_python itself.""" load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive", _http_file = "http_file") +load("@bazel_tools//tools/build_defs/repo:local.bzl", "local_repository") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=bzl-visibility @@ -42,6 +43,11 @@ def rules_python_internal_deps(): """ internal_config_repo(name = "rules_python_internal") + local_repository( + name = "other", + path = "tests/modules/other", + ) + http_archive( name = "bazel_skylib", sha256 = "bc283cdfcd526a52c3201279cda4bc298652efa898b10b4db0837dc51652756f", diff --git a/python/BUILD.bazel b/python/BUILD.bazel index c52e772666..a699c81cc4 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -79,6 +79,9 @@ bzl_library( bzl_library( name = "features_bzl", srcs = ["features.bzl"], + deps = [ + "@rules_python_internal//:rules_python_config_bzl", + ], ) bzl_library( diff --git a/python/features.bzl b/python/features.bzl index a7098f4710..f5b1a0b2bb 100644 --- a/python/features.bzl +++ b/python/features.bzl @@ -19,8 +19,49 @@ load("@rules_python_internal//:rules_python_config.bzl", "config") # See https://git-scm.com/docs/git-archive/2.29.0#Documentation/git-archive.txt-export-subst _VERSION_PRIVATE = "$Format:%(describe:tags=true)$" +def _features_typedef(): + """Information about features rules_python has implemented. + + ::::{field} precompile + :type: bool + + True if the precompile attributes are available. + + :::{versionadded} 0.33.0 + ::: + :::: + + ::::{field} site_packages_root_attr + :type: bool + + True if the {obj}`site_packages_root` attribute is available. + + :::{versionadded} VERSION_NEXT_FEATURE + ::: + :::: + + ::::{field} uses_builtin_rules + :type: bool + + True if the rules are using the Bazel-builtin implementation. + + :::{versionadded} 1.1.0 + ::: + :::: + + ::::{field} version + :type: str + + The rules_python version. This is a semver format, e.g. `X.Y.Z` with + optional trailing `-rcN`. For unreleased versions, it is an empty string. + :::{versionadded} 0.38.0 + :::: + """ + features = struct( - version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "", + TYPEDEF = _features_typedef, precompile = True, + site_packages_root_attr = True, uses_builtin_rules = not config.enable_pystar, + version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "", ) diff --git a/python/private/attributes.bzl b/python/private/attributes.bzl index b57e275406..bd2d596585 100644 --- a/python/private/attributes.bzl +++ b/python/private/attributes.bzl @@ -236,6 +236,8 @@ that depend on this rule. The strings are repo-runfiles-root relative, Absolute paths (paths that start with `/`) and paths that references a path above the execution root are not allowed and will result in an error. + +This attribute is mutually exclusive with the {attr}`site_packages_root` attribute. """, ), } @@ -262,6 +264,17 @@ These are typically `py_library` rules. Targets that only provide data files used at runtime belong in the `data` attribute. + +:::{note} +The order of this list can matter because it affects the order that information +from dependencies is merged in, which can be relevant depending on the ordering +mode of depsets that are merged. + +* {obj}`PyInfo.site_packages_symlinks` uses topological ordering. + +See {obj}`PyInfo` for more information about the ordering of its depsets and +how its fields are merged. +::: """, ), "precompile": lambda: attrb.String( diff --git a/python/private/builders.bzl b/python/private/builders.bzl index 50aa3ed91a..54d46c2af2 100644 --- a/python/private/builders.bzl +++ b/python/private/builders.bzl @@ -15,12 +15,19 @@ load("@bazel_skylib//lib:types.bzl", "types") -def _DepsetBuilder(): - """Create a builder for a depset.""" +def _DepsetBuilder(order = None): + """Create a builder for a depset. + + Args: + order: {type}`str | None` The order to initialize the depset to, if any. + + Returns: + {type}`DepsetBuilder` + """ # buildifier: disable=uninitialized self = struct( - _order = [None], + _order = [order], add = lambda *a, **k: _DepsetBuilder_add(self, *a, **k), build = lambda *a, **k: _DepsetBuilder_build(self, *a, **k), direct = [], diff --git a/python/private/common.bzl b/python/private/common.bzl index 48e2653ebb..072a1bb296 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -30,6 +30,16 @@ PackageSpecificationInfo = getattr(py_internal, "PackageSpecificationInfo", None # Extensions without the dot _PYTHON_SOURCE_EXTENSIONS = ["py"] +# Extensions that mean a file is relevant to Python +PYTHON_FILE_EXTENSIONS = [ + "dll", # Python C modules, Windows specific + "dylib", # Python C modules, Mac specific + "py", + "pyc", + "pyi", + "so", # Python C modules, usually Linux +] + def create_binary_semantics_struct( *, create_executable, @@ -367,7 +377,8 @@ def create_py_info( required_pyc_files, implicit_pyc_files, implicit_pyc_source_files, - imports): + imports, + site_packages_symlinks = []): """Create PyInfo provider. Args: @@ -385,6 +396,9 @@ def create_py_info( implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files that a binary can choose to include. imports: depset of strings; the import path values to propagate. + site_packages_symlinks: {type}`list[tuple[str, str]]` tuples of + `(runfiles_path, site_packages_path)` for symlinks to create + in the consuming binary's venv site packages. Returns: A tuple of the PyInfo instance and a depset of the @@ -392,6 +406,7 @@ def create_py_info( necessary for deprecated extra actions support). """ py_info = PyInfoBuilder() + py_info.site_packages_symlinks.add(site_packages_symlinks) py_info.direct_original_sources.add(original_sources) py_info.direct_pyc_files.add(required_pyc_files) py_info.direct_pyi_files.add(ctx.files.pyi_srcs) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index d0ac3146ac..13686c0867 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -619,15 +619,89 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): }, computed_substitutions = computed_subs, ) + site_packages_symlinks = _create_site_packages_symlinks(ctx, site_packages) return struct( interpreter = interpreter, recreate_venv_at_runtime = not venvs_use_declare_symlink_enabled, # Runfiles root relative path or absolute path interpreter_actual_path = interpreter_actual_path, - files_without_interpreter = [pyvenv_cfg, pth, site_init], + files_without_interpreter = [pyvenv_cfg, pth, site_init] + site_packages_symlinks, ) +def _create_site_packages_symlinks(ctx, site_packages): + """Creates symlinks within site-packages. + + Args: + ctx: current rule ctx + site_packages: runfiles-root-relative path to the site-packages directory + + Returns: + {type}`list[File]` list of the File symlink objects created. + """ + + # maps site-package symlink to the runfiles path it should point to + entries = depset( + # NOTE: Topological ordering is used so that dependencies closer to the + # binary have precedence in creating their symlinks. This allows the + # binary a modicum of control over the result. + order = "topological", + transitive = [ + dep[PyInfo].site_packages_symlinks + for dep in ctx.attr.deps + if PyInfo in dep + ], + ).to_list() + link_map = _build_link_map(entries) + + sp_files = [] + for sp_dir_path, link_to in link_map.items(): + sp_link = ctx.actions.declare_symlink(paths.join(site_packages, sp_dir_path)) + sp_link_rf_path = runfiles_root_path(ctx, sp_link.short_path) + rel_path = relative_path( + # dirname is necessary because a relative symlink is relative to + # the directory the symlink resides within. + from_ = paths.dirname(sp_link_rf_path), + to = link_to, + ) + ctx.actions.symlink(output = sp_link, target_path = rel_path) + sp_files.append(sp_link) + return sp_files + +def _build_link_map(entries): + link_map = {} + for link_to_runfiles_path, site_packages_path in entries: + if site_packages_path in link_map: + # We ignore duplicates by design. The dependency closer to the + # binary gets precedence due to the topological ordering. + continue + else: + link_map[site_packages_path] = link_to_runfiles_path + + # An empty link_to value means to not create the site package symlink. + # Because of the topological ordering, this allows binaries to remove + # entries by having an earlier dependency produce empty link_to values. + for sp_dir_path, link_to in link_map.items(): + if not link_to: + link_map.pop(sp_dir_path) + + # Remove entries that would be a child path of a created symlink. + # Earlier entries have precedence to match how exact matches are handled. + keep_link_map = {} + for _ in range(len(link_map)): + if not link_map: + break + dirname, value = link_map.popitem() + keep_link_map[dirname] = value + + prefix = dirname + "/" # Add slash to prevent /X matching /XY + for maybe_suffix in link_map.keys(): + maybe_suffix += "/" # Add slash to prevent /X matching /XY + if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix): + link_map.pop(maybe_suffix) + + return keep_link_map + def _map_each_identity(v): return v diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl index ef654c303e..ae326278a9 100644 --- a/python/private/py_info.bzl +++ b/python/private/py_info.bzl @@ -42,7 +42,8 @@ def _PyInfo_init( direct_original_sources = depset(), transitive_original_sources = depset(), direct_pyi_files = depset(), - transitive_pyi_files = depset()): + transitive_pyi_files = depset(), + site_packages_symlinks = depset()): _check_arg_type("transitive_sources", "depset", transitive_sources) # Verify it's postorder compatible, but retain is original ordering. @@ -70,6 +71,7 @@ def _PyInfo_init( "has_py2_only_sources": has_py2_only_sources, "has_py3_only_sources": has_py2_only_sources, "imports": imports, + "site_packages_symlinks": site_packages_symlinks, "transitive_implicit_pyc_files": transitive_implicit_pyc_files, "transitive_implicit_pyc_source_files": transitive_implicit_pyc_source_files, "transitive_original_sources": transitive_original_sources, @@ -140,6 +142,31 @@ A depset of import path strings to be added to the `PYTHONPATH` of executable Python targets. These are accumulated from the transitive `deps`. The order of the depset is not guaranteed and may be changed in the future. It is recommended to use `default` order (the default). +""", + "site_packages_symlinks": """ +:type: depset[tuple[str | None, str]] + +A depset with `topological` ordering. + +Tuples of `(runfiles_path, site_packages_path)`. Where +* `runfiles_path` is a runfiles-root relative path. It is the path that + has the code to make importable. If `None` or empty string, then it means + to not create a site packages directory with the `site_packages_path` + name. +* `site_packages_path` is a path relative to the site-packages directory of + the venv for whatever creates the venv (typically py_binary). It makes + the code in `runfiles_path` available for import. Note that this + is created as a "raw" symlink (via `declare_symlink`). + +:::{tip} +The topological ordering means dependencies earlier and closer to the consumer +have precedence. This allows e.g. a binary to add dependencies that override +values from further way dependencies, such as forcing symlinks to point to +specific paths or preventing symlinks from being created. +::: + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, "transitive_implicit_pyc_files": """ :type: depset[File] @@ -266,6 +293,7 @@ def PyInfoBuilder(): transitive_pyc_files = builders.DepsetBuilder(), transitive_pyi_files = builders.DepsetBuilder(), transitive_sources = builders.DepsetBuilder(), + site_packages_symlinks = builders.DepsetBuilder(order = "topological"), ) return self @@ -351,6 +379,7 @@ def _PyInfoBuilder_merge_all(self, transitive, *, direct = []): self.transitive_original_sources.add(info.transitive_original_sources) self.transitive_pyc_files.add(info.transitive_pyc_files) self.transitive_pyi_files.add(info.transitive_pyi_files) + self.site_packages_symlinks.add(info.site_packages_symlinks) return self @@ -400,6 +429,7 @@ def _PyInfoBuilder_build(self): transitive_original_sources = self.transitive_original_sources.build(), transitive_pyc_files = self.transitive_pyc_files.build(), transitive_pyi_files = self.transitive_pyi_files.build(), + site_packages_symlinks = self.site_packages_symlinks.build(), ) else: kwargs = {} diff --git a/python/private/py_library.bzl b/python/private/py_library.bzl index 7b024a0f07..c999f73ad2 100644 --- a/python/private/py_library.bzl +++ b/python/private/py_library.bzl @@ -14,6 +14,7 @@ """Common code for implementing py_library rules.""" load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load(":attr_builders.bzl", "attrb") load( @@ -25,7 +26,21 @@ load( "REQUIRED_EXEC_GROUP_BUILDERS", ) load(":builders.bzl", "builders") -load(":common.bzl", "collect_cc_info", "collect_imports", "collect_runfiles", "create_instrumented_files_info", "create_library_semantics_struct", "create_output_group_info", "create_py_info", "filter_to_py_srcs", "get_imports") +load( + ":common.bzl", + "PYTHON_FILE_EXTENSIONS", + "collect_cc_info", + "collect_imports", + "collect_runfiles", + "create_instrumented_files_info", + "create_library_semantics_struct", + "create_output_group_info", + "create_py_info", + "filter_to_py_srcs", + "get_imports", + "runfiles_root_path", + "union_attrs", +) load(":flags.bzl", "AddSrcsToRunfilesFlag", "PrecompileFlag") load(":precompile.bzl", "maybe_precompile") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") @@ -44,6 +59,47 @@ LIBRARY_ATTRS = dicts.add( PY_SRCS_ATTRS, IMPORTS_ATTRS, { + "site_packages_root": lambda: attrb.String( + doc = """ +Package relative prefix to remove from `srcs` for site-packages layouts. + +This attribute is mutually exclusive with the {attr}`imports` attribute. + +When set, `srcs` are interpreted to have a file layout as if they were installed +in site-packages. This attribute specifies the directory within `srcs` to treat +as the site-packages root so the correct site-packages relative paths for +the files can be computed. + +:::{note} +This string is relative to the target's *Bazel package*. e.g. Relative to the +directory with the BUILD file that defines the target (the same as how e.g. +`srcs`). +::: + +For example, given `srcs=["site-packages/foo/bar.py"]`, specifying +`site_packages_root="site-packages/" means `foo/bar.py` is the file path +under the binary's venv site-packages directory that should be made available. + +`__init__.py` files are treated specially to provide basic support for [implicit +namespace packages]( +https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#native-namespace-packages). +However, the *content* of the files cannot be taken into account, merely their +presence or absense. Stated another way: [pkgutil-style namespace packages]( +https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages) +won't be understood as namespace packages; they'll be seen as regular packages. This will +likely lead to conflicts with other targets that contribute to the namespace. + +:::{tip} +This attributes populates {obj}`PyInfo.site_packages_symlinks`, which is +a topologically ordered depset. This means dependencies closer and earlier +to a consumer have precedence. See {obj}`PyInfo.site_packages_symlinks` for +more information. +::: + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), "_add_srcs_to_runfiles_flag": lambda: attrb.Label( default = "//python/config_settings:add_srcs_to_runfiles", ), @@ -98,6 +154,19 @@ def py_library_impl(ctx, *, semantics): runfiles.add(collect_runfiles(ctx)) runfiles = runfiles.build(ctx) + imports = [] + site_packages_symlinks = [] + if ctx.attr.imports and ctx.attr.site_packages_root: + fail(("Only one of the `imports` or `site_packages_root` attributes " + + "can be set: site_packages_root={}, imports={}").format( + ctx.attr.site_packages_root, + ctx.attr.imports, + )) + elif ctx.attr.site_packages_root: + site_packages_symlinks = _get_site_packages_symlinks(ctx) + elif ctx.attr.imports: + imports = collect_imports(ctx, semantics) + cc_info = semantics.get_cc_info_for_library(ctx) py_info, deps_transitive_sources, builtins_py_info = create_py_info( ctx, @@ -106,7 +175,8 @@ def py_library_impl(ctx, *, semantics): required_pyc_files = required_pyc_files, implicit_pyc_files = implicit_pyc_files, implicit_pyc_source_files = implicit_pyc_source_files, - imports = collect_imports(ctx, semantics), + imports = imports, + site_packages_symlinks = site_packages_symlinks, ) # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 @@ -144,6 +214,73 @@ Source files are no longer added to the runfiles directly. ::: """ +def _get_site_packages_symlinks(ctx): + if not ctx.attr.site_packages_root: + return [] + + # We have to build a list of (runfiles path, site-packages path) pairs of + # the files to create in the consuming binary's venv site-packages directory. + # To minimize the number of files to create, we just return the paths + # to the directories containing the code of interest. + # + # However, namespace packages complicate matters: multiple + # distributions install in the same directory in site-packages. This + # works out because they don't overlap in their files. Typically, they + # install to different directories within the namespace package + # directory. Namespace package directories are simply directories + # within site-packages that *don't* have an `__init__.py` file, which + # can be arbitrarily deep. Thus, we simply have to look for the + # directories that _do_ have an `__init__.py` file and treat those as + # the path to symlink to. + + site_packages_root = paths.join(ctx.label.package, ctx.attr.site_packages_root) + repo_runfiles_dirname = None + dirs_with_init = {} # dirname -> runfile path + for src in ctx.files.srcs: + if src.extension not in PYTHON_FILE_EXTENSIONS: + continue + path = _repo_relative_short_path(src.short_path) + if not path.startswith(site_packages_root): + continue + path = path.removeprefix(site_packages_root) + dir_name, _, filename = path.rpartition("/") + if not dir_name: + # This would be e.g. `site-packages/__init__.py`, which isn't valid. + # Apparently, the pypi integration adds such a file? + continue + + if filename.startswith("__init__."): + dirs_with_init[dir_name] = None + repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0] + + # Sort so that we encounter `foo` before `foo/bar`. This ensures we + # see the top-most explicit package first. + dirnames = sorted(dirs_with_init.keys()) + first_level_explicit_packages = [] + for d in dirnames: + is_sub_package = False + for existing in first_level_explicit_packages: + # Suffix with / to prevent foo matching foobar + if d.startswith(existing + "/"): + is_sub_package = True + break + if not is_sub_package: + first_level_explicit_packages.append(d) + + site_packages_symlinks = [] + for dirname in first_level_explicit_packages: + site_packages_symlinks.append(( + paths.join(repo_runfiles_dirname, site_packages_root, dirname), + dirname, + )) + return site_packages_symlinks + +def _repo_relative_short_path(short_path): + if short_path.startswith("../"): + return short_path[3:].partition("/")[2] + else: + return short_path + # NOTE: Exported publicaly def create_py_library_rule_builder(): """Create a rule builder for a py_library. diff --git a/tests/modules/other/BUILD.bazel b/tests/modules/other/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/modules/other/MODULE.bazel b/tests/modules/other/MODULE.bazel new file mode 100644 index 0000000000..7cd3118b81 --- /dev/null +++ b/tests/modules/other/MODULE.bazel @@ -0,0 +1,3 @@ +module(name = "other") + +bazel_dep(name = "rules_python", version = "0") diff --git a/tests/modules/other/nspkg_delta/BUILD.bazel b/tests/modules/other/nspkg_delta/BUILD.bazel new file mode 100644 index 0000000000..e3b005e4b2 --- /dev/null +++ b/tests/modules/other/nspkg_delta/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "nspkg_delta", + srcs = glob(["site-packages/**/*.py"]), + site_packages_root = "site-packages/", +) diff --git a/tests/modules/other/nspkg_delta/site-packages/nspkg/subnspkg/delta/__init__.py b/tests/modules/other/nspkg_delta/site-packages/nspkg/subnspkg/delta/__init__.py new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/tests/modules/other/nspkg_delta/site-packages/nspkg/subnspkg/delta/__init__.py @@ -0,0 +1 @@ +# Intentionally empty diff --git a/tests/modules/other/nspkg_gamma/BUILD.bazel b/tests/modules/other/nspkg_gamma/BUILD.bazel new file mode 100644 index 0000000000..dbbd7a7c04 --- /dev/null +++ b/tests/modules/other/nspkg_gamma/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "nspkg_gamma", + srcs = glob(["site-packages/**/*.py"]), + site_packages_root = "site-packages/", +) diff --git a/tests/modules/other/nspkg_gamma/site-packages/nspkg/subnspkg/gamma/__init__.py b/tests/modules/other/nspkg_gamma/site-packages/nspkg/subnspkg/gamma/__init__.py new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/tests/modules/other/nspkg_gamma/site-packages/nspkg/subnspkg/gamma/__init__.py @@ -0,0 +1 @@ +# Intentionally empty diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel new file mode 100644 index 0000000000..22b7d9fa97 --- /dev/null +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -0,0 +1,16 @@ +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") +load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") + +py_reconfig_test( + name = "venv_site_packages_libs_test", + srcs = ["bin.py"], + bootstrap_impl = "script", + main = "bin.py", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + deps = [ + "//tests/venv_site_packages_libs/nspkg_alpha", + "//tests/venv_site_packages_libs/nspkg_beta", + "@other//nspkg_delta", + "@other//nspkg_gamma", + ], +) diff --git a/tests/venv_site_packages_libs/bin.py b/tests/venv_site_packages_libs/bin.py new file mode 100644 index 0000000000..b944be69e3 --- /dev/null +++ b/tests/venv_site_packages_libs/bin.py @@ -0,0 +1,32 @@ +import importlib +import os +import sys +import unittest + + +class VenvSitePackagesLibraryTest(unittest.TestCase): + def setUp(self): + super().setUp() + if sys.prefix == sys.base_prefix: + raise AssertionError("Not running under a venv") + self.venv = sys.prefix + + def assert_imported_from_venv(self, module_name): + module = importlib.import_module(module_name) + self.assertEqual(module.__name__, module_name) + self.assertTrue( + module.__file__.startswith(self.venv), + f"\n{module_name} was imported, but not from the venv.\n" + + f"venv : {self.venv}\n" + + f"actual: {module.__file__}", + ) + + def test_imported_from_venv(self): + self.assert_imported_from_venv("nspkg.subnspkg.alpha") + self.assert_imported_from_venv("nspkg.subnspkg.beta") + self.assert_imported_from_venv("nspkg.subnspkg.gamma") + self.assert_imported_from_venv("nspkg.subnspkg.delta") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel b/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel new file mode 100644 index 0000000000..2a53839048 --- /dev/null +++ b/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "nspkg_alpha", + srcs = glob(["site-packages/**/*.py"]), + site_packages_root = "site-packages/", +) diff --git a/tests/venv_site_packages_libs/nspkg_alpha/site-packages/nspkg/subnspkg/alpha/__init__.py b/tests/venv_site_packages_libs/nspkg_alpha/site-packages/nspkg/subnspkg/alpha/__init__.py new file mode 100644 index 0000000000..b5ee093672 --- /dev/null +++ b/tests/venv_site_packages_libs/nspkg_alpha/site-packages/nspkg/subnspkg/alpha/__init__.py @@ -0,0 +1 @@ +whoami = "alpha" diff --git a/tests/venv_site_packages_libs/nspkg_beta/BUILD.bazel b/tests/venv_site_packages_libs/nspkg_beta/BUILD.bazel new file mode 100644 index 0000000000..b0bfdfceba --- /dev/null +++ b/tests/venv_site_packages_libs/nspkg_beta/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "nspkg_beta", + srcs = glob(["site-packages/**/*.py"]), + site_packages_root = "site-packages/", +) diff --git a/tests/venv_site_packages_libs/nspkg_beta/site-packages/nspkg/subnspkg/beta/__init__.py b/tests/venv_site_packages_libs/nspkg_beta/site-packages/nspkg/subnspkg/beta/__init__.py new file mode 100644 index 0000000000..a2a65910c7 --- /dev/null +++ b/tests/venv_site_packages_libs/nspkg_beta/site-packages/nspkg/subnspkg/beta/__init__.py @@ -0,0 +1 @@ +whoami = "beta" diff --git a/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py b/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py new file mode 100644 index 0000000000..519b258044 --- /dev/null +++ b/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py @@ -0,0 +1,36 @@ +import os +import sys +import unittest + + +class VenvSitePackagesLibraryTest(unittest.TestCase): + def test_imported_from_venv(self): + self.assertNotEqual(sys.prefix, sys.base_prefix, "Not running under a venv") + venv = sys.prefix + + from nspkg.subnspkg import alpha + + self.assertEqual(alpha.whoami, "alpha") + self.assertEqual(alpha.__name__, "nspkg.subnspkg.alpha") + + self.assertTrue( + alpha.__file__.startswith(sys.prefix), + f"\nalpha was imported, not from within the venv.\n" + + f"venv : {venv}\n" + + f"actual: {alpha.__file__}", + ) + + from nspkg.subnspkg import beta + + self.assertEqual(beta.whoami, "beta") + self.assertEqual(beta.__name__, "nspkg.subnspkg.beta") + self.assertTrue( + beta.__file__.startswith(sys.prefix), + f"\nbeta was imported, not from within the venv.\n" + + f"venv : {venv}\n" + + f"actual: {beta.__file__}", + ) + + +if __name__ == "__main__": + unittest.main()