From 26419705b20cb35d4c37173491a6a177119a2bac Mon Sep 17 00:00:00 2001
From: Stephen Tridgell <stephen.tridgell@cruxml.com.au>
Date: Fri, 25 Oct 2024 14:53:59 +1100
Subject: [PATCH 1/4] type(feat): Allow parsing of wheel file paths in
requirements.in inside a bazel repo.
---
.../pypi/dependency_resolver/dependency_resolver.py | 10 +++++++++-
python/private/pypi/pip_compile.bzl | 3 ++-
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py
index 0ff9b2fb7c..8df48fe7c8 100644
--- a/python/private/pypi/dependency_resolver/dependency_resolver.py
+++ b/python/private/pypi/dependency_resolver/dependency_resolver.py
@@ -20,6 +20,7 @@
import sys
from pathlib import Path
from typing import Optional, Tuple
+import multiprocessing
import click
import piptools.writer as piptools_writer
@@ -97,6 +98,7 @@ def main(
extra_args: Tuple[str, ...],
) -> None:
bazel_runfiles = runfiles.Create()
+ runfiles_dir = Path(bazel_runfiles._python_runfiles_root) / "_main"
requirements_file = _select_golden_requirements_file(
requirements_txt=requirements_txt,
@@ -182,10 +184,16 @@ def main(
resolved_requirements_file, requirements_file_tree
)
)
- cli(argv)
+ # Run in a process as pip tools is calling sys.exit()
+ piptools_process = multiprocessing.Process(target=cli, args=[argv])
+ piptools_process.start()
+ piptools_process.join()
requirements_file_relative_path = Path(requirements_file_relative)
content = requirements_file_relative_path.read_text()
content = content.replace(absolute_path_prefix, "")
+ # For windows or linux.
+ content = content.replace(str(runfiles_dir) + "/", "")
+ content = content.replace(str(runfiles_dir) + "\\", "")
requirements_file_relative_path.write_text(content)
else:
# cli will exit(0) on success
diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl
index dc5b186a6a..da1c1319cf 100644
--- a/python/private/pypi/pip_compile.bzl
+++ b/python/private/pypi/pip_compile.bzl
@@ -27,6 +27,7 @@ def pip_compile(
src = None,
extra_args = [],
extra_deps = [],
+ extra_data = [],
generate_hashes = True,
py_binary = _py_binary,
py_test = _py_test,
@@ -99,7 +100,7 @@ def pip_compile(
visibility = visibility,
)
- data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None]
+ data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + extra_data
# Use the Label constructor so this is expanded in the context of the file
# where it appears, which is to say, in @rules_python
From 76c57c18397675fb65ec9759a4a7cdf4c337ba9d Mon Sep 17 00:00:00 2001
From: Stephen Tridgell <stephen.tridgell@cruxml.com.au>
Date: Fri, 25 Oct 2024 15:01:23 +1100
Subject: [PATCH 2/4] type(feat): Document extra_data for pip_compile
---
python/private/pypi/pip_compile.bzl | 1 +
1 file changed, 1 insertion(+)
diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl
index da1c1319cf..885aebf76e 100644
--- a/python/private/pypi/pip_compile.bzl
+++ b/python/private/pypi/pip_compile.bzl
@@ -67,6 +67,7 @@ def pip_compile(
[PEP621](https://peps.python.org/pep-0621/).
extra_args: passed to pip-compile.
extra_deps: extra dependencies passed to pip-compile.
+ extra_data: extra data passed to pip-compile.
generate_hashes: whether to put hashes in the requirements_txt file.
py_binary: the py_binary rule to be used.
py_test: the py_test rule to be used.
From 21185530a9dddf23df3855cc9293d365861c1f26 Mon Sep 17 00:00:00 2001
From: Stephen Tridgell <stephen.tridgell@cruxml.com.au>
Date: Fri, 25 Oct 2024 17:01:11 +1100
Subject: [PATCH 3/4] type(feat): This is a bit ugly having to provide whl file
again. Is there a better way to do this?
---
.../pypi/requirements.bzl.tmpl.workspace | 31 +++++++++++++------
1 file changed, 21 insertions(+), 10 deletions(-)
diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace
index 2f4bcd6916..a76002c562 100644
--- a/python/private/pypi/requirements.bzl.tmpl.workspace
+++ b/python/private/pypi/requirements.bzl.tmpl.workspace
@@ -35,7 +35,7 @@ def _get_annotation(requirement):
name = requirement.split(" ")[0].split("=")[0].split("[")[0]
return _annotations.get(name)
-def install_deps(**whl_library_kwargs):
+def install_deps(whl_files = {}, **whl_library_kwargs):
"""Repository rule macro. Install dependencies from `pip_parse`.
Args:
@@ -61,12 +61,23 @@ def install_deps(**whl_library_kwargs):
for name, requirement in _packages:
group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", ""))
group_deps = all_requirement_groups.get(group_name, [])
-
- whl_library(
- name = name,
- requirement = requirement,
- group_name = group_name,
- group_deps = group_deps,
- annotation = _get_annotation(requirement),
- **whl_config
- )
+ if "@ file://" in requirement:
+ whl_file = whl_files[name]
+ whl_library(
+ name = name,
+ requirement = requirement,
+ group_name = group_name,
+ group_deps = group_deps,
+ annotation = _get_annotation(requirement),
+ whl_file = whl_file,
+ **whl_config
+ )
+ else:
+ whl_library(
+ name = name,
+ requirement = requirement,
+ group_name = group_name,
+ group_deps = group_deps,
+ annotation = _get_annotation(requirement),
+ **whl_config
+ )
From 705ed917bf7d9ebd6ed0f98428fda11d49d30ff7 Mon Sep 17 00:00:00 2001
From: Stephen Tridgell <stephen.tridgell@cruxml.com.au>
Date: Wed, 30 Oct 2024 13:53:34 +1100
Subject: [PATCH 4/4] Refactor and fix indentation.
---
.../pypi/requirements.bzl.tmpl.workspace | 29 +++++++------------
1 file changed, 11 insertions(+), 18 deletions(-)
diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace
index a76002c562..03736e5fef 100644
--- a/python/private/pypi/requirements.bzl.tmpl.workspace
+++ b/python/private/pypi/requirements.bzl.tmpl.workspace
@@ -62,22 +62,15 @@ def install_deps(whl_files = {}, **whl_library_kwargs):
group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", ""))
group_deps = all_requirement_groups.get(group_name, [])
if "@ file://" in requirement:
- whl_file = whl_files[name]
- whl_library(
- name = name,
- requirement = requirement,
- group_name = group_name,
- group_deps = group_deps,
- annotation = _get_annotation(requirement),
- whl_file = whl_file,
- **whl_config
- )
+ whl_file = whl_files[name]
else:
- whl_library(
- name = name,
- requirement = requirement,
- group_name = group_name,
- group_deps = group_deps,
- annotation = _get_annotation(requirement),
- **whl_config
- )
+ whl_file = None
+ whl_library(
+ name = name,
+ requirement = requirement,
+ group_name = group_name,
+ group_deps = group_deps,
+ annotation = _get_annotation(requirement),
+ whl_file = whl_file,
+ **whl_config
+ )