From c7a86840825ed0260b1a6ecc80bbee39355c5203 Mon Sep 17 00:00:00 2001 From: "Joshua \"jag\" Ginsberg" Date: Sun, 7 May 2023 14:58:59 -0400 Subject: [PATCH] PR review feedback. Updated README. --- README.rst | 22 ++++++++++ piptools/locations.py | 3 ++ piptools/scripts/compile.py | 12 +++--- piptools/scripts/sync.py | 8 ++-- piptools/utils.py | 84 ++++++++++++++++++++++++------------- pyproject.toml | 3 +- tests/conftest.py | 16 ++++--- tests/test_utils.py | 66 +++++++++++++++++------------ 8 files changed, 144 insertions(+), 70 deletions(-) diff --git a/README.rst b/README.rst index c04947d14..7a3b398a6 100644 --- a/README.rst +++ b/README.rst @@ -306,6 +306,28 @@ Any valid ``pip`` flags or arguments may be passed on with ``pip-compile``'s Configuration ------------- +You can define project-level defaults for ``pip-compile`` and ``pip-sync`` by +writing them to a configuration file in the same directory as your requirements +input file. By default, both ``pip-compile`` and ``pip-sync`` will look first +for a ``.pip-tools.toml`` file and then in your ``pyproject.toml``. You can +also specify an alternate TOML configuration file with the ``--config`` option. + +For example, to by default generate ``pip`` hashes in the resulting +requirements file output, you can specify in a configuration file + +.. code-block:: toml + # In a .pip-tools.toml file + [pip-tools] + generate-hashes = true + + # In a pyproject.toml file + [tool.pip-tools] + generate-hashes = true + +Options to ``pip-compile`` and ``pip-sync`` that may be used more than once +must be defined as lists in a configuration file, even if they only have one +value. + You might be wrapping the ``pip-compile`` command in another script. To avoid confusing consumers of your custom script you can override the update command generated at the top of requirements files by setting the diff --git a/piptools/locations.py b/piptools/locations.py index bf757603f..f31891cae 100644 --- a/piptools/locations.py +++ b/piptools/locations.py @@ -4,3 +4,6 @@ # The user_cache_dir helper comes straight from pip itself CACHE_DIR = user_cache_dir("pip-tools") + +# The project defaults specific to pip-tools should be written to this filename +CONFIG_FILE_NAME = ".pip-tools.toml" diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index bed6f0c20..2fee2c375 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -8,7 +8,6 @@ from typing import IO, Any, BinaryIO, cast import click -from build import BuildBackendException from build.util import project_wheel_metadata from click.utils import LazyFile, safecall from pip._internal.commands import create_command @@ -16,22 +15,24 @@ from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url +from build import BuildBackendException + from .._compat import parse_requirements from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError -from ..locations import CACHE_DIR +from ..locations import CACHE_DIR, CONFIG_FILE_NAME from ..logging import log from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import BacktrackingResolver, LegacyResolver from ..utils import ( UNSAFE_PACKAGES, + callback_config_file_defaults, dedup, drop_extras, is_pinned_requirement, key_from_ireq, parse_requirements_from_wheel_metadata, - pyproject_toml_defaults_cb, ) from ..writer import OutputWriter @@ -313,9 +314,10 @@ def _determine_linesep( allow_dash=False, path_type=str, ), - help="Path to a pyproject.toml file with specialized defaults for pip-tools", + help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " + "pyproject.toml.", is_eager=True, - callback=pyproject_toml_defaults_cb, + callback=callback_config_file_defaults, ) def cli( ctx: click.Context, diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 2a7c60a6a..c09fd082f 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -17,14 +17,15 @@ from .._compat import parse_requirements from .._compat.pip_compat import Distribution from ..exceptions import PipToolsError +from ..locations import CONFIG_FILE_NAME from ..logging import log from ..repositories import PyPIRepository from ..utils import ( + callback_config_file_defaults, flat_map, get_pip_version_for_python_executable, get_required_pip_specification, get_sys_path_for_python_executable, - pyproject_toml_defaults_cb, ) DEFAULT_REQUIREMENTS_FILE = "requirements.txt" @@ -97,9 +98,10 @@ allow_dash=False, path_type=str, ), - help="Path to a pyproject.toml file with specialized defaults for pip-tools", + help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " + "pyproject.toml.", is_eager=True, - callback=pyproject_toml_defaults_cb, + callback=callback_config_file_defaults, ) def cli( ask: bool, diff --git a/piptools/utils.py b/piptools/utils.py index 4509f8e37..e750903bd 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -8,11 +8,16 @@ import os import re import shlex +import sys from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypeVar, cast import click -import toml + +if sys.version_info >= (3, 11): + import tomllib +else: + import toml from click.utils import LazyFile from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line, parse_req_from_line @@ -25,6 +30,7 @@ from pip._vendor.pkg_resources import Distribution, Requirement, get_distribution from piptools._compat import PIP_VERSION +from piptools.locations import CONFIG_FILE_NAME from piptools.subprocess_utils import run_python_snippet if TYPE_CHECKING: @@ -408,9 +414,9 @@ def get_required_pip_specification() -> SpecifierSet: Returns pip version specifier requested by current pip-tools installation. """ project_dist = get_distribution("pip-tools") - requirement = next( # pragma: no branch + requirement = next( (r for r in project_dist.requires() if r.name == "pip"), None - ) + ) # pragma: no branch assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" @@ -527,30 +533,34 @@ def parse_requirements_from_wheel_metadata( ) -def pyproject_toml_defaults_cb( +def callback_config_file_defaults( ctx: click.Context, param: click.Parameter, value: str | None ) -> str | None: """ - Defaults for `click.Command` parameters should be override-able in pyproject.toml + Returns the path to the config file with defaults being used, or `None` if no such file is + found. - Returns the path to the configuration file found, or None if no such file is found. + Defaults for `click.Command` parameters should be override-able in a config file. `pip-tools` + will use the first file found, searching in this order: an explicitly given config file, a + `.pip-tools.toml`, a `pyproject.toml` file. Those files are searched for in the same directory + as the requirements input file. """ if value is None: - config_file = find_pyproject_toml(ctx.params.get("src_files", ())) + config_file = select_config_file(ctx.params.get("src_files", ())) if config_file is None: return None else: - config_file = value + config_file = Path(value) try: - config = parse_pyproject_toml(config_file) + config = parse_config_file(config_file) except OSError as e: raise click.FileError( - filename=config_file, hint=f"Could not read '{config_file}': {e}" + filename=str(config_file), hint=f"Could not read '{config_file}': {e}" ) except ValueError as e: raise click.FileError( - filename=config_file, hint=f"Could not parse '{config_file}': {e}" + filename=str(config_file), hint=f"Could not parse '{config_file}': {e}" ) if not config: @@ -560,28 +570,34 @@ def pyproject_toml_defaults_cb( defaults.update(config) ctx.default_map = defaults - return config_file + return str(config_file) -def find_pyproject_toml(src_files: tuple[str, ...]) -> str | None: +def select_config_file(src_files: tuple[str, ...]) -> Path | None: + """ + Returns the config file to use for defaults given `src_files` provided. + """ if not src_files: # If no src_files were specified, we consider the current directory the only candidate - candidates = [Path.cwd()] + candidate_dirs = [Path.cwd()] else: # Collect the candidate directories based on the src_file arguments provided src_files_as_paths = [ Path(Path.cwd(), src_file).resolve() for src_file in src_files ] - candidates = [src if src.is_dir() else src.parent for src in src_files_as_paths] - pyproject_toml_path = next( + candidate_dirs = [ + src if src.is_dir() else src.parent for src in src_files_as_paths + ] + config_file_path = next( ( - str(candidate / "pyproject.toml") - for candidate in candidates - if (candidate / "pyproject.toml").is_file() + candidate_dir / config_file + for candidate_dir in candidate_dirs + for config_file in (CONFIG_FILE_NAME, "pyproject.toml") + if (candidate_dir / config_file).is_file() ), None, ) - return pyproject_toml_path + return config_file_path # Some of the defined click options have different `dest` values than the defaults @@ -593,8 +609,8 @@ def find_pyproject_toml(src_files: tuple[str, ...]) -> str | None: } -def mutate_option_to_click_dest(option_name: str) -> str: - "Mutates an option from how click/pyproject.toml expect them to the click `dest` value" +def get_click_dest_for_option(option_name: str) -> str: + """Returns the click `dest` value for the given option name.""" # Format the keys properly option_name = option_name.lstrip("-").replace("-", "_").lower() # Some options have dest values that are overrides from the click generated default @@ -614,15 +630,27 @@ def mutate_option_to_click_dest(option_name: str) -> str: @functools.lru_cache() -def parse_pyproject_toml(config_file: str) -> dict[str, Any]: - pyproject_toml = toml.load(config_file) - config: dict[str, Any] = pyproject_toml.get("tool", {}).get("pip-tools", {}) - config = {mutate_option_to_click_dest(k): v for k, v in config.items()} +def parse_config_file(config_file: Path) -> dict[str, Any]: + if sys.version_info >= (3, 11): + # Python 3.11 stdlib tomllib load() requires a binary file object + with config_file.open("rb") as ifs: + config = tomllib.load(ifs) + else: + # Before 3.11, using the external toml library, load requires the filename + config = toml.load(str(config_file)) + # In a pyproject.toml file, we expect the config to be under `[tool.pip-tools]`, but in our + # native configuration, it would be just `[pip-tools]`. + if config_file.name == "pyproject.toml": + config = config.get("tool", {}) + piptools_config: dict[str, Any] = config.get("pip-tools", {}) + piptools_config = { + get_click_dest_for_option(k): v for k, v in piptools_config.items() + } # Any option with multiple values needs to be a list in the pyproject.toml for mv_option in MULTIPLE_VALUE_OPTIONS: - if not isinstance(config.get(mv_option), (list, type(None))): + if not isinstance(piptools_config.get(mv_option), (list, type(None))): original_option = mv_option.replace("_", "-") raise click.BadOptionUsage( original_option, f"Config key '{original_option}' must be a list" ) - return config + return piptools_config diff --git a/pyproject.toml b/pyproject.toml index 7bf1a8b0d..f084bcd17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,9 +40,9 @@ dependencies = [ "build", "click >= 8", "pip >= 22.2", + "toml >= 0.10.1; python_version<'3.11'", # indirect dependencies "setuptools", # typically needed when pip-tools invokes setup.py - "toml >= 0.10.1", "wheel", # pip plugin needed by pip-tools ] @@ -58,6 +58,7 @@ testing = [ "pytest >= 7.2.0", "pytest-rerunfailures", "pytest-xdist", + "toml >= 0.10.1", # build deps for tests "flit_core >=2,<4", "poetry_core>=1.0.0", diff --git a/tests/conftest.py b/tests/conftest.py index d6536b22d..166f86fa3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ from piptools._compat.pip_compat import PIP_VERSION, uses_pkg_resources from piptools.cache import DependencyCache from piptools.exceptions import NoCandidateFound +from piptools.locations import CONFIG_FILE_NAME from piptools.logging import log from piptools.repositories import PyPIRepository from piptools.repositories.base import BaseRepository @@ -455,15 +456,18 @@ def _reset_log(): @pytest.fixture -def make_pyproject_toml_conf(tmpdir_cwd): - def _maker(pyproject_param, new_default): - # Make a pyproject.toml with this one config default override +def make_config_file(tmpdir_cwd): + def _maker(pyproject_param, new_default, config_file_name=CONFIG_FILE_NAME): + # Make a config file with this one config default override config_path = Path(tmpdir_cwd) / pyproject_param - config_file = config_path / "pyproject.toml" - config_path.mkdir() + config_file = config_path / config_file_name + config_path.mkdir(exist_ok=True) + config_to_dump = {"pip-tools": {pyproject_param: new_default}} + if config_file_name == "pyproject.toml": + config_to_dump = {"tool": config_to_dump} with open(config_file, "w") as ofs: - toml.dump({"tool": {"pip-tools": {pyproject_param: new_default}}}, ofs) + toml.dump(config_to_dump, ofs) return config_file return _maker diff --git a/tests/test_utils.py b/tests/test_utils.py index 9e91dbb44..e2bee3e33 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,7 @@ import os import shlex import sys +from pathlib import Path import pip import pytest @@ -14,11 +15,13 @@ from piptools.scripts.compile import cli as compile_cli from piptools.utils import ( as_tuple, + callback_config_file_defaults, dedup, drop_extras, flat_map, format_requirement, format_specifier, + get_click_dest_for_option, get_compile_command, get_hashes_from_ireq, get_pip_version_for_python_executable, @@ -28,8 +31,6 @@ key_from_ireq, lookup_table, lookup_table_from_tuples, - mutate_option_to_click_dest, - pyproject_toml_defaults_cb, ) @@ -590,17 +591,15 @@ def test_get_sys_path_for_python_executable(): ("unsafe-package", ["changed"]), ), ) -def test_pyproject_toml_defaults_cb( - pyproject_param, new_default, make_pyproject_toml_conf -): - config_file = make_pyproject_toml_conf(pyproject_param, new_default) - # Create a "compile" run example pointing to the pyproject.toml +def test_callback_config_file_defaults(pyproject_param, new_default, make_config_file): + config_file = make_config_file(pyproject_param, new_default) + # Create a "compile" run example pointing to the config file ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - found_config_file = pyproject_toml_defaults_cb(ctx, "config", None) + found_config_file = callback_config_file_defaults(ctx, "config", None) assert found_config_file == str(config_file) # Make sure the default has been updated - lookup_param = mutate_option_to_click_dest(pyproject_param) + lookup_param = get_click_dest_for_option(pyproject_param) assert ctx.default_map[lookup_param] == new_default @@ -615,30 +614,43 @@ def test_pyproject_toml_defaults_cb( "trusted-host", ), ) -def test_pyproject_toml_defaults_cb_multi_value_options( - mv_option, make_pyproject_toml_conf -): - config_file = make_pyproject_toml_conf(mv_option, "not-a-list") +def test_callback_config_file_defaults_multi_value_options(mv_option, make_config_file): + config_file = make_config_file(mv_option, "not-a-list") ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - pytest.raises(BadOptionUsage, pyproject_toml_defaults_cb, ctx, "config", None) + with pytest.raises(BadOptionUsage, match="must be a list"): + callback_config_file_defaults(ctx, "config", None) -def test_pyproject_toml_defaults_cb_bad_toml(make_pyproject_toml_conf): - config_file = make_pyproject_toml_conf("verbose", True) - config_text = open(config_file).read() - open(config_file, "w").write(config_text[::-1]) +def test_callback_config_file_defaults_bad_toml(make_config_file): + config_file = make_config_file("verbose", True) + # Simple means of making invalid TOML: have duplicate keys + with Path(config_file).open("r+") as fs: + config_text_lines = fs.readlines() + fs.write(config_text_lines[-1]) ctx = Context(compile_cli) ctx.params["src_files"] = (str(config_file),) - pytest.raises(FileError, pyproject_toml_defaults_cb, ctx, "config", None) + with pytest.raises(FileError, match="Could not parse "): + callback_config_file_defaults(ctx, "config", None) -def test_pyproject_toml_defaults_cb_unreadable_toml(make_pyproject_toml_conf): +def test_callback_config_file_defaults_precedence(make_config_file): + piptools_config_file = make_config_file("newline", "LF") + project_config_file = make_config_file("newline", "CRLF", "pyproject.toml") ctx = Context(compile_cli) - pytest.raises( - FileError, - pyproject_toml_defaults_cb, - ctx, - "config", - "/path/does/not/exist/pyproject.toml", - ) + ctx.params["src_files"] = (str(project_config_file),) + found_config_file = callback_config_file_defaults(ctx, "config", None) + # The pip-tools specific config file should take precedence over pyproject.toml + assert found_config_file == str(piptools_config_file) + lookup_param = get_click_dest_for_option("newline") + assert ctx.default_map[lookup_param] == "LF" + + +def test_callback_config_file_defaults_unreadable_toml(make_config_file): + ctx = Context(compile_cli) + with pytest.raises(FileError, match="Could not read "): + callback_config_file_defaults( + ctx, + "config", + "/path/does/not/exist/my-config.toml", + )