From 672f9b6de5d6899ffcb2821cdb2e814cf6a1562c Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Sun, 23 Feb 2025 11:18:46 +0000 Subject: [PATCH 1/8] chore: Add argument and env to specific compress level --- src/auditwheel/main_repair.py | 15 ++++++++++++ src/auditwheel/tools.py | 24 ++++++++++++++++++- src/auditwheel/wheeltools.py | 45 ++++++++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index 65e356ee..c1058815 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -2,10 +2,12 @@ import argparse import logging +import zlib from pathlib import Path from auditwheel.patcher import Patchelf +from . import tools from .policy import WheelPolicies from .tools import EnvironmentDefault @@ -40,6 +42,18 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("WHEEL_FILE", type=Path, help="Path to wheel file.", nargs="+") + parser.add_argument( + "-z", + "--zip-level", + action=EnvironmentDefault, + metavar="zip", + env="AUDITWHEEL_ZIP_LEVEL", + dest="zip", + type=int, + help="Compress level to be used to create zip file.", + choices=list(range(zlib.Z_NO_COMPRESSION, zlib.Z_BEST_COMPRESSION + 1)), + default=zlib.Z_DEFAULT_COMPRESSION, + ) parser.add_argument( "--plat", action=EnvironmentDefault, @@ -119,6 +133,7 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: wheel_dir: Path = args.WHEEL_DIR.absolute() wheel_files: list[Path] = args.WHEEL_FILE wheel_policy = WheelPolicies() + tools._COMPRESS_LEVEL = args.zip for wheel_file in wheel_files: if not wheel_file.is_file(): diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 0592ad2d..fade9668 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -1,9 +1,11 @@ from __future__ import annotations import argparse +import logging import os import subprocess import zipfile +import zlib from collections.abc import Generator, Iterable from datetime import datetime, timezone from pathlib import Path @@ -11,6 +13,15 @@ _T = TypeVar("_T") +logger = logging.getLogger(__name__) + +# Default: zlib.Z_DEFAULT_COMPRESSION (-1 aka. level 6) balances speed and size. +# Maintained for typical builds where iteration speed outweighs distribution savings. +# Override via AUDITWHEEL_ZIP_LEVEL/--zip-level for: +# - some test builds that needs no compression at all (0) +# - bandwidth-constrained or large amount of downloads (9) +_COMPRESS_LEVEL = zlib.Z_DEFAULT_COMPRESSION + def unique_by_index(sequence: Iterable[_T]) -> list[_T]: """unique elements in `sequence` in the order in which they occur @@ -90,6 +101,7 @@ def zip2dir(zip_fname: Path, out_dir: Path) -> None: out_dir : str Directory path containing files to go in the zip archive """ + start = datetime.now() with zipfile.ZipFile(zip_fname, "r") as z: for name in z.namelist(): member = z.getinfo(name) @@ -102,6 +114,9 @@ def zip2dir(zip_fname: Path, out_dir: Path) -> None: attr &= 511 # only keep permission bits attr |= 6 << 6 # at least read/write for current user os.chmod(extracted_path, attr) + logger.debug( + "zip2dir from %s to %s takes %s", zip_fname, out_dir, datetime.now() - start + ) def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> None: @@ -120,6 +135,7 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> date_time : Optional[datetime] Time stamp to set on each file in the archive """ + start = datetime.now() in_dir = in_dir.resolve(strict=True) if date_time is None: st = in_dir.stat() @@ -140,7 +156,10 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> zinfo.date_time = date_time_args zinfo.compress_type = compression with open(fname, "rb") as fp: - z.writestr(zinfo, fp.read()) + z.writestr(zinfo, fp.read(), compresslevel=_COMPRESS_LEVEL) + logger.debug( + "dir2zip from %s to %s takes %s", in_dir, zip_fname, datetime.now() - start + ) def tarbz2todir(tarbz2_fname: Path, out_dir: Path) -> None: @@ -157,11 +176,14 @@ def __init__( required: bool = True, default: str | None = None, choices: Iterable[str] | None = None, + type: type | None = None, **kwargs: Any, ) -> None: self.env_default = os.environ.get(env) self.env = env if self.env_default: + if type: + self.env_default = type(self.env_default) default = self.env_default if default: required = False diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 828fbeee..7bd5bfcf 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -15,7 +15,9 @@ from itertools import product from os.path import splitext from pathlib import Path +from tempfile import TemporaryDirectory from types import TracebackType +from typing import Any, ClassVar from packaging.utils import parse_wheel_filename @@ -98,8 +100,13 @@ class InWheel(InTemporaryDirectory): On entering, you'll find yourself in the root tree of the wheel. If you've asked for an output wheel, then on exit we'll rewrite the wheel record and pack stuff up for you. + + If `out_wheel` is None, we assume the wheel won't be modified and we can + cache the unpacked wheel for future use. """ + _whl_cache: ClassVar[dict[Path, TemporaryDirectory[Any]]] = {} + def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: """Initialize in-wheel context manager @@ -113,9 +120,35 @@ def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: """ self.in_wheel = in_wheel.absolute() self.out_wheel = None if out_wheel is None else out_wheel.absolute() - super().__init__() + self.read_only = out_wheel is None + self.use_cache = self.in_wheel in self._whl_cache + if self.use_cache and not Path(self._whl_cache[self.in_wheel].name).exists(): + self.use_cache = False + logger.debug( + "Wheel ctx %s for %s is no longer valid", + self._whl_cache.pop(self.in_wheel), + self.in_wheel, + ) + + if self.use_cache: + logger.debug( + "Reuse %s for %s", self._whl_cache[self.in_wheel], self.in_wheel + ) + self._tmpdir = self._whl_cache[self.in_wheel] + if not self.read_only: + self._whl_cache.pop(self.in_wheel) + else: + super().__init__() + if self.read_only: + self._whl_cache[self.in_wheel] = self._tmpdir def __enter__(self) -> Path: + if self.use_cache or self.read_only: + if not self.use_cache: + zip2dir(self.in_wheel, self.name) + self._pwd = Path.cwd() + os.chdir(self.name) + return Path(self.name) zip2dir(self.in_wheel, self.name) return super().__enter__() @@ -132,6 +165,16 @@ def __exit__( if timestamp: date_time = datetime.fromtimestamp(int(timestamp), tz=timezone.utc) dir2zip(self.name, self.out_wheel, date_time) + if self.use_cache or self.read_only: + logger.debug( + "Exiting reused %s for %s", + self._whl_cache[self.in_wheel], + self.in_wheel, + ) + os.chdir(self._pwd) + if not self.read_only: + super().__exit__(exc, value, tb) + return None return super().__exit__(exc, value, tb) From 2ff0add22159a139a75736bcdb38c385887f16be Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Sun, 23 Feb 2025 20:18:19 +0800 Subject: [PATCH 2/8] fix test --- tests/integration/test_bundled_wheels.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index 33d76a3d..a77130e7 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -138,6 +138,7 @@ def test_wheel_source_date_epoch(tmp_path, monkeypatch): func=Mock(), prog="auditwheel", verbose=1, + zip=None, ) monkeypatch.setenv("SOURCE_DATE_EPOCH", "650203200") # patchelf might not be available as we aren't running in a manylinux container From 85721f96a77a32025e5a1884e2f2424ab4a953b8 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Sun, 23 Feb 2025 20:46:53 +0800 Subject: [PATCH 3/8] cleanup unexpected changes --- src/auditwheel/wheeltools.py | 45 +----------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 7bd5bfcf..828fbeee 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -15,9 +15,7 @@ from itertools import product from os.path import splitext from pathlib import Path -from tempfile import TemporaryDirectory from types import TracebackType -from typing import Any, ClassVar from packaging.utils import parse_wheel_filename @@ -100,13 +98,8 @@ class InWheel(InTemporaryDirectory): On entering, you'll find yourself in the root tree of the wheel. If you've asked for an output wheel, then on exit we'll rewrite the wheel record and pack stuff up for you. - - If `out_wheel` is None, we assume the wheel won't be modified and we can - cache the unpacked wheel for future use. """ - _whl_cache: ClassVar[dict[Path, TemporaryDirectory[Any]]] = {} - def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: """Initialize in-wheel context manager @@ -120,35 +113,9 @@ def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: """ self.in_wheel = in_wheel.absolute() self.out_wheel = None if out_wheel is None else out_wheel.absolute() - self.read_only = out_wheel is None - self.use_cache = self.in_wheel in self._whl_cache - if self.use_cache and not Path(self._whl_cache[self.in_wheel].name).exists(): - self.use_cache = False - logger.debug( - "Wheel ctx %s for %s is no longer valid", - self._whl_cache.pop(self.in_wheel), - self.in_wheel, - ) - - if self.use_cache: - logger.debug( - "Reuse %s for %s", self._whl_cache[self.in_wheel], self.in_wheel - ) - self._tmpdir = self._whl_cache[self.in_wheel] - if not self.read_only: - self._whl_cache.pop(self.in_wheel) - else: - super().__init__() - if self.read_only: - self._whl_cache[self.in_wheel] = self._tmpdir + super().__init__() def __enter__(self) -> Path: - if self.use_cache or self.read_only: - if not self.use_cache: - zip2dir(self.in_wheel, self.name) - self._pwd = Path.cwd() - os.chdir(self.name) - return Path(self.name) zip2dir(self.in_wheel, self.name) return super().__enter__() @@ -165,16 +132,6 @@ def __exit__( if timestamp: date_time = datetime.fromtimestamp(int(timestamp), tz=timezone.utc) dir2zip(self.name, self.out_wheel, date_time) - if self.use_cache or self.read_only: - logger.debug( - "Exiting reused %s for %s", - self._whl_cache[self.in_wheel], - self.in_wheel, - ) - os.chdir(self._pwd) - if not self.read_only: - super().__exit__(exc, value, tb) - return None return super().__exit__(exc, value, tb) From 71e79cd51be8ce0f781be52005a7a7f71a243692 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 6 Mar 2025 20:37:05 +0800 Subject: [PATCH 4/8] Add test --- src/auditwheel/tools.py | 15 ++++++++++++++- tests/unit/test_tools.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index fade9668..7dda8906 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -183,7 +183,20 @@ def __init__( self.env = env if self.env_default: if type: - self.env_default = type(self.env_default) + try: + self.env_default = type(self.env_default) + except Exception: + self.option_strings = kwargs["option_strings"] + args = { + "value": self.env_default, + "type": type, + "env": self.env, + } + msg = ( + "invalid type: %(value)r from environment variable " + "%(env)r cannot be converted to %(type)r" + ) + raise argparse.ArgumentError(self, msg % args) from None default = self.env_default if default: required = False diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index 64874af0..ac3e994c 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -3,6 +3,7 @@ import argparse import lzma import zipfile +import zlib from pathlib import Path import pytest @@ -44,7 +45,7 @@ def test_environment_action( assert expected == args.PLAT -def test_environment_action_invalid_env(monkeypatch: pytest.MonkeyPatch) -> None: +def test_environment_action_invalid_plat_env(monkeypatch: pytest.MonkeyPatch) -> None: choices = ["linux", "manylinux1", "manylinux2010"] monkeypatch.setenv("AUDITWHEEL_PLAT", "foo") p = argparse.ArgumentParser() @@ -59,6 +60,39 @@ def test_environment_action_invalid_env(monkeypatch: pytest.MonkeyPatch) -> None ) +def test_environment_action_invalid_zip_env(monkeypatch: pytest.MonkeyPatch) -> None: + choices = list(range(zlib.Z_NO_COMPRESSION, zlib.Z_BEST_COMPRESSION + 1)) + monkeypatch.setenv("AUDITWHEEL_ZIP_LEVEL", "foo") + p = argparse.ArgumentParser() + with pytest.raises(argparse.ArgumentError): + p.add_argument( + "-z", + "--zip-level", + action=EnvironmentDefault, + metavar="zip", + env="AUDITWHEEL_ZIP_LEVEL", + dest="zip", + type=int, + help="Compress level to be used to create zip file.", + choices=choices, + default=zlib.Z_DEFAULT_COMPRESSION, + ) + monkeypatch.setenv("AUDITWHEEL_ZIP_LEVEL", "10") + with pytest.raises(argparse.ArgumentError): + p.add_argument( + "-z", + "--zip-level", + action=EnvironmentDefault, + metavar="zip", + env="AUDITWHEEL_ZIP_LEVEL", + dest="zip", + type=int, + help="Compress level to be used to create zip file.", + choices=choices, + default=zlib.Z_DEFAULT_COMPRESSION, + ) + + def _write_test_permissions_zip(path: Path) -> None: source_zip_xz = Path(__file__).parent / "test-permissions.zip.xz" with lzma.open(source_zip_xz) as f: From 8a84867911df7f468e375aacd7c486d905d2b182 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 6 Mar 2025 21:15:54 +0800 Subject: [PATCH 5/8] add positive tests --- src/auditwheel/tools.py | 15 +++++++++---- tests/unit/test_tools.py | 47 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 7dda8906..323f2845 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -198,9 +198,11 @@ def __init__( ) raise argparse.ArgumentError(self, msg % args) from None default = self.env_default - if default: - required = False - if self.env_default and choices is not None and self.env_default not in choices: + if ( + self.env_default is not None + and choices is not None + and self.env_default not in choices + ): self.option_strings = kwargs["option_strings"] args = { "value": self.env_default, @@ -213,7 +215,12 @@ def __init__( ) raise argparse.ArgumentError(self, msg % args) - super().__init__(default=default, required=required, choices=choices, **kwargs) + if default is not None: + required = False + + super().__init__( + default=default, required=required, choices=choices, type=type, **kwargs + ) def __call__( self, diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index ac3e994c..caf22935 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -20,7 +20,7 @@ ("manylinux2010", "linux", "linux"), ], ) -def test_environment_action( +def test_plat_environment_action( monkeypatch: pytest.MonkeyPatch, environ: str | None, passed: str | None, @@ -45,6 +45,49 @@ def test_environment_action( assert expected == args.PLAT +_all_zip_level: list[int] = list( + range(zlib.Z_NO_COMPRESSION, zlib.Z_BEST_COMPRESSION + 1) +) + + +@pytest.mark.parametrize( + ("environ", "passed", "expected"), + [ + (None, None, -1), + (0, None, 0), + (0, 1, 1), + (6, 1, 1), + ], +) +def test_zip_environment_action( + monkeypatch: pytest.MonkeyPatch, + environ: int | None, + passed: int | None, + expected: int, +) -> None: + choices = _all_zip_level + argv = [] + if passed is not None: + argv = ["--zip-level", str(passed)] + if environ is not None: + monkeypatch.setenv("AUDITWHEEL_ZIP_LEVEL", str(environ)) + p = argparse.ArgumentParser() + p.add_argument( + "-z", + "--zip-level", + action=EnvironmentDefault, + metavar="zip", + env="AUDITWHEEL_ZIP_LEVEL", + dest="zip", + type=int, + help="Compress level to be used to create zip file.", + choices=choices, + default=zlib.Z_DEFAULT_COMPRESSION, + ) + args = p.parse_args(argv) + assert expected == args.zip + + def test_environment_action_invalid_plat_env(monkeypatch: pytest.MonkeyPatch) -> None: choices = ["linux", "manylinux1", "manylinux2010"] monkeypatch.setenv("AUDITWHEEL_PLAT", "foo") @@ -61,7 +104,7 @@ def test_environment_action_invalid_plat_env(monkeypatch: pytest.MonkeyPatch) -> def test_environment_action_invalid_zip_env(monkeypatch: pytest.MonkeyPatch) -> None: - choices = list(range(zlib.Z_NO_COMPRESSION, zlib.Z_BEST_COMPRESSION + 1)) + choices = _all_zip_level monkeypatch.setenv("AUDITWHEEL_ZIP_LEVEL", "foo") p = argparse.ArgumentParser() with pytest.raises(argparse.ArgumentError): From c5f90cead63f62fc4af9f332382a06aab942ebd8 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Mar 2025 07:03:56 +0100 Subject: [PATCH 6/8] chore: rename zip-level to zip-compression-level while a bit more verbose, it's also much clearer. --- src/auditwheel/main_repair.py | 10 +++++----- src/auditwheel/tools.py | 4 ++-- tests/integration/test_bundled_wheels.py | 2 +- tests/unit/test_tools.py | 18 +++++++++--------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index c1058815..f532a614 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -44,11 +44,11 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] parser.add_argument("WHEEL_FILE", type=Path, help="Path to wheel file.", nargs="+") parser.add_argument( "-z", - "--zip-level", + "--zip-compression-level", action=EnvironmentDefault, - metavar="zip", - env="AUDITWHEEL_ZIP_LEVEL", - dest="zip", + metavar="ZIP_COMPRESSION_LEVEL", + env="AUDITWHEEL_ZIP_COMPRESSION_LEVEL", + dest="ZIP_COMPRESSION_LEVEL", type=int, help="Compress level to be used to create zip file.", choices=list(range(zlib.Z_NO_COMPRESSION, zlib.Z_BEST_COMPRESSION + 1)), @@ -133,7 +133,7 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: wheel_dir: Path = args.WHEEL_DIR.absolute() wheel_files: list[Path] = args.WHEEL_FILE wheel_policy = WheelPolicies() - tools._COMPRESS_LEVEL = args.zip + tools._ZIP_COMPRESSION_LEVEL = args.ZIP_COMPRESSION_LEVEL for wheel_file in wheel_files: if not wheel_file.is_file(): diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 323f2845..210d46ff 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -20,7 +20,7 @@ # Override via AUDITWHEEL_ZIP_LEVEL/--zip-level for: # - some test builds that needs no compression at all (0) # - bandwidth-constrained or large amount of downloads (9) -_COMPRESS_LEVEL = zlib.Z_DEFAULT_COMPRESSION +_ZIP_COMPRESSION_LEVEL = zlib.Z_DEFAULT_COMPRESSION def unique_by_index(sequence: Iterable[_T]) -> list[_T]: @@ -156,7 +156,7 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> zinfo.date_time = date_time_args zinfo.compress_type = compression with open(fname, "rb") as fp: - z.writestr(zinfo, fp.read(), compresslevel=_COMPRESS_LEVEL) + z.writestr(zinfo, fp.read(), compresslevel=_ZIP_COMPRESSION_LEVEL) logger.debug( "dir2zip from %s to %s takes %s", in_dir, zip_fname, datetime.now() - start ) diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index a77130e7..4605ec54 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -134,11 +134,11 @@ def test_wheel_source_date_epoch(tmp_path, monkeypatch): WHEEL_FILE=[wheel_path], EXCLUDE=[], DISABLE_ISA_EXT_CHECK=False, + ZIP_COMPRESSION_LEVEL=6, cmd="repair", func=Mock(), prog="auditwheel", verbose=1, - zip=None, ) monkeypatch.setenv("SOURCE_DATE_EPOCH", "650203200") # patchelf might not be available as we aren't running in a manylinux container diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index caf22935..8dd209c5 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -68,16 +68,16 @@ def test_zip_environment_action( choices = _all_zip_level argv = [] if passed is not None: - argv = ["--zip-level", str(passed)] + argv = ["--zip-compression-level", str(passed)] if environ is not None: - monkeypatch.setenv("AUDITWHEEL_ZIP_LEVEL", str(environ)) + monkeypatch.setenv("AUDITWHEEL_ZIP_COMPRESSION_LEVEL", str(environ)) p = argparse.ArgumentParser() p.add_argument( "-z", "--zip-level", action=EnvironmentDefault, metavar="zip", - env="AUDITWHEEL_ZIP_LEVEL", + env="AUDITWHEEL_ZIP_COMPRESSION_LEVEL", dest="zip", type=int, help="Compress level to be used to create zip file.", @@ -105,29 +105,29 @@ def test_environment_action_invalid_plat_env(monkeypatch: pytest.MonkeyPatch) -> def test_environment_action_invalid_zip_env(monkeypatch: pytest.MonkeyPatch) -> None: choices = _all_zip_level - monkeypatch.setenv("AUDITWHEEL_ZIP_LEVEL", "foo") + monkeypatch.setenv("AUDITWHEEL_ZIP_COMPRESSION_LEVEL", "foo") p = argparse.ArgumentParser() with pytest.raises(argparse.ArgumentError): p.add_argument( "-z", - "--zip-level", + "--zip-compression-level", action=EnvironmentDefault, metavar="zip", - env="AUDITWHEEL_ZIP_LEVEL", + env="AUDITWHEEL_ZIP_COMPRESSION_LEVEL", dest="zip", type=int, help="Compress level to be used to create zip file.", choices=choices, default=zlib.Z_DEFAULT_COMPRESSION, ) - monkeypatch.setenv("AUDITWHEEL_ZIP_LEVEL", "10") + monkeypatch.setenv("AUDITWHEEL_ZIP_COMPRESSION_LEVEL", "10") with pytest.raises(argparse.ArgumentError): p.add_argument( "-z", - "--zip-level", + "--zip-compression-level", action=EnvironmentDefault, metavar="zip", - env="AUDITWHEEL_ZIP_LEVEL", + env="AUDITWHEEL_ZIP_COMPRESSION_LEVEL", dest="zip", type=int, help="Compress level to be used to create zip file.", From 7c8a28749513269c252dbf106921f1acc376de27 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Mar 2025 07:26:07 +0100 Subject: [PATCH 7/8] Don't use _ZIP_COMPRESSION_LEVEL module variable --- src/auditwheel/main_repair.py | 3 +-- src/auditwheel/repair.py | 4 +++- src/auditwheel/tools.py | 21 +++++++++++---------- src/auditwheel/wheeltools.py | 4 +++- tests/unit/test_tools.py | 6 +++--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index f532a614..47c09157 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -7,7 +7,6 @@ from auditwheel.patcher import Patchelf -from . import tools from .policy import WheelPolicies from .tools import EnvironmentDefault @@ -133,7 +132,6 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: wheel_dir: Path = args.WHEEL_DIR.absolute() wheel_files: list[Path] = args.WHEEL_FILE wheel_policy = WheelPolicies() - tools._ZIP_COMPRESSION_LEVEL = args.ZIP_COMPRESSION_LEVEL for wheel_file in wheel_files: if not wheel_file.is_file(): @@ -212,6 +210,7 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: patcher=patcher, exclude=exclude, strip=args.STRIP, + zip_compression_level=args.ZIP_COMPRESSION_LEVEL, ) if out_wheel is not None: diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 964676cc..82cf6084 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -41,7 +41,8 @@ def repair_wheel( update_tags: bool, patcher: ElfPatcher, exclude: frozenset[str], - strip: bool = False, + strip: bool, + zip_compression_level: int, ) -> Path | None: elf_data = get_wheel_elfdata(wheel_policy, wheel_path, exclude) external_refs_by_fn = elf_data.full_external_refs @@ -57,6 +58,7 @@ def repair_wheel( with InWheelCtx(wheel_path) as ctx: ctx.out_wheel = out_dir / wheel_fname + ctx.zip_compression_level = zip_compression_level match = WHEEL_INFO_RE(wheel_fname) if not match: diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 210d46ff..70d30ab0 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -5,7 +5,6 @@ import os import subprocess import zipfile -import zlib from collections.abc import Generator, Iterable from datetime import datetime, timezone from pathlib import Path @@ -15,13 +14,6 @@ logger = logging.getLogger(__name__) -# Default: zlib.Z_DEFAULT_COMPRESSION (-1 aka. level 6) balances speed and size. -# Maintained for typical builds where iteration speed outweighs distribution savings. -# Override via AUDITWHEEL_ZIP_LEVEL/--zip-level for: -# - some test builds that needs no compression at all (0) -# - bandwidth-constrained or large amount of downloads (9) -_ZIP_COMPRESSION_LEVEL = zlib.Z_DEFAULT_COMPRESSION - def unique_by_index(sequence: Iterable[_T]) -> list[_T]: """unique elements in `sequence` in the order in which they occur @@ -119,7 +111,12 @@ def zip2dir(zip_fname: Path, out_dir: Path) -> None: ) -def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> None: +def dir2zip( + in_dir: Path, + zip_fname: Path, + zip_compression_level: int, + date_time: datetime | None, +) -> None: """Make a zip file `zip_fname` with contents of directory `in_dir` The recorded filenames are relative to `in_dir`, so doing a standard zip @@ -132,6 +129,10 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> Directory path containing files to go in the zip archive zip_fname : Path Filename of zip archive to write + zip_compression_level: int + zlib.Z_DEFAULT_COMPRESSION (-1 aka. level 6) balances speed and size. + zlib.Z_NO_COMPRESSION (O) for some test builds that needs no compression at all + zlib.Z_BEST_COMPRESSION (9) for bandwidth-constrained or large amount of downloads date_time : Optional[datetime] Time stamp to set on each file in the archive """ @@ -156,7 +157,7 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> zinfo.date_time = date_time_args zinfo.compress_type = compression with open(fname, "rb") as fp: - z.writestr(zinfo, fp.read(), compresslevel=_ZIP_COMPRESSION_LEVEL) + z.writestr(zinfo, fp.read(), compresslevel=zip_compression_level) logger.debug( "dir2zip from %s to %s takes %s", in_dir, zip_fname, datetime.now() - start ) diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 828fbeee..3435a873 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -9,6 +9,7 @@ import hashlib import logging import os +import zlib from base64 import urlsafe_b64encode from collections.abc import Generator, Iterable from datetime import datetime, timezone @@ -113,6 +114,7 @@ def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: """ self.in_wheel = in_wheel.absolute() self.out_wheel = None if out_wheel is None else out_wheel.absolute() + self.zip_compression_level = zlib.Z_DEFAULT_COMPRESSION super().__init__() def __enter__(self) -> Path: @@ -131,7 +133,7 @@ def __exit__( timestamp = os.environ.get("SOURCE_DATE_EPOCH") if timestamp: date_time = datetime.fromtimestamp(int(timestamp), tz=timezone.utc) - dir2zip(self.name, self.out_wheel, date_time) + dir2zip(self.name, self.out_wheel, self.zip_compression_level, date_time) return super().__exit__(exc, value, tb) diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index 8dd209c5..aeec2f1c 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -169,7 +169,7 @@ def test_zip2dir_round_trip_permissions(tmp_path: Path) -> None: _write_test_permissions_zip(source_zip) extract_path = tmp_path / "unzip2" zip2dir(source_zip, tmp_path / "unzip1") - dir2zip(tmp_path / "unzip1", tmp_path / "tmp.zip") + dir2zip(tmp_path / "unzip1", tmp_path / "tmp.zip", zlib.Z_DEFAULT_COMPRESSION, None) zip2dir(tmp_path / "tmp.zip", extract_path) _check_permissions(extract_path) @@ -181,7 +181,7 @@ def test_dir2zip_deflate(tmp_path: Path) -> None: input_file = input_dir / "zeros.bin" input_file.write_bytes(buffer) output_file = tmp_path / "ouput.zip" - dir2zip(input_dir, output_file) + dir2zip(input_dir, output_file, zlib.Z_DEFAULT_COMPRESSION, None) assert output_file.stat().st_size < len(buffer) / 4 @@ -194,7 +194,7 @@ def test_dir2zip_folders(tmp_path: Path) -> None: empty_folder = input_dir / "dummy" / "empty" empty_folder.mkdir(parents=True) output_file = tmp_path / "output.zip" - dir2zip(input_dir, output_file) + dir2zip(input_dir, output_file, zlib.Z_DEFAULT_COMPRESSION, None) expected_dirs = {"dummy/", "dummy/empty/", "dummy-1.0.dist-info/"} with zipfile.ZipFile(output_file, "r") as z: assert len(z.filelist) == 4 From a08481fa6d7b640ebee5443542ff31543bcec3fe Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Mar 2025 07:29:31 +0100 Subject: [PATCH 8/8] fix argument name --- tests/unit/test_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index aeec2f1c..d5c022b9 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -74,7 +74,7 @@ def test_zip_environment_action( p = argparse.ArgumentParser() p.add_argument( "-z", - "--zip-level", + "--zip-compression-level", action=EnvironmentDefault, metavar="zip", env="AUDITWHEEL_ZIP_COMPRESSION_LEVEL",