From 49d2711e5d891feb620fde4545d08fe5bea646c8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 22 Feb 2025 23:53:47 +0530 Subject: [PATCH 01/16] Grab `SOURCE_DATE_EPOCH` --- pyodide_build/recipe/builder.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index ad604830..f46a8c36 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +import time from collections.abc import Iterator from datetime import datetime from email.message import Message @@ -46,6 +47,18 @@ from pyodide_build.recipe.spec import MetaConfig, _SourceSpec +def _get_source_epoch() -> int: + """Get SOURCE_DATE_EPOCH from environment or fallback to current time. + Uses 315532800, i.e., 1980-01-01 00:00:00 UTC as minimum timestamp (as + this is the zipfile limit). + """ + try: + source_epoch = int(os.environ.get("SOURCE_DATE_EPOCH", time.time())) + return max(315532800, source_epoch) + except ValueError: + return int(time.time()) + + def _make_whlfile( *args: Any, owner: int | None = None, group: int | None = None, **kwargs: Any ) -> str: From 07f7f2f863a7a166a74907beab00eca3cffebc19 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 22 Feb 2025 23:54:22 +0530 Subject: [PATCH 02/16] Pass UTC time to `_make_whlfile` --- pyodide_build/recipe/builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index f46a8c36..d2dc3b3d 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -62,6 +62,9 @@ def _get_source_epoch() -> int: def _make_whlfile( *args: Any, owner: int | None = None, group: int | None = None, **kwargs: Any ) -> str: + filetime = _get_source_epoch() + # gtime() ensures UTC + kwargs["date_time"] = time.gmtime(filetime)[:6] return shutil._make_zipfile(*args, **kwargs) # type: ignore[attr-defined] From 2d86a54f5712b9a54ab9209ac27d1726aa3236e8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 22 Feb 2025 23:54:34 +0530 Subject: [PATCH 03/16] Fix typo --- pyodide_build/recipe/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index d2dc3b3d..00ef4aa8 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -547,7 +547,7 @@ def _package_wheel( ) -> None: """Package a wheel - This unpacks the wheel, unvendors tests if necessary, runs and "build.post" + This unpacks the wheel, unvendors tests if necessary, and runs the "build.post" script, and then repacks the wheel. Parameters From e86a39d918df3002997848afaf7ab662c0dc4186 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 23 Feb 2025 00:01:54 +0530 Subject: [PATCH 04/16] Extend "data" filter with `SOURCE_DATE_EPOCH` --- pyodide_build/recipe/builder.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index 00ef4aa8..1ade1a3b 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -345,11 +345,29 @@ def _download_and_extract(self) -> None: # is too large for the chown() call. This behavior can lead to "Permission denied" errors # (missing x bit) or random strange `make` behavior (due to wrong mtime order) in the CI # pipeline. - shutil.unpack_archive( - tarballpath, - self.build_dir, - filter=None if tarballpath.suffix == ".zip" else "data", - ) + if tarballpath.suffix == ".zip": + shutil.unpack_archive(tarballpath, self.build_dir, filter=None) + else: + filetime = ( + _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None + ) + + def reproducible_filter(tarinfo): + """Filter that preserves permissions but normalizes ownership and optionally + timestamps. This is similar to the "data" filter but injects SOURCE_DATE_EPOCH.""" + + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + + # set timestamp from SOURCE_DATE_EPOCH if available + if filetime is not None: + tarinfo.mtime = filetime + + return tarinfo + + shutil.unpack_archive( + tarballpath, self.build_dir, filter=reproducible_filter + ) extract_dir_name = self.source_metadata.extract_dir if extract_dir_name is None: From 25fa1473cb1c0c824f1b4d4e31f5d31e14d0dc8a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 23 Feb 2025 00:02:19 +0530 Subject: [PATCH 05/16] Make tests archive reproducible --- pyodide_build/recipe/builder.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index 1ade1a3b..04244f3a 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -627,7 +627,24 @@ def _package_wheel( ) if nmoved: with chdir(self.src_dist_dir): - shutil.make_archive(f"{self.name}-tests", "tar", test_dir) + filetime = _get_source_epoch() + make_archive_kwargs = { + "root_dir": "tests", + "owner": "root", + "group": "root", + } + + if "SOURCE_DATE_EPOCH" in os.environ: + + def _set_time(tarinfo): + tarinfo.mtime = filetime + return tarinfo + + make_archive_kwargs["filter"] = _set_time + + shutil.make_archive( + f"{self.name}-tests", "tar", **make_archive_kwargs + ) finally: shutil.rmtree(test_dir, ignore_errors=True) From 4bd74be7842cf660bf532c3528b223d6179f9fb5 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 23 Feb 2025 00:06:10 +0530 Subject: [PATCH 06/16] Add type annotation for `reproducible_filter` --- pyodide_build/recipe/builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index 04244f3a..06feca8d 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +import tarfile import time from collections.abc import Iterator from datetime import datetime @@ -352,7 +353,7 @@ def _download_and_extract(self) -> None: _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None ) - def reproducible_filter(tarinfo): + def reproducible_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: """Filter that preserves permissions but normalizes ownership and optionally timestamps. This is similar to the "data" filter but injects SOURCE_DATE_EPOCH.""" From 2cbde964aba8a0146cc4928158bfe8f95ad2b491 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 23 Feb 2025 00:06:31 +0530 Subject: [PATCH 07/16] Trigger [integration] tests From f12e7ad6a948e2bcd916ace913a69baf9b839006 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sun, 23 Feb 2025 00:24:51 +0530 Subject: [PATCH 08/16] Fix `reproducible_filter`'s signature [integration] --- pyodide_build/recipe/builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index 06feca8d..7e10ca10 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -353,7 +353,9 @@ def _download_and_extract(self) -> None: _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None ) - def reproducible_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: + def reproducible_filter( + tarinfo: tarfile.TarInfo, path: str | Path | None = None + ) -> tarfile.TarInfo: """Filter that preserves permissions but normalizes ownership and optionally timestamps. This is similar to the "data" filter but injects SOURCE_DATE_EPOCH.""" From d6c857558057427d17099a2ca89087a4b6e929a4 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 03:27:54 +0530 Subject: [PATCH 09/16] Add some tests for reproducible filter, archiving --- pyodide_build/tests/recipe/test_builder.py | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/pyodide_build/tests/recipe/test_builder.py b/pyodide_build/tests/recipe/test_builder.py index a1f90658..4fa6d842 100644 --- a/pyodide_build/tests/recipe/test_builder.py +++ b/pyodide_build/tests/recipe/test_builder.py @@ -1,6 +1,9 @@ +import os import shutil import subprocess +import tarfile import time +from contextlib import contextmanager from pathlib import Path from typing import Self @@ -351,3 +354,114 @@ def test_extract_tarballname(): for header, tarballname in zip(headers, tarballnames, strict=True): assert _builder._extract_tarballname(url, header) == tarballname + + +# Some reproducibility tests. These are not exhaustive, but should catch +# some common issues for basics like timestamps and file contents. They +# test the behavior of the builder functions that are most likely to be +# affected by SOURCE_DATE_EPOCH. + + +from pyodide_build.recipe.builder import _get_source_epoch + + +@contextmanager +def source_date_epoch(value=None): + old_value = os.environ.get("SOURCE_DATE_EPOCH") + try: + if value is None: + if "SOURCE_DATE_EPOCH" in os.environ: + del os.environ["SOURCE_DATE_EPOCH"] + else: + os.environ["SOURCE_DATE_EPOCH"] = str(value) + yield + finally: + if old_value is None: + if "SOURCE_DATE_EPOCH" in os.environ: + del os.environ["SOURCE_DATE_EPOCH"] + else: + os.environ["SOURCE_DATE_EPOCH"] = old_value + + +def test_get_source_epoch_reproducibility(): + with source_date_epoch("1735689600"): # 2025-01-01 + assert _get_source_epoch() == 1735689600 + + with source_date_epoch("invalid"): + assert _get_source_epoch() > 0 # should fall back to current time + + with source_date_epoch("0"): + assert ( + _get_source_epoch() == 315532800 + ) # should fall back to minimum ZIP timestamp + + +def test_make_whlfile_reproducibility(monkeypatch, tmp_path): + """Test that _make_whlfile is passing the correct timestamp to _make_zipfile.""" + from pyodide_build.recipe.builder import _make_whlfile + + test_epoch = 1735689600 # 2025-01-01 + + def mock_make_zipfile( + base_name, base_dir, verbose=0, dry_run=0, logger=None, date_time=None + ): + assert date_time == time.gmtime(test_epoch)[:6] + + monkeypatch.setattr(shutil, "_make_zipfile", mock_make_zipfile) + + with source_date_epoch(test_epoch): + _make_whlfile("archive.whl", "base_dir", ["file1.py"], b"content") + + +def test_set_archive_time_reproducibility(tmp_path): + """Test that archive creation using _set_time sets correct mtime.""" + import tarfile + + from pyodide_build.recipe.builder import _get_source_epoch + + # Create a test tarfile with a specific timestamp + test_file = tmp_path / "test.txt" + test_file.write_text("test content") + test_epoch = 1735689600 # 2025-01-01 + + with source_date_epoch(test_epoch): + with tarfile.open(tmp_path / "archive.tar", "w") as tar: + tarinfo = tar.gettarinfo(str(test_file)) + tarinfo.mtime = _get_source_epoch() + tar.addfile(tarinfo, open(test_file, "rb")) + + # Now, verify this timestamp in the archive + with tarfile.open(tmp_path / "archive.tar") as tar: + info = tar.getmembers()[0] + assert info.mtime == test_epoch + + +def test_reproducible_tar_filter(monkeypatch, tmp_path): + """Test that our reproducible_filter function sets the timestamp correctly.""" + + test_epoch = 1735689600 # 2025-01-01 + + class MockTarInfo: + def __init__(self, name): + self.name = name + self.uid = 1000 + self.gid = 1000 + self.uname = None + self.gname = None + self.mtime = int(time.time()) + + monkeypatch.setattr(tarfile, "TarInfo", MockTarInfo) + monkeypatch.setattr(os.path, "getmtime", lambda *args: test_epoch) + + with source_date_epoch(test_epoch): + # Create and check a tarinfo object + tarinfo = tarfile.TarInfo("test.txt") + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + tarinfo.mtime = test_epoch + + assert tarinfo.mtime == test_epoch + assert tarinfo.uid == 0 + assert tarinfo.gid == 0 + assert tarinfo.uname == "root" + assert tarinfo.gname == "root" From 7e4a7e1f62ebd80d6521d238535747aa1600846e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 04:06:08 +0530 Subject: [PATCH 10/16] Make tests unvendoring reproducible [integration] --- pyodide_build/recipe/builder.py | 75 +++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index 7e10ca10..3d0f04c6 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -60,6 +60,23 @@ def _get_source_epoch() -> int: return int(time.time()) +def _update_recursive_timestamp(path: Path, timestamp: int | None = None) -> None: + """Update timestamps recursively for all directories and files. If + SOURCE_DATE_EPOCH is set, uses that, otherwise keeps original ones.""" + + if timestamp is None and "SOURCE_DATE_EPOCH" not in os.environ: + return + + if timestamp is None: + timestamp = _get_source_epoch() + + # Update directory, subdirectories, and files + os.utime(path, (timestamp, timestamp)) + if path.is_dir(): + for child in path.iterdir(): + _update_recursive_timestamp(child, timestamp) + + def _make_whlfile( *args: Any, owner: int | None = None, group: int | None = None, **kwargs: Any ) -> str: @@ -631,22 +648,29 @@ def _package_wheel( if nmoved: with chdir(self.src_dist_dir): filetime = _get_source_epoch() - make_archive_kwargs = { - "root_dir": "tests", - "owner": "root", - "group": "root", - } - - if "SOURCE_DATE_EPOCH" in os.environ: - - def _set_time(tarinfo): - tarinfo.mtime = filetime - return tarinfo - - make_archive_kwargs["filter"] = _set_time - shutil.make_archive( - f"{self.name}-tests", "tar", **make_archive_kwargs + f"{self.name}-tests", + format="tar", + root_dir="tests", + owner="root", + group="root", + ) + if filetime is not None: + with tarfile.open(f"{self.name}-tests.tar", "r") as src: + with tarfile.open( + f"{self.name}-tests.new.tar", "w" + ) as dst: + for member in src.getmembers(): + member.mtime = filetime + if member.isfile(): + dst.addfile( + member, src.extractfile(member) + ) + else: + dst.addfile(member) + # replace original with timestamped version + os.replace( + f"{self.name}-tests.new.tar", f"{self.name}-tests.tar" ) finally: shutil.rmtree(test_dir, ignore_errors=True) @@ -803,14 +827,20 @@ def unvendor_tests( n_moved = 0 out_files = [] shutil.rmtree(test_install_prefix, ignore_errors=True) + + filetime = _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None + for root, _dirs, files in os.walk(install_prefix): root_rel = Path(root).relative_to(install_prefix) if root_rel.name == "__pycache__" or root_rel.name.endswith(".egg_info"): continue if root_rel.name in ["test", "tests"]: # This is a test folder - (test_install_prefix / root_rel).parent.mkdir(exist_ok=True, parents=True) - shutil.move(install_prefix / root_rel, test_install_prefix / root_rel) + target = test_install_prefix / root_rel + target.parent.mkdir(exist_ok=True, parents=True) + shutil.move(install_prefix / root_rel, target) + if filetime is not None: + _update_recursive_timestamp(target, filetime) n_moved += 1 continue out_files.append(root) @@ -822,11 +852,12 @@ def unvendor_tests( ): if any(fnmatch.fnmatchcase(fpath, pat) for pat in retain_test_patterns): continue - (test_install_prefix / root_rel).mkdir(exist_ok=True, parents=True) - shutil.move( - install_prefix / root_rel / fpath, - test_install_prefix / root_rel / fpath, - ) + target_dir = test_install_prefix / root_rel + target_dir.mkdir(exist_ok=True, parents=True) + target = target_dir / fpath + shutil.move(install_prefix / root_rel / fpath, target) + if filetime is not None: + os.utime(target, (filetime, filetime)) n_moved += 1 return n_moved From 4c55ea8d5816cbf7336b1f5b210a7a0d6b074d02 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 13:05:52 +0530 Subject: [PATCH 11/16] Make reproducible filter private --- pyodide_build/recipe/builder.py | 38 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index 3d0f04c6..a2b2c4fd 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -308,6 +308,23 @@ def ignore(path: str, names: list[str]) -> list[str]: self.src_dist_dir.mkdir(parents=True, exist_ok=True) + def _reproducible_filter( + tarinfo: tarfile.TarInfo, path: str | Path | None = None + ) -> tarfile.TarInfo: + """Filter that preserves permissions but normalizes ownership and optionally + timestamps. This is similar to the "data" filter but injects SOURCE_DATE_EPOCH.""" + + # set timestamp from SOURCE_DATE_EPOCH if available + filetime = _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None + + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + + if filetime is not None: + tarinfo.mtime = filetime + + return tarinfo + def _download_and_extract(self) -> None: """ Download the source from specified in the package metadata, @@ -366,27 +383,8 @@ def _download_and_extract(self) -> None: if tarballpath.suffix == ".zip": shutil.unpack_archive(tarballpath, self.build_dir, filter=None) else: - filetime = ( - _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None - ) - - def reproducible_filter( - tarinfo: tarfile.TarInfo, path: str | Path | None = None - ) -> tarfile.TarInfo: - """Filter that preserves permissions but normalizes ownership and optionally - timestamps. This is similar to the "data" filter but injects SOURCE_DATE_EPOCH.""" - - tarinfo.uid = tarinfo.gid = 0 - tarinfo.uname = tarinfo.gname = "root" - - # set timestamp from SOURCE_DATE_EPOCH if available - if filetime is not None: - tarinfo.mtime = filetime - - return tarinfo - shutil.unpack_archive( - tarballpath, self.build_dir, filter=reproducible_filter + tarballpath, self.build_dir, filter=self._reproducible_filter ) extract_dir_name = self.source_metadata.extract_dir From 4f3f4067ca49e236976c27604305148f82cdc5fe Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 13:06:43 +0530 Subject: [PATCH 12/16] Fix rust toolchain mention from #103 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9b73bb4..33922f32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- The Rust toolchain version has been updated to `nightly-2025-01-18`. +- The Rust toolchain version has been updated to `nightly-2025-02-01`. [#103](https://github.com/pyodide/pyodide-build/pull/103) ### Fixed From f2fa5b5363716e7dd2c74827e8dab7719d4aa6eb Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 13:08:26 +0530 Subject: [PATCH 13/16] Add CHANGELOG entry for #109 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33922f32..a8e5ea59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed Pyodide venv `sys_platform` marker evaluation with pip >= 25. [#108](https://github.com/pyodide/pyodide-build/pull/108) +- `pyodide-build` now respects `SOURCE_DATE_EPOCH` to enable reproducible + builds on a best-effort basis. + [#109](https://github.com/pyodide/pyodide-build/pull/109) + ## [0.29.3] - 2025/02/04 ### Added From de729afbc571de0a5a1088e613317c8d418db947 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:07:42 +0530 Subject: [PATCH 14/16] Move `source_date_epoch` to `common` utilities --- pyodide_build/build_env.py | 12 ++++++++++- pyodide_build/common.py | 13 ++++++++++++ pyodide_build/recipe/builder.py | 24 +++++++--------------- pyodide_build/tests/recipe/test_builder.py | 12 +++++------ 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/pyodide_build/build_env.py b/pyodide_build/build_env.py index db36c0f3..810cce40 100644 --- a/pyodide_build/build_env.py +++ b/pyodide_build/build_env.py @@ -14,7 +14,12 @@ from packaging.tags import Tag, compatible_tags, cpython_tags from pyodide_build import __version__ -from pyodide_build.common import search_pyproject_toml, to_bool, xbuildenv_dirname +from pyodide_build.common import ( + get_source_epoch, + search_pyproject_toml, + to_bool, + xbuildenv_dirname, +) from pyodide_build.config import ConfigManager, CrossBuildEnvConfigManager RUST_BUILD_PRELUDE = """ @@ -35,6 +40,11 @@ class BuildArgs: target_install_dir: str = "" # The path to the target Python installation host_install_dir: str = "" # Directory for installing built host packages. builddir: str = "" # The path to run pypa/build + env: dict[str, str] | None = None + + def __post_init__(self) -> None: + self.env = self.env or {} + self.env["SOURCE_DATE_EPOCH"] = str(get_source_epoch()) def init_environment(*, quiet: bool = False) -> None: diff --git a/pyodide_build/common.py b/pyodide_build/common.py index c30c0e25..2e65bc6f 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -9,6 +9,7 @@ import subprocess import sys import textwrap +import time import tomllib import warnings import zipfile @@ -211,6 +212,18 @@ def get_num_cores() -> int: return cpu_count() +def get_source_epoch() -> int: + """Get SOURCE_DATE_EPOCH from environment or fallback to current time. + Uses 315532800, i.e., 1980-01-01 00:00:00 UTC as minimum timestamp (as + this is the zipfile limit). + """ + try: + source_epoch = int(os.environ.get("SOURCE_DATE_EPOCH", time.time())) + return max(315532800, source_epoch) + except ValueError: + return int(time.time()) + + def make_zip_archive( archive_path: Path, input_dir: Path, diff --git a/pyodide_build/recipe/builder.py b/pyodide_build/recipe/builder.py index a2b2c4fd..3822ff90 100755 --- a/pyodide_build/recipe/builder.py +++ b/pyodide_build/recipe/builder.py @@ -36,6 +36,7 @@ chdir, exit_with_stdio, find_matching_wheels, + get_source_epoch, make_zip_archive, modify_wheel, retag_wheel, @@ -48,18 +49,6 @@ from pyodide_build.recipe.spec import MetaConfig, _SourceSpec -def _get_source_epoch() -> int: - """Get SOURCE_DATE_EPOCH from environment or fallback to current time. - Uses 315532800, i.e., 1980-01-01 00:00:00 UTC as minimum timestamp (as - this is the zipfile limit). - """ - try: - source_epoch = int(os.environ.get("SOURCE_DATE_EPOCH", time.time())) - return max(315532800, source_epoch) - except ValueError: - return int(time.time()) - - def _update_recursive_timestamp(path: Path, timestamp: int | None = None) -> None: """Update timestamps recursively for all directories and files. If SOURCE_DATE_EPOCH is set, uses that, otherwise keeps original ones.""" @@ -68,7 +57,7 @@ def _update_recursive_timestamp(path: Path, timestamp: int | None = None) -> Non return if timestamp is None: - timestamp = _get_source_epoch() + timestamp = get_source_epoch() # Update directory, subdirectories, and files os.utime(path, (timestamp, timestamp)) @@ -80,7 +69,7 @@ def _update_recursive_timestamp(path: Path, timestamp: int | None = None) -> Non def _make_whlfile( *args: Any, owner: int | None = None, group: int | None = None, **kwargs: Any ) -> str: - filetime = _get_source_epoch() + filetime = get_source_epoch() # gtime() ensures UTC kwargs["date_time"] = time.gmtime(filetime)[:6] return shutil._make_zipfile(*args, **kwargs) # type: ignore[attr-defined] @@ -308,6 +297,7 @@ def ignore(path: str, names: list[str]) -> list[str]: self.src_dist_dir.mkdir(parents=True, exist_ok=True) + @staticmethod def _reproducible_filter( tarinfo: tarfile.TarInfo, path: str | Path | None = None ) -> tarfile.TarInfo: @@ -315,7 +305,7 @@ def _reproducible_filter( timestamps. This is similar to the "data" filter but injects SOURCE_DATE_EPOCH.""" # set timestamp from SOURCE_DATE_EPOCH if available - filetime = _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None + filetime = get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None tarinfo.uid = tarinfo.gid = 0 tarinfo.uname = tarinfo.gname = "root" @@ -645,7 +635,7 @@ def _package_wheel( ) if nmoved: with chdir(self.src_dist_dir): - filetime = _get_source_epoch() + filetime = get_source_epoch() shutil.make_archive( f"{self.name}-tests", format="tar", @@ -826,7 +816,7 @@ def unvendor_tests( out_files = [] shutil.rmtree(test_install_prefix, ignore_errors=True) - filetime = _get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None + filetime = get_source_epoch() if "SOURCE_DATE_EPOCH" in os.environ else None for root, _dirs, files in os.walk(install_prefix): root_rel = Path(root).relative_to(install_prefix) diff --git a/pyodide_build/tests/recipe/test_builder.py b/pyodide_build/tests/recipe/test_builder.py index 4fa6d842..29a03010 100644 --- a/pyodide_build/tests/recipe/test_builder.py +++ b/pyodide_build/tests/recipe/test_builder.py @@ -362,7 +362,7 @@ def test_extract_tarballname(): # affected by SOURCE_DATE_EPOCH. -from pyodide_build.recipe.builder import _get_source_epoch +from pyodide_build.common import get_source_epoch @contextmanager @@ -385,14 +385,14 @@ def source_date_epoch(value=None): def test_get_source_epoch_reproducibility(): with source_date_epoch("1735689600"): # 2025-01-01 - assert _get_source_epoch() == 1735689600 + assert get_source_epoch() == 1735689600 with source_date_epoch("invalid"): - assert _get_source_epoch() > 0 # should fall back to current time + assert get_source_epoch() > 0 # should fall back to current time with source_date_epoch("0"): assert ( - _get_source_epoch() == 315532800 + get_source_epoch() == 315532800 ) # should fall back to minimum ZIP timestamp @@ -417,8 +417,6 @@ def test_set_archive_time_reproducibility(tmp_path): """Test that archive creation using _set_time sets correct mtime.""" import tarfile - from pyodide_build.recipe.builder import _get_source_epoch - # Create a test tarfile with a specific timestamp test_file = tmp_path / "test.txt" test_file.write_text("test content") @@ -427,7 +425,7 @@ def test_set_archive_time_reproducibility(tmp_path): with source_date_epoch(test_epoch): with tarfile.open(tmp_path / "archive.tar", "w") as tar: tarinfo = tar.gettarinfo(str(test_file)) - tarinfo.mtime = _get_source_epoch() + tarinfo.mtime = get_source_epoch() tar.addfile(tarinfo, open(test_file, "rb")) # Now, verify this timestamp in the archive From 17eca17144eb30162777a72b0e166214ca4586e1 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:08:28 +0530 Subject: [PATCH 15/16] Try to generate lockfile in [integration] tests --- .gitignore | 1 + integration_tests/Makefile | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e7ce9657..5f15df41 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ venv/ # integration tests integration_tests/**/build.log integration_tests/**/.libs +/integration_tests/recipes-installed diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 0bb7c8bf..984021b7 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,3 +1,5 @@ +export SOURCE_DATE_EPOCH=1735689600 + all: @echo "Please specify a target" @exit 1 @@ -6,7 +8,7 @@ all: test-recipe: check @echo "... Running integration tests for building recipes" - pyodide build-recipes --recipe-dir=recipes --force-rebuild "*" + pyodide build-recipes --recipe-dir=recipes --force-rebuild "*" --install --install-dir=recipes-installed @echo "... Passed" From 6e65fd0cba11979ab326f2f2fba91ea1817b06b7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 24 Feb 2025 16:04:17 +0530 Subject: [PATCH 16/16] Don't filter `SOURCE_DATE_EPOCH` out Co-Authored-By: Hood Chatham --- pyodide_build/build_env.py | 6 ------ pyodide_build/config.py | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pyodide_build/build_env.py b/pyodide_build/build_env.py index 810cce40..cfe1645e 100644 --- a/pyodide_build/build_env.py +++ b/pyodide_build/build_env.py @@ -15,7 +15,6 @@ from pyodide_build import __version__ from pyodide_build.common import ( - get_source_epoch, search_pyproject_toml, to_bool, xbuildenv_dirname, @@ -40,11 +39,6 @@ class BuildArgs: target_install_dir: str = "" # The path to the target Python installation host_install_dir: str = "" # Directory for installing built host packages. builddir: str = "" # The path to run pypa/build - env: dict[str, str] | None = None - - def __post_init__(self) -> None: - self.env = self.env or {} - self.env["SOURCE_DATE_EPOCH"] = str(get_source_epoch()) def init_environment(*, quiet: bool = False) -> None: diff --git a/pyodide_build/config.py b/pyodide_build/config.py index 97862209..a54ddb48 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -193,6 +193,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]: "ldflags_base": "LDFLAGS_BASE", "home": "HOME", "path": "PATH", + "source_date_epoch": "SOURCE_DATE_EPOCH", "zip_compression_level": "PYODIDE_ZIP_COMPRESSION_LEVEL", "skip_emscripten_version_check": "SKIP_EMSCRIPTEN_VERSION_CHECK", "build_dependency_index_url": "BUILD_DEPENDENCY_INDEX_URL",