From 8f490a9b9a7abded552b54c04c68934bc8d5bfdf Mon Sep 17 00:00:00 2001 From: jax authors Date: Thu, 2 Oct 2025 13:08:40 -0700 Subject: [PATCH] Speed up unpacking `.tar.xz` files in the hermetic toolchain by using multithreaded `xz` execution. Note that the multithreaded feature should be available in non-hermetic `xz` tool. The next step will be to make `tar` and `xz` tools available in hermetic toolchains. Linux x86 platform operations | repository creation time, s :-----------: | :-----------: download clang.tar.xz and extract with bazel | 137 download clang.tar and extract with bazel | 48 download clang.tar.xz and extract with non-hermetic tar (with multithreaded xz) | 19 download clang.tar and extract with non-hermetic tar | 38 Parameters of the Linux machine that was used to collect the data: ``` Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Address sizes: 48 bits physical, 48 bits virtual Byte Order: Little Endian CPU(s): 128 On-line CPU(s) list: 0-127 ``` PiperOrigin-RevId: 814354233 --- .bazelrc | 3 --- WORKSPACE | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.bazelrc b/.bazelrc index 3d84113f9867..0d79226bdab2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -413,9 +413,6 @@ common:rbe_linux_x86_64_base --platforms="@ml_build_config_platform//:platform" common:rbe_linux_x86_64 --config=rbe_linux_x86_64_base common:rbe_linux_x86_64 --config=ci_linux_x86_64 -common:use_tar_archive_files --repo_env=USE_CUDA_TAR_ARCHIVE_FILES=1 -common:use_tar_archive_files --repo_env=USE_NVSHMEM_TAR_ARCHIVE_FILES=1 - common:rbe_linux_x86_64_cuda_common --config=rbe_linux_x86_64_base common:rbe_linux_x86_64_cuda_common --repo_env=REMOTE_GPU_TESTING=1 # Speed up CUDA repos creation by downloading ".tar" dists from the mirror. diff --git a/WORKSPACE b/WORKSPACE index 25026ba10d1b..2b065758f148 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -17,10 +17,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Details: https://github.com/google-ml-infra/rules_ml_toolchain http_archive( name = "rules_ml_toolchain", - sha256 = "bbe18d4948cd4845255ee8d400d2f29854b64d1434523052e304c1d41e90f2ec", - strip_prefix = "rules_ml_toolchain-3a77adf9473cf958967bc42eb9c1a663cd7ea878", + sha256 = "d5aef14096f367bae6f2deb5946a31892b06badd660f8c30f6565a0f9a99185f", + strip_prefix = "rules_ml_toolchain-18ba88facf7f0d7203adbaccb982315936b1ac7a", urls = [ - "https://github.com/google-ml-infra/rules_ml_toolchain/archive/3a77adf9473cf958967bc42eb9c1a663cd7ea878.tar.gz", + "https://github.com/google-ml-infra/rules_ml_toolchain/archive/18ba88facf7f0d7203adbaccb982315936b1ac7a.tar.gz", ], )