From 3c8855f36d5efbd65224b84917d385548b8ff1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20R=C3=B6ttsches?= Date: Thu, 16 Mar 2023 16:54:48 +0200 Subject: [PATCH] Add fetch tool for retrieving Chromium's custom-built Rust toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapt the approach [1] that Chromium uses for fetching its Rust toolchain to Skia. Take update_rust.py and the minimally required helper functions from Chromium's clang update script to fetch a specific, fully functional and self-contained Rust build from Chromium's CDS bucket. In these buckets, Chromium stores its Clang and Rust toolchain builds. Maintaining and building a tip-of-tree up-to-date rust toolchain requires a level of constant effort that can be avoided on the Skia side if we rely on Chromium's work here. Currently, this script needs manual rolling to the respective latest build in [2]. Issue skia:14191 tracks changing that so that the script is upgraded to be able to automatically roll to the latest known-good revision. [1] https://source.chromium.org/chromium/chromium/src/+/main:tools/rust/update_rust.py [2] https://commondatastorage.googleapis.com/chromium-browser-clang/index.html?path=Linux_x64/rust-toolchain- Bug: skia:14185 Change-Id: I66657a4943251b15d1c2ce69044bbfbb5403b4d4 Reviewed-on: https://skia-review.googlesource.com/c/skia/+/657157 Reviewed-by: Kevin Lubick Commit-Queue: Dominik Röttsches --- .gitignore | 1 + bin/clang_update.py | 147 +++++++++++++++++++++++++++++++++++++++ bin/fetch-rust-toolchain | 119 +++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 bin/clang_update.py create mode 100755 bin/fetch-rust-toolchain diff --git a/.gitignore b/.gitignore index 13a92b5faefd..90cfd190265e 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ bin/sk bin/sk.exe bin/sk.version third_party/ninja +third_party/rust-toolchain bin/ninja.exe bin/ninja.version diff --git a/bin/clang_update.py b/bin/clang_update.py new file mode 100644 index 000000000000..9562e4fc74ae --- /dev/null +++ b/bin/clang_update.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +""" +Bits and pieces of Chromium's [1] tools/clang/scripts/update.py needed +for fetching Rust toolchain, see fetch-rust-toolchain. + +[1] https://source.chromium.org/chromium/chromium/src/+/main:tools/clang/scripts/update.py + +""" + +import argparse +import os +import platform +import shutil +import stat +import tarfile +import tempfile +import time +import urllib.request +import urllib.error +import zipfile +import zlib +import sys + + +CDS_URL = os.environ.get( + "CDS_CLANG_BUCKET_OVERRIDE", + "https://commondatastorage.googleapis.com/chromium-browser-clang", +) + + +def EnsureDirExists(path): + if not os.path.exists(path): + os.makedirs(path) + + +def DownloadAndUnpack(url, output_dir, path_prefixes=None): + """Download an archive from url and extract into output_dir. If path_prefixes + is not None, only extract files whose paths within the archive start with + any prefix in path_prefixes.""" + with tempfile.TemporaryFile() as f: + DownloadUrl(url, f) + f.seek(0) + EnsureDirExists(output_dir) + if url.endswith(".zip"): + assert path_prefixes is None + zipfile.ZipFile(f).extractall(path=output_dir) + else: + t = tarfile.open(mode="r:*", fileobj=f) + members = None + if path_prefixes is not None: + members = [ + m + for m in t.getmembers() + if any(m.name.startswith(p) for p in path_prefixes) + ] + t.extractall(path=output_dir, members=members) + + +def GetPlatformUrlPrefix(host_os): + _HOST_OS_URL_MAP = { + "linux": "Linux_x64", + "mac": "Mac", + "mac-arm64": "Mac_arm64", + "win": "Win", + } + return CDS_URL + "/" + _HOST_OS_URL_MAP[host_os] + "/" + + +def DownloadUrl(url, output_file): + """Download url into output_file.""" + CHUNK_SIZE = 4096 + TOTAL_DOTS = 10 + num_retries = 3 + retry_wait_s = 5 # Doubled at each retry. + + while True: + try: + sys.stdout.write("Downloading %s " % url) + sys.stdout.flush() + request = urllib.request.Request(url) + request.add_header("Accept-Encoding", "gzip") + response = urllib.request.urlopen(request) + total_size = None + if "Content-Length" in response.headers: + total_size = int(response.headers["Content-Length"].strip()) + + is_gzipped = response.headers.get("Content-Encoding", "").strip() == "gzip" + if is_gzipped: + gzip_decode = zlib.decompressobj(zlib.MAX_WBITS + 16) + + bytes_done = 0 + dots_printed = 0 + while True: + chunk = response.read(CHUNK_SIZE) + if not chunk: + break + bytes_done += len(chunk) + + if is_gzipped: + chunk = gzip_decode.decompress(chunk) + output_file.write(chunk) + + if total_size is not None: + num_dots = TOTAL_DOTS * bytes_done // total_size + sys.stdout.write("." * (num_dots - dots_printed)) + sys.stdout.flush() + dots_printed = num_dots + if total_size is not None and bytes_done != total_size: + raise urllib.error.URLError( + "only got %d of %d bytes" % (bytes_done, total_size) + ) + if is_gzipped: + output_file.write(gzip_decode.flush()) + print(" Done.") + return + except urllib.error.URLError as e: + sys.stdout.write("\n") + print(e) + if ( + num_retries == 0 + or isinstance(e, urllib.error.HTTPError) + and e.code == 404 + ): + raise e + num_retries -= 1 + output_file.seek(0) + output_file.truncate() + print("Retrying in %d s ..." % retry_wait_s) + sys.stdout.flush() + time.sleep(retry_wait_s) + retry_wait_s *= 2 + + +def GetDefaultHostOs(): + _PLATFORM_HOST_OS_MAP = { + "darwin": "mac", + "cygwin": "win", + "linux2": "linux", + "win32": "win", + } + default_host_os = _PLATFORM_HOST_OS_MAP.get(sys.platform, sys.platform) + if default_host_os == "mac" and platform.machine() == "arm64": + default_host_os = "mac-arm64" + return default_host_os diff --git a/bin/fetch-rust-toolchain b/bin/fetch-rust-toolchain new file mode 100755 index 000000000000..9b423e8eebd5 --- /dev/null +++ b/bin/fetch-rust-toolchain @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Update in-tree checkout of Rust toolchain + +Minimal Skia adapation of [1] to fetch a Rust revision manually +specified in MANUAL_REVISION below. Frequently roll this to latest +version available in [2]. + +[1] https://source.chromium.org/chromium/chromium/src/+/main:tools/rust/update_rust.py +[2] https://commondatastorage.googleapis.com/chromium-browser-clang/index.html?path=Linux_x64/rust-toolchain- + +""" + +import argparse +import os +import re +import shutil +import sys +import tempfile +import urllib + +from pathlib import Path + +# Chromium's Rust builds (for Linux) that worked are found at: +# https://commondatastorage.googleapis.com/chromium-browser-clang/index.html?path=Linux_x64/rust-toolchain- +# The latest builds are prefixed with a date, such as `20230101-1`. +# To update, roll this to versions that Chromium's tools/rust/build-rust.py has produced and which are found from the CDS url. +MANUAL_REVISION = ( + "ac4379fea9e83465d814bb05005689f49bd2141e-1-llvmorg-17-init-3874-g93a2fecc-1" +) + +THIS_DIR = os.path.abspath(os.path.dirname(__file__)) +SKIA_DIR = os.path.abspath(os.path.join(THIS_DIR, "..")) +THIRD_PARTY_DIR = os.path.join(SKIA_DIR, "third_party") +RUST_TOOLCHAIN_OUT_DIR = os.path.join(THIRD_PARTY_DIR, "rust-toolchain") +VERSION_STAMP_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, "VERSION") + + +def GetDownloadPackageVersion(): + # TODO(https://crbug.com/14191): This is hardcoded in Skia right + # now, as Skia does not rebuild rust on its own and can't directly + # access the most recent version that's been built for + # Chromium. Could the Chromium side-build publish something like a + # symbolic link to last-known-good? + return MANUAL_REVISION + + +# Get the version of the toolchain package we already have. +def GetStampVersion(): + if os.path.exists(RUST_TOOLCHAIN_OUT_DIR): + with open(VERSION_STAMP_PATH) as version_file: + existing_stamp = version_file.readline().rstrip() + version_re = re.compile(r"rustc [0-9.]+ [0-9a-f]+ \((.+?) chromium\)") + match = version_re.fullmatch(existing_stamp) + if match is None: + return None + return match.group(1) + + return None + + +def main(): + parser = argparse.ArgumentParser(description="Update Rust package") + parser.add_argument( + "--print-package-version", + action="store_true", + help="Print Rust package version (including both the " + "Rust and Clang revisions) and quit.", + ) + args = parser.parse_args() + + if args.print_package_version: + print(GetDownloadPackageVersion()) + return 0 + + from clang_update import DownloadAndUnpack, GetDefaultHostOs, GetPlatformUrlPrefix + + # Exit early if the existing package is up-to-date. Note that we cannot + # simply call DownloadAndUnpack() every time: aside from unnecessarily + # downloading the toolchain if it hasn't changed, it also leads to multiple + # versions of the same rustlibs. build/rust/std/find_std_rlibs.py chokes in + # this case. + if os.path.exists(RUST_TOOLCHAIN_OUT_DIR): + if GetDownloadPackageVersion() == GetStampVersion(): + return 0 + + if os.path.exists(RUST_TOOLCHAIN_OUT_DIR): + shutil.rmtree(RUST_TOOLCHAIN_OUT_DIR) + + try: + host_os = GetDefaultHostOs() + # TODO(https://crbug.com/skia/14190): Enable this on additional + # platforms. + if not "linux" in host_os: + print( + "Unsupported platform, Rust support only available on Linux " + "at the moment, see https://crbug.com/skia/14190" + ) + return 1 + platform_prefix = GetPlatformUrlPrefix(host_os) + version = GetDownloadPackageVersion() + url = f"{platform_prefix}rust-toolchain-{version}.tgz" + DownloadAndUnpack(url, THIRD_PARTY_DIR) + except urllib.error.HTTPError as e: + # Fail softly for now. This can happen if a Rust package was not + # produced, e.g. if the Rust build failed upon a Clang update, or if a + # Rust roll and a Clang roll raced against each other. + # + # TODO(https://crbug.com/1245714): Reconsider how to handle this. + print(f"warning: could not download Rust package") + + # Ensure the newly extracted package has the correct version. + assert GetDownloadPackageVersion() == GetStampVersion() + + +if __name__ == "__main__": + sys.exit(main())