Zephyr: Update to Zephyr 4.4

.ci/docker/common/install_zephyr.sh

@@ -1,4 +1,3 @@
-
 #!/bin/bash
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
@@ -9,27 +8,27 @@
 
 set -ex
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 # shellcheck source=/dev/null
-source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+source "${SCRIPT_DIR}/utils.sh"
 
-# Double check if the NDK version is set
+# Double check if Zephyr setup is enabled.
 [ -n "${ZEPHYR_SDK}" ]
 
-install_prerequiresites() {
+install_prerequisites() {
     rm /var/lib/dpkg/info/libc-bin.*
     apt-get clean
     apt-get -y update
     apt-get install -y libc-bin
     apt-get -y update
     apt-get clean
-    apt-get install --no-install-recommends -y dos2unix
     apt-get install --no-install-recommends -y ca-certificates
     apt-get install -y --reinstall libc-bin
     apt-get install --no-install-recommends -y file
     apt-get install --no-install-recommends -y locales
     apt-get install --no-install-recommends -y git
     apt-get install --no-install-recommends -y build-essential
-    apt-get install --no-install-recommends -y cmake
     apt-get install --no-install-recommends -y ninja-build gperf
     apt-get install --no-install-recommends -y device-tree-compiler
     apt-get install --no-install-recommends -y wget
@@ -49,16 +48,14 @@ install_prerequiresites() {
     apt install -y python3.12 python3.12-dev python3.12-venv python3-pip
     update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
 
-    # Upgrade cmake ot 3.24
-    apt update
-    apt install cmake
-    apt install software-properties-common lsb-release
+    # Upgrade cmake to 3.24 or newer.
+    apt install -y software-properties-common lsb-release
     apt update
     test -f /usr/share/doc/kitware-archive-keyring/copyright || \
         wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
-    "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list > /dev/null
+    echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list > /dev/null
     apt update
-    apt install cmake
+    apt install -y cmake
 
     # Install additional required software for Zephyr
     apt install --no-install-recommends -y ccache \
@@ -77,19 +74,19 @@ install_prerequiresites() {
     apt-get clean -y
     apt-get autoremove --purge -y
     rm -rf /var/lib/apt/lists/*
-    wget https://apt.kitware.com/kitware-archive.sh && \
-        chmod +x kitware-archive.sh && \
-        ./kitware-archive.sh && \
-        rm -f kitware-archive.sh
     pip_install --no-cache-dir west
     pip_install pyelftools
 
-    # Zephyr SDK
-    wget https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.17.4/zephyr-sdk-0.17.4_linux-x86_64.tar.xz
-    tar -xf zephyr-sdk-0.17.4_linux-x86_64.tar.xz
-    rm -f zephyr-sdk-0.17.4_linux-x86_64.tar.xz*
-    # Save setup to later and this get symlinked in to another folder in the test in trunk.yml
-    #./zephyr-sdk-0.17.4/setup.sh -c -t arm-zephyr-eabi
+    # Cache the Zephyr SDK release assets in the image. CI still runs
+    # `west sdk install` in its workspace, but the local proxy fallback can serve
+    # these files without downloading them on every job.
+    local zephyr_sdk_version
+    zephyr_sdk_version="$(python3 "${SCRIPT_DIR}/zephyr_sdk_release_proxy.py" --print-version)"
+    local zephyr_sdk_cache_dir="${ZEPHYR_SDK_RELEASE_PROXY_CACHE_DIR:-/opt/zephyr-sdk-cache/v${zephyr_sdk_version}}"
+    python3 "${SCRIPT_DIR}/zephyr_sdk_release_proxy.py" \
+        --version "${zephyr_sdk_version}" \
+        --cache-dir "${zephyr_sdk_cache_dir}" \
+        --populate-cache
 }
 
-install_prerequiresites
+install_prerequisites

.ci/docker/common/zephyr_sdk_release_proxy.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+# Copyright 2026 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Local Zephyr SDK release proxy for `west sdk install`.
+
+This avoids GitHub releases API rate limits by serving a tiny synthetic releases
+API locally. Asset requests are served from a cache directory and populated with
+`wget` on cache miss.
+
+Examples:
+
+  # Prepopulate CI cache only.
+  .ci/docker/common/zephyr_sdk_release_proxy.py \
+      --version <version> \
+      --cache-dir .cache/zephyr-sdk/v<version> \
+      --populate-cache
+
+  # Serve cached assets and release metadata.
+  .ci/docker/common/zephyr_sdk_release_proxy.py \
+      --version <version> \
+      --cache-dir .cache/zephyr-sdk/v<version> \
+      --port 8765
+
+  west sdk install \
+      --version <version> \
+      --api-url http://127.0.0.1:8765/releases \
+      --gnu-toolchains arm-zephyr-eabi
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import http.server
+import json
+import os
+import platform
+import re
+import subprocess
+import sys
+import urllib.parse
+from pathlib import Path
+
+
+DEFAULT_SDK_VERSION = "1.0.1"
+
+
+class AssetVerificationError(Exception):
+    pass
+
+
+def default_sdk_version() -> str:
+    return (
+        os.environ.get("ZEPHYR_SDK_VERSION")
+        or os.environ.get("SDK_VERSION")
+        or DEFAULT_SDK_VERSION
+    )
+
+
+def host_tuple() -> tuple[str, str]:
+    system = platform.system()
+    machine = platform.machine()
+
+    if system != "Linux":
+        raise SystemExit(f"Unsupported host OS: {system}")
+
+    if machine in ("x86_64", "AMD64"):
+        return "linux", "x86_64"
+    if machine in ("aarch64", "arm64"):
+        return "linux", "aarch64"
+
+    raise SystemExit(f"Unsupported host architecture: {machine}")
+
+
+def release_base_url(version: str) -> str:
+    return f"https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v{version}"
+
+
+def asset_names(version: str, toolchain: str) -> list[str]:
+    host_os, host_arch = host_tuple()
+    return [
+        "sha256.sum",
+        f"zephyr-sdk-{version}_{host_os}-{host_arch}_minimal.tar.xz",
+        f"hosttools_{host_os}-{host_arch}.tar.xz",
+        f"toolchain_gnu_{host_os}-{host_arch}_{toolchain}.tar.xz",
+    ]
+
+
+def run_wget(url: str, output: Path) -> None:
+    output.parent.mkdir(parents=True, exist_ok=True)
+    partial = output.with_suffix(output.suffix + ".tmp")
+    cmd = [
+        "wget",
+        "--continue",
+        "--progress=bar:force:noscroll",
+        "--output-document",
+        str(partial),
+        url,
+    ]
+    subprocess.run(cmd, check=True)
+    partial.replace(output)
+
+
+def ensure_asset(cache_dir: Path, version: str, filename: str) -> Path:
+    path = cache_dir / filename
+    if path.exists():
+        return path
+
+    url = f"{release_base_url(version)}/{filename}"
+    print(f"Downloading {url}", flush=True)
+    run_wget(url, path)
+    return path
+
+
+def parse_sha256_sum(path: Path) -> dict[str, str]:
+    checksums: dict[str, str] = {}
+    for line in path.read_text().splitlines():
+        match = re.match(r"^([0-9a-fA-F]{64})\s+(.+)$", line.strip())
+        if match:
+            checksums[match.group(2)] = match.group(1).lower()
+    return checksums
+
+
+def verify_asset(cache_dir: Path, checksums: dict[str, str], filename: str) -> None:
+    if filename == "sha256.sum":
+        return
+
+    expected = checksums.get(filename)
+    if expected is None:
+        raise AssetVerificationError(f"No checksum entry for {filename}")
+
+    hasher = hashlib.sha256()
+    with (cache_dir / filename).open("rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    digest = hasher.hexdigest()
+
+    if digest != expected:
+        raise AssetVerificationError(
+            f"Checksum mismatch for {filename}: expected {expected}, got {digest}"
+        )
+
+
+def ensure_verified_asset(
+    cache_dir: Path, version: str, checksums: dict[str, str], filename: str
+) -> Path:
+    path = ensure_asset(cache_dir, version, filename)
+    try:
+        verify_asset(cache_dir, checksums, filename)
+    except AssetVerificationError:
+        path.unlink(missing_ok=True)
+        path = ensure_asset(cache_dir, version, filename)
+        verify_asset(cache_dir, checksums, filename)
+    return path
+
+
+def populate_cache(cache_dir: Path, version: str, toolchain: str) -> None:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    names = asset_names(version, toolchain)
+
+    sha_file = ensure_asset(cache_dir, version, "sha256.sum")
+    checksums = parse_sha256_sum(sha_file)
+
+    for name in names:
+        ensure_verified_asset(cache_dir, version, checksums, name)
+
+
+def release_json(version: str, toolchain: str, base_url: str) -> bytes:
+    assets = [
+        {
+            "name": name,
+            "browser_download_url": f"{base_url}/assets/{name}",
+        }
+        for name in asset_names(version, toolchain)
+    ]
+    return json.dumps([{"tag_name": f"v{version}", "assets": assets}]).encode()
+
+
+class SdkProxyHandler(http.server.SimpleHTTPRequestHandler):
+    version: str
+    toolchain: str
+    cache_dir: Path
+
+    def log_message(self, fmt: str, *args: object) -> None:
+        print(f"{self.address_string()} - {fmt % args}", file=sys.stderr)
+
+    def send_bytes(self, data: bytes, content_type: str) -> None:
+        self.send_response(200)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_GET(self) -> None:
+        parsed = urllib.parse.urlparse(self.path)
+
+        if parsed.path == "/releases":
+            query = urllib.parse.parse_qs(parsed.query)
+            page = query.get("page", ["1"])[0]
+            if page == "1":
+                host = self.headers.get("Host", "127.0.0.1")
+                data = release_json(self.version, self.toolchain, f"http://{host}")
+            else:
+                data = b"[]"
+            self.send_bytes(data, "application/json")
+            return
+
+        if parsed.path.startswith("/assets/"):
+            filename = Path(urllib.parse.unquote(parsed.path[len("/assets/") :])).name
+            allowed = set(asset_names(self.version, self.toolchain))
+            if filename not in allowed:
+                self.send_error(404, f"Unknown asset: {filename}")
+                return
+
+            try:
+                if filename != "sha256.sum":
+                    sha_file = ensure_asset(self.cache_dir, self.version, "sha256.sum")
+                    path = ensure_verified_asset(
+                        self.cache_dir,
+                        self.version,
+                        parse_sha256_sum(sha_file),
+                        filename,
+                    )
+                else:
+                    path = ensure_asset(self.cache_dir, self.version, filename)
+            except (
+                AssetVerificationError,
+                subprocess.CalledProcessError,
+                OSError,
+            ) as error:
+                self.send_error(500, str(error))
+                return
+
+            self.send_response(200)
+            self.send_header("Content-Type", "application/octet-stream")
+            self.send_header("Content-Length", str(path.stat().st_size))
+            self.end_headers()
+            with path.open("rb") as f:
+                while chunk := f.read(1024 * 1024):
+                    self.wfile.write(chunk)
+            return
+
+        self.send_error(404, "Not found")
+
+
+def default_cache_dir(version: str) -> Path:
+    root = os.environ.get("XDG_CACHE_HOME")
+    if root:
+        return Path(root) / "zephyr-sdk" / f"v{version}"
+    return Path.home() / ".cache" / "zephyr-sdk" / f"v{version}"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--version", default=default_sdk_version())
+    parser.add_argument("--toolchain", default="arm-zephyr-eabi")
+    parser.add_argument("--cache-dir", type=Path)
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8765)
+    parser.add_argument(
+        "--print-version",
+        action="store_true",
+        help="Print the effective default SDK version and exit.",
+    )
+    parser.add_argument(
+        "--download-only",
+        "--populate-cache",
+        dest="download_only",
+        action="store_true",
+        help="Download and verify release assets, then exit without serving.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+
+    if args.print_version:
+        print(args.version)
+        return
+
+    cache_dir = args.cache_dir or default_cache_dir(args.version)
+
+    if args.download_only:
+        try:
+            populate_cache(cache_dir, args.version, args.toolchain)
+        except AssetVerificationError as error:
+            raise SystemExit(str(error)) from error
+        print(f"Cached Zephyr SDK assets in {cache_dir}")
+        return
+
+    SdkProxyHandler.version = args.version
+    SdkProxyHandler.toolchain = args.toolchain
+    SdkProxyHandler.cache_dir = cache_dir
+
+    server = http.server.ThreadingHTTPServer((args.host, args.port), SdkProxyHandler)
+    print(
+        f"Serving Zephyr SDK release metadata at http://{args.host}:{args.port}/releases",
+        flush=True,
+    )
+    print(f"Cache directory: {cache_dir}", flush=True)
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()