From d9d748c71fbf3e035ceb9c2bd0696af632fba858 Mon Sep 17 00:00:00 2001 From: Goren G Date: Thu, 22 Feb 2024 12:51:56 +0800 Subject: [PATCH] feat: support to specific the version of golang base image - generate osx sdk of macos-11, macos-12 and macos-13 - install cmake v3.28.3 - install clang 16 Signed-off-by: Goren G --- .github/workflows/builder.yml | 13 +- .github/workflows/osx-sdk.yaml | 19 ++- .github/workflows/release-golang-cross.yml | 92 ++++++----- Dockerfile | 4 +- Dockerfile.builder | 55 +++++-- Dockerfile.osx_sdk | 3 +- scripts/llvm.sh | 177 +++++++++++++++++++++ 7 files changed, 298 insertions(+), 65 deletions(-) create mode 100755 scripts/llvm.sh diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index ebe44d0..634290d 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -5,25 +5,29 @@ on: inputs: builder-tag: description: golang cross builder tag name - default: v1.22.0 + default: v1.22.0-0 golang-version: description: golang version default: "1.22.0" osxcross-git-hash: description: git commit hash of osx-cross project - default: "3dcc13644cfaa3d7ea6a959acbe0f1a23cf2df72" + default: "ff8d100f3f026b4ffbe4ce96d8aac4ce06f1278b" osx-min-version: description: minimal macOS SDK deployment target - default: "10.12" + default: "10.13" jobs: build: + strategy: + matrix: + codename: [bullseye, bookworm] runs-on: ubuntu-latest + continue-on-error: true permissions: id-token: write packages: write contents: read env: - GOLANG_CROSS_TAG: ${{ github.event.inputs.builder-tag }} + GOLANG_CROSS_TAG: ${{ github.event.inputs.builder-tag }}-${{ matrix.codename }} DOCKER_REGISTRY: "ghcr.io" steps: @@ -59,6 +63,7 @@ jobs: GO_VERSION=${{ github.event.inputs.golang-version }} OSX_VERSION_MIN=${{ github.event.inputs.osx-min-version }} OSX_CROSS_COMMIT=${{ github.event.inputs.osxcross-git-hash }} + OS_CODENAME=${{ matrix.codename }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha diff --git a/.github/workflows/osx-sdk.yaml b/.github/workflows/osx-sdk.yaml index 2db59c9..1b5965f 100644 --- a/.github/workflows/osx-sdk.yaml +++ b/.github/workflows/osx-sdk.yaml @@ -5,13 +5,16 @@ on: jobs: build: - runs-on: macos-13 + strategy: + matrix: + os: [macos-11, macos-12, macos-13] + runs-on: ${{ matrix.os }} + continue-on-error: true permissions: id-token: write packages: write contents: read env: - OSX_TAG: v13 DOCKER_REGISTRY: "ghcr.io" steps: # - name: Setup Xcode @@ -37,12 +40,16 @@ jobs: uses: actions/checkout@v4 with: repository: "tpoechtrager/osxcross" - ref: "3dcc13644cfaa3d7ea6a959acbe0f1a23cf2df72" + ref: "ff8d100f3f026b4ffbe4ce96d8aac4ce06f1278b" path: "osxcross" - name: Build OSX SDK + id: build_sdk_file run: | ./osxcross/tools/gen_sdk_package.sh - find . -type f -name "*.tar.xz" | xargs shasum -a 256 | tee sha256sum.txt + find . -type f -name "*.tar.xz" -print0 | xargs shasum -a 256 | tee sha256sum.txt + sdk_file=$(find . -name "*.tar.xz" | head -n 1) + [ -f "$sdk_file" ] && echo "sdk_file: $sdk_file" + echo "sdk_file=${sdk_file}" >> $GITHUB_OUTPUT shell: bash - name: Get Repo Owner id: get_repo_owner @@ -54,7 +61,7 @@ jobs: images: ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/osx-sdk tags: | type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=${{ env.OSX_TAG }} + type=raw,value=${{ matrix.os }} - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' uses: docker/login-action@v3 @@ -68,6 +75,8 @@ jobs: with: context: . file: Dockerfile.osx_sdk + build-args: | + SDK_FILE=${{ steps.build_sdk_file.outputs.sdk_file }} push: ${{ github.event_name != 'pull_request' }} platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/release-golang-cross.yml b/.github/workflows/release-golang-cross.yml index e8ca525..d28541b 100644 --- a/.github/workflows/release-golang-cross.yml +++ b/.github/workflows/release-golang-cross.yml @@ -7,51 +7,59 @@ on: jobs: build: + strategy: + matrix: + codename: [bullseye, bookworm] permissions: id-token: write packages: write contents: read runs-on: ubuntu-latest + continue-on-error: false steps: - - uses: actions/checkout@v4 - - uses: sigstore/cosign-installer@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Get Repo Owner - id: get_repo_owner - run: echo "repo_owner=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/golang-cross - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=semver,pattern={{raw}} - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - name: Login to GitHub Container Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ steps.get_repo_owner.outputs.repo_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v5 - id: buildpush - with: - context: . - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Sign the images - run: | - echo "sign ${{ steps.buildpush.outputs.digest }}" - cosign sign --yes "ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/golang-cross@${{ steps.buildpush.outputs.digest }}" - shell: bash - env: - DOCKER_PASSWD: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 + - uses: sigstore/cosign-installer@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Get Repo Owner + id: get_repo_owner + run: echo "repo_owner=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/golang-cross + tags: | + type=raw,value=latest,enable={{#if is_default_branch}}${{matrix.codename == 'bookwarm'}}{{else}}false{{/if}} + type=semver,pattern={{raw}},enable=${{matrix.codename == 'bookwarm'}} + type=semver,pattern={{version}},enable=${{matrix.codename == 'bookwarm'}} + type=semver,pattern={{major}}.{{minor}},enable=${{matrix.codename == 'bookwarm'}} + type=semver,pattern={{major}},enable=${{matrix.codename == 'bookwarm'}} + type=semver,pattern={{raw}},enable=${{matrix.codename != 'bookwarm'}},suffix=-${{matrix.codename}} + type=semver,pattern={{version}},enable=${{matrix.codename != 'bookwarm'}},suffix=-${{matrix.codename}} + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ steps.get_repo_owner.outputs.repo_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + id: buildpush + with: + context: . + build-args: | + OS_CODENAME=${{ matrix.codename }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Sign the images + run: | + echo "sign ${{ steps.buildpush.outputs.digest }}" + cosign sign --yes "ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/golang-cross@${{ steps.buildpush.outputs.digest }}" + shell: bash + env: + DOCKER_PASSWD: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 3c3d891..dde7efa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM ghcr.io/gythialy/golang-cross-builder:v1.22.0-0 +ARG OS_CODENAME=bookworm + +FROM ghcr.io/gythialy/golang-cross-builder:v1.22.0-0-${OS_CODENAME:-bookworm} LABEL maintainer="Goren G" LABEL org.opencontainers.image.source https://github.com/gythialy/golang-cross diff --git a/Dockerfile.builder b/Dockerfile.builder index 04f15bd..6203430 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,12 +1,14 @@ # golang parameters ARG GO_VERSION=1.22.0 +ARG OS_CODENAME=bookworm +ARG OSK_SDK=macos-13 -FROM ghcr.io/gythialy/osx-sdk:v13 AS osx-sdk +FROM ghcr.io/gythialy/osx-sdk:${OSK_SDK:-macos-13} AS osx-sdk -FROM golang:${GO_VERSION:-1.22.0}-bookworm AS base +FROM golang:${GO_VERSION:-1.22.0}-${OS_CODENAME:-bookworm} AS base # osxcross parameters -ARG OSX_VERSION_MIN=10.12 +ARG OSX_VERSION_MIN=10.13 ARG OSX_CROSS_COMMIT=ff8d100f3f026b4ffbe4ce96d8aac4ce06f1278b # ARG APT_MIRROR # RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ @@ -36,7 +38,7 @@ RUN set -x; echo "Starting image build for Debian " \ binutils-multiarch \ binutils-multiarch-dev \ build-essential \ - clang \ + # clang \ crossbuild-essential-arm64 \ crossbuild-essential-armel \ crossbuild-essential-armhf \ @@ -56,7 +58,7 @@ RUN set -x; echo "Starting image build for Debian " \ subversion \ wget \ xz-utils \ - cmake \ + # cmake \ qemu-user-static \ libxml2-dev \ lzma-dev \ @@ -64,6 +66,9 @@ RUN set -x; echo "Starting image build for Debian " \ mingw-w64 \ musl-tools \ libssl-dev \ + unzip \ + gnupg \ + lsb-release \ && apt -y autoremove \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -71,23 +76,49 @@ RUN set -x; echo "Starting image build for Debian " \ # FIXME: install gcc-multilib # FIXME: add mips and powerpc architectures +ENV PATH=/usr/local/bin:${OSX_CROSS_PATH}/target/bin:$PATH + WORKDIR "${OSX_CROSS_PATH}" + # install osxcross: -RUN git clone https://github.com/tpoechtrager/osxcross.git . \ - && git checkout -q "${OSX_CROSS_COMMIT:-3dcc13644cfaa3d7ea6a959acbe0f1a23cf2df72}" +RUN \ + git clone https://github.com/tpoechtrager/osxcross.git . \ + && git checkout -q "${OSX_CROSS_COMMIT:-ff8d100f3f026b4ffbe4ce96d8aac4ce06f1278b}" # install osx sdk COPY --from=osx-sdk "${OSX_CROSS_PATH}/." "${OSX_CROSS_PATH}" +# install cmake +ARG CMAKE_VERSION=3.28.3 +RUN \ + # wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz \ + # && tar -xf cmake-${CMAKE_VERSION}.tar.gz \ + # && cd cmake-${CMAKE_VERSION} \ + # && ./bootstrap \ + # && make \ + # && make install \ + # && cmake --version \ + # && cd .. \ + # && rm -rf cmake-${CMAKE_VERSION}.tar.gz cmake-${CMAKE_VERSION} + wget -qO- "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local \ + && cmake --version + # https://github.com/tpoechtrager/osxcross/issues/313 COPY patch/osxcross-08-52-08.patch "${OSX_CROSS_PATH}/" RUN patch -p1 < osxcross-08-52-08.patch +COPY scripts/llvm.sh "${OSX_CROSS_PATH}/" RUN \ - UNATTENDED=yes OSX_VERSION_MIN=${OSX_VERSION_MIN:-10.12} ./build.sh \ - && ./build_compiler_rt.sh \ + # install clang-16 + ./llvm.sh 16 \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 100 \ + && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 100 \ + && clang --version \ + && clang++ --version \ + && UNATTENDED=yes OSX_VERSION_MIN=${OSX_VERSION_MIN:-10.13} ./build.sh \ + && DISABLE_PARALLEL_ARCH_BUILD=1 ./build_compiler_rt.sh \ && rm -rf *~ build *.tar.xz \ - && rm -rf ./.git && \ - ls -al "${OSX_CROSS_PATH}/target/bin" + && rm -rf ./.git \ + && ls -al "${OSX_CROSS_PATH}/target/bin" \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -ENV PATH=${OSX_CROSS_PATH}/target/bin:$PATH diff --git a/Dockerfile.osx_sdk b/Dockerfile.osx_sdk index 55d95ed..863936f 100644 --- a/Dockerfile.osx_sdk +++ b/Dockerfile.osx_sdk @@ -1,7 +1,8 @@ FROM scratch +ARG SDK_FILE ENV OSX_CROSS_PATH=/osxcross WORKDIR "${OSX_CROSS_PATH}" -COPY "./MacOSX13.sdk.tar.xz" "${OSX_CROSS_PATH}/tarballs/" \ No newline at end of file +COPY "./${SDK_FILE}" "${OSX_CROSS_PATH}/tarballs/" \ No newline at end of file diff --git a/scripts/llvm.sh b/scripts/llvm.sh new file mode 100755 index 0000000..a927cea --- /dev/null +++ b/scripts/llvm.sh @@ -0,0 +1,177 @@ +#!/bin/bash +################################################################################ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +################################################################################ +# +# This script will install the llvm toolchain on the different +# Debian and Ubuntu versions + +set -eux + +usage() { + set +x + echo "Usage: $0 [llvm_major_version] [all] [OPTIONS]" 1>&2 + echo -e "all\t\t\tInstall all packages." 1>&2 + echo -e "-n=code_name\t\tSpecifies the distro codename, for example bionic" 1>&2 + echo -e "-h\t\t\tPrints this help." 1>&2 + echo -e "-m=repo_base_url\tSpecifies the base URL from which to download." 1>&2 + exit 1; +} + +CURRENT_LLVM_STABLE=18 +BASE_URL="http://apt.llvm.org" + +# Check for required tools +needed_binaries=(lsb_release wget add-apt-repository gpg) +missing_binaries=() +for binary in "${needed_binaries[@]}"; do + if ! which $binary &>/dev/null ; then + missing_binaries+=($binary) + fi +done +if [[ ${#missing_binaries[@]} -gt 0 ]] ; then + echo "You are missing some tools this script requires: ${missing_binaries[@]}" + echo "(hint: apt install lsb-release wget software-properties-common gnupg)" + exit 4 +fi + +# Set default values for commandline arguments +# We default to the current stable branch of LLVM +LLVM_VERSION=$CURRENT_LLVM_STABLE +ALL=0 +DISTRO=$(lsb_release -is) +VERSION=$(lsb_release -sr) +UBUNTU_CODENAME="" +CODENAME_FROM_ARGUMENTS="" +# Obtain VERSION_CODENAME and UBUNTU_CODENAME (for Ubuntu and its derivatives) +source /etc/os-release +DISTRO=${DISTRO,,} +case ${DISTRO} in + debian) + # Debian Trixie has a workaround because of + # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1038383 + if [[ "${VERSION}" == "unstable" ]] || [[ "${VERSION}" == "testing" ]] || [[ "${VERSION_CODENAME}" == "trixie" ]]; then + CODENAME=unstable + LINKNAME= + else + # "stable" Debian release + CODENAME=${VERSION_CODENAME} + LINKNAME=-${CODENAME} + fi + ;; + *) + # ubuntu and its derivatives + if [[ -n "${UBUNTU_CODENAME}" ]]; then + CODENAME=${UBUNTU_CODENAME} + if [[ -n "${CODENAME}" ]]; then + LINKNAME=-${CODENAME} + fi + fi + ;; +esac + +# read optional command line arguments +if [ "$#" -ge 1 ] && [ "${1::1}" != "-" ]; then + if [ "$1" != "all" ]; then + LLVM_VERSION=$1 + else + # special case for ./llvm.sh all + ALL=1 + fi + OPTIND=2 + if [ "$#" -ge 2 ]; then + if [ "$2" == "all" ]; then + # Install all packages + ALL=1 + OPTIND=3 + fi + fi +fi + +while getopts ":hm:n:" arg; do + case $arg in + h) + usage + ;; + m) + BASE_URL=${OPTARG} + ;; + n) + CODENAME=${OPTARG} + if [[ "${CODENAME}" == "unstable" ]]; then + # link name does not apply to unstable repository + LINKNAME= + else + LINKNAME=-${CODENAME} + fi + CODENAME_FROM_ARGUMENTS="true" + ;; + esac +done + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root!" + exit 1 +fi + +declare -A LLVM_VERSION_PATTERNS +LLVM_VERSION_PATTERNS[9]="-9" +LLVM_VERSION_PATTERNS[10]="-10" +LLVM_VERSION_PATTERNS[11]="-11" +LLVM_VERSION_PATTERNS[12]="-12" +LLVM_VERSION_PATTERNS[13]="-13" +LLVM_VERSION_PATTERNS[14]="-14" +LLVM_VERSION_PATTERNS[15]="-15" +LLVM_VERSION_PATTERNS[16]="-16" +LLVM_VERSION_PATTERNS[17]="-17" +LLVM_VERSION_PATTERNS[18]="-18" +LLVM_VERSION_PATTERNS[19]="" + +if [ ! ${LLVM_VERSION_PATTERNS[$LLVM_VERSION]+_} ]; then + echo "This script does not support LLVM version $LLVM_VERSION" + exit 3 +fi + +LLVM_VERSION_STRING=${LLVM_VERSION_PATTERNS[$LLVM_VERSION]} + +# join the repository name +if [[ -n "${CODENAME}" ]]; then + REPO_NAME="deb ${BASE_URL}/${CODENAME}/ llvm-toolchain${LINKNAME}${LLVM_VERSION_STRING} main" + + # check if the repository exists for the distro and version + if ! wget -q --method=HEAD ${BASE_URL}/${CODENAME} &> /dev/null; then + if [[ -n "${CODENAME_FROM_ARGUMENTS}" ]]; then + echo "Specified codename '${CODENAME}' is not supported by this script." + else + echo "Distribution '${DISTRO}' in version '${VERSION}' is not supported by this script." + fi + exit 2 + fi +fi + + +# install everything + +if [[ ! -f /etc/apt/trusted.gpg.d/apt.llvm.org.asc ]]; then + # download GPG key once + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc +fi + +if [[ -z "`apt-key list 2> /dev/null | grep -i llvm`" ]]; then + # Delete the key in the old format + apt-key del AF4F7421 +fi +add-apt-repository "${REPO_NAME}" -y +apt-get update -y +PKG="clang-$LLVM_VERSION lldb-$LLVM_VERSION lld-$LLVM_VERSION clangd-$LLVM_VERSION" +if [[ $ALL -eq 1 ]]; then + # same as in test-install.sh + # No worries if we have dups + PKG="$PKG clang-tidy-$LLVM_VERSION clang-format-$LLVM_VERSION clang-tools-$LLVM_VERSION llvm-$LLVM_VERSION-dev lld-$LLVM_VERSION lldb-$LLVM_VERSION llvm-$LLVM_VERSION-tools libomp-$LLVM_VERSION-dev libc++-$LLVM_VERSION-dev libc++abi-$LLVM_VERSION-dev libclang-common-$LLVM_VERSION-dev libclang-$LLVM_VERSION-dev libclang-cpp$LLVM_VERSION-dev libunwind-$LLVM_VERSION-dev" + if test $LLVM_VERSION -gt 14; then + PKG="$PKG libclang-rt-$LLVM_VERSION-dev libpolly-$LLVM_VERSION-dev" + fi +fi +apt-get install -y $PKG