From 31e1b22c670a9add3686f7b4f293e6e86292c197 Mon Sep 17 00:00:00 2001 From: Marvin Giessing Date: Thu, 25 Aug 2022 22:40:54 +0200 Subject: [PATCH 1/7] Better support for ppc64le Signed-off-by: Marvin Giessing --- build_container/build_container_ubuntu.sh | 27 ++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/build_container/build_container_ubuntu.sh b/build_container/build_container_ubuntu.sh index eb844d0e..40d9ba47 100755 --- a/build_container/build_container_ubuntu.sh +++ b/build_container/build_container_ubuntu.sh @@ -20,17 +20,7 @@ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ # docker-ce-cli curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -case $ARCH in - 'ppc64le' ) - add-apt-repository "deb [arch=ppc64le] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - ;; - 'x86_64' ) - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - ;; - 'aarch64' ) - add-apt-repository "deb [arch=arm64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - ;; -esac +add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" # CMake curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - @@ -53,7 +43,6 @@ PACKAGES=( gdb git gnupg2 - google-cloud-sdk graphviz jq libffi-dev @@ -86,24 +75,31 @@ case $ARCH in 'ppc64le' ) LLVM_DISTRO=powerpc64le-linux-ubuntu-18.04 LLVM_SHA256SUM=2d504c4920885c86b306358846178bc2232dfac83b47c3b1d05861a8162980e6 + apt-get install -y --no-install-recommends \ + libtinfo5 # LLVM dependencies on Ubuntu 20.04 ;; 'x86_64' ) LLVM_DISTRO=x86_64-linux-gnu-ubuntu-18.04 LLVM_SHA256SUM=61582215dafafb7b576ea30cc136be92c877ba1f1c31ddbbd372d6d65622fef5 + apt-get install -y --no-install-recommends \ + google-cloud-sdk \ + libtinfo5 # LLVM dependencies on Ubuntu 20.04 ;; 'aarch64' ) LLVM_DISTRO=aarch64-linux-gnu LLVM_SHA256SUM=1792badcd44066c79148ffeb1746058422cc9d838462be07e3cb19a4b724a1ee - apt-get install -y --no-install-recommends libtinfo5 # LLVM dependencies on Ubuntu 20.04 + apt-get install -y --no-install-recommends \ + google-cloud-sdk \ + libtinfo5 # LLVM dependencies on Ubuntu 20.04 ;; esac # Bazel and related dependencies. case $ARCH in 'ppc64le' ) - BAZEL_LATEST="$(curl https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_16.04/latest/ 2>&1 \ + BAZEL_LATEST="$(curl https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_$(lsb_release -r | awk '{print $2}')/latest/ 2>&1 \ | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep '^bazel' | head -n 1)" - curl -fSL https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_16.04/latest/${BAZEL_LATEST} \ + curl -fSL https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_$(lsb_release -r | awk '{print $2}')/latest/${BAZEL_LATEST} \ -o /usr/local/bin/bazel chmod +x /usr/local/bin/bazel ;; @@ -132,6 +128,7 @@ pip3 install -U pyyaml virtualenv source ./build_container_common.sh # Soft link the gcc compiler (required by python env) +ARCH=${ARCH/ppc64le/powerpc64le} # For ppc64le the ARCH variable must be renamed accordingly update-alternatives --install "/usr/bin/${ARCH}-linux-gnu-gcc" "${ARCH}-linux-gnu-gcc" "/usr/bin/${ARCH}-linux-gnu-gcc-9" 1 apt-get clean From 1acc43ec46a08130080d22f4d70022aac3b65cd9 Mon Sep 17 00:00:00 2001 From: Marvin Giessing Date: Thu, 25 Aug 2022 23:22:17 +0200 Subject: [PATCH 2/7] Add logic to build gn if no binary Signed-off-by: Marvin Giessing --- build_container/build_container_common.sh | 28 +++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/build_container/build_container_common.sh b/build_container/build_container_common.sh index 85b8236a..494f2c77 100755 --- a/build_container/build_container_common.sh +++ b/build_container/build_container_common.sh @@ -11,21 +11,19 @@ function download_and_check () { function install_gn(){ # Install gn tools which will be used for building wee8 - case "$(uname -m)" in - "x86_64") - GN_ARCH=amd64 - ;; - - "aarch64") - GN_ARCH=arm64 - ;; - esac - - wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-${GN_ARCH}/+/latest" - unzip gntool.zip -d gntool - cp gntool/gn /usr/local/bin/gn - chmod +x /usr/local/bin/gn - rm -rf gntool* + # amd64 & arm64 install binary, else compile from source + if [ $(uname -m) = x64_64 ] || [ $(uname -m) = aarch64 ]; then + wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-$(dpkg --print-architecture)/+/latest" + unzip gntool.zip -d gntool + cp gntool/gn /usr/local/bin/gn + chmod +x /usr/local/bin/gn + rm -rf gntool* + else + git clone https://gn.googlesource.com/gn && cd gn + CC=/opt/llvm/bin/clang CXX=/opt/llvm/bin/clang++ python3 build/gen.py && \ + ninja -C out && \ + ln -sf out/gn /usr/local/bin/gn + fi } if [[ "$(uname -m)" == "x86_64" ]]; then From f15ef90c3d7950cf8170c4438d1578884c7a16ff Mon Sep 17 00:00:00 2001 From: Marvin Giessing Date: Thu, 25 Aug 2022 23:40:18 +0200 Subject: [PATCH 3/7] Misspelled arch Signed-off-by: Marvin Giessing --- build_container/build_container_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_container/build_container_common.sh b/build_container/build_container_common.sh index 494f2c77..66f75ad4 100755 --- a/build_container/build_container_common.sh +++ b/build_container/build_container_common.sh @@ -12,7 +12,7 @@ function download_and_check () { function install_gn(){ # Install gn tools which will be used for building wee8 # amd64 & arm64 install binary, else compile from source - if [ $(uname -m) = x64_64 ] || [ $(uname -m) = aarch64 ]; then + if [ $(uname -m) = x86_64 ] || [ $(uname -m) = aarch64 ]; then wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-$(dpkg --print-architecture)/+/latest" unzip gntool.zip -d gntool cp gntool/gn /usr/local/bin/gn From e0a2c7448baa95440b78935029eb2147c93067c8 Mon Sep 17 00:00:00 2001 From: Marvin Giessing Date: Fri, 26 Aug 2022 00:12:25 +0200 Subject: [PATCH 4/7] Fix arch issue & add build target Signed-off-by: Marvin Giessing --- build_container/build_container_common.sh | 6 ++++-- build_container/docker_build_linux.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build_container/build_container_common.sh b/build_container/build_container_common.sh index 66f75ad4..4755ee9b 100755 --- a/build_container/build_container_common.sh +++ b/build_container/build_container_common.sh @@ -12,8 +12,10 @@ function download_and_check () { function install_gn(){ # Install gn tools which will be used for building wee8 # amd64 & arm64 install binary, else compile from source - if [ $(uname -m) = x86_64 ] || [ $(uname -m) = aarch64 ]; then - wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-$(dpkg --print-architecture)/+/latest" + if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then + GN_ARCH=${ARCH/x86_64/amd64} + GN_ARCH=${ARCH/aarch64/arm64} + wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-${GN_ARCH}/+/latest" unzip gntool.zip -d gntool cp gntool/gn /usr/local/bin/gn chmod +x /usr/local/bin/gn diff --git a/build_container/docker_build_linux.sh b/build_container/docker_build_linux.sh index bd9094b3..303cd724 100755 --- a/build_container/docker_build_linux.sh +++ b/build_container/docker_build_linux.sh @@ -18,7 +18,7 @@ config_env() { if [[ -z "${BUILD_TOOLS_PLATFORMS}" ]]; then if [[ "${OS_DISTRO}" == "ubuntu" ]]; then - export BUILD_TOOLS_PLATFORMS=linux/arm64,linux/amd64 + export BUILD_TOOLS_PLATFORMS=linux/arm64,linux/amd64,linux/ppc64le else export BUILD_TOOLS_PLATFORMS=linux/amd64 fi From 9d406956ecf4ff5e47c8e59c41786c13e4d2967e Mon Sep 17 00:00:00 2001 From: Marvin Giessing Date: Fri, 26 Aug 2022 00:31:14 +0200 Subject: [PATCH 5/7] Fix arch Signed-off-by: Marvin Giessing --- build_container/build_container_common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_container/build_container_common.sh b/build_container/build_container_common.sh index 4755ee9b..ad5c6ef2 100755 --- a/build_container/build_container_common.sh +++ b/build_container/build_container_common.sh @@ -12,7 +12,8 @@ function download_and_check () { function install_gn(){ # Install gn tools which will be used for building wee8 # amd64 & arm64 install binary, else compile from source - if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then + ARCH=$(uname -m) + if [[ "${ARCH}" == "x86_64" ]] || [[ "${ARCH}" == "aarch64" ]]; then GN_ARCH=${ARCH/x86_64/amd64} GN_ARCH=${ARCH/aarch64/arm64} wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-${GN_ARCH}/+/latest" From 199a19d86620696816d62e24ca5c3dca41adb7ef Mon Sep 17 00:00:00 2001 From: Marvin Giessing Date: Fri, 26 Aug 2022 00:43:29 +0200 Subject: [PATCH 6/7] Fix build issue Signed-off-by: Marvin Giessing --- build_container/build_container_common.sh | 24 +++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/build_container/build_container_common.sh b/build_container/build_container_common.sh index ad5c6ef2..80a6c259 100755 --- a/build_container/build_container_common.sh +++ b/build_container/build_container_common.sh @@ -12,21 +12,33 @@ function download_and_check () { function install_gn(){ # Install gn tools which will be used for building wee8 # amd64 & arm64 install binary, else compile from source - ARCH=$(uname -m) - if [[ "${ARCH}" == "x86_64" ]] || [[ "${ARCH}" == "aarch64" ]]; then - GN_ARCH=${ARCH/x86_64/amd64} - GN_ARCH=${ARCH/aarch64/arm64} + case "$(uname -m)" in + "x86_64") + GN_ARCH=amd64 wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-${GN_ARCH}/+/latest" unzip gntool.zip -d gntool cp gntool/gn /usr/local/bin/gn chmod +x /usr/local/bin/gn rm -rf gntool* - else + ;; + + "aarch64") + GN_ARCH=arm64 + wget -O gntool.zip "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-${GN_ARCH}/+/latest" + unzip gntool.zip -d gntool + cp gntool/gn /usr/local/bin/gn + chmod +x /usr/local/bin/gn + rm -rf gntool* + ;; + + "ppc64le") git clone https://gn.googlesource.com/gn && cd gn CC=/opt/llvm/bin/clang CXX=/opt/llvm/bin/clang++ python3 build/gen.py && \ ninja -C out && \ ln -sf out/gn /usr/local/bin/gn - fi + ;; + + esac } if [[ "$(uname -m)" == "x86_64" ]]; then From 3a867ae13cc4da1426955646eab996a7d44fcda8 Mon Sep 17 00:00:00 2001 From: Marvin Giessing Date: Thu, 8 Sep 2022 14:15:01 +0200 Subject: [PATCH 7/7] Simplified install logic Signed-off-by: Marvin Giessing --- build_container/build_container_ubuntu.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/build_container/build_container_ubuntu.sh b/build_container/build_container_ubuntu.sh index 40d9ba47..d8b7f719 100755 --- a/build_container/build_container_ubuntu.sh +++ b/build_container/build_container_ubuntu.sh @@ -47,6 +47,7 @@ PACKAGES=( jq libffi-dev libncurses-dev + libtinfo5 libtool make ninja-build @@ -67,6 +68,10 @@ PACKAGES=( xz-utils zip) +if [[ "${ARCH}" == "x86_64" || "${ARCH}" == "aarch64" ]]; then + PACKAGES+=("google-cloud-sdk") +fi + apt-get install -y --no-install-recommends "${PACKAGES[@]}" # Set LLVM version for each cpu architecture. @@ -75,22 +80,14 @@ case $ARCH in 'ppc64le' ) LLVM_DISTRO=powerpc64le-linux-ubuntu-18.04 LLVM_SHA256SUM=2d504c4920885c86b306358846178bc2232dfac83b47c3b1d05861a8162980e6 - apt-get install -y --no-install-recommends \ - libtinfo5 # LLVM dependencies on Ubuntu 20.04 ;; 'x86_64' ) LLVM_DISTRO=x86_64-linux-gnu-ubuntu-18.04 LLVM_SHA256SUM=61582215dafafb7b576ea30cc136be92c877ba1f1c31ddbbd372d6d65622fef5 - apt-get install -y --no-install-recommends \ - google-cloud-sdk \ - libtinfo5 # LLVM dependencies on Ubuntu 20.04 ;; 'aarch64' ) LLVM_DISTRO=aarch64-linux-gnu LLVM_SHA256SUM=1792badcd44066c79148ffeb1746058422cc9d838462be07e3cb19a4b724a1ee - apt-get install -y --no-install-recommends \ - google-cloud-sdk \ - libtinfo5 # LLVM dependencies on Ubuntu 20.04 ;; esac