From 6d68cb58079fb0e54a2daf9e967c0cf5db53ae4a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 12 Mar 2025 13:17:10 -0700 Subject: [PATCH 01/10] Upgrade Protobuf C++ to 22.5 --- COMPILING.md | 4 +- buildscripts/grpc-java-artifacts/Dockerfile | 1 + .../Dockerfile.multiarch.base | 1 + .../Dockerfile.ubuntu2004.base | 1 + buildscripts/kokoro/android-interop.sh | 4 +- buildscripts/kokoro/android.sh | 4 +- buildscripts/kokoro/unix.sh | 4 +- buildscripts/make_dependencies.bat | 7 +-- buildscripts/make_dependencies.sh | 54 ++++++++++++------- buildscripts/toolchain.cmake | 9 ++++ compiler/build.gradle | 2 +- 11 files changed, 59 insertions(+), 32 deletions(-) create mode 100644 buildscripts/toolchain.cmake diff --git a/COMPILING.md b/COMPILING.md index de3cbb026c1..b7df1319beb 100644 --- a/COMPILING.md +++ b/COMPILING.md @@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code generation. Most users only need to use `skipCodegen=true` as discussed above. ### Build Protobuf -The codegen plugin is C++ code and requires protobuf 21.7 or later. +The codegen plugin is C++ code and requires protobuf 22.5 or later. For Linux, Mac and MinGW: ``` -$ PROTOBUF_VERSION=21.7 +$ PROTOBUF_VERSION=22.5 $ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz $ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz $ cd protobuf-$PROTOBUF_VERSION diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index 736babe9d8e..6df0a39bc47 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -3,6 +3,7 @@ FROM almalinux:8 RUN yum install -y \ autoconf \ automake \ + cmake \ diffutils \ gcc-c++ \ glibc-devel \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index 8f7cfae2f52..f9d4fb4fcc3 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ + cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 2d11d76c373..126a417b58f 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ + cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/kokoro/android-interop.sh b/buildscripts/kokoro/android-interop.sh index b4adc8bed43..f987aea85f6 100755 --- a/buildscripts/kokoro/android-interop.sh +++ b/buildscripts/kokoro/android-interop.sh @@ -7,8 +7,8 @@ set -exu -o pipefail cd github/grpc-java -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS=-I/tmp/protobuf/include +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib export OS_NAME=$(uname) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 13983e747b7..08e564458a3 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -9,8 +9,8 @@ BASE_DIR="$(pwd)" cd "$BASE_DIR/github/grpc-java" -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS=-I/tmp/protobuf/include +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib export OS_NAME=$(uname) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index 1b88b56ab40..9d0cd5ac493 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -51,9 +51,9 @@ fi export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" # Make protobuf discoverable by :grpc-compiler +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS="-I/tmp/protobuf/include" ./gradlew grpc-compiler:clean $GRADLE_FLAGS diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 2bbfd394d46..64d475f8982 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,5 +1,6 @@ -set PROTOBUF_VER=21.7 -set CMAKE_NAME=cmake-3.3.2-win32-x86 +@rem set PROTOBUF_VER=21.7 +set PROTOBUF_VER=22.5 +set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" ( call :installProto || exit /b 1 @@ -20,7 +21,7 @@ if not exist "%CMAKE_NAME%" ( set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin :hasCmake @rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled -powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip mkdir protobuf-%PROTOBUF_VER%\build diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 3d02a72f4eb..13164f9dbd4 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -3,13 +3,15 @@ # Build protoc set -evux -o pipefail -PROTOBUF_VERSION=21.7 +PROTOBUF_VERSION=22.5 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" DOWNLOAD_DIR=/tmp/source INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH" +BUILDSCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" mkdir -p $DOWNLOAD_DIR +cd "$DOWNLOAD_DIR" # Start with a sane default NUM_CPU=4 @@ -26,27 +28,39 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then - curl -Ls https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR + if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then + curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz fi - pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION} + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" # install here so we don't need sudo if [[ "$ARCH" == x86* ]]; then - ./configure CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} --disable-shared \ - --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == aarch* ]]; then - ./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == ppc* ]]; then - ./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == s390* ]]; then - ./configure --disable-shared --host=s390x-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == loongarch* ]]; then - ./configure --disable-shared --host=loongarch64-unknown-linux-gnu --prefix="$INSTALL_DIR" + CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 + else + if [[ "$ARCH" == aarch_64 ]]; then + GCC_ARCH=aarch64-linux-gnu + elif [[ "$ARCH" == ppcle_64 ]]; then + GCC_ARCH=powerpc64le-linux-gnu + elif [[ "$ARCH" == s390_64 ]]; then + GCC_ARCH=s390x-linux-gnu + elif [[ "$ARCH" == loongarch_64 ]]; then + GCC_ARCH=loongarch64-unknown-linux-gnu + else + echo "Unknown architecture: $ARCH" + exit 1 + fi + cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ + -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - make clean - make V=0 -j$NUM_CPU - make install + cmake --build . -j "$NUM_CPU" + cmake --install . + [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib" popd fi @@ -60,7 +74,7 @@ ln -s "$INSTALL_DIR" /tmp/protobuf cat < Date: Wed, 12 Mar 2025 20:15:51 -0700 Subject: [PATCH 02/10] Re-add downloading of absl --- buildscripts/make_dependencies.bat | 6 ++++++ buildscripts/make_dependencies.sh | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 64d475f8982..5c801fa56f9 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,5 +1,6 @@ @rem set PROTOBUF_VER=21.7 set PROTOBUF_VER=22.5 +set ABSL_VERSION=20230125.4 set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" ( @@ -24,6 +25,11 @@ set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1 +del absl.zip +rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp +rename abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp mkdir protobuf-%PROTOBUF_VER%\build pushd protobuf-%PROTOBUF_VER%\build diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 13164f9dbd4..32d8f8db69f 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -4,6 +4,7 @@ set -evux -o pipefail PROTOBUF_VERSION=22.5 +ABSL_VERSION=20230125.4 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" @@ -30,6 +31,9 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then else if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz + curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz + rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" + mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" fi # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" From b76908fa6ff65df0c254b88b0d2d9813aa1c4d44 Mon Sep 17 00:00:00 2001 From: deadEternally Date: Thu, 13 Mar 2025 21:46:01 +0530 Subject: [PATCH 03/10] Set LIBRARY_PATH to /tmp/protobuf/lib --- buildscripts/kokoro/unix.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index 9d0cd5ac493..c237b4ce041 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -54,6 +54,7 @@ export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib +export LIBRARY_PATH=/tmp/protobuf/lib ./gradlew grpc-compiler:clean $GRADLE_FLAGS From 76a613a963d281ea9b395690a2a6dae2936eacd0 Mon Sep 17 00:00:00 2001 From: deadEternally Date: Thu, 13 Mar 2025 22:02:49 +0530 Subject: [PATCH 04/10] Remove LD_LIBRARY_PATH to try fix "found unexpected dependencies" in the :grpc-compiler:checkArtifacts step --- buildscripts/kokoro/unix.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index c237b4ce041..e65825cac01 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -53,7 +53,6 @@ export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" # Make protobuf discoverable by :grpc-compiler export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" -export LD_LIBRARY_PATH=/tmp/protobuf/lib export LIBRARY_PATH=/tmp/protobuf/lib ./gradlew grpc-compiler:clean $GRADLE_FLAGS From 707a50a5d8c53774607e9f37742f8acfb9526e98 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 04:06:01 +0000 Subject: [PATCH 05/10] Add librt.so.1 to needed dependencies. --- compiler/check-artifact.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh index 4d0c2fa6286..768a67b763b 100755 --- a/compiler/check-artifact.sh +++ b/compiler/check-artifact.sh @@ -114,7 +114,7 @@ checkDependencies () white_list="KERNEL32\.dll\|msvcrt\.dll\|USER32\.dll" elif [[ "$OS" == linux ]]; then dump_cmd='objdump -x '"$1"' | grep "NEEDED"' - white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6" + white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6|librt\.so\.1" if [[ "$ARCH" == x86_32 ]]; then white_list="${white_list}\|libm\.so\.6" elif [[ "$ARCH" == x86_64 ]]; then From 948511329baf4d6bb7536ed1aebea1b038cf8653 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 05:52:18 +0000 Subject: [PATCH 06/10] Add libm.so.6 as an expected dependency for linux. --- buildscripts/make_dependencies.sh | 2 +- compiler/check-artifact.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 32d8f8db69f..17e1675d4f7 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -80,5 +80,5 @@ To compile with the build dependencies: export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" -export LD_LIBRARY_PATH=/tmp/protobuf/lib +export LIBRARY_PATH=/tmp/protobuf/lib EOF diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh index 768a67b763b..12d7709a2a8 100755 --- a/compiler/check-artifact.sh +++ b/compiler/check-artifact.sh @@ -114,7 +114,7 @@ checkDependencies () white_list="KERNEL32\.dll\|msvcrt\.dll\|USER32\.dll" elif [[ "$OS" == linux ]]; then dump_cmd='objdump -x '"$1"' | grep "NEEDED"' - white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6|librt\.so\.1" + white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6\|librt\.so\.1\|libm\.so\.6" if [[ "$ARCH" == x86_32 ]]; then white_list="${white_list}\|libm\.so\.6" elif [[ "$ARCH" == x86_64 ]]; then From 5a3e3cc585f2450e5c58b8cee957de10a1b4a454 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 09:19:31 +0000 Subject: [PATCH 07/10] Replace parallel compilation option -j with env variable for the purpose since it doesn't work on multiarch build arch64. --- buildscripts/make_dependencies.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 17e1675d4f7..9fec0dad5b8 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -62,7 +62,8 @@ else -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake fi - cmake --build . -j "$NUM_CPU" + export CMAKE_BUILD_PARALLEL_LEVEL="$NUM_CPU" + cmake --build . cmake --install . [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib" popd From 8d8e03c291f97965fec53c18400e503f0b32ed7a Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 13:30:01 +0000 Subject: [PATCH 08/10] On multiarch arch64, it requires cmake 3.13 to allow using abseil module instead of as a package, and this version is not available to install. Hence building cmake from source. Building the latest source of cmake needs openssl location to be specified (or have openssl turned off) so using the older version 3.13.4 --- buildscripts/grpc-java-artifacts/Dockerfile | 1 - .../grpc-java-artifacts/Dockerfile.multiarch.base | 1 - .../grpc-java-artifacts/Dockerfile.ubuntu2004.base | 1 - buildscripts/make_dependencies.sh | 14 +++++++++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index 6df0a39bc47..736babe9d8e 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -3,7 +3,6 @@ FROM almalinux:8 RUN yum install -y \ autoconf \ automake \ - cmake \ diffutils \ gcc-c++ \ glibc-devel \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index f9d4fb4fcc3..8f7cfae2f52 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -4,7 +4,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 126a417b58f..2d11d76c373 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -4,7 +4,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 9fec0dad5b8..af9665e21db 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -5,6 +5,7 @@ set -evux -o pipefail PROTOBUF_VERSION=22.5 ABSL_VERSION=20230125.4 +CMAKE_VERSION=3.13.4 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" @@ -29,6 +30,16 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else + if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then + curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz + fi + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" + cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" + ./bootstrap + make + make install + ln -s /usr/local/bin/cmake /usr/bin/cmake if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz @@ -60,7 +71,8 @@ else cmake .. \ -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ - -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake + -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake \ + -B. fi export CMAKE_BUILD_PARALLEL_LEVEL="$NUM_CPU" cmake --build . From a5ec1ae427e208add36696dfca74e7ec0374cbdf Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 17 Mar 2025 14:00:01 +0000 Subject: [PATCH 09/10] Fix mistake in current directory used. --- buildscripts/make_dependencies.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index af9665e21db..1bbcde5cfa9 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -40,6 +40,7 @@ else make make install ln -s /usr/local/bin/cmake /usr/bin/cmake + cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz From 54c962ae7bc26e339e73d33be078127ad1e0e132 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Wed, 19 Mar 2025 12:44:01 +0000 Subject: [PATCH 10/10] Fix missing software deps --- buildscripts/grpc-java-artifacts/Dockerfile | 5 ++++- .../grpc-java-artifacts/Dockerfile.multiarch.base | 10 ++++++++++ .../Dockerfile.ubuntu2004.base | 9 +++++++++ buildscripts/make_dependencies.sh | 15 +++------------ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index 736babe9d8e..e43e2c0bf3d 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -27,7 +27,10 @@ RUN mkdir -p "$ANDROID_HOME/cmdline-tools" && \ mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" && \ yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + # Install Maven RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ tar xz -C /var/local -ENV PATH /var/local/apache-maven-3.8.8/bin:$PATH +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index 8f7cfae2f52..27c3f9a154c 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -10,5 +10,15 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-aarch64-linux-gnu \ g++-powerpc64le-linux-gnu \ openjdk-8-jdk \ + pkg-config \ && \ rm -rf /var/lib/apt/lists/* + +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + +# Install Maven +RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ + tar xz -C /var/local +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH + diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 2d11d76c373..40999502b2c 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -9,5 +9,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ curl \ g++-s390x-linux-gnu \ openjdk-8-jdk \ + pkg-config \ && \ rm -rf /var/lib/apt/lists/* + +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + +# Install Maven +RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ + tar xz -C /var/local +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 1bbcde5cfa9..e3abb0f39cc 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -5,7 +5,7 @@ set -evux -o pipefail PROTOBUF_VERSION=22.5 ABSL_VERSION=20230125.4 -CMAKE_VERSION=3.13.4 +CMAKE_VERSION=3.26.3 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" @@ -30,16 +30,6 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "cmake-${CMAKE_VERSION}" ]]; then - curl -Ls "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz" | tar xz - fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - rm -rf "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}/bin" - cd "$DOWNLOAD_DIR/cmake-${CMAKE_VERSION}" - ./bootstrap - make - make install - ln -s /usr/local/bin/cmake /usr/bin/cmake cd "$DOWNLOAD_DIR" if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz @@ -55,7 +45,8 @@ else if [[ "$ARCH" == x86* ]]; then CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \ -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 \ + -B. else if [[ "$ARCH" == aarch_64 ]]; then GCC_ARCH=aarch64-linux-gnu