diff --git a/COMPILING.md b/COMPILING.md index de3cbb026c1..b7df1319beb 100644 --- a/COMPILING.md +++ b/COMPILING.md @@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code generation. Most users only need to use `skipCodegen=true` as discussed above. ### Build Protobuf -The codegen plugin is C++ code and requires protobuf 21.7 or later. +The codegen plugin is C++ code and requires protobuf 22.5 or later. For Linux, Mac and MinGW: ``` -$ PROTOBUF_VERSION=21.7 +$ PROTOBUF_VERSION=22.5 $ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz $ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz $ cd protobuf-$PROTOBUF_VERSION diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index 736babe9d8e..e43e2c0bf3d 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -27,7 +27,10 @@ RUN mkdir -p "$ANDROID_HOME/cmdline-tools" && \ mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" && \ yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + # Install Maven RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ tar xz -C /var/local -ENV PATH /var/local/apache-maven-3.8.8/bin:$PATH +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index 8f7cfae2f52..27c3f9a154c 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -10,5 +10,15 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ g++-aarch64-linux-gnu \ g++-powerpc64le-linux-gnu \ openjdk-8-jdk \ + pkg-config \ && \ rm -rf /var/lib/apt/lists/* + +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + +# Install Maven +RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ + tar xz -C /var/local +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH + diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 2d11d76c373..40999502b2c 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -9,5 +9,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ curl \ g++-s390x-linux-gnu \ openjdk-8-jdk \ + pkg-config \ && \ rm -rf /var/lib/apt/lists/* + +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local + +# Install Maven +RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \ + tar xz -C /var/local +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH diff --git a/buildscripts/kokoro/android-interop.sh b/buildscripts/kokoro/android-interop.sh index b4adc8bed43..f987aea85f6 100755 --- a/buildscripts/kokoro/android-interop.sh +++ b/buildscripts/kokoro/android-interop.sh @@ -7,8 +7,8 @@ set -exu -o pipefail cd github/grpc-java -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS=-I/tmp/protobuf/include +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib export OS_NAME=$(uname) diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 13983e747b7..08e564458a3 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -9,8 +9,8 @@ BASE_DIR="$(pwd)" cd "$BASE_DIR/github/grpc-java" -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS=-I/tmp/protobuf/include +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" export LD_LIBRARY_PATH=/tmp/protobuf/lib export OS_NAME=$(uname) diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index 1b88b56ab40..e65825cac01 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -51,9 +51,9 @@ fi export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" # Make protobuf discoverable by :grpc-compiler -export LD_LIBRARY_PATH=/tmp/protobuf/lib -export LDFLAGS=-L/tmp/protobuf/lib -export CXXFLAGS="-I/tmp/protobuf/include" +export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)" +export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)" +export LIBRARY_PATH=/tmp/protobuf/lib ./gradlew grpc-compiler:clean $GRADLE_FLAGS diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index ffd4d3b99a6..cfa9e87168a 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -15,7 +15,6 @@ set ESCWORKSPACE=%WORKSPACE:\=\\% @rem Clear JAVA_HOME to prevent a different Java version from being used set JAVA_HOME= -set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH% mkdir grpc-java-helper32 cd grpc-java-helper32 @@ -26,9 +25,12 @@ cd "%WORKSPACE%" SET TARGET_ARCH=x86_32 SET FAIL_ON_WARNINGS=true -SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\Release -SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\include -SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET PROTOBUF_VER=22.5 +SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig +SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib +SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include +call :Get_Libs +SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% build" @@ -50,3 +52,28 @@ IF NOT %GRADLEEXIT% == 0 ( cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:publish" || exit /b 1 + +goto :eof +:Get_Libs +SetLocal EnableDelayedExpansion +set "libs_list=" +for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( + for %%b in (%%a) do ( + set lib=%%b + set libfirst2char=!lib:~0,2! + if !libfirst2char!==-l ( + @rem remove the leading -l + set lib=!lib:~2! + @rem remove spaces + set lib=!lib: =! + @rem Because protobuf is specified as libprotobuf and elsewhere + if !lib! NEQ protobuf ( + set lib=!lib!.lib, + set libs_list=!libs_list! !lib! + ) + ) + ) +) +EndLocal & set "VC_PROTOBUF_LIBS=%libs_list%" +exit /b 0 + diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 8542f1c0536..26457cf7fe1 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -1,36 +1,63 @@ -@rem ########################################################################## -@rem -@rem Builds artifacts for x86_64 into %WORKSPACE%\artifacts\ -@rem -@rem ########################################################################## - -type c:\VERSION - -@rem Enter repo root -cd /d %~dp0\..\.. - -set WORKSPACE=T:\src\github\grpc-java -set ESCWORKSPACE=%WORKSPACE:\=\\% - -@rem Clear JAVA_HOME to prevent a different Java version from being used -set JAVA_HOME= -set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH% - -mkdir grpc-java-helper64 -cd grpc-java-helper64 -call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" || exit /b 1 -call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 - -cd "%WORKSPACE%" - -SET TARGET_ARCH=x86_64 -SET FAIL_ON_WARNINGS=true -SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\Release -SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\include -SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true -SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" - -@rem make sure no daemons have any files open -cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" - -cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1 +@rem ########################################################################## +@rem +@rem Builds artifacts for x86_64 into %WORKSPACE%\artifacts\ +@rem +@rem ########################################################################## + +type c:\VERSION + +@rem Enter repo root +cd /d %~dp0\..\.. + +set WORKSPACE=T:\src\github\grpc-java +set ESCWORKSPACE=%WORKSPACE:\=\\% + +@rem Clear JAVA_HOME to prevent a different Java version from being used +set JAVA_HOME= + +mkdir grpc-java-helper64 +cd grpc-java-helper64 +call "%VS140COMNTOOLS%\vsvars64.bat" || exit /b 1 +call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1 + +cd "%WORKSPACE%" + +SET TARGET_ARCH=x86_64 +SET FAIL_ON_WARNINGS=true +SET PROTOBUF_VER=22.5 +SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig +SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib +SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include +call :Get_Libs +SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" + +@rem make sure no daemons have any files open +cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" + +cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1 + +goto :eof +:Get_Libs +SetLocal EnableDelayedExpansion +set "libs_list=" +for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do ( + for %%b in (%%a) do ( + set lib=%%b + set libfirst2char=!lib:~0,2! + if !libfirst2char!==-l ( + @rem remove the leading -l + set lib=!lib:~2! + @rem remove spaces + set lib=!lib: =! + @rem Because protobuf is specified as libprotobuf and elsewhere + if !lib! NEQ protobuf ( + set lib=!lib!.lib, + set libs_list=!libs_list! !lib! + ) + ) + ) +) +EndLocal & set "VC_PROTOBUF_LIBS=%libs_list%" +exit /b 0 + diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 2bbfd394d46..bfc1005fad9 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,12 +1,18 @@ -set PROTOBUF_VER=21.7 -set CMAKE_NAME=cmake-3.3.2-win32-x86 +echo on +@rem set PROTOBUF_VER=21.7 +choco install -y gradle git curl pkgconfiglite +choco install -y openjdk --version=21.0 +set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-21\bin" +set JAVA_HOME= +set PROTOBUF_VER=22.5 +set ABSL_VERSION=20230125.4 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" ( call :installProto || exit /b 1 ) echo Compile gRPC-Java with something like: -echo -PtargetArch=x86_32 -PvcProtobufLibs=%cd%\protobuf-%PROTOBUF_VER%\build\Release -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\include +echo -PtargetArch=x86_32 -PvcProtobufLibPath=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\lib -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\include -PvcProtobufLibs=insert-list-of-libs-from-pkg-config-output-here goto :eof @@ -14,31 +20,40 @@ goto :eof where /q cmake if not ERRORLEVEL 1 goto :hasCmake +set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "%CMAKE_NAME%" ( call :installCmake || exit /b 1 ) set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin :hasCmake @rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled -powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1 +del absl.zip +rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp +move abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp mkdir protobuf-%PROTOBUF_VER%\build pushd protobuf-%PROTOBUF_VER%\build -@rem Workaround https://github.com/protocolbuffers/protobuf/issues/10174 -powershell -command "(Get-Content ..\cmake\extract_includes.bat.in) -replace '\.\.\\', '' | Out-File -encoding ascii ..\cmake\extract_includes.bat.in" @rem cmake does not detect x86_64 from the vcvars64.bat variables. @rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS -if "%PLATFORM%" == "X64" ( +if "%PLATFORM%" == "x64" ( @rem Note the space - SET CMAKE_VSARCH= Win64 + SET CMAKE_VSARCH=-A x64 ) else ( SET CMAKE_VSARCH= ) -cmake -Dprotobuf_BUILD_TESTS=OFF -G "Visual Studio %VisualStudioVersion:~0,2%%CMAKE_VSARCH%" .. || exit /b 1 -msbuild /maxcpucount /p:Configuration=Release /verbosity:minimal libprotoc.vcxproj || exit /b 1 -call extract_includes.bat || exit /b 1 +for /f "tokens=4 delims=\" %%a in ("%VCINSTALLDIR%") do ( + SET VC_YEAR=%%a +) +for /f "tokens=1 delims=." %%a in ("%VisualStudioVersion%") do ( + SET visual_studio_major_version=%%a +) +cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% .. +cmake --build . --config Release --target install popd goto :eof @@ -49,3 +64,4 @@ powershell -command "$ErrorActionPreference = 'stop'; & { iwr https://cmake.org/ powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('cmake.zip', '.') }" || exit /b 1 del cmake.zip goto :eof + diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 3d02a72f4eb..e3abb0f39cc 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -3,13 +3,17 @@ # Build protoc set -evux -o pipefail -PROTOBUF_VERSION=21.7 +PROTOBUF_VERSION=22.5 +ABSL_VERSION=20230125.4 +CMAKE_VERSION=3.26.3 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" DOWNLOAD_DIR=/tmp/source INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH" +BUILDSCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" mkdir -p $DOWNLOAD_DIR +cd "$DOWNLOAD_DIR" # Start with a sane default NUM_CPU=4 @@ -26,27 +30,46 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then - curl -Ls https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR + cd "$DOWNLOAD_DIR" + if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then + curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz + curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz + rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" + mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" fi - pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION} + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" # install here so we don't need sudo if [[ "$ARCH" == x86* ]]; then - ./configure CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} --disable-shared \ - --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == aarch* ]]; then - ./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == ppc* ]]; then - ./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == s390* ]]; then - ./configure --disable-shared --host=s390x-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == loongarch* ]]; then - ./configure --disable-shared --host=loongarch64-unknown-linux-gnu --prefix="$INSTALL_DIR" + CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 \ + -B. + else + if [[ "$ARCH" == aarch_64 ]]; then + GCC_ARCH=aarch64-linux-gnu + elif [[ "$ARCH" == ppcle_64 ]]; then + GCC_ARCH=powerpc64le-linux-gnu + elif [[ "$ARCH" == s390_64 ]]; then + GCC_ARCH=s390x-linux-gnu + elif [[ "$ARCH" == loongarch_64 ]]; then + GCC_ARCH=loongarch64-unknown-linux-gnu + else + echo "Unknown architecture: $ARCH" + exit 1 + fi + cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ + -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake \ + -B. fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - make clean - make V=0 -j$NUM_CPU - make install + export CMAKE_BUILD_PARALLEL_LEVEL="$NUM_CPU" + cmake --build . + cmake --install . + [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib" popd fi @@ -60,7 +83,7 @@ ln -s "$INSTALL_DIR" /tmp/protobuf cat < - def value = System.getenv(env) - if (value != null) { - value.split(' +').each() { it -> argList.add(it) } - } -} - -// Adds corresponding "-l" option to the argList if libName is not found in -// LDFLAGS. This is only used for Mac because when building for uploadArchives -// artifacts, we add the ".a" files directly to LDFLAGS and without "-l" in -// order to get statically linked, otherwise we add the libraries through "-l" -// so that they can be searched for in default search paths. -def addLibraryIfNotLinked = { libName, argList -> - def ldflags = System.env.LDFLAGS - if (ldflags == null || !ldflags.contains('lib' + libName + '.a')) { - argList.add('-l' + libName) - } -} - -def String arch = rootProject.hasProperty('targetArch') ? rootProject.targetArch : osdetector.arch -def boolean vcDisable = rootProject.hasProperty('vcDisable') ? rootProject.vcDisable : false -def boolean usingVisualCpp // Whether VisualCpp is actually available and selected - -model { - toolChains { - // If you have both VC and Gcc installed, VC will be selected, unless you - // set 'vcDisable=true' - if (!vcDisable) { - visualCpp(VisualCpp) { - // Prefer vcvars-provided environment over registry-discovered environment - def String vsDir = System.getenv("VSINSTALLDIR") - def String winDir = System.getenv("WindowsSdkDir") - if (vsDir != null && winDir != null) { - installDir = vsDir - windowsSdkDir = winDir - } - } - } - gcc(Gcc) { - target("ppcle_64") { - cppCompiler.executable = 'powerpc64le-linux-gnu-g++' - linker.executable = 'powerpc64le-linux-gnu-g++' - } - target("aarch_64") { - cppCompiler.executable = 'aarch64-linux-gnu-g++' - linker.executable = 'aarch64-linux-gnu-g++' - } - target("s390_64") { - cppCompiler.executable = 's390x-linux-gnu-g++' - linker.executable = 's390x-linux-gnu-g++' - } - target("loongarch_64") - } - clang(Clang) { - target("aarch_64") {} - } - } - - platforms { - x86_32 { architecture "x86" } - x86_64 { architecture "x86_64" } - ppcle_64 { architecture "ppcle_64" } - aarch_64 { architecture "aarch_64" } - s390_64 { architecture "s390_64" } - loongarch_64 { architecture "loongarch_64" } - } - - components { - java_plugin(NativeExecutableSpec) { - if (arch in [ - 'x86_32', - 'x86_64', - 'ppcle_64', - 'aarch_64', - 's390_64', - 'loongarch_64' - ]) { - // If arch is not within the defined platforms, we do not specify the - // targetPlatform so that Gradle will choose what is appropriate. - targetPlatform arch - } - baseName "$protocPluginBaseName" - } - } - - binaries { - all { - if (toolChain in Gcc || toolChain in Clang) { - cppCompiler.define("GRPC_VERSION", version) - cppCompiler.args "--std=c++0x" - addEnvArgs("CXXFLAGS", cppCompiler.args) - addEnvArgs("CPPFLAGS", cppCompiler.args) - if (osdetector.os == "osx") { - cppCompiler.args "-mmacosx-version-min=10.7", "-stdlib=libc++" - addLibraryIfNotLinked('protoc', linker.args) - addLibraryIfNotLinked('protobuf', linker.args) - } else if (osdetector.os == "windows") { - linker.args "-static", "-lprotoc", "-lprotobuf", "-static-libgcc", "-static-libstdc++", - "-s" - } else if (osdetector.arch == "ppcle_64") { - linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-Wl,-Bdynamic", "-lpthread", "-s" - } else { - // Link protoc, protobuf, libgcc and libstdc++ statically. - // Link other (system) libraries dynamically. - // Clang under OSX doesn't support these options. - linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-static-libgcc", - "-Wl,-Bdynamic", "-lpthread", "-s" - } - addEnvArgs("LDFLAGS", linker.args) - } else if (toolChain in VisualCpp) { - usingVisualCpp = true - cppCompiler.define("GRPC_VERSION", version) - cppCompiler.args "/EHsc", "/MT" - if (rootProject.hasProperty('vcProtobufInclude')) { - cppCompiler.args "/I${rootProject.vcProtobufInclude}" - } - linker.args "libprotobuf.lib", "libprotoc.lib" - if (rootProject.hasProperty('vcProtobufLibs')) { - linker.args "/LIBPATH:${rootProject.vcProtobufLibs}" - } - } - } - } -} - -sourceSets { - testLite { - proto { setSrcDirs(['src/test/proto']) } - } -} - -dependencies { - testImplementation project(':grpc-protobuf'), - project(':grpc-stub'), - libraries.javax.annotation - testLiteImplementation project(':grpc-protobuf-lite'), - project(':grpc-stub'), - libraries.javax.annotation -} - -tasks.named("compileTestJava").configure { - options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" -} - -tasks.named("compileTestLiteJava").configure { - options.compilerArgs = compileTestJava.options.compilerArgs - options.compilerArgs += [ - "-Xlint:-cast" - ] - options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" -} - -tasks.named("checkstyleTestLite").configure { - enabled = false -} - -protobuf { - protoc { - if (project.hasProperty('protoc')) { - path = project.protoc - } else { - artifact = libs.protobuf.protoc.get() - } - } - plugins { - grpc { path = javaPluginPath } - } - generateProtoTasks { - all().configureEach { - dependsOn 'java_pluginExecutable' - inputs.file javaPluginPath - } - ofSourceSet('test').configureEach { - plugins { grpc {} } - } - ofSourceSet('testLite').configureEach { - builtins { - java { option 'lite' } - } - plugins { - grpc { - option 'lite' - option '@generated=omit' - } - } - } - } -} - -println "*** Building codegen requires Protobuf" -println "*** Please refer to https://github.com/grpc/grpc-java/blob/master/COMPILING.md#how-to-build-code-generation-plugin" - -tasks.register("buildArtifacts", Copy) { - dependsOn 'java_pluginExecutable' - from("$buildDir/exe") { - if (osdetector.os != 'windows') { - rename 'protoc-gen-grpc-java', '$0.exe' - } - } - into artifactStagingPath -} - -base { - archivesName = "$protocPluginBaseName" -} - -def checkArtifacts = tasks.register("checkArtifacts") { - dependsOn buildArtifacts - doLast { - if (!usingVisualCpp) { - def ret = exec { - executable 'bash' - args 'check-artifact.sh', osdetector.os, arch - } - if (ret.exitValue != 0) { - throw new GradleException("check-artifact.sh exited with " + ret.exitValue) - } - } else { - def exeName = "$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" - def os = new ByteArrayOutputStream() - def ret = exec { - executable 'dumpbin' - args '/nologo', '/dependents', exeName - standardOutput = os - } - if (ret.exitValue != 0) { - throw new GradleException("dumpbin exited with " + ret.exitValue) - } - def dlls = os.toString() =~ /Image has the following dependencies:\s+(.*)\s+Summary/ - if (dlls[0][1] != "KERNEL32.dll") { - throw new Exception("unexpected dll deps: " + dlls[0][1]); - } - os.reset() - ret = exec { - executable 'dumpbin' - args '/nologo', '/headers', exeName - standardOutput = os - } - if (ret.exitValue != 0) { - throw new GradleException("dumpbin exited with " + ret.exitValue) - } - def machine = os.toString() =~ / machine \(([^)]+)\)/ - def expectedArch = [x86_32: "x86", x86_64: "x64"][arch] - if (machine[0][1] != expectedArch) { - throw new Exception("unexpected architecture: " + machine[0][1]); - } - } - } -} - -// Exe files are skipped by Maven by default. Override it. -// Also skip jar files that is generated by the java plugin. -publishing { - publications { - maven(MavenPublication) { - // Removes all artifacts since grpc-compiler doesn't generates any Jar - artifacts = [] - artifactId 'protoc-gen-grpc-java' - artifact("$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" as File) { - classifier osdetector.os + "-" + arch - extension "exe" - builtBy checkArtifacts - } - pom.withXml { - // This isn't any sort of Java archive artifact, and OSSRH doesn't enforce - // javadoc for 'pom' packages. 'exe' would be a more appropriate packaging - // value, but it isn't clear how that will be interpreted. In addition, - // 'pom' is typically the value used when building an exe with Maven. - asNode().project.packaging*.value = 'pom' - } - } - } -} - -def configureTestTask(Task task, String dep, String serviceName) { - def genDir = files(tasks.named("generateTest${dep}Proto")).singleFile - def genFile = "${genDir}/grpc/io/grpc/testing/compiler/${serviceName}Grpc.java" - task.dependsOn "generateTest${dep}Proto" - task.inputs.file genFile - if (osdetector.os != 'windows') { - task.executable "diff" - task.args "-u" - } else { - task.executable "fc" - } - task.args layout.projectDirectory.file("src/test${dep}/golden/${serviceName}.java.txt") - task.args genFile - // Register an output to allow up-to-date checking - task.outputs.file(layout.buildDirectory.file(task.name)) -} - -def testGolden = tasks.register("testGolden", Exec) { - configureTestTask(it, '', 'TestService') -} -def testLiteGolden = tasks.register("testLiteGolden", Exec) { - configureTestTask(it, 'Lite', 'TestService') -} -def testDeprecatedGolden = tasks.register("testDeprecatedGolden", Exec) { - configureTestTask(it, '', 'TestDeprecatedService') -} -def testDeprecatedLiteGolden = tasks.register("testDeprecatedLiteGolden", Exec) { - configureTestTask(it, 'Lite', 'TestDeprecatedService') -} -tasks.named("test").configure { - dependsOn testGolden - dependsOn testLiteGolden - dependsOn testDeprecatedGolden - dependsOn testDeprecatedLiteGolden -} +plugins { + id "cpp" + id "java" + id "maven-publish" + + id "com.google.protobuf" +} + +description = 'The protoc plugin for gRPC Java' + +def artifactStagingPath = "$buildDir/artifacts" as File +// Adds space-delimited arguments from the environment variable env to the +// argList. +def addEnvArgs = { env, argList -> + def value = System.getenv(env) + if (value != null) { + value.split(' +').each() { it -> argList.add(it) } + } +} + +// Adds corresponding "-l" option to the argList if libName is not found in +// LDFLAGS. This is only used for Mac because when building for uploadArchives +// artifacts, we add the ".a" files directly to LDFLAGS and without "-l" in +// order to get statically linked, otherwise we add the libraries through "-l" +// so that they can be searched for in default search paths. +def addLibraryIfNotLinked = { libName, argList -> + def ldflags = System.env.LDFLAGS + if (ldflags == null || !ldflags.contains('lib' + libName + '.a')) { + argList.add('-l' + libName) + } +} + +def String arch = rootProject.hasProperty('targetArch') ? rootProject.targetArch : osdetector.arch +def boolean vcDisable = rootProject.hasProperty('vcDisable') ? rootProject.vcDisable : false +def boolean usingVisualCpp // Whether VisualCpp is actually available and selected + +model { + toolChains { + // If you have both VC and Gcc installed, VC will be selected, unless you + // set 'vcDisable=true' + if (!vcDisable) { + visualCpp(VisualCpp) { + // Prefer vcvars-provided environment over registry-discovered environment + def String vsDir = System.getenv("VSINSTALLDIR") + def String winDir = System.getenv("WindowsSdkDir") + if (vsDir != null && winDir != null) { + installDir = vsDir + windowsSdkDir = winDir + } + } + } + gcc(Gcc) { + target("ppcle_64") { + cppCompiler.executable = 'powerpc64le-linux-gnu-g++' + linker.executable = 'powerpc64le-linux-gnu-g++' + } + target("aarch_64") { + cppCompiler.executable = 'aarch64-linux-gnu-g++' + linker.executable = 'aarch64-linux-gnu-g++' + } + target("s390_64") { + cppCompiler.executable = 's390x-linux-gnu-g++' + linker.executable = 's390x-linux-gnu-g++' + } + target("loongarch_64") + } + clang(Clang) { + target("aarch_64") {} + } + } + + platforms { + x86_32 { architecture "x86" } + x86_64 { architecture "x86_64" } + ppcle_64 { architecture "ppcle_64" } + aarch_64 { architecture "aarch_64" } + s390_64 { architecture "s390_64" } + loongarch_64 { architecture "loongarch_64" } + } + + components { + java_plugin(NativeExecutableSpec) { + if (arch in [ + 'x86_32', + 'x86_64', + 'ppcle_64', + 'aarch_64', + 's390_64', + 'loongarch_64' + ]) { + // If arch is not within the defined platforms, we do not specify the + // targetPlatform so that Gradle will choose what is appropriate. + targetPlatform arch + } + baseName "$protocPluginBaseName" + } + } + + binaries { + all { + if (toolChain in Gcc || toolChain in Clang) { + cppCompiler.define("GRPC_VERSION", version) + cppCompiler.args "--std=c++0x" + addEnvArgs("CXXFLAGS", cppCompiler.args) + addEnvArgs("CPPFLAGS", cppCompiler.args) + if (osdetector.os == "osx") { + cppCompiler.args "-mmacosx-version-min=10.7", "-stdlib=libc++" + addLibraryIfNotLinked('protoc', linker.args) + addLibraryIfNotLinked('protobuf', linker.args) + } else if (osdetector.os == "windows") { + linker.args "-static", "-lprotoc", "-lprotobuf", "-static-libgcc", "-static-libstdc++", + "-s" + } else if (osdetector.arch == "ppcle_64") { + linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-Wl,-Bdynamic", "-lpthread", "-s" + } else { + // Link protoc, protobuf, libgcc and libstdc++ statically. + // Link other (system) libraries dynamically. + // Clang under OSX doesn't support these options. + linker.args "-Wl,-Bstatic", "-lprotoc", "-lprotobuf", "-static-libgcc", + "-Wl,-Bdynamic", "-lpthread", "-s" + } + addEnvArgs("LDFLAGS", linker.args) + } else if (toolChain in VisualCpp) { + usingVisualCpp = true + cppCompiler.define("GRPC_VERSION", version) + cppCompiler.args "/EHsc", "/MT" + if (rootProject.hasProperty('vcProtobufInclude')) { + cppCompiler.args "/I${rootProject.vcProtobufInclude}" + } + if (rootProject.hasProperty('vcProtobufLibs')) { + String libsList = rootProject.property('vcProtobufLibs') as String + linker.args.add("libprotoc.lib") + linker.args.add("libprotobuf.lib") + libsList.split(',').each() { lib -> linker.args.add(lib) } + } + if (rootProject.hasProperty('vcProtobufLibPaths')) { + linker.args "/LIBPATH:${rootProject.vcProtobufLibPaths}" + } + } + } + } +} + +sourceSets { + testLite { + proto { setSrcDirs(['src/test/proto']) } + } +} + +dependencies { + testImplementation project(':grpc-protobuf'), + project(':grpc-stub'), + libraries.javax.annotation + testLiteImplementation project(':grpc-protobuf-lite'), + project(':grpc-stub'), + libraries.javax.annotation +} + +tasks.named("compileTestJava").configure { + options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" +} + +tasks.named("compileTestLiteJava").configure { + options.compilerArgs = compileTestJava.options.compilerArgs + options.compilerArgs += [ + "-Xlint:-cast" + ] + options.errorprone.excludedPaths = ".*/build/generated/source/proto/.*" +} + +tasks.named("checkstyleTestLite").configure { + enabled = false +} + +protobuf { + protoc { + if (project.hasProperty('protoc')) { + path = project.protoc + } else { + artifact = libs.protobuf.protoc.get() + } + } + plugins { + grpc { path = javaPluginPath } + } + generateProtoTasks { + all().configureEach { + dependsOn 'java_pluginExecutable' + inputs.file javaPluginPath + } + ofSourceSet('test').configureEach { + plugins { grpc {} } + } + ofSourceSet('testLite').configureEach { + builtins { + java { option 'lite' } + } + plugins { + grpc { + option 'lite' + option '@generated=omit' + } + } + } + } +} + +println "*** Building codegen requires Protobuf" +println "*** Please refer to https://github.com/grpc/grpc-java/blob/master/COMPILING.md#how-to-build-code-generation-plugin" + +tasks.register("buildArtifacts", Copy) { + dependsOn 'java_pluginExecutable' + from("$buildDir/exe") { + if (osdetector.os != 'windows') { + rename 'protoc-gen-grpc-java', '$0.exe' + } + } + into artifactStagingPath +} + +base { + archivesName = "$protocPluginBaseName" +} + +def checkArtifacts = tasks.register("checkArtifacts") { + dependsOn buildArtifacts + doLast { + if (!usingVisualCpp) { + def ret = exec { + executable 'bash' + args 'check-artifact.sh', osdetector.os, arch + } + if (ret.exitValue != 0) { + throw new GradleException("check-artifact.sh exited with " + ret.exitValue) + } + } else { + def exeName = "$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" + def os = new ByteArrayOutputStream() + def ret = exec { + executable 'dumpbin' + args '/nologo', '/dependents', exeName + standardOutput = os + } + if (ret.exitValue != 0) { + throw new GradleException("dumpbin exited with " + ret.exitValue) + } + def dlls = os.toString() =~ /Image has the following dependencies:\s+(.*)\s+Summary/ + if (dlls[0][1] != "KERNEL32.dll") { + throw new Exception("unexpected dll deps: " + dlls[0][1]); + } + os.reset() + ret = exec { + executable 'dumpbin' + args '/nologo', '/headers', exeName + standardOutput = os + } + if (ret.exitValue != 0) { + throw new GradleException("dumpbin exited with " + ret.exitValue) + } + def machine = os.toString() =~ / machine \(([^)]+)\)/ + def expectedArch = [x86_32: "x86", x86_64: "x64"][arch] + if (machine[0][1] != expectedArch) { + throw new Exception("unexpected architecture: " + machine[0][1]); + } + } + } +} + +// Exe files are skipped by Maven by default. Override it. +// Also skip jar files that is generated by the java plugin. +publishing { + publications { + maven(MavenPublication) { + // Removes all artifacts since grpc-compiler doesn't generates any Jar + artifacts = [] + artifactId 'protoc-gen-grpc-java' + artifact("$artifactStagingPath/java_plugin/${protocPluginBaseName}.exe" as File) { + classifier osdetector.os + "-" + arch + extension "exe" + builtBy checkArtifacts + } + pom.withXml { + // This isn't any sort of Java archive artifact, and OSSRH doesn't enforce + // javadoc for 'pom' packages. 'exe' would be a more appropriate packaging + // value, but it isn't clear how that will be interpreted. In addition, + // 'pom' is typically the value used when building an exe with Maven. + asNode().project.packaging*.value = 'pom' + } + } + } +} + +def configureTestTask(Task task, String dep, String serviceName) { + def genDir = files(tasks.named("generateTest${dep}Proto")).singleFile + def genFile = "${genDir}/grpc/io/grpc/testing/compiler/${serviceName}Grpc.java" + task.dependsOn "generateTest${dep}Proto" + task.inputs.file genFile + if (osdetector.os != 'windows') { + task.executable "diff" + task.args "-u" + } else { + task.executable "fc" + } + task.args layout.projectDirectory.file("src/test${dep}/golden/${serviceName}.java.txt") + task.args genFile + // Register an output to allow up-to-date checking + task.outputs.file(layout.buildDirectory.file(task.name)) +} + +def testGolden = tasks.register("testGolden", Exec) { + configureTestTask(it, '', 'TestService') +} +def testLiteGolden = tasks.register("testLiteGolden", Exec) { + configureTestTask(it, 'Lite', 'TestService') +} +def testDeprecatedGolden = tasks.register("testDeprecatedGolden", Exec) { + configureTestTask(it, '', 'TestDeprecatedService') +} +def testDeprecatedLiteGolden = tasks.register("testDeprecatedLiteGolden", Exec) { + configureTestTask(it, 'Lite', 'TestDeprecatedService') +} +tasks.named("test").configure { + dependsOn testGolden + dependsOn testLiteGolden + dependsOn testDeprecatedGolden + dependsOn testDeprecatedLiteGolden +} diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh index 4d0c2fa6286..12d7709a2a8 100755 --- a/compiler/check-artifact.sh +++ b/compiler/check-artifact.sh @@ -114,7 +114,7 @@ checkDependencies () white_list="KERNEL32\.dll\|msvcrt\.dll\|USER32\.dll" elif [[ "$OS" == linux ]]; then dump_cmd='objdump -x '"$1"' | grep "NEEDED"' - white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6" + white_list="libpthread\.so\.0\|libstdc++\.so\.6\|libc\.so\.6\|librt\.so\.1\|libm\.so\.6" if [[ "$ARCH" == x86_32 ]]; then white_list="${white_list}\|libm\.so\.6" elif [[ "$ARCH" == x86_64 ]]; then