Fix build issues reported by weekly jobs

.github/workflows/velox_weekly.yml

@@ -35,45 +35,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-on-centos-7:
-    strategy:
-      fail-fast: false
-    if: ${{ startsWith(github.repository, 'apache/') }}
-    runs-on: ubuntu-22.04
-    container: centos:7
-    steps:
-      - name: Build Gluten native libraries and package
-        run: |
-          sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true
-          sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true
-          yum update -y
-          yum install -y epel-release sudo dnf
-          yum install -y centos-release-scl
-          rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
-          sed -i \
-          -e 's/^mirrorlist/#mirrorlist/' \
-          -e 's/^#baseurl/baseurl/' \
-          -e 's/mirror\.centos\.org/vault.centos.org/' \
-          /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-          yum install -y devtoolset-11
-          source /opt/rh/devtoolset-11/enable
-          yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip
-          export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
-          export PATH=$JAVA_HOME/bin:$PATH
-
-          # action/checkout does not work centos7 anymore, so we clone the branch instead.
-          git clone -b main --depth=1 https://github.com/apache/gluten.git && cd gluten/
-          if [ ${{ github.event_name }} = "pull_request" ]; then
-            git fetch origin ${{ github.ref }}:pr_branch && git checkout pr_branch
-          fi
-          ./dev/package.sh --spark_version=3.5
 
-  build-on-centos:
+  weekly-build-on-centos:
     strategy:
       fail-fast: false
       matrix:
         os: [ "centos:8", "quay.io/centos/centos:stream9" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
@@ -87,17 +55,7 @@ jobs:
         run: |
           yum update -y
           yum install -y epel-release sudo dnf
-          if [ "${{ matrix.os }}" = "centos:7" ]; then
-            yum install -y centos-release-scl
-            rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
-            sed -i \
-            -e 's/^mirrorlist/#mirrorlist/' \
-            -e 's/^#baseurl/baseurl/' \
-            -e 's/mirror\.centos\.org/vault.centos.org/' \
-            /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-            yum install -y devtoolset-11
-            source /opt/rh/devtoolset-11/enable
-          elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
+          if [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
             dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
             source /opt/rh/gcc-toolset-12/enable || exit 1
           else
@@ -107,15 +65,15 @@ jobs:
           yum install -y java-1.8.0-openjdk-devel patch wget git perl
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
           export PATH=$JAVA_HOME/bin:$PATH
-
+          export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
 
-  build-on-ubuntu:
+  weekly-build-on-ubuntu:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu:22.04" ] # skip "ubuntu:20.04" due to not enough disk space
-    if: ${{ startsWith(github.repository, 'apache/') }}
+        os: [ "ubuntu:22.04" ]
+
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
@@ -125,35 +83,40 @@ jobs:
           # sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list
           # To avoid the prompt for region selection during installing tzdata.
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update && apt-get install -y sudo maven wget git curl
-          if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then
-            sudo apt install -y software-properties-common
-            sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-            sudo apt update && sudo apt install -y gcc-11 g++-11 build-essential cmake
-            export CC=/usr/bin/gcc-11
-            export CXX=/usr/bin/g++-11
-            wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
-            tar xf v0.8.1.tar.gz
-            cd xxHash-0.8.1/
-            make && make install
-            cd /tmp
-            wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
-            tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
-            export CXXFLAGS=-fPIC
-            cmake . && make install -j
-            apt autoremove -y
-            rm -rf /tmp/*
-          fi
+          apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake
+          cd /tmp
+          wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz
+          tar -xzf gflags.tar.gz && cd gflags-2.2.2
+          cmake -B build \
+            -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+            -DBUILD_SHARED_LIBS=ON \
+            -DBUILD_STATIC_LIBS=ON \
+            -DBUILD_gflags_LIB=ON \
+            -DCMAKE_INSTALL_PREFIX=/usr/local \
+            -DCMAKE_BUILD_TYPE=Release
+          sudo cmake --build build -j4 --target install
+          wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
+          tar xf v0.8.1.tar.gz
+          cd xxHash-0.8.1/
+          make && make install
+          cd /tmp
+          wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
+          tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
+          export CXXFLAGS=-fPIC
+          cmake . && make install -j
+          apt autoremove -y
+          rm -rf /tmp/*
           sudo apt-get install -y openjdk-8-jdk python3-pip cmake
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+          export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
 
-  build-on-openeuler:
+  weekly-build-on-openeuler:
     strategy:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -177,12 +140,12 @@ jobs:
           $TPCH_TEST --extra-conf=spark.gluten.loadLibFromJar=true \
             --extra-conf=spark.gluten.loadLibOS=openEuler --extra-conf=spark.gluten.loadLibOSVersion=24.03
 
-  build-on-openeuler-vcpkg:
+  weekly-build-on-openeuler-vcpkg:
     strategy:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -205,10 +168,10 @@ jobs:
           cd $GITHUB_WORKSPACE/tools/gluten-it
           $TPCH_TEST
 
-  build-on-macos:
+  weekly-build-on-macos:
     strategy:
       fail-fast: false
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: macos-15
     env:
       INSTALL_PREFIX: /tmp/deps-install

dev/build-thirdparty.sh

@@ -38,7 +38,9 @@ function process_setup_ubuntu_2204 {
 function process_setup_centos_9 {
   cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/
   cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/
-  cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/
+  cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/
+  find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
+  find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
 }
 
 function process_setup_centos_8 {

dev/vcpkg/setup-build-depends.sh

@@ -202,7 +202,7 @@ install_openeuler_24.03() {
         gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
         java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static
 
-    pip install cmake==3.28.3
+    pip install cmake==3.31.4
 
     install_maven_from_source
 }

ep/build-velox/src/setup-centos8.sh

@@ -172,7 +172,10 @@ function install_wangle {
 
 function install_fbthrift {
   wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
-  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
+  cd ${DEPENDENCY_DIR}/fbthrift
+  git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true
+  cd ${DEPENDENCY_DIR}
+  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_mvfst {

ep/build-velox/src/setup-openeuler24.sh

@@ -36,6 +36,7 @@ NPROC=$(getconf _NPROCESSORS_ONLN)
 export CFLAGS=$(get_cxx_flags $CPU_TARGET)  # Used by LZO.
 export CXXFLAGS=$CFLAGS  # Used by boost.
 export CPPFLAGS=$CFLAGS  # Used by LZO.
+EXTRA_PKG_CXXFLAGS=" -isystem ${INSTALL_PREFIX}/include"
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so.
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
@@ -44,8 +45,8 @@ VERSION=$(cat /etc/os-release | grep VERSION_ID)
 export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
-FB_OS_VERSION="v2024.07.01.00"
-FMT_VERSION="10.1.1"
+FB_OS_VERSION="v2026.01.05.00"
+FMT_VERSION="11.2.0"
 BOOST_VERSION="boost-1.84.0"
 DUCKDB_VERSION="v0.8.1"
 GEOS_VERSION="3.10.7"
@@ -65,7 +66,7 @@ function install_build_prerequisites {
   dnf_install autoconf automake python3-devel python3-pip libtool
   dnf_install libxml2-devel libgsasl-devel libuuid-devel
 
-  pip install cmake==3.28.3
+  pip install cmake==3.31.4
 }
 
 # Install dependencies from the package managers.
@@ -110,11 +111,23 @@ function install_boost {
   )
 }
 
+function install_xxhash {
+  wget_and_untar https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.2.tar.gz xxHash
+  cd ${DEPENDENCY_DIR}/xxHash
+  make "-j${NPROC}"
+  make install PREFIX=${INSTALL_PREFIX}
+}
+
 function install_snappy {
   wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
   cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF
 }
 
+function install_fast_float {
+  wget_and_untar https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.2.tar.gz fast_float
+  cmake_install_dir fast_float -DBUILD_TESTS=OFF
+}
+
 function install_fmt {
   wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
   cmake_install_dir fmt -DFMT_TEST=OFF
@@ -133,7 +146,7 @@ function install_protobuf {
 
 function install_fizz {
   wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
-  cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF
+  cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF
 }
 
 function install_folly {
@@ -143,11 +156,14 @@ function install_folly {
 
 function install_wangle {
   wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
-  cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF
+  cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF
 }
 
 function install_fbthrift {
   wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
+  cd ${DEPENDENCY_DIR}/fbthrift
+  git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true
+  cd ${DEPENDENCY_DIR}
   cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
@@ -198,8 +214,10 @@ function install_velox_deps {
   run_and_time install_gflags
   run_and_time install_glog
   run_and_time install_lzo
+  run_and_time install_xxhash
   run_and_time install_snappy
   run_and_time install_boost
+  run_and_time install_fast_float
   run_and_time install_protobuf
   run_and_time install_fmt
   run_and_time install_folly
@@ -233,4 +251,4 @@ function install_velox_deps {
     echo "All dependencies for Velox installed!"
     dnf clean all
   fi
-)
+)