diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 1e0cd10155e..0d4ab1998e8 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -35,87 +35,46 @@ concurrency: cancel-in-progress: true jobs: - build-on-centos-7: - strategy: - fail-fast: false - if: ${{ startsWith(github.repository, 'apache/') }} - runs-on: ubuntu-22.04 - container: centos:7 - steps: - - name: Build Gluten native libraries and package - run: | - sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true - sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true - yum update -y - yum install -y epel-release sudo dnf - yum install -y centos-release-scl - rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f - sed -i \ - -e 's/^mirrorlist/#mirrorlist/' \ - -e 's/^#baseurl/baseurl/' \ - -e 's/mirror\.centos\.org/vault.centos.org/' \ - /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo - yum install -y devtoolset-11 - source /opt/rh/devtoolset-11/enable - yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ - export PATH=$JAVA_HOME/bin:$PATH - - # action/checkout does not work centos7 anymore, so we clone the branch instead. - git clone -b main --depth=1 https://github.com/apache/gluten.git && cd gluten/ - if [ ${{ github.event_name }} = "pull_request" ]; then - git fetch origin ${{ github.ref }}:pr_branch && git checkout pr_branch - fi - ./dev/package.sh --spark_version=3.5 - build-on-centos: + weekly-build-on-centos: strategy: fail-fast: false matrix: os: [ "centos:8", "quay.io/centos/centos:stream9" ] - if: ${{ startsWith(github.repository, 'apache/') }} + runs-on: ubuntu-22.04 container: ${{ matrix.os }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Update mirror list - run: | - sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true - sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: build run: | - yum update -y - yum install -y epel-release sudo dnf - if [ "${{ matrix.os }}" = "centos:7" ]; then - yum install -y centos-release-scl - rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f - sed -i \ - -e 's/^mirrorlist/#mirrorlist/' \ - -e 's/^#baseurl/baseurl/' \ - -e 's/mirror\.centos\.org/vault.centos.org/' \ - /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo - yum install -y devtoolset-11 - source /opt/rh/devtoolset-11/enable - elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then - dnf install -y --setopt=install_weak_deps=False gcc-toolset-12 - source /opt/rh/gcc-toolset-12/enable || exit 1 - else - dnf install -y --setopt=install_weak_deps=False gcc-toolset-11 - source /opt/rh/gcc-toolset-11/enable || exit 1 + if [ "${{ matrix.os }}" = "centos:8" ]; then + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + dnf update -y + dnf install -y centos-release-stream + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true + dnf swap -y --allowerasing centos-{linux,stream}-repos + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true fi + dnf install -y epel-release sudo dnf + dnf install -y --setopt=install_weak_deps=False gcc-toolset-12 + source /opt/rh/gcc-toolset-12/enable || exit 1 yum install -y java-1.8.0-openjdk-devel patch wget git perl export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ export PATH=$JAVA_HOME/bin:$PATH - + export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 - build-on-ubuntu: + weekly-build-on-ubuntu: strategy: fail-fast: false matrix: - os: [ "ubuntu:22.04" ] # skip "ubuntu:20.04" due to not enough disk space - if: ${{ startsWith(github.repository, 'apache/') }} + os: [ "ubuntu:22.04" ] + runs-on: ubuntu-22.04 container: ${{ matrix.os }} steps: @@ -125,35 +84,42 @@ jobs: # sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list # To avoid the prompt for region selection during installing tzdata. export DEBIAN_FRONTEND=noninteractive - apt-get update && apt-get install -y sudo maven wget git curl - if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then - sudo apt install -y software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install -y gcc-11 g++-11 build-essential cmake - export CC=/usr/bin/gcc-11 - export CXX=/usr/bin/g++-11 - wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz - tar xf v0.8.1.tar.gz - cd xxHash-0.8.1/ - make && make install - cd /tmp - wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz - tar xf v1.13.0.tar.gz && cd googletest-1.13.0/ - export CXXFLAGS=-fPIC - cmake . && make install -j - apt autoremove -y - rm -rf /tmp/* - fi - sudo apt-get install -y openjdk-8-jdk python3-pip cmake + export INSTALL_PREFIX=/usr/ + apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake + cd /tmp + wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz + tar -xzf gflags.tar.gz && cd gflags-2.2.2 + cmake -B build \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_STATIC_LIBS=ON \ + -DBUILD_gflags_LIB=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=Release + cmake --build build -j4 --target install + wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz + tar xf v0.8.1.tar.gz + cd xxHash-0.8.1/ + make && make install + cd /tmp + wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz + tar xf v1.13.0.tar.gz && cd googletest-1.13.0/ + export CXXFLAGS=-fPIC + cmake . && make install -j + apt autoremove -y + rm -rf /tmp/* + apt-get install -y openjdk-8-jdk python3-pip + pip3 install cmake==3.31.4 export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 - build-on-openeuler: + weekly-build-on-openeuler: strategy: fail-fast: false matrix: os: [ "openeuler/openeuler:24.03" ] - if: ${{ startsWith(github.repository, 'apache/') }} + runs-on: ubuntu-24.04-arm container: ${{ matrix.os }} steps: @@ -177,12 +143,12 @@ jobs: $TPCH_TEST --extra-conf=spark.gluten.loadLibFromJar=true \ --extra-conf=spark.gluten.loadLibOS=openEuler --extra-conf=spark.gluten.loadLibOSVersion=24.03 - build-on-openeuler-vcpkg: + weekly-build-on-openeuler-vcpkg: strategy: fail-fast: false matrix: os: [ "openeuler/openeuler:24.03" ] - if: ${{ startsWith(github.repository, 'apache/') }} + runs-on: ubuntu-24.04-arm container: ${{ matrix.os }} steps: @@ -205,10 +171,10 @@ jobs: cd $GITHUB_WORKSPACE/tools/gluten-it $TPCH_TEST - build-on-macos: + weekly-build-on-macos: strategy: fail-fast: false - if: ${{ startsWith(github.repository, 'apache/') }} + runs-on: macos-15 env: INSTALL_PREFIX: /tmp/deps-install diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh index e0968b316d2..4374b995796 100755 --- a/dev/build-thirdparty.sh +++ b/dev/build-thirdparty.sh @@ -38,7 +38,9 @@ function process_setup_ubuntu_2204 { function process_setup_centos_9 { cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/ cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/ - cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/ + cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/ + find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null + find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } function process_setup_centos_8 { diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh index 473f8260aac..be164957c1f 100755 --- a/dev/vcpkg/setup-build-depends.sh +++ b/dev/vcpkg/setup-build-depends.sh @@ -202,7 +202,7 @@ install_openeuler_24.03() { gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \ java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static - pip install cmake==3.28.3 + pip install cmake==3.31.4 install_maven_from_source } diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 9841157539a..3aa6f922a9b 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -39,8 +39,8 @@ export CPPFLAGS=$CFLAGS # Used by LZO. CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" BUILD_DUCKDB="${BUILD_DUCKDB:-true}" BUILD_GEOS="${BUILD_GEOS:-true}" -export CC=/opt/rh/gcc-toolset-11/root/bin/gcc -export CXX=/opt/rh/gcc-toolset-11/root/bin/g++ +export CC=/opt/rh/gcc-toolset-12/root/bin/gcc +export CXX=/opt/rh/gcc-toolset-12/root/bin/g++ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} FB_OS_VERSION="v2026.01.05.00" @@ -57,9 +57,10 @@ function dnf_install { function install_build_prerequisites { dnf update -y dnf_install epel-release dnf-plugins-core # For ccache, ninja - dnf config-manager --set-enabled powertools + dnf config-manager --set-enabled powertools || true # For centos8, powertools is needed for some dependencies. + dnf config-manager --set-enabled Stream-PowerTools || true dnf update -y - dnf_install ninja-build curl ccache gcc-toolset-11 git wget which expat-devel gettext-devel + dnf_install ninja-build curl ccache gcc-toolset-12 git wget which expat-devel gettext-devel dnf_install yasm dnf_install autoconf automake python39 python39-devel python39-pip libtool pip3.9 install cmake==3.28.3 @@ -172,7 +173,10 @@ function install_wangle { function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + cd ${DEPENDENCY_DIR}/fbthrift + git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true + cd ${DEPENDENCY_DIR} + cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } function install_mvfst { @@ -228,7 +232,7 @@ function install_velox_deps { ( if [[ $# -ne 0 ]]; then # Activate gcc11; enable errors on unset variables afterwards. - source /opt/rh/gcc-toolset-11/enable || exit 1 + source /opt/rh/gcc-toolset-12/enable || exit 1 set -u for cmd in "$@"; do run_and_time "${cmd}" @@ -242,7 +246,7 @@ function install_velox_deps { echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" fi # Activate gcc11; enable errors on unset variables afterwards. - source /opt/rh/gcc-toolset-11/enable || exit 1 + source /opt/rh/gcc-toolset-12/enable || exit 1 set -u install_velox_deps echo "All dependencies for Velox installed!" diff --git a/ep/build-velox/src/setup-openeuler24.sh b/ep/build-velox/src/setup-openeuler24.sh index 29204bbbf8e..80a66917b22 100755 --- a/ep/build-velox/src/setup-openeuler24.sh +++ b/ep/build-velox/src/setup-openeuler24.sh @@ -36,6 +36,7 @@ NPROC=$(getconf _NPROCESSORS_ONLN) export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. export CXXFLAGS=$CFLAGS # Used by boost. export CPPFLAGS=$CFLAGS # Used by LZO. +EXTRA_PKG_CXXFLAGS=" -isystem ${INSTALL_PREFIX}/include" CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so. BUILD_DUCKDB="${BUILD_DUCKDB:-true}" @@ -44,8 +45,8 @@ VERSION=$(cat /etc/os-release | grep VERSION_ID) export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} -FB_OS_VERSION="v2024.07.01.00" -FMT_VERSION="10.1.1" +FB_OS_VERSION="v2026.01.05.00" +FMT_VERSION="11.2.0" BOOST_VERSION="boost-1.84.0" DUCKDB_VERSION="v0.8.1" GEOS_VERSION="3.10.7" @@ -65,7 +66,7 @@ function install_build_prerequisites { dnf_install autoconf automake python3-devel python3-pip libtool dnf_install libxml2-devel libgsasl-devel libuuid-devel - pip install cmake==3.28.3 + pip install cmake==3.31.4 } # Install dependencies from the package managers. @@ -110,11 +111,23 @@ function install_boost { ) } +function install_xxhash { + wget_and_untar https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.2.tar.gz xxHash + cd ${DEPENDENCY_DIR}/xxHash + make "-j${NPROC}" + make install PREFIX=${INSTALL_PREFIX} +} + function install_snappy { wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF } +function install_fast_float { + wget_and_untar https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.2.tar.gz fast_float + cmake_install_dir fast_float -DBUILD_TESTS=OFF +} + function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt cmake_install_dir fmt -DFMT_TEST=OFF @@ -133,7 +146,7 @@ function install_protobuf { function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF + cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF } function install_folly { @@ -143,11 +156,14 @@ function install_folly { function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF + cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF } function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + cd ${DEPENDENCY_DIR}/fbthrift + git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true + cd ${DEPENDENCY_DIR} cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } @@ -198,8 +214,10 @@ function install_velox_deps { run_and_time install_gflags run_and_time install_glog run_and_time install_lzo + run_and_time install_xxhash run_and_time install_snappy run_and_time install_boost + run_and_time install_fast_float run_and_time install_protobuf run_and_time install_fmt run_and_time install_folly @@ -233,4 +251,4 @@ function install_velox_deps { echo "All dependencies for Velox installed!" dnf clean all fi -) +) \ No newline at end of file