[VL] Respect INSTALL_PREFIX in Velox native builds

cpp/velox/CMakeLists.txt

@@ -413,16 +413,26 @@ if(DEFINED VCPKG_INSTALLED_DIR
     PRIVATE ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a
             ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a)
 else()
-  message(STATUS "Using system thrift libraries from /usr/local/lib")
+  set(THRIFT_LIBRARY_DIRS)
+  if(CMAKE_INSTALL_PREFIX)
+    list(APPEND THRIFT_LIBRARY_DIRS "${CMAKE_INSTALL_PREFIX}/lib")
+  endif()
+  foreach(PREFIX_PATH IN LISTS CMAKE_PREFIX_PATH)
+    list(APPEND THRIFT_LIBRARY_DIRS "${PREFIX_PATH}/lib" "${PREFIX_PATH}")
+  endforeach()
+  list(APPEND THRIFT_LIBRARY_DIRS /usr/local/lib)
+  list(REMOVE_DUPLICATES THRIFT_LIBRARY_DIRS)
+
+  message(STATUS "Searching thrift libraries in ${THRIFT_LIBRARY_DIRS}")
   find_library(
     THRIFTCPP2_LIB
     NAMES thriftcpp2
-    PATHS /usr/local/lib
+    PATHS ${THRIFT_LIBRARY_DIRS}
     NO_DEFAULT_PATH)
   find_library(
     THRIFTPROTOCOL_LIB
     NAMES thriftprotocol
-    PATHS /usr/local/lib
+    PATHS ${THRIFT_LIBRARY_DIRS}
     NO_DEFAULT_PATH)
 
   if(THRIFTCPP2_LIB AND THRIFTPROTOCOL_LIB)

dev/build-helper-functions.sh

@@ -181,6 +181,7 @@ function cmake_install {
     if [[ "${INSTALL_PREFIX:-}" == "/usr/local" || "${INSTALL_PREFIX:-}" == /usr/local/* ]]; then
       echo "INFO: INSTALL_PREFIX=${INSTALL_PREFIX} is under /usr/local; keeping /usr/local visible to CMake." >&2
     else
+      COMPILER_FLAGS="$COMPILER_FLAGS -isystem /usr/local/include"
       MACOS_ISOLATION_FLAGS="-DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON \
         -DCMAKE_IGNORE_PREFIX_PATH=/usr/local \
         -DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake \

dev/builddeps-veloxbe.sh

@@ -163,6 +163,8 @@ if [[ "$(uname)" == "Darwin" ]]; then
     if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" == /usr/local/* ]]; then
         echo "INFO: INSTALL_PREFIX=$INSTALL_PREFIX is under /usr/local; keeping /usr/local visible to CMake." >&2
     fi
+elif [ -n "${INSTALL_PREFIX:-}" ]; then
+    export INSTALL_PREFIX
 fi
 
 function concat_velox_param {
@@ -244,34 +246,39 @@ function build_gluten_cpp {
   mkdir build
   cd build
 
-  GLUTEN_CMAKE_OPTIONS="-DBUILD_VELOX_BACKEND=ON \
-    -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-    -DVELOX_HOME=$VELOX_HOME \
-    -DBUILD_TESTS=$BUILD_TESTS \
-    -DBUILD_EXAMPLES=$BUILD_EXAMPLES \
-    -DBUILD_BENCHMARKS=$BUILD_BENCHMARKS \
-    -DENABLE_JEMALLOC_STATS=$ENABLE_JEMALLOC_STATS \
-    -DENABLE_QAT=$ENABLE_QAT \
-    -DENABLE_GCS=$ENABLE_GCS \
-    -DENABLE_S3=$ENABLE_S3 \
-    -DENABLE_HDFS=$ENABLE_HDFS \
-    -DENABLE_ABFS=$ENABLE_ABFS \
-    -DENABLE_GPU=$ENABLE_GPU \
-    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-    -DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES"
+  GLUTEN_CMAKE_OPTIONS=(
+    "-DBUILD_VELOX_BACKEND=ON"
+    "-DCMAKE_BUILD_TYPE=$BUILD_TYPE"
+    "-DVELOX_HOME=$VELOX_HOME"
+    "-DBUILD_TESTS=$BUILD_TESTS"
+    "-DBUILD_EXAMPLES=$BUILD_EXAMPLES"
+    "-DBUILD_BENCHMARKS=$BUILD_BENCHMARKS"
+    "-DENABLE_JEMALLOC_STATS=$ENABLE_JEMALLOC_STATS"
+    "-DENABLE_QAT=$ENABLE_QAT"
+    "-DENABLE_GCS=$ENABLE_GCS"
+    "-DENABLE_S3=$ENABLE_S3"
+    "-DENABLE_HDFS=$ENABLE_HDFS"
+    "-DENABLE_ABFS=$ENABLE_ABFS"
+    "-DENABLE_GPU=$ENABLE_GPU"
+    "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
+    "-DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES"
+  )
 
+  if [ -n "${INSTALL_PREFIX:-}" ]; then
+    GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
+    GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX")
+  fi
   if [ $OS == 'Darwin' ]; then
-    GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX"
-    if [[ "$INSTALL_PREFIX" != "/usr/local" && "$INSTALL_PREFIX" != /usr/local/* ]]; then
-      GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON"
-      GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_IGNORE_PREFIX_PATH=/usr/local"
-      GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake"
-      GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_SYSTEM_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake"
+    if [[ "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then
+      GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON")
+      GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_IGNORE_PREFIX_PATH=/usr/local")
+      GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake")
+      GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_SYSTEM_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake")
     fi
-    GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_CXX_FLAGS=-Wno-inconsistent-missing-override -Wno-macro-redefined"
+    GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_CXX_FLAGS=-Wno-inconsistent-missing-override -Wno-macro-redefined")
   fi
 
-  cmake -G Ninja $GLUTEN_CMAKE_OPTIONS ..
+  cmake -G Ninja "${GLUTEN_CMAKE_OPTIONS[@]}" ..
   ninja -j $NUM_THREADS
 }
 

ep/build-velox/src/build-velox.sh

@@ -114,12 +114,18 @@ function compile {
     -Wno-error=uninitialized -Wno-unknown-warning-option -Wno-deprecated-declarations'
   if [[ "$(uname)" == "Darwin" ]]; then
     CXX_FLAGS="$CXX_FLAGS -Wno-inconsistent-missing-override -Wno-macro-redefined"
+    if [[ "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then
+      CXX_FLAGS="$CXX_FLAGS -isystem /usr/local/include"
+    fi
   fi
 
   COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=OFF \
       -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF -DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON \
       -DVELOX_ENABLE_GEO=OFF"
-  if [[ "$(uname)" == "Darwin" && "$INSTALL_PREFIX" != "/usr/local" && "$INSTALL_PREFIX" != /usr/local/* ]]; then
+  if [ -n "${INSTALL_PREFIX:-}" ]; then
+    COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}"
+  fi
+  if [[ "$(uname)" == "Darwin" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then
     COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON"
     COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PREFIX_PATH=/usr/local"
     COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PATH=/usr/local\;/usr/local/include\;/usr/local/lib\;/usr/local/lib/cmake"
@@ -213,6 +219,8 @@ if [ "$OS" == 'Darwin' ]; then
   if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" == /usr/local/* ]]; then
     echo "INFO: INSTALL_PREFIX=$INSTALL_PREFIX is under /usr/local; keeping /usr/local visible to CMake." >&2
   fi
+elif [ -n "${INSTALL_PREFIX:-}" ]; then
+  export INSTALL_PREFIX
 fi
 
 echo "Start building Velox..."

ep/build-velox/src/get-velox.sh

@@ -89,6 +89,22 @@ function process_setup_tencentos32 {
   sed -i "/^[[:space:]]*#/!s/.*dnf config-manager --set-enabled powertools/#&/" ${CURRENT_DIR}/setup-centos8.sh
 }
 
+# Keep macOS dependency builds on INSTALL_PREFIX even when /usr/local is present.
+# Apple clang injects /usr/local/include as a normal include path before CMake's
+# imported system includes, which can mix /usr/local headers with INSTALL_PREFIX
+# libraries. Demote it to system include order. Folly also enables jemalloc from
+# Homebrew headers but does not link libjemalloc, so keep Folly's Linux-equivalent
+# no-jemalloc behavior here; Gluten's own jemalloc build is independent of this.
+function process_setup_macos {
+  if [[ "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]] &&
+      ! grep -Fq '/usr/local/include' scripts/setup-macos.sh; then
+    sed -i '' 's|OS_CXXFLAGS=" -isystem $(brew --prefix)/include "|OS_CXXFLAGS=" -isystem $(brew --prefix)/include -isystem /usr/local/include "|' scripts/setup-macos.sh
+  fi
+  if ! grep -Fq 'FOLLY_USE_JEMALLOC=OFF' scripts/setup-common.sh; then
+    sed -i '' 's/local FOLLY_FLAGS=(/local FOLLY_FLAGS=(-DFOLLY_USE_JEMALLOC=OFF /' scripts/setup-common.sh
+  fi
+}
+
 function prepare_velox_source_code {
   echo "Preparing Velox source code..."
 
@@ -215,7 +231,7 @@ function apply_setup_fixes() {
   if [ $OS == 'Linux' ]; then
     setup_linux
   elif [ $OS == 'Darwin' ]; then
-    :
+    process_setup_macos
   else
     echo "Unsupported kernel: $OS"
     exit 1