@@ -1867,82 +1867,117 @@ endif()
18671867# ----------------------------------------------------------------------
18681868# Protocol Buffers (required for ORC, Flight and Substrait libraries)
18691869
1870- macro (build_protobuf)
1871- message (STATUS "Building Protocol Buffers from source" )
1872- set (PROTOBUF_VENDORED TRUE )
1873- set (PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR} /protobuf_ep-install" )
1870+ function (build_protobuf)
1871+ list (APPEND CMAKE_MESSAGE_INDENT "Protobuf: " )
1872+ message (STATUS "Building Protocol Buffers from source using FetchContent" )
1873+ set (PROTOBUF_VENDORED
1874+ TRUE
1875+ PARENT_SCOPE)
1876+ set (PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR} /protobuf_fc-install" )
1877+ set (PROTOBUF_PREFIX
1878+ "${PROTOBUF_PREFIX} "
1879+ PARENT_SCOPE)
18741880 set (PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX} /include" )
1881+ set (PROTOBUF_INCLUDE_DIR
1882+ "${PROTOBUF_INCLUDE_DIR} "
1883+ PARENT_SCOPE)
1884+
1885+ fetchcontent_declare(protobuf
1886+ URL ${PROTOBUF_SOURCE_URL}
1887+ URL_HASH "SHA256=${ARROW_PROTOBUF_BUILD_SHA256_CHECKSUM} "
1888+ SOURCE_SUBDIR cmake)
1889+
1890+ prepare_fetchcontent()
1891+
18751892 # This flag is based on what the user initially requested but if
18761893 # we've fallen back to building protobuf we always build it statically
18771894 # so we need to reset the flag so that we can link against it correctly
18781895 # later.
18791896 set (Protobuf_USE_STATIC_LIBS ON )
1880- # Newer protobuf releases always have a lib prefix independent from CMAKE_STATIC_LIBRARY_PREFIX
1881- set (PROTOBUF_STATIC_LIB
1882- "${PROTOBUF_PREFIX} /lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX} " )
1883- set (PROTOC_STATIC_LIB "${PROTOBUF_PREFIX} /lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX} " )
1884- set (Protobuf_PROTOC_LIBRARY "${PROTOC_STATIC_LIB} " )
1885- set (PROTOBUF_COMPILER "${PROTOBUF_PREFIX} /bin/protoc" )
18861897
18871898 # Strip lto flags (which may be added by dh_auto_configure)
18881899 # See https://github.com/protocolbuffers/protobuf/issues/7092
1889- set (PROTOBUF_C_FLAGS ${EP_C_FLAGS} )
1890- set (PROTOBUF_CXX_FLAGS ${EP_CXX_FLAGS} )
1891- string (REPLACE "-flto=auto" "" PROTOBUF_C_FLAGS "${PROTOBUF_C_FLAGS} " )
1892- string (REPLACE "-ffat-lto-objects" "" PROTOBUF_C_FLAGS "${PROTOBUF_C_FLAGS} " )
1893- string (REPLACE "-flto=auto" "" PROTOBUF_CXX_FLAGS "${PROTOBUF_CXX_FLAGS} " )
1894- string (REPLACE "-ffat-lto-objects" "" PROTOBUF_CXX_FLAGS "${PROTOBUF_CXX_FLAGS} " )
1895- set (PROTOBUF_CMAKE_ARGS
1896- ${EP_COMMON_CMAKE_ARGS}
1897- "-DCMAKE_CXX_FLAGS=${PROTOBUF_CXX_FLAGS} "
1898- "-DCMAKE_C_FLAGS=${PROTOBUF_C_FLAGS} "
1899- "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX} "
1900- -Dprotobuf_BUILD_TESTS=OFF
1901- -Dprotobuf_DEBUG_POSTFIX =)
1900+ string (REPLACE "-flto=auto" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS} " )
1901+ string (REPLACE "-ffat-lto-objects" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS} " )
1902+ string (REPLACE "-flto=auto" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} " )
1903+ string (REPLACE "-ffat-lto-objects" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} " )
1904+
1905+ set (protobuf_BUILD_TESTS OFF )
19021906 if (MSVC AND NOT ARROW_USE_STATIC_CRT)
1903- list ( APPEND PROTOBUF_CMAKE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME= OFF" )
1907+ set (protobuf_MSVC_STATIC_RUNTIME OFF )
19041908 endif ()
1905- if (ZLIB_ROOT)
1906- list (APPEND PROTOBUF_CMAKE_ARGS "-DZLIB_ROOT=${ZLIB_ROOT} " )
1907- endif ()
1908- set (PROTOBUF_EXTERNAL_PROJECT_ADD_ARGS CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS} SOURCE_SUBDIR
1909- "cmake" )
19101909
1911- externalproject_add(protobuf_ep
1912- ${EP_COMMON_OPTIONS} ${PROTOBUF_EXTERNAL_PROJECT_ADD_ARGS}
1913- BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB} " "${PROTOBUF_COMPILER} "
1914- BUILD_IN_SOURCE 1
1915- URL ${PROTOBUF_SOURCE_URL}
1916- URL_HASH "SHA256=${ARROW_PROTOBUF_BUILD_SHA256_CHECKSUM} " )
1910+ # Unity build causes some build errors
1911+ set (CMAKE_UNITY_BUILD OFF )
19171912
1918- file (MAKE_DIRECTORY "${PROTOBUF_INCLUDE_DIR} " )
1913+ fetchcontent_makeavailable(protobuf)
1914+
1915+ # Get the actual include directory from the Protobuf target.
1916+ # For FetchContent, this points to the source directory which contains the .proto files.
1917+ set (PROTOBUF_INCLUDE_DIR "${protobuf_SOURCE_DIR} /src" )
19191918 # For compatibility of CMake's FindProtobuf.cmake.
19201919 set (Protobuf_INCLUDE_DIRS "${PROTOBUF_INCLUDE_DIR} " )
1920+ set (Protobuf_INCLUDE_DIRS
1921+ "${Protobuf_INCLUDE_DIRS} "
1922+ PARENT_SCOPE)
1923+ # Set import dirs so protoc can find well-known types like timestamp.proto
1924+ set (Protobuf_IMPORT_DIRS "${PROTOBUF_INCLUDE_DIR} " )
1925+ set (Protobuf_IMPORT_DIRS
1926+ "${Protobuf_IMPORT_DIRS} "
1927+ PARENT_SCOPE)
1928+
1929+ # For FetchContent builds, protoc and libprotoc are regular targets, not imported.
1930+ # We can't get their locations at configure time, so we set placeholders.
1931+ # The actual locations will be resolved at build time or by the install step.
1932+ set (PROTOBUF_COMPILER "$<TARGET_FILE:protobuf::protoc>" )
1933+ set (PROTOC_STATIC_LIB "$<TARGET_FILE:protobuf::libprotoc>" )
1934+ set (Protobuf_PROTOC_LIBRARY "${PROTOC_STATIC_LIB} " )
1935+
1936+ # gRPC requires Protobuf to be installed to a known location.
1937+ # We have to do this in two steps to avoid double installation of Protobuf
1938+ # when Arrow is installed.
1939+ # This custom target ensures Protobuf is built before we install
1940+ add_custom_target (protobuf_built
1941+ DEPENDS protobuf::libprotobuf protobuf::libprotobuf-lite
1942+ protobuf::libprotoc protobuf::protoc)
1943+
1944+ # Disable Protobuf's install script after it's built to prevent double installation
1945+ add_custom_command (OUTPUT "${protobuf_BINARY_DIR} /cmake_install.cmake.saved"
1946+ COMMAND ${CMAKE_COMMAND} -E copy_if_different
1947+ "${protobuf_BINARY_DIR} /cmake_install.cmake"
1948+ "${protobuf_BINARY_DIR} /cmake_install.cmake.saved"
1949+ COMMAND ${CMAKE_COMMAND} -E echo
1950+ "# Protobuf install disabled to prevent double installation with Arrow"
1951+ > "${protobuf_BINARY_DIR} /cmake_install.cmake"
1952+ DEPENDS protobuf_built
1953+ COMMENT "Disabling Protobuf install to prevent double installation"
1954+ VERBATIM )
19211955
1922- add_library (arrow::protobuf::libprotobuf STATIC IMPORTED )
1923- set_target_properties (arrow::protobuf::libprotobuf PROPERTIES IMPORTED_LOCATION
1924- "${PROTOBUF_STATIC_LIB} " )
1925- target_include_directories (arrow::protobuf::libprotobuf BEFORE
1926- INTERFACE "${PROTOBUF_INCLUDE_DIR} " )
1927- add_library (arrow::protobuf::libprotoc STATIC IMPORTED )
1928- set_target_properties (arrow::protobuf::libprotoc PROPERTIES IMPORTED_LOCATION
1929- "${PROTOC_STATIC_LIB} " )
1930- target_include_directories (arrow::protobuf::libprotoc BEFORE
1931- INTERFACE "${PROTOBUF_INCLUDE_DIR} " )
1932- add_executable (arrow::protobuf::protoc IMPORTED )
1933- set_target_properties (arrow::protobuf::protoc PROPERTIES IMPORTED_LOCATION
1934- "${PROTOBUF_COMPILER} " )
1935-
1936- add_dependencies (arrow::protobuf::libprotobuf protobuf_ep)
1937- add_dependencies (arrow::protobuf::protoc protobuf_ep)
1938-
1939- list (APPEND ARROW_BUNDLED_STATIC_LIBS arrow::protobuf::libprotobuf)
1956+ add_custom_target (protobuf_install_disabled ALL
1957+ DEPENDS "${protobuf_BINARY_DIR} /cmake_install.cmake.saved" )
1958+
1959+ # Install Protobuf to PROTOBUF_PREFIX for dependendants to find
1960+ add_custom_command (OUTPUT "${PROTOBUF_PREFIX} /.protobuf_installed"
1961+ COMMAND ${CMAKE_COMMAND} -E copy_if_different
1962+ "${protobuf_BINARY_DIR} /cmake_install.cmake.saved"
1963+ "${protobuf_BINARY_DIR} /cmake_install.cmake.tmp"
1964+ COMMAND ${CMAKE_COMMAND} -DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}
1965+ -DCMAKE_INSTALL_CONFIG_NAME=$<CONFIG> -P
1966+ "${protobuf_BINARY_DIR} /cmake_install.cmake.tmp" ||
1967+ ${CMAKE_COMMAND} -E true
1968+ COMMAND ${CMAKE_COMMAND} -E touch
1969+ "${PROTOBUF_PREFIX} /.protobuf_installed"
1970+ DEPENDS protobuf_install_disabled
1971+ COMMENT "Installing Protobuf to ${PROTOBUF_PREFIX} for gRPC"
1972+ VERBATIM )
1973+
1974+ # Make protobuf_fc depend on the install completion marker
1975+ add_custom_target (protobuf_fc DEPENDS "${PROTOBUF_PREFIX} /.protobuf_installed" )
1976+ list (APPEND ARROW_BUNDLED_STATIC_LIBS protobuf::libprotobuf)
19401977
19411978 if (CMAKE_CROSSCOMPILING )
19421979 # If we are cross compiling, we need to build protoc for the host
19431980 # system also, as it is used when building Arrow
1944- # We do this by calling CMake as a child process
1945- # with CXXFLAGS / CFLAGS and CMake flags cleared.
19461981 set (PROTOBUF_HOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR} /protobuf_ep_host-install" )
19471982 set (PROTOBUF_HOST_COMPILER "${PROTOBUF_HOST_PREFIX} /bin/protoc" )
19481983
@@ -1953,21 +1988,23 @@ macro(build_protobuf)
19531988 -Dprotobuf_BUILD_TESTS=OFF
19541989 -Dprotobuf_DEBUG_POSTFIX =)
19551990
1991+ # We reuse the FetchContent downloaded source but build it with host compiler
19561992 externalproject_add(protobuf_ep_host
19571993 ${EP_COMMON_OPTIONS}
19581994 CMAKE_ARGS ${PROTOBUF_HOST_CMAKE_ARGS}
1995+ SOURCE_DIR "${protobuf_SOURCE_DIR} "
1996+ SOURCE_SUBDIR cmake
19591997 BUILD_BYPRODUCTS "${PROTOBUF_HOST_COMPILER} "
1960- BUILD_IN_SOURCE 1
1961- URL ${PROTOBUF_SOURCE_URL}
1962- URL_HASH "SHA256=${ARROW_PROTOBUF_BUILD_SHA256_CHECKSUM} " )
1998+ DOWNLOAD_COMMAND "" DOWNLOAD_EXTRACT_TIMESTAMP TRUE )
19631999
19642000 add_executable (arrow::protobuf::host_protoc IMPORTED )
19652001 set_target_properties (arrow::protobuf::host_protoc
19662002 PROPERTIES IMPORTED_LOCATION "${PROTOBUF_HOST_COMPILER} " )
19672003
19682004 add_dependencies (arrow::protobuf::host_protoc protobuf_ep_host)
19692005 endif ()
1970- endmacro ()
2006+ list (POP_BACK CMAKE_MESSAGE_INDENT)
2007+ endfunction ()
19712008
19722009if (ARROW_WITH_PROTOBUF)
19732010 if (ARROW_FLIGHT_SQL)
@@ -2017,7 +2054,10 @@ if(ARROW_WITH_PROTOBUF)
20172054 REQUIRED_VERSION
20182055 ${ARROW_PROTOBUF_REQUIRED_VERSION} )
20192056
2020- if (NOT Protobuf_USE_STATIC_LIBS AND MSVC_TOOLCHAIN)
2057+ # If PROTOBUF_VENDORED we build static protobuf from source via FetchContent
2058+ if (NOT PROTOBUF_VENDORED
2059+ AND NOT Protobuf_USE_STATIC_LIBS
2060+ AND MSVC_TOOLCHAIN)
20212061 add_definitions (-DPROTOBUF_USE_DLLS)
20222062 endif ()
20232063
@@ -2128,8 +2168,12 @@ macro(build_substrait)
21282168 SKIP_UNITY_BUILD_INCLUSION TRUE )
21292169 list (APPEND SUBSTRAIT_PROTO_GEN_ALL "${SUBSTRAIT_PROTO_GEN} .${EXT} " )
21302170 endforeach ()
2171+ set (SUBSTRAIT_PROTOC_INCLUDES "-I${SUBSTRAIT_LOCAL_DIR} /proto" )
2172+ if (PROTOBUF_VENDORED AND Protobuf_INCLUDE_DIRS)
2173+ list (APPEND SUBSTRAIT_PROTOC_INCLUDES "-I${Protobuf_INCLUDE_DIRS} " )
2174+ endif ()
21312175 add_custom_command (OUTPUT "${SUBSTRAIT_PROTO_GEN} .cc" "${SUBSTRAIT_PROTO_GEN} .h"
2132- COMMAND ${ARROW_PROTOBUF_PROTOC} "-I ${SUBSTRAIT_LOCAL_DIR} /proto"
2176+ COMMAND ${ARROW_PROTOBUF_PROTOC} ${SUBSTRAIT_PROTOC_INCLUDES}
21332177 "--cpp_out=${SUBSTRAIT_CPP_DIR} "
21342178 "${SUBSTRAIT_LOCAL_DIR} /proto/substrait/${SUBSTRAIT_PROTO} .proto"
21352179 DEPENDS ${PROTO_DEPENDS} substrait_ep)
@@ -2147,10 +2191,11 @@ macro(build_substrait)
21472191 SKIP_UNITY_BUILD_INCLUSION TRUE )
21482192 list (APPEND SUBSTRAIT_PROTO_GEN_ALL "${ARROW_SUBSTRAIT_PROTO_GEN} .${EXT} " )
21492193 endforeach ()
2194+ set (ARROW_SUBSTRAIT_PROTOC_INCLUDES ${SUBSTRAIT_PROTOC_INCLUDES}
2195+ "-I${ARROW_SUBSTRAIT_PROTOS_DIR} " )
21502196 add_custom_command (OUTPUT "${ARROW_SUBSTRAIT_PROTO_GEN} .cc"
21512197 "${ARROW_SUBSTRAIT_PROTO_GEN} .h"
2152- COMMAND ${ARROW_PROTOBUF_PROTOC} "-I${SUBSTRAIT_LOCAL_DIR} /proto"
2153- "-I${ARROW_SUBSTRAIT_PROTOS_DIR} "
2198+ COMMAND ${ARROW_PROTOBUF_PROTOC} ${ARROW_SUBSTRAIT_PROTOC_INCLUDES}
21542199 "--cpp_out=${SUBSTRAIT_CPP_DIR} "
21552200 "${ARROW_SUBSTRAIT_PROTOS_DIR} /substrait/${ARROW_SUBSTRAIT_PROTO} .proto"
21562201 DEPENDS ${PROTO_DEPENDS} substrait_ep)
@@ -3257,6 +3302,9 @@ macro(build_grpc)
32573302 if (CARES_VENDORED)
32583303 add_dependencies (grpc_dependencies cares_fc)
32593304 endif ()
3305+ if (PROTOBUF_VENDORED)
3306+ add_dependencies (grpc_dependencies protobuf_fc)
3307+ endif ()
32603308
32613309 if (GFLAGS_VENDORED)
32623310 add_dependencies (grpc_dependencies gflags_ep)
@@ -3269,9 +3317,15 @@ macro(build_grpc)
32693317 add_dependencies (grpc_dependencies ${ARROW_PROTOBUF_LIBPROTOBUF} c-ares::cares
32703318 ZLIB::ZLIB)
32713319
3272- get_target_property (GRPC_PROTOBUF_INCLUDE_DIR ${ARROW_PROTOBUF_LIBPROTOBUF}
3273- INTERFACE_INCLUDE_DIRECTORIES )
3274- get_filename_component (GRPC_PB_ROOT "${GRPC_PROTOBUF_INCLUDE_DIR} " DIRECTORY )
3320+ # For FetchContent Protobuf, use the install prefix directly
3321+ if (PROTOBUF_VENDORED)
3322+ set (GRPC_PB_ROOT "${PROTOBUF_PREFIX} " )
3323+ else ()
3324+ get_target_property (GRPC_PROTOBUF_INCLUDE_DIR ${ARROW_PROTOBUF_LIBPROTOBUF}
3325+ INTERFACE_INCLUDE_DIRECTORIES )
3326+ get_filename_component (GRPC_PB_ROOT "${GRPC_PROTOBUF_INCLUDE_DIR} " DIRECTORY )
3327+ endif ()
3328+
32753329 get_target_property (GRPC_Protobuf_PROTOC_LIBRARY ${ARROW_PROTOBUF_LIBPROTOC}
32763330 IMPORTED_LOCATION )
32773331
@@ -3946,11 +4000,6 @@ function(build_orc)
39464000 INTERFACE_INCLUDE_DIRECTORIES )
39474001 get_filename_component (Protobuf_ROOT "${PROTOBUF_INCLUDE_DIR} " DIRECTORY )
39484002 set (PROTOBUF_HOME ${Protobuf_ROOT} )
3949- # ORC uses this.
3950- if (PROTOBUF_VENDORED)
3951- target_include_directories (${ARROW_PROTOBUF_LIBPROTOC}
3952- INTERFACE "${PROTOBUF_INCLUDE_DIR} " )
3953- endif ()
39544003 set (PROTOBUF_EXECUTABLE ${ARROW_PROTOBUF_PROTOC} )
39554004 set (PROTOBUF_LIBRARY ${ARROW_PROTOBUF_LIBPROTOBUF} )
39564005 set (PROTOC_LIBRARY ${ARROW_PROTOBUF_LIBPROTOC} )
@@ -4194,10 +4243,8 @@ macro(build_opentelemetry)
41944243
41954244 get_target_property (OPENTELEMETRY_PROTOBUF_INCLUDE_DIR ${ARROW_PROTOBUF_LIBPROTOBUF}
41964245 INTERFACE_INCLUDE_DIRECTORIES )
4197- get_target_property (OPENTELEMETRY_PROTOBUF_LIBRARY ${ARROW_PROTOBUF_LIBPROTOBUF}
4198- IMPORTED_LOCATION )
4199- get_target_property (OPENTELEMETRY_PROTOC_EXECUTABLE ${ARROW_PROTOBUF_PROTOC}
4200- IMPORTED_LOCATION )
4246+ set (OPENTELEMETRY_PROTOBUF_LIBRARY "$<TARGET_FILE:${ARROW_PROTOBUF_LIBPROTOBUF} >" )
4247+ set (OPENTELEMETRY_PROTOC_EXECUTABLE "$<TARGET_FILE:${ARROW_PROTOBUF_PROTOC} >" )
42014248 list (APPEND
42024249 OPENTELEMETRY_CMAKE_ARGS
42034250 -DWITH_OTLP_HTTP=ON
@@ -4234,6 +4281,11 @@ macro(build_opentelemetry)
42344281 add_dependencies (opentelemetry_dependencies nlohmann_json::nlohmann_json
42354282 opentelemetry_proto_ep ${ARROW_PROTOBUF_LIBPROTOBUF} )
42364283
4284+ # Ensure vendored protobuf is installed before OpenTelemetry builds
4285+ if (PROTOBUF_VENDORED)
4286+ add_dependencies (opentelemetry_dependencies protobuf_fc)
4287+ endif ()
4288+
42374289 string (JOIN "${EP_LIST_SEPARATOR} " OPENTELEMETRY_PREFIX_PATH
42384290 ${OPENTELEMETRY_PREFIX_PATH_LIST} )
42394291 list (APPEND OPENTELEMETRY_CMAKE_ARGS "-DCMAKE_PREFIX_PATH=${OPENTELEMETRY_PREFIX_PATH} " )
0 commit comments