From 8e5bb4b4489d6f55592908c2e14a93b672bf8219 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 16 Apr 2025 09:45:20 -0400 Subject: [PATCH 1/5] GH-46165: [C++] Add cuda option to Meson configuration --- ci/docker/conda-cpp.dockerfile | 2 + ci/scripts/cpp_build.sh | 3 ++ cpp/meson.build | 2 + cpp/meson.options | 5 +++ cpp/src/arrow/gpu/meson.build | 76 ++++++++++++++++++++++++++++++++++ cpp/src/arrow/meson.build | 4 ++ 6 files changed, 92 insertions(+) create mode 100644 cpp/src/arrow/gpu/meson.build diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index a387fb266990..d5a8aaba02d6 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -38,6 +38,8 @@ RUN mamba install -q -y \ doxygen \ libnuma \ mold \ + nvidia::cuda \ + nvidia::cuda-toolkit \ python=${python} \ valgrind && \ mamba clean --all --yes diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 2f02f8c14966..ff49612c56fe 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -140,6 +140,9 @@ if [ "${ARROW_USE_MESON:-OFF}" = "ON" ]; then fi fi fi + + export CUDA_PATH="${CONDA_PREFIX}" + env meson setup \ --prefix=${MESON_PREFIX:-${ARROW_HOME}} \ --buildtype=${ARROW_BUILD_TYPE:-debug} \ diff --git a/cpp/meson.build b/cpp/meson.build index ae1f391a884d..80b37f19d6a8 100644 --- a/cpp/meson.build +++ b/cpp/meson.build @@ -76,6 +76,7 @@ needs_integration = get_option('integration').enabled() needs_tests = get_option('tests').enabled() needs_acero = get_option('acero').enabled() or needs_dataset needs_compute = get_option('compute').enabled() or needs_acero +needs_cuda = get_option('cuda').enabled() needs_flight_sql = false needs_flight = get_option('flight').enabled() or needs_flight_sql needs_gandiva = false @@ -83,6 +84,7 @@ needs_ipc = (get_option('ipc').enabled() or needs_tests or needs_acero or needs_benchmarks + or needs_cuda or needs_flight or needs_parquet or needs_substrait diff --git a/cpp/meson.options b/cpp/meson.options index 3124bb61fc66..bdf552449d1b 100644 --- a/cpp/meson.options +++ b/cpp/meson.options @@ -39,6 +39,11 @@ option( description: 'Build all Arrow Compute kernels', ) option('csv', type: 'feature', description: 'Build the Arrow CSV Parser Module') +option( + 'cuda', + type: 'feature', + description: 'Build the Arrow CUDA extensions (requires CUDA toolkit)', +) option( 'dataset', type: 'feature', diff --git a/cpp/src/arrow/gpu/meson.build b/cpp/src/arrow/gpu/meson.build new file mode 100644 index 000000000000..3a5f2c0b8ef3 --- /dev/null +++ b/cpp/src/arrow/gpu/meson.build @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cuda_dep = dependency('cuda', modules: ['cuda']) + +arrow_cuda_srcs = [ + 'cuda_arrow_ipc.cc', + 'cuda_context.cc', + 'cuda_internal.cc', + 'cuda_memory.cc', +] + +arrow_cuda_lib = library( + 'arrow-cuda', + sources: [arrow_cuda_srcs], + dependencies: [arrow_dep, cuda_dep, flatbuffers_dep], + cpp_args: ['-DARROW_CUDA_EXPORTING'], +) + +arrow_cuda_dep = declare_dependency( + link_with: [arrow_cuda_lib], + dependencies: [cuda_dep], +) + +cuda_version = cuda_dep.version() +cuda_split_version = cuda_version.split('.') +cuda_conf_data = configuration_data() +cuda_conf_data.set('CUDA_VERSION_MAJOR', cuda_split_version[0]) +cuda_conf_data.set('CUDA_VERSION_MINOR', cuda_split_version[1]) + +configure_file( + input: 'cuda_version.h.in', + output: 'cuda_version.h', + configuration: cuda_conf_data, + install_dir: 'arrow/gpu', +) + +install_headers( + [ + 'cuda_api.h', + 'cuda_arrow_ipc.h', + 'cuda_context.h', + 'cuda_internal.h', + 'cuda_memory.h', + 'visibility.h', + ], + subdir: 'arrow/gpu', +) + +exc = executable( + 'arrow-cuda-test', + sources: ['cuda_test.cc'], + dependencies: [arrow_test_dep, arrow_cuda_dep], +) +test('arrow-cuda-test', exc) + +exc = executable( + 'arrow-cuda-benchmark', + sources: ['cuda_benchmark.cc'], + dependencies: [arrow_benchmark_dep, arrow_cuda_dep], +) +benchmark('arrow-gpu-cuda-benchmark', exc) diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index 43050aa15977..b8506c739133 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -918,6 +918,10 @@ if needs_acero subdir('acero') endif +if needs_cuda + subdir('gpu') +endif + if needs_filesystem subdir('filesystem') endif From dad59c7f01de4d4a2b956eed53c2ce15d7146c1e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 10 Sep 2025 15:46:48 -0400 Subject: [PATCH 2/5] Try LD_LIBRARY_PATH hack --- ci/scripts/cpp_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index a0b77b11be2e..dfa9d65e25a5 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -31,7 +31,7 @@ binary_output_dir=${build_dir}/${ARROW_BUILD_TYPE:-debug} export ARROW_TEST_DATA=${arrow_dir}/testing/data export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data -export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${CUDA_PATH}/lib/stubs:${LD_LIBRARY_PATH} # By default, aws-sdk tries to contact a non-existing local ip host # to retrieve metadata. Disable this so that S3FileSystem tests run faster. From aa1d1f80a64759b7c28c0e32426abaf4079c1ed9 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 13 Nov 2025 10:47:38 -0500 Subject: [PATCH 3/5] Revert LD_LIBRARY_PATH hack --- ci/scripts/cpp_build.sh | 3 --- ci/scripts/cpp_test.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index ff49612c56fe..2f02f8c14966 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -140,9 +140,6 @@ if [ "${ARROW_USE_MESON:-OFF}" = "ON" ]; then fi fi fi - - export CUDA_PATH="${CONDA_PREFIX}" - env meson setup \ --prefix=${MESON_PREFIX:-${ARROW_HOME}} \ --buildtype=${ARROW_BUILD_TYPE:-debug} \ diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index dfa9d65e25a5..a0b77b11be2e 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -31,7 +31,7 @@ binary_output_dir=${build_dir}/${ARROW_BUILD_TYPE:-debug} export ARROW_TEST_DATA=${arrow_dir}/testing/data export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data -export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${CUDA_PATH}/lib/stubs:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH} # By default, aws-sdk tries to contact a non-existing local ip host # to retrieve metadata. Disable this so that S3FileSystem tests run faster. From 5c0a3876f43c3f035b1889c39f1e56febfaf5e3d Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 13 Nov 2025 11:08:54 -0500 Subject: [PATCH 4/5] Specify CUDA lower bound --- cpp/src/arrow/gpu/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/gpu/meson.build b/cpp/src/arrow/gpu/meson.build index 3a5f2c0b8ef3..2bd7cea25d10 100644 --- a/cpp/src/arrow/gpu/meson.build +++ b/cpp/src/arrow/gpu/meson.build @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cuda_dep = dependency('cuda', modules: ['cuda']) +cuda_dep = dependency('cuda', modules: ['cuda'], version: '>=11.7.1') arrow_cuda_srcs = [ 'cuda_arrow_ipc.cc', From 52b01f9078e08fc7fbb91333d4ac540ed6c31fc0 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 13 Nov 2025 11:20:21 -0500 Subject: [PATCH 5/5] Try LD_LIBRARY_PATH hack again --- ci/scripts/cpp_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index a0b77b11be2e..dfa9d65e25a5 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -31,7 +31,7 @@ binary_output_dir=${build_dir}/${ARROW_BUILD_TYPE:-debug} export ARROW_TEST_DATA=${arrow_dir}/testing/data export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data -export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${CUDA_PATH}/lib/stubs:${LD_LIBRARY_PATH} # By default, aws-sdk tries to contact a non-existing local ip host # to retrieve metadata. Disable this so that S3FileSystem tests run faster.