From dc4bef8a6ef0ea81d53ccb59d700d11d53624b55 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Tue, 24 Feb 2026 15:48:22 -0800 Subject: [PATCH 01/12] Build ODBC on Linux CI remotely Next: resolve the string conversion issues in tests. Write simple helper function to convert wstring (or wchar_t*) to `std::vector` should solve the issue. Disable ODBC test build on Linux - Building Linux test in ODBC will be in a separate PR - Reducing dockerfile will be in a separate PR Resolve sqlucode.h build issue * Doesn't impact macOS. Check CI if it impacts Windows ODBC. Resolve `sign-compare` warnings - Need to have same type during comparison. - Now `Built target arrow-odbc-spi-impl-test` is successful. Add forward function header Resolve system.cc build errors Still have some wsigns stuff to fix later. In-progress fix with unicode issues on system_dsn.cc TODO - still need to make the `ToSqlWCharPtr` function safe. Fix Linux build errors with configuration.cc * Finish fixing wchar related build errors at configuration.cc * Still have errors at `system_dsn.cc` Remove some log messages This partially reverts commit 568f72d2a307d2605c2f80a3aad683a206b6f348. All tests on macOS passing locally. In-progress fix read SQLWCHAR string In-progress Fix ODBC Build errors on Linux - Mar2 week - Add `[[fallthrough]];` to indicate intentional fallthrough. - Add `default` handling case. - Add casts for `record_number` compares Attempt to fix Windows build from odbc_statement.cc header change In-progress Fix ODBC Build errors on Linux - remove `std::move(options);` - Remove `boost` library install and add boost::headers to cmake lists, which resolves a lot of the boost issues I think - Continue on Monday. Add wrappers for `reinterpret_cast` This was causing an issue on Linux Replace `boost::lexicographical_compare` with `std::lexicographical_compare` Restore boost usages and add `BOOST_NO_CXX98_FUNCTION_BASE` Undo Set build level warning to production It didn't resolve the build errors related to boost on Linux (or any other build errors) Set build level warning to production * If this acts as a workaround for boost, then we don't have to redo boost In-progress Fix the `LPCWSTR` convert error on Linux - Plus formatting fix - Plus build fix for casting More boost disables I think the solution would be to replace boost completely. Now the build works on macOS. Continue in-progress fix odbc build Temp boost disables In-progress fix ODBC build - Add `unixodbc-dev` to dockerfile - There are changes to macOS static build PR that I will need to incorporate back to `apache-odbc` later Enable ODBC mac and Windows in case of errors Enable ODBC build on Linux Add ODBC=ON to dockerfile Fix `arm64` cannot found error from typo Add `DOCKER_VOLUME_PREFIX` and other environment settings Add Linux docker to `compose.yaml` Disable mac and Windows ODBC for faster build More disable Draft ODBC Linux implementation Comment out non-ODBC items --- .github/workflows/cpp_extra.yml | 500 ++++++++++-------- ci/docker/ubuntu-24.04-cpp-odbc.dockerfile | 216 ++++++++ compose.yaml | 25 + cpp/cmake_modules/BuildUtils.cmake | 1 + cpp/cmake_modules/DefineOptions.cmake | 4 - cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- cpp/src/arrow/CMakeLists.txt | 1 - cpp/src/arrow/flight/sql/odbc/CMakeLists.txt | 5 +- cpp/src/arrow/flight/sql/odbc/odbc_api.cc | 2 + .../arrow/flight/sql/odbc/odbc_api_internal.h | 3 +- .../flight/sql/odbc/odbc_impl/CMakeLists.txt | 21 +- .../accessors/binary_array_accessor_test.cc | 2 +- .../accessors/boolean_array_accessor_test.cc | 2 +- .../accessors/decimal_array_accessor_test.cc | 2 +- .../primitive_array_accessor_test.cc | 2 +- .../accessors/string_array_accessor_test.cc | 4 +- .../sql/odbc/odbc_impl/accessors/types.h | 10 +- .../sql/odbc/odbc_impl/attribute_utils.h | 10 + .../sql/odbc/odbc_impl/blocking_queue.h | 2 +- .../odbc/odbc_impl/config/configuration.cc | 63 ++- .../odbc/odbc_impl/flight_sql_connection.cc | 7 +- .../odbc/odbc_impl/flight_sql_result_set.cc | 2 +- .../flight_sql_result_set_accessors.cc | 7 + .../odbc/odbc_impl/flight_sql_statement.cc | 4 +- .../sql/odbc/odbc_impl/get_info_cache.cc | 13 +- .../arrow/flight/sql/odbc/odbc_impl/main.cc | 6 +- .../sql/odbc/odbc_impl/odbc_connection.cc | 9 +- .../sql/odbc/odbc_impl/odbc_descriptor.cc | 14 +- .../sql/odbc/odbc_impl/odbc_statement.cc | 15 +- .../sql/odbc/odbc_impl/odbc_statement.h | 2 +- .../sql/odbc/odbc_impl/spi/connection.h | 7 +- .../flight/sql/odbc/odbc_impl/spi/statement.h | 1 + .../flight/sql/odbc/odbc_impl/system_dsn.cc | 37 +- .../arrow/flight/sql/odbc/odbc_impl/util.cc | 3 + 34 files changed, 688 insertions(+), 316 deletions(-) create mode 100644 ci/docker/ubuntu-24.04-cpp-odbc.dockerfile diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 8ebf3b9dfbfe..dd758058d9e9 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -91,10 +91,255 @@ jobs: with: parent-workflow: cpp_extra - docker: + # docker: + # needs: check-labels + # name: ${{ matrix.title }} + # runs-on: ${{ matrix.runs-on }} + # if: >- + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + # timeout-minutes: 75 + # strategy: + # fail-fast: false + # matrix: + # include: + # - image: alpine-linux-cpp + # runs-on: ubuntu-latest + # title: AMD64 Alpine Linux + # - image: conda-cpp + # run-options: >- + # -e ARROW_USE_MESON=ON + # runs-on: ubuntu-latest + # title: AMD64 Ubuntu Meson + # # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved + # - continue-on-error: true + # envs: + # - DEBIAN=13 + # image: debian-cpp + # run-options: >- + # -e CMAKE_CXX_STANDARD=23 + # runs-on: ubuntu-latest + # title: AMD64 Debian C++23 + # env: + # ARCHERY_DEBUG: 1 + # ARROW_ENABLE_TIMING_TESTS: OFF + # DOCKER_VOLUME_PREFIX: ".docker/" + # steps: + # - name: Checkout Arrow + # uses: actions/checkout@v6 + # with: + # fetch-depth: 0 + # submodules: recursive + # - name: Cache Docker Volumes + # uses: actions/cache@v5 + # with: + # path: .docker + # key: extra-ubuntu-cpp-odbc-${{ hashFiles('cpp/**') }} + # restore-keys: extra-ubuntu-cpp-odbc- + # - name: Setup Python + # uses: actions/setup-python@v6 + # with: + # python-version: 3 + # - name: Setup Archery + # run: python3 -m pip install -e dev/archery[docker] + # - name: Execute Docker Build + # continue-on-error: ${{ matrix.continue-on-error || false }} + # env: + # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + # ENVS: ${{ toJSON(matrix.envs) }} + # run: | + # # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes + # sudo sysctl -w vm.mmap_rnd_bits=28 + # source ci/scripts/util_enable_core_dumps.sh + # if [ "${ENVS}" != "null" ]; then + # echo "${ENVS}" | jq -r '.[]' | while read env; do + # echo "${env}" >> .env + # done + # fi + # archery docker run ${{ matrix.run-options || '' }} ubuntu-cpp-odbc + # - name: Docker Push + # if: >- + # success() && + # github.event_name == 'push' && + # github.repository == 'apache/arrow' && + # github.ref_name == 'main' + # env: + # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + # continue-on-error: true + # run: archery docker push ubuntu-cpp-odbc + + # msvc-arm64: + # needs: check-labels + # if: >- + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + # name: ARM64 Windows 11 MSVC + # uses: ./.github/workflows/cpp_windows.yml + # with: + # arch: arm64 + # os: windows-11-arm + # simd-level: NONE + + # jni-linux: + # needs: check-labels + # name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} + # runs-on: ${{ matrix.platform.runs-on }} + # if: >- + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + # timeout-minutes: 240 + # permissions: + # # This is for using GitHub Packages for vcpkg cache + # packages: write + # strategy: + # fail-fast: false + # matrix: + # platform: + # - arch: "amd64" + # runs-on: ubuntu-latest + # - arch: "arm64v8" + # runs-on: ubuntu-24.04-arm + # env: + # ARCH: ${{ matrix.platform.arch }} + # REPO: ghcr.io/${{ github.repository }}-dev + # steps: + # - name: Checkout Arrow + # uses: actions/checkout@v6 + # with: + # fetch-depth: 0 + # submodules: recursive + # - name: Free up disk space + # run: | + # ci/scripts/util_free_space.sh + # - name: Cache Docker Volumes + # uses: actions/cache@v5 + # with: + # path: .docker + # key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} + # restore-keys: jni-${{ matrix.platform.runs-on }}- + # - name: Setup Python + # uses: actions/setup-python@v6 + # with: + # python-version: 3 + # - name: Setup Archery + # run: python3 -m pip install -e dev/archery[docker] + # - name: Execute Docker Build + # env: + # ARCHERY_DOCKER_USER: ${{ github.actor }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" + # run: | + # source ci/scripts/util_enable_core_dumps.sh + # archery docker run cpp-jni + # - name: Docker Push + # if: >- + # success() && + # github.event_name == 'push' && + # github.ref_name == 'main' + # env: + # ARCHERY_DOCKER_USER: ${{ github.actor }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + # continue-on-error: true + # run: archery docker push cpp-jni + + # jni-macos: + # needs: check-labels + # name: JNI macOS + # runs-on: macos-14 + # if: >- + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + # timeout-minutes: 45 + # env: + # MACOSX_DEPLOYMENT_TARGET: "14.0" + # steps: + # - name: Checkout Arrow + # uses: actions/checkout@v6 + # with: + # fetch-depth: 0 + # submodules: recursive + # - name: Install dependencies + # run: | + # brew bundle --file=cpp/Brewfile + # # We want to link aws-sdk-cpp statically but Homebrew's + # # aws-sdk-cpp provides only shared library. If we have + # # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + # brew uninstall aws-sdk-cpp + # # We want to use bundled RE2 for static linking. If + # # Homebrew's RE2 is installed, its header file may be used. + # # We uninstall Homebrew's RE2 to ensure using bundled RE2. + # brew uninstall grpc || : # gRPC depends on RE2 + # brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + # brew uninstall re2 + # # We want to use bundled Protobuf for static linking. If + # # Homebrew's Protobuf is installed, its library file may be + # # used on test We uninstall Homebrew's Protobuf to ensure using + # # bundled Protobuf. + # brew uninstall protobuf + # - name: Prepare ccache + # run: | + # echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + # - name: Cache ccache + # uses: actions/cache@v5 + # with: + # path: ccache + # key: jni-macos-${{ hashFiles('cpp/**') }} + # restore-keys: jni-macos- + # - name: CMake + # run: | + # cmake \ + # -S cpp \ + # -B cpp.build \ + # --preset=ninja-release-jni-macos \ + # -DARROW_BUILD_TESTS=ON \ + # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + # - name: Build + # run: | + # cmake --build cpp.build + # - name: Install + # run: | + # cmake --install cpp.build + # - name: Test + # env: + # ARROW_TEST_DATA: ${{ github.workspace }}/testing/data + # PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data + # run: | + # # MinIO is required + # exclude_tests="arrow-s3fs-test" + # # unstable + # exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" + # exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" + # ctest \ + # --exclude-regex "${exclude_tests}" \ + # --label-regex unittest \ + # --output-on-failure \ + # --parallel "$(sysctl -n hw.ncpu)" \ + # --test-dir "cpp.build" \ + # --timeout 300 + # - name: Build example + # run: | + # cmake \ + # -S cpp/examples/minimal_build/ \ + # -B cpp/examples/minimal_build.build \ + # -GNinja \ + # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + # cmake --build cpp/examples/minimal_build.build + # cd cpp/examples/minimal_build + # ../minimal_build.build/arrow-example + + odbc-linux: needs: check-labels - name: ${{ matrix.title }} - runs-on: ${{ matrix.runs-on }} + name: ODBC Linux + runs-on: ubuntu-latest if: >- needs.check-labels.outputs.force == 'true' || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || @@ -102,26 +347,13 @@ jobs: timeout-minutes: 75 strategy: fail-fast: false - matrix: - include: - - image: alpine-linux-cpp - runs-on: ubuntu-latest - title: AMD64 Alpine Linux - - image: conda-cpp - run-options: >- - -e ARROW_USE_MESON=ON - runs-on: ubuntu-latest - title: AMD64 Ubuntu Meson - # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved - - continue-on-error: true - envs: - - DEBIAN=13 - image: debian-cpp - run-options: >- - -e CMAKE_CXX_STANDARD=23 - runs-on: ubuntu-latest - title: AMD64 Debian C++23 env: + ARCH: amd64 + CLANG_TOOLS: 18 + LLVM: 18 + UBUNTU: 24.04 + + # -AL- todo make the list alphabetical ARCHERY_DEBUG: 1 ARROW_ENABLE_TIMING_TESTS: OFF DOCKER_VOLUME_PREFIX: ".docker/" @@ -135,30 +367,23 @@ jobs: uses: actions/cache@v5 with: path: .docker - key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: extra-${{ matrix.image }}- - - name: Setup Python + key: ubuntu-cpp-odbc-${{ hashFiles('cpp/**') }} + restore-keys: ubuntu-cpp-odbc- + - name: Setup Python on hosted runner uses: actions/setup-python@v6 with: python-version: 3 - name: Setup Archery run: python3 -m pip install -e dev/archery[docker] - name: Execute Docker Build - continue-on-error: ${{ matrix.continue-on-error || false }} env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - ENVS: ${{ toJSON(matrix.envs) }} run: | # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes sudo sysctl -w vm.mmap_rnd_bits=28 source ci/scripts/util_enable_core_dumps.sh - if [ "${ENVS}" != "null" ]; then - echo "${ENVS}" | jq -r '.[]' | while read env; do - echo "${env}" >> .env - done - fi - archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} + archery docker run ubuntu-cpp-odbc - name: Docker Push if: >- success() && @@ -169,172 +394,7 @@ jobs: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} continue-on-error: true - run: archery docker push ${{ matrix.image }} - - msvc-arm64: - needs: check-labels - if: >- - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - name: ARM64 Windows 11 MSVC - uses: ./.github/workflows/cpp_windows.yml - with: - arch: arm64 - os: windows-11-arm - simd-level: NONE - - jni-linux: - needs: check-labels - name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} - runs-on: ${{ matrix.platform.runs-on }} - if: >- - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - timeout-minutes: 240 - permissions: - # This is for using GitHub Packages for vcpkg cache - packages: write - strategy: - fail-fast: false - matrix: - platform: - - arch: "amd64" - runs-on: ubuntu-latest - - arch: "arm64v8" - runs-on: ubuntu-24.04-arm - env: - ARCH: ${{ matrix.platform.arch }} - REPO: ghcr.io/${{ github.repository }}-dev - steps: - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v5 - with: - path: .docker - key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} - restore-keys: jni-${{ matrix.platform.runs-on }}- - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ github.actor }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run cpp-jni - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ github.actor }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - continue-on-error: true - run: archery docker push cpp-jni - - jni-macos: - needs: check-labels - name: JNI macOS - runs-on: macos-14 - if: >- - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - timeout-minutes: 45 - env: - MACOSX_DEPLOYMENT_TARGET: "14.0" - steps: - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Install dependencies - run: | - brew bundle --file=cpp/Brewfile - # We want to link aws-sdk-cpp statically but Homebrew's - # aws-sdk-cpp provides only shared library. If we have - # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - brew uninstall aws-sdk-cpp - # We want to use bundled RE2 for static linking. If - # Homebrew's RE2 is installed, its header file may be used. - # We uninstall Homebrew's RE2 to ensure using bundled RE2. - brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 - # We want to use bundled Protobuf for static linking. If - # Homebrew's Protobuf is installed, its library file may be - # used on test We uninstall Homebrew's Protobuf to ensure using - # bundled Protobuf. - brew uninstall protobuf - - name: Prepare ccache - run: | - echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - - name: Cache ccache - uses: actions/cache@v5 - with: - path: ccache - key: jni-macos-${{ hashFiles('cpp/**') }} - restore-keys: jni-macos- - - name: CMake - run: | - cmake \ - -S cpp \ - -B cpp.build \ - --preset=ninja-release-jni-macos \ - -DARROW_BUILD_TESTS=ON \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - - name: Build - run: | - cmake --build cpp.build - - name: Install - run: | - cmake --install cpp.build - - name: Test - env: - ARROW_TEST_DATA: ${{ github.workspace }}/testing/data - PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data - run: | - # MinIO is required - exclude_tests="arrow-s3fs-test" - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(sysctl -n hw.ncpu)" \ - --test-dir "cpp.build" \ - --timeout 300 - - name: Build example - run: | - cmake \ - -S cpp/examples/minimal_build/ \ - -B cpp/examples/minimal_build.build \ - -GNinja \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - cmake --build cpp/examples/minimal_build.build - cd cpp/examples/minimal_build - ../minimal_build.build/arrow-example + run: archery docker push ubuntu-cpp-odbc odbc-macos: needs: check-labels @@ -627,37 +687,6 @@ jobs: mkdir odbc-installer mv *.msi odbc-installer/ - # Add `dev-yyyy-mm-dd` to ODBC MSI before `win64.msi`: - # Apache Arrow Flight SQL ODBC-24.0.0-win64.msi -> - # Apache Arrow Flight SQL ODBC-24.0.0-dev-2026-02-06-win64.msi - cd odbc-installer - msi_name=$(ls *.msi) - dev_msi_name=$(echo ${msi_name} | sed -e "s/win64\.msi$/dev-$(date +%Y-%m-%d)-win64.msi/") - mv "${msi_name}" "${dev_msi_name}" - cd .. - - tree odbc-installer - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 1 - path: arrow - repository: apache/arrow - ref: main - submodules: recursive - - name: Sync to Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - upload: true - switches: -avzh --update --delete --progress - local_path: odbc-installer - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/odbc - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} - odbc-release: needs: odbc-msvc name: ODBC release @@ -694,12 +723,13 @@ jobs: report-extra-cpp: if: github.event_name == 'schedule' && always() needs: - - docker - - jni-linux - - jni-macos - - msvc-arm64 + # - docker + # - jni-linux + # - jni-macos + # - msvc-arm64 + - odbc-linux - odbc-macos - odbc-msvc - - odbc-nightly + # - odbc-nightly uses: ./.github/workflows/report_ci.yml secrets: inherit diff --git a/ci/docker/ubuntu-24.04-cpp-odbc.dockerfile b/ci/docker/ubuntu-24.04-cpp-odbc.dockerfile new file mode 100644 index 000000000000..1e5d7bd0c91b --- /dev/null +++ b/ci/docker/ubuntu-24.04-cpp-odbc.dockerfile @@ -0,0 +1,216 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:24.04 +FROM ${base} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +# Installs LLVM toolchain, for Gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build. +ARG clang_tools +ARG llvm +RUN latest_system_llvm=18 && \ + if [ ${llvm} -gt ${latest_system_llvm} -o \ + ${clang_tools} -gt ${latest_system_llvm} ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + lsb-release \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + code_name=$(lsb_release --codename --short) && \ + if [ ${llvm} -gt 10 ]; then \ + echo "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list; \ + fi && \ + if [ ${clang_tools} -ne ${llvm} -a \ + ${clang_tools} -gt ${latest_system_llvm} ]; then \ + echo "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi; \ + fi && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + clang-${clang_tools} \ + clang-${llvm} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + libclang-rt-${llvm}-dev \ + llvm-${llvm}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + curl \ + gdb \ + git \ + libbenchmark-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libidn2-dev \ + libkrb5-dev \ + libldap-dev \ + liblz4-dev \ + libnghttp2-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libpsl-dev \ + libradospp-dev \ + libre2-dev \ + librtmp-dev \ + libsnappy-dev \ + libsqlite3-dev \ + libssh-dev \ + libssh2-1-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libxml2-dev \ + libzstd-dev \ + make \ + mold \ + ninja-build \ + nlohmann-json3-dev \ + npm \ + patch \ + pkg-config \ + protobuf-compiler \ + protobuf-compiler-grpc \ + python3-dev \ + python3-pip \ + python3-rados \ + python3-venv \ + rados-objclass-dev \ + rapidjson-dev \ + rsync \ + tzdata \ + tzdata-legacy \ + unixodbc-dev \ + uuid-runtime \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +ARG gcc="" +RUN if [ "${gcc}" = "" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + g++ \ + gcc; \ + else \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + g++-${gcc} \ + gcc-${gcc} && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \ + update-alternatives --install \ + /usr/bin/$(uname --machine)-linux-gnu-gcc \ + $(uname --machine)-linux-gnu-gcc \ + /usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc} 100 && \ + update-alternatives --install \ + /usr/bin/$(uname --machine)-linux-gnu-g++ \ + $(uname --machine)-linux-gnu-g++ \ + /usr/bin/$(uname --machine)-linux-gnu-g++-${gcc} 100 && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \ + update-alternatives --set cc /usr/bin/gcc && \ + update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \ + update-alternatives --set c++ /usr/bin/g++; \ + fi + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh latest /usr/local + +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default + +COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_azurite.sh + +COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_ceph.sh + +COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin + +# Prioritize system packages and local installation. +# +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - opentelemetry-cpp-dev is not packaged +ENV ARROW_ACERO=ON \ + ARROW_AZURE=ON \ + ARROW_BUILD_STATIC=ON \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=BUNDLED \ + ARROW_DEPENDENCY_USE_SHARED=OFF \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=ON \ + ARROW_FLIGHT_SQL=ON \ + ARROW_FLIGHT_SQL_ODBC=ON \ + ARROW_GANDIVA=ON \ + ARROW_GCS=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_JEMALLOC=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_S3=ON \ + ARROW_SUBSTRAIT=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_MOLD=ON \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_OPENTELEMETRY=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \ + AWSSDK_SOURCE=BUNDLED \ + Azure_SOURCE=BUNDLED \ + google_cloud_cpp_storage_SOURCE=BUNDLED \ + opentelemetry_cpp_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 \ + xsimd_SOURCE=BUNDLED diff --git a/compose.yaml b/compose.yaml index c799059fe254..3f3b1509a7c5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -151,6 +151,7 @@ x-hierarchy: - ubuntu-r-only-r - ubuntu-cpp-bundled - ubuntu-cpp-bundled-offline + - ubuntu-cpp-odbc - ubuntu-cpp-minimal - ubuntu-cuda-cpp: - ubuntu-cuda-python @@ -496,6 +497,30 @@ services: volumes: *ubuntu-volumes command: *cpp-command + ubuntu-cpp-odbc: + # -AL- todo check this later + # Arrow Flight SQL ODBC build with BUNDLED dependencies with downloaded dependencies. + image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-odbc + build: + context: . + dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-odbc.dockerfile + cache_from: + - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-odbc + args: + arch: ${ARCH} + base: "${ARCH}/ubuntu:${UBUNTU}" + llvm: ${LLVM} + shm_size: *shm-size + ulimits: *ulimits + environment: + <<: [*common, *ccache, *cpp] + ARROW_DEPENDENCY_SOURCE: BUNDLED + # Apache ORC always uses external orc-format. + ARROW_ORC: OFF + CMAKE_GENERATOR: "Unix Makefiles" + volumes: *ubuntu-volumes + command: *cpp-command + ubuntu-cpp-minimal: # Arrow build with minimal components/dependencies image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index d9e06b1657bc..9efa2009dd8e 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -722,6 +722,7 @@ function(ADD_TEST_CASE REL_TEST_NAME) set(TEST_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_NAME}") add_executable(${TEST_NAME} ${SOURCES}) + message(STATUS "-AL- TEST_NAME: ${TEST_NAME}") # With OSX and conda, we need to set the correct RPATH so that dependencies # are found. The installed libraries with conda have an RPATH that matches diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index 5d34ff50e35c..017a5a6efb26 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -107,10 +107,6 @@ macro(tsort_bool_option_dependencies) endmacro() macro(resolve_option_dependencies) - # Arrow Flight SQL ODBC is available only for Windows and macOS for now. - if(NOT WIN32 AND NOT APPLE) - set(ARROW_FLIGHT_SQL_ODBC OFF) - endif() if(MSVC_TOOLCHAIN) set(ARROW_USE_GLOG OFF) endif() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index edf2c939aa40..1bdf7c5712ec 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1090,7 +1090,7 @@ function(build_boost) set(ARROW_BOOST_NEED_MULTIPRECISION FALSE) endif() if(ARROW_ENABLE_THREADING) - if(ARROW_WITH_THRIFT OR (ARROW_FLIGHT_SQL_ODBC AND MSVC)) + if(ARROW_WITH_THRIFT OR ARROW_FLIGHT_SQL_ODBC) list(APPEND BOOST_INCLUDE_LIBRARIES locale) endif() if(ARROW_BOOST_NEED_MULTIPRECISION) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index eee63b11ca1c..d18be5766824 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -259,7 +259,6 @@ function(ADD_ARROW_TEST REL_TEST_NAME) else() set(LABELS "arrow-tests") endif() - add_test_case(${REL_TEST_NAME} PREFIX ${PREFIX} diff --git a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt index 2560cdccd09e..2a7a31733969 100644 --- a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt +++ b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt @@ -28,7 +28,10 @@ else() endif() add_subdirectory(odbc_impl) -add_subdirectory(tests) +if(WIN32 OR APPLE) + # TODO: Enable Linux test build in separate PR + add_subdirectory(tests) +endif() arrow_install_all_headers("arrow/flight/sql/odbc") diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_api.cc b/cpp/src/arrow/flight/sql/odbc/odbc_api.cc index d0451a551e56..e2358e9a9f8f 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_api.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_api.cc @@ -37,6 +37,8 @@ #endif // defined(_WIN32) namespace arrow::flight::sql::odbc { +void LoadPropertiesFromDSN(const std::string& dsn, Connection::ConnPropertyMap& props); + SQLRETURN SQLAllocHandle(SQLSMALLINT type, SQLHANDLE parent, SQLHANDLE* result) { ARROW_LOG(DEBUG) << "SQLAllocHandle called with type: " << type << ", parent: " << parent diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_api_internal.h b/cpp/src/arrow/flight/sql/odbc/odbc_api_internal.h index f9d8d887cb87..0f581d734e1d 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_api_internal.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_api_internal.h @@ -20,8 +20,7 @@ #include "arrow/flight/sql/odbc/odbc_impl/platform.h" #include -#include -#include +#include // \file odbc_api_internal.h // diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt b/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt index 74c60cd91632..f8bbe5c5141e 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt @@ -130,7 +130,13 @@ if(WIN32) win_system_dsn.cc) endif() -if(APPLE) +if(WIN32) + find_package(ODBC REQUIRED) + target_include_directories(arrow_odbc_spi_impl PUBLIC ${ODBC_INCLUDE_DIR}) + target_link_libraries(arrow_odbc_spi_impl + PUBLIC arrow_flight_sql_shared arrow_compute_shared Boost::locale + ${ODBCINST}) +elseif(APPLE) target_include_directories(arrow_odbc_spi_impl SYSTEM BEFORE PUBLIC ${ODBC_INCLUDE_DIR}) target_link_libraries(arrow_odbc_spi_impl PUBLIC arrow_flight_sql_static @@ -139,11 +145,16 @@ if(APPLE) Boost::headers RapidJSON) else() - find_package(ODBC REQUIRED) - target_include_directories(arrow_odbc_spi_impl PUBLIC ${ODBC_INCLUDE_DIR}) + # -AL- Linux build + message(status "-AL- ODBC_INCLUDE_DIR on Linux: ${ODBC_INCLUDE_DIR}") + target_include_directories(arrow_odbc_spi_impl SYSTEM BEFORE PUBLIC ${ODBC_INCLUDE_DIR}) target_link_libraries(arrow_odbc_spi_impl - PUBLIC arrow_flight_sql_shared arrow_compute_shared Boost::locale - ${ODBCINST}) + PUBLIC arrow_flight_sql_static + arrow_compute_static + Boost::locale + Boost::headers + RapidJSON + ${ODBCINST}) # -AL- adding ${ODBCINST} as attempt to fix CI remotely endif() # CLI diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/binary_array_accessor_test.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/binary_array_accessor_test.cc index 423870eb3bed..502eaf730689 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/binary_array_accessor_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/binary_array_accessor_test.cc @@ -43,7 +43,7 @@ TEST(BinaryArrayAccessor, TestCDataTypeBinaryBasic) { accessor.GetColumnarData(&binding, 0, values.size(), value_offset, false, diagnostics, nullptr)); - for (int i = 0; i < values.size(); ++i) { + for (size_t i = 0; i < values.size(); ++i) { ASSERT_EQ(values[i].length(), str_len_buffer[i]); // Beware that CDataType_BINARY values are not null terminated. // It's safe to create a std::string from this data because we know it's diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/boolean_array_accessor_test.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/boolean_array_accessor_test.cc index b3f402dd7c1f..9299ee7602f0 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/boolean_array_accessor_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/boolean_array_accessor_test.cc @@ -40,7 +40,7 @@ TEST(BooleanArrayFlightSqlAccessor, TestBooleanArrayCDataTypeBit) { accessor.GetColumnarData(&binding, 0, values.size(), value_offset, false, diagnostics, nullptr)); - for (int i = 0; i < values.size(); ++i) { + for (size_t i = 0; i < values.size(); ++i) { ASSERT_EQ(sizeof(unsigned char), str_len_buffer[i]); ASSERT_EQ(values[i] ? 1 : 0, buffer[i]); } diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/decimal_array_accessor_test.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/decimal_array_accessor_test.cc index 6664b2d6e60a..1b589b41c049 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/decimal_array_accessor_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/decimal_array_accessor_test.cc @@ -84,7 +84,7 @@ void AssertNumericOutput(int input_precision, int input_scale, accessor.GetColumnarData(&binding, 0, values.size(), value_offset, false, diagnostics, nullptr)); - for (int i = 0; i < values.size(); ++i) { + for (size_t i = 0; i < values.size(); ++i) { ASSERT_EQ(sizeof(NUMERIC_STRUCT), str_len_buffer[i]); ASSERT_EQ(output_precision, buffer[i].precision); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/primitive_array_accessor_test.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/primitive_array_accessor_test.cc index a5ce05fb717c..521a9c48b9aa 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/primitive_array_accessor_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/primitive_array_accessor_test.cc @@ -47,7 +47,7 @@ void TestPrimitiveArraySqlAccessor() { accessor.GetColumnarData(&binding, 0, values.size(), value_offset, false, diagnostics, nullptr)); - for (int i = 0; i < values.size(); ++i) { + for (size_t i = 0; i < values.size(); ++i) { ASSERT_EQ(sizeof(c_type), str_len_buffer[i]); ASSERT_EQ(values[i], buffer[i]); } diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/string_array_accessor_test.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/string_array_accessor_test.cc index 4d0e13934072..6950f3351b7c 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/string_array_accessor_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/string_array_accessor_test.cc @@ -44,7 +44,7 @@ TEST(StringArrayAccessor, TestCDataTypeCharBasic) { accessor.GetColumnarData(&binding, 0, values.size(), value_offset, false, diagnostics, nullptr)); - for (int i = 0; i < values.size(); ++i) { + for (size_t i = 0; i < values.size(); ++i) { ASSERT_EQ(values[i].length(), str_len_buffer[i]); ASSERT_EQ(values[i], std::string(buffer.data() + i * max_str_len)); } @@ -103,7 +103,7 @@ TEST(StringArrayAccessor, TestCDataTypeWcharBasic) { accessor->GetColumnarData(&binding, 0, values.size(), value_offset, false, diagnostics, nullptr)); - for (int i = 0; i < values.size(); ++i) { + for (size_t i = 0; i < values.size(); ++i) { ASSERT_EQ(values[i].length() * GetSqlWCharSize(), str_len_buffer[i]); std::vector expected; Utf8ToWcs(values[i].c_str(), &expected); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/types.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/types.h index c0084a5ab155..7cd52af1bd02 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/types.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/types.h @@ -43,12 +43,12 @@ struct ColumnBinding { ColumnBinding(CDataType target_type, int precision, int scale, void* buffer, size_t buffer_length, ssize_t* str_len_buffer) - : target_type(target_type), - precision(precision), - scale(scale), - buffer(buffer), + : buffer(buffer), + str_len_buffer(str_len_buffer), buffer_length(buffer_length), - str_len_buffer(str_len_buffer) {} + target_type(target_type), + precision(precision), + scale(scale) {} }; /// \brief Accessor interface meant to provide a way of populating data of a diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/attribute_utils.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/attribute_utils.h index cfda4cf29a74..e659d6ef9dfa 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/attribute_utils.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/attribute_utils.h @@ -160,11 +160,21 @@ inline void SetAttributeSQLWCHAR(SQLPOINTER new_value, SQLINTEGER input_length_i thread_local std::vector utf8_str; if (input_length_in_bytes == SQL_NTS) { WcsToUtf8(new_value, &utf8_str); + } else if (input_length_in_bytes <= 0) { + // empty string + attribute_to_write.clear(); + return; } else { WcsToUtf8(new_value, input_length_in_bytes / arrow::flight::sql::odbc::GetSqlWCharSize(), &utf8_str); } + + // add null-terminator + if (utf8_str.back() != '\0') { + utf8_str.push_back('\0'); + } + attribute_to_write.assign((char*)utf8_str.data()); } diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/blocking_queue.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/blocking_queue.h index e52c305e4616..55ec3a7beaec 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/blocking_queue.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/blocking_queue.h @@ -49,7 +49,7 @@ class BlockingQueue { void AddProducer(Supplier supplier) { active_threads_++; - threads_.emplace_back([=] { + threads_.emplace_back([this, supplier] { while (!closed_) { // Block while queue is full std::unique_lock unique_lock(mtx_); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc index 1d51f10be69e..e0c7baa3a5f6 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc @@ -15,9 +15,12 @@ // specific language governing permissions and limitations // under the License. +// flight_sql_connection.h needs to be included first due to conflicts with windows.h +#include "arrow/flight/sql/odbc/odbc_impl/flight_sql_connection.h" + #include "arrow/flight/sql/odbc/odbc_impl/config/configuration.h" -#include "arrow/flight/sql/odbc/odbc_impl/flight_sql_connection.h" +#include "arrow/flight/sql/odbc/odbc_impl/attribute_utils.h" #include "arrow/flight/sql/odbc/odbc_impl/util.h" #include "arrow/result.h" #include "arrow/util/utf8.h" @@ -28,6 +31,10 @@ #include #include +#include "arrow/util/logging.h" // -AL- TEMP + +using ODBC::SetAttributeSQLWCHAR; + namespace arrow::flight::sql::odbc { namespace config { static const char DEFAULT_DSN[] = "Apache Arrow Flight SQL"; @@ -42,22 +49,37 @@ std::string ReadDsnString(const std::string& dsn, std::string_view key, CONVERT_WIDE_STR(const std::wstring wkey, key); CONVERT_WIDE_STR(const std::wstring wdflt, dflt); + // -AL- found workaround for `cannot convert 'const wchar_t*' to 'LPCWSTR' {aka + // 'const short unsigned int*'}` on Linux. + + // Via CONVERT_WIDE_STR, Arrow correctly converts to UFT-32 on Unix systems, + // so the conversion from wchar_t to short unsigned int* will work on Linux. + + // -AL- I just need to wrap `reinterpret_cast()` on all string args for + // SQLGetPrivateProfileString. + #define BUFFER_SIZE (1024) - std::vector buf(BUFFER_SIZE); - int ret = - SQLGetPrivateProfileString(wdsn.c_str(), wkey.c_str(), wdflt.c_str(), buf.data(), - static_cast(buf.size()), L"ODBC.INI"); + std::vector buf(BUFFER_SIZE); + int ret = SQLGetPrivateProfileString( + reinterpret_cast(wdsn.c_str()), reinterpret_cast(wkey.c_str()), + reinterpret_cast(wdflt.c_str()), buf.data(), static_cast(buf.size()), + reinterpret_cast(L"ODBC.INI")); if (ret > BUFFER_SIZE) { // If there wasn't enough space, try again with the right size buffer. buf.resize(ret + 1); - ret = - SQLGetPrivateProfileString(wdsn.c_str(), wkey.c_str(), wdflt.c_str(), buf.data(), - static_cast(buf.size()), L"ODBC.INI"); + ret = SQLGetPrivateProfileString( + reinterpret_cast(wdsn.c_str()), reinterpret_cast(wkey.c_str()), + reinterpret_cast(wdflt.c_str()), buf.data(), + static_cast(buf.size()), reinterpret_cast(L"ODBC.INI")); } - std::wstring wresult = std::wstring(buf.data(), ret); - CONVERT_UTF8_STR(const std::string result, wresult); + std::string result(""); + ARROW_LOG(DEBUG) << "-AL- ReadDsnString key: " << key; + ARROW_LOG(DEBUG) << "-AL- ReadDsnString result before: " << result; + SetAttributeSQLWCHAR(buf.data(), ret * GetSqlWCharSize(), result); + ARROW_LOG(DEBUG) << "-AL- ReadDsnString result: " << result; + ARROW_LOG(DEBUG) << "-AL- ReadDsnString ret: " << ret; return result; } @@ -77,30 +99,35 @@ void RemoveAllKnownKeys(std::vector& keys) { std::vector ReadAllKeys(const std::string& dsn) { CONVERT_WIDE_STR(const std::wstring wdsn, dsn); - std::vector buf(BUFFER_SIZE); + std::vector buf(BUFFER_SIZE); - int ret = SQLGetPrivateProfileString(wdsn.c_str(), NULL, L"", buf.data(), - static_cast(buf.size()), L"ODBC.INI"); + int ret = SQLGetPrivateProfileString( + reinterpret_cast(wdsn.c_str()), NULL, reinterpret_cast(L""), + buf.data(), static_cast(buf.size()), reinterpret_cast(L"ODBC.INI")); if (ret > BUFFER_SIZE) { // If there wasn't enough space, try again with the right size buffer. buf.resize(ret + 1); - ret = SQLGetPrivateProfileString(wdsn.c_str(), NULL, L"", buf.data(), - static_cast(buf.size()), L"ODBC.INI"); + ret = SQLGetPrivateProfileString( + reinterpret_cast(wdsn.c_str()), NULL, reinterpret_cast(L""), + buf.data(), static_cast(buf.size()), reinterpret_cast(L"ODBC.INI")); } // When you pass NULL to SQLGetPrivateProfileString it gives back a \0 delimited list of // all the keys. The below loop simply tokenizes all the keys and places them into a // vector. std::vector keys; - wchar_t* begin = buf.data(); + SQLWCHAR* begin = buf.data(); while (begin && *begin != '\0') { - wchar_t* cur; + SQLWCHAR* cur; for (cur = begin; *cur != '\0'; ++cur) { } - CONVERT_UTF8_STR(const std::string key, std::wstring(begin, cur)); + std::string key(""); + SQLINTEGER key_len = cur - begin; + SetAttributeSQLWCHAR(begin, key_len * GetSqlWCharSize(), key); keys.emplace_back(key); + ARROW_LOG(DEBUG) << "-AL- ReadAllKeys key: " << key; begin = ++cur; } return keys; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_connection.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_connection.cc index abe82b36c454..e33268476d23 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_connection.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_connection.cc @@ -27,6 +27,7 @@ #include "arrow/flight/sql/odbc/odbc_impl/util.h" #include "arrow/flight/types.h" +#define BOOST_NO_CXX98_FUNCTION_BASE // ARROW-17805 #include #include #include @@ -298,7 +299,7 @@ FlightClientOptions FlightSqlConnection::BuildFlightClientOptions( } } - return std::move(options); + return options; } Location FlightSqlConnection::BuildLocation( @@ -410,9 +411,9 @@ Connection::Info FlightSqlConnection::GetInfo(uint16_t info_type) { FlightSqlConnection::FlightSqlConnection(OdbcVersion odbc_version, const std::string& driver_version) - : diagnostics_("Apache Arrow", "Flight SQL", odbc_version), + : info_(client_options_, call_options_, sql_client_, driver_version), + diagnostics_("Apache Arrow", "Flight SQL", odbc_version), odbc_version_(odbc_version), - info_(client_options_, call_options_, sql_client_, driver_version), closed_(true) { attribute_[CONNECTION_DEAD] = static_cast(SQL_TRUE); attribute_[LOGIN_TIMEOUT] = static_cast(0); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc index 56e5bb973f75..6a366ac5d001 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc @@ -81,7 +81,7 @@ size_t FlightSqlResultSet::Move(size_t rows, size_t bind_offset, size_t bind_typ } // Reset GetData value offsets. - if (num_binding_ != get_data_offsets_.size() && reset_get_data_) { + if ((size_t)num_binding_ != get_data_offsets_.size() && reset_get_data_) { std::fill(get_data_offsets_.begin(), get_data_offsets_.end(), 0); } diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set_accessors.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set_accessors.cc index f309ab6156ea..e34b36ab0d43 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set_accessors.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set_accessors.cc @@ -29,6 +29,13 @@ #include "arrow/flight/sql/odbc/odbc_impl/accessors/timestamp_array_accessor.h" #include "arrow/flight/sql/odbc/odbc_impl/platform.h" +// Workaround for ODBC `BOOL` def conflict on Linux +#ifdef __linux__ +# ifdef BOOL +# undef BOOL +# endif // BOOL +#endif // __linux__ + namespace arrow::flight::sql::odbc { typedef std::pair SourceAndTargetPair; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_statement.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_statement.cc index ef652d34fac1..c75631222605 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_statement.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_statement.cc @@ -58,9 +58,9 @@ FlightSqlStatement::FlightSqlStatement(const Diagnostics& diagnostics, const MetadataSettings& metadata_settings) : diagnostics_("Apache Arrow", diagnostics.GetDataSourceComponent(), diagnostics.GetOdbcVersion()), - sql_client_(sql_client), client_options_(std::move(client_options)), call_options_(std::move(call_options)), + sql_client_(sql_client), metadata_settings_(metadata_settings) { attribute_[METADATA_ID] = static_cast(SQL_FALSE); attribute_[MAX_LENGTH] = static_cast(0); @@ -88,8 +88,8 @@ bool FlightSqlStatement::SetAttribute(StatementAttributeId attribute, TimeoutDuration{static_cast(std::get(value))}; } else { call_options_.timeout = TimeoutDuration{-1}; - // Intentional fall-through. } + [[fallthrough]]; default: attribute_[attribute] = value; return true; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc index 7d6239f24a7e..55e8d72c9b7c 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc @@ -395,8 +395,10 @@ bool GetInfoCache::LoadInfoFromServer() { // Unused by ODBC. break; case SqlInfoOptions::SQL_DDL_SCHEMA: { - bool supports_schema_ddl = - reinterpret_cast(scalar->child_value().get())->value; + // bool supports_schema_ddl = + // reinterpret_cast(scalar->child_value().get())->value; + // -AL- todo raise GitHub issues for finishing this work. + // Note: this is a bitmask and we can't describe cascade or restrict // flags. info_[SQL_DROP_SCHEMA] = static_cast(SQL_DS_DROP_SCHEMA); @@ -407,8 +409,10 @@ bool GetInfoCache::LoadInfoFromServer() { break; } case SqlInfoOptions::SQL_DDL_TABLE: { - bool supports_table_ddl = - reinterpret_cast(scalar->child_value().get())->value; + // bool supports_table_ddl = + // reinterpret_cast(scalar->child_value().get())->value; + // -AL- todo raise GitHub issues for finishing this work. + // This is a bitmask and we cannot describe all clauses. info_[SQL_CREATE_TABLE] = static_cast(SQL_CT_CREATE_TABLE); info_[SQL_DROP_TABLE] = static_cast(SQL_DT_DROP_TABLE); @@ -697,7 +701,6 @@ bool GetInfoCache::LoadInfoFromServer() { break; } case SqlInfoOptions::SQL_DEFAULT_TRANSACTION_ISOLATION: { - constexpr int32_t NONE = 0; constexpr int32_t READ_UNCOMMITTED = 1; constexpr int32_t READ_COMMITTED = 2; constexpr int32_t REPEATABLE_READ = 3; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/main.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/main.cc index 3336e0160e1f..47929830067a 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/main.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/main.cc @@ -69,7 +69,7 @@ void TestBindColumn(const std::shared_ptr& connection) { total += fetched_rows; std::cout << "Total:" << total << std::endl; - for (int i = 0; i < fetched_rows; ++i) { + for (size_t i = 0; i < fetched_rows; ++i) { ARROW_LOG(DEBUG) << "Row[" << i << "] incidnt_num: '" << incidnt_num[i] << "', Category: '" << category[i] << "'"; } @@ -138,7 +138,7 @@ void TestBindColumnBigInt(const std::shared_ptr& connection) { total += fetched_rows; ARROW_LOG(DEBUG) << "Total:" << total; - for (int i = 0; i < fetched_rows; ++i) { + for (size_t i = 0; i < fetched_rows; ++i) { ARROW_LOG(DEBUG) << "Row[" << i << "] incidnt_num: '" << incidnt_num[i] << "', " << "double_field: '" << double_field[i] << "', " << "category: '" << category[i] << "'"; @@ -183,7 +183,7 @@ void TestGetColumnsV3(const std::shared_ptr& connection) { ssize_t result_length; while (result_set->Move(1, 0, 0, nullptr) == 1) { - for (int i = 0; i < column_count; ++i) { + for (size_t i = 0; i < column_count; ++i) { result_set->GetData(1 + i, arrow::flight::sql::odbc::CDataType_CHAR, 0, 0, result.data(), buffer_length, &result_length); std::cout << (result_length != -1 ? result.data() : "NULL") << '\t'; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_connection.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_connection.cc index a2464aa00e36..c92012cc84f9 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_connection.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_connection.cc @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/flight/sql/odbc/odbc_impl/odbc_connection.h" - #include "arrow/result.h" #include "arrow/util/utf8.h" +// Include ODBC headers after arrow fwd type header to avoid conflicts +#include "arrow/flight/sql/odbc/odbc_impl/odbc_connection.h" + #include "arrow/flight/sql/odbc/odbc_impl/attribute_utils.h" #include "arrow/flight/sql/odbc/odbc_impl/config/configuration.h" #include "arrow/flight/sql/odbc/odbc_impl/exceptions.h" @@ -30,10 +31,12 @@ #include "arrow/flight/sql/odbc/odbc_impl/spi/statement.h" #include "arrow/flight/sql/odbc/odbc_impl/util.h" -// Include ODBC headers after arrow headers to avoid conflicts with sql_info_undef.h +// Include ODBC headers after arrow headers to avoid conflicts #include #include #include + +#define BOOST_NO_CXX98_FUNCTION_BASE // ARROW-17805 #include #include #include diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_descriptor.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_descriptor.cc index 8c856fdbd6bc..ebb69720c887 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_descriptor.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_descriptor.cc @@ -17,8 +17,6 @@ #include "arrow/flight/sql/odbc/odbc_impl/odbc_descriptor.h" -#include -#include #include #include "arrow/flight/sql/odbc/odbc_impl/attribute_utils.h" #include "arrow/flight/sql/odbc/odbc_impl/exceptions.h" @@ -28,6 +26,10 @@ #include "arrow/flight/sql/odbc/odbc_impl/spi/statement.h" #include "arrow/flight/sql/odbc/odbc_impl/type_utilities.h" +// Include ODBC headers after arrow headers to avoid conflicts +#include +#include + using ODBC::DescriptorRecord; using ODBC::ODBCConnection; using ODBC::ODBCDescriptor; @@ -155,7 +157,7 @@ void ODBCDescriptor::SetField(SQLSMALLINT record_number, SQLSMALLINT field_ident throw DriverException("Bookmarks are unsupported.", "07009"); } - if (record_number > records_.size()) { + if (static_cast(record_number) > records_.size()) { throw DriverException("Invalid descriptor index", "HY009"); } @@ -308,7 +310,7 @@ void ODBCDescriptor::GetField(SQLSMALLINT record_number, SQLSMALLINT field_ident throw DriverException("Bookmarks are unsupported.", "07009"); } - if (record_number > records_.size()) { + if (static_cast(record_number) > records_.size()) { throw DriverException("Invalid descriptor index", "07009"); } @@ -541,7 +543,7 @@ void ODBCDescriptor::BindCol(SQLSMALLINT record_number, SQLSMALLINT c_type, assert(is_writable_); // The set of records auto-expands to the supplied record number. - if (records_.size() < record_number) { + if (records_.size() < static_cast(record_number)) { records_.resize(record_number); } @@ -561,7 +563,7 @@ void ODBCDescriptor::BindCol(SQLSMALLINT record_number, SQLSMALLINT c_type, } void ODBCDescriptor::SetDataPtrOnRecord(SQLPOINTER data_ptr, SQLSMALLINT record_number) { - assert(record_number <= records_.size()); + assert(static_cast(record_number) <= records_.size()); DescriptorRecord& record = records_[record_number - 1]; if (data_ptr) { record.CheckConsistency(); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc index c9cedfe3b3f3..f7701ddc36c7 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc @@ -15,17 +15,22 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/flight/sql/odbc/odbc_impl/odbc_statement.h" +// platform.h platform.h includes windows.h so it needs to be included first +// -AL- this is an attempt to fix the windows build +#include "arrow/flight/sql/odbc/odbc_impl/platform.h" + +#include "arrow/type.h" +// Include ODBC headers after arrow fwd type header to avoid conflicts #include "arrow/flight/sql/odbc/odbc_impl/attribute_utils.h" #include "arrow/flight/sql/odbc/odbc_impl/exceptions.h" #include "arrow/flight/sql/odbc/odbc_impl/odbc_connection.h" #include "arrow/flight/sql/odbc/odbc_impl/odbc_descriptor.h" +#include "arrow/flight/sql/odbc/odbc_impl/odbc_statement.h" #include "arrow/flight/sql/odbc/odbc_impl/spi/result_set.h" #include "arrow/flight/sql/odbc/odbc_impl/spi/result_set_metadata.h" #include "arrow/flight/sql/odbc/odbc_impl/spi/statement.h" #include "arrow/flight/sql/odbc/odbc_impl/types.h" -#include "arrow/type.h" #include #include @@ -737,7 +742,7 @@ SQLRETURN ODBCStatement::GetData(SQLSMALLINT record_number, SQLSMALLINT c_type, SQLLEN* indicator_ptr) { if (record_number == 0) { throw DriverException("Bookmarks are not supported", "07009"); - } else if (record_number > ird_->GetRecords().size()) { + } else if (static_cast(record_number) > ird_->GetRecords().size()) { throw DriverException("Invalid column index: " + std::to_string(record_number), "07009"); } @@ -752,7 +757,7 @@ SQLRETURN ODBCStatement::GetData(SQLSMALLINT record_number, SQLSMALLINT c_type, int scale = ird_record.scale; if (c_type == SQL_ARD_TYPE) { - if (record_number > current_ard_->GetRecords().size()) { + if (static_cast(record_number) > current_ard_->GetRecords().size()) { throw DriverException("Invalid column index: " + std::to_string(record_number), "07009"); } @@ -765,7 +770,7 @@ SQLRETURN ODBCStatement::GetData(SQLSMALLINT record_number, SQLSMALLINT c_type, // Note: this is intentionally not an else if, since the type can be SQL_C_DEFAULT in // the ARD. if (evaluated_c_type == SQL_C_DEFAULT) { - if (record_number <= current_ard_->GetRecords().size()) { + if (static_cast(record_number) <= current_ard_->GetRecords().size()) { const DescriptorRecord& ard_record = current_ard_->GetRecords()[record_number - 1]; precision = ard_record.precision; scale = ard_record.scale; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.h index a223b1867e05..f2226e84f6c9 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.h @@ -85,7 +85,7 @@ class ODBCStatement : public ODBCHandle { /// \brief Return number of columns from data set void GetColumnCount(SQLSMALLINT* column_count_ptr); - /// \brief Return number of rows affected by an UPDATE, INSERT, or DELETE statement\ + /// \brief Return number of rows affected by an UPDATE, INSERT, or DELETE statement /// /// -1 is returned as driver only supports SELECT statement void GetRowCount(SQLLEN* row_count_ptr); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/connection.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/connection.h index 00a7b511ef20..3e4c08c9cdd2 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/connection.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/connection.h @@ -17,6 +17,7 @@ #pragma once +#define BOOST_NO_CXX98_FUNCTION_BASE // ARROW-17805 #include #include #include @@ -36,7 +37,11 @@ struct CaseInsensitiveComparator { using is_transparent = std::true_type; bool operator()(std::string_view s1, std::string_view s2) const { - return boost::lexicographical_compare(s1, s2, boost::is_iless()); + return std::lexicographical_compare( + s1.begin(), s1.end(), s2.begin(), s2.end(), [](char a, char b) { + return std::tolower(static_cast(a)) < + std::tolower(static_cast(b)); + }); } }; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/statement.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/statement.h index 2c58371d1080..f5157b00bc2c 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/statement.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/spi/statement.h @@ -22,6 +22,7 @@ #include #include +#include "arrow/flight/sql/odbc/odbc_impl/diagnostics.h" #include "arrow/flight/sql/odbc/odbc_impl/type_fwd.h" namespace arrow::flight::sql::odbc { diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc index 7b974cc35a3e..ae97bf875676 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc @@ -23,6 +23,13 @@ #include #include +#ifdef __linux__ +# define GET_SQWCHAR_PTR(wstring_var) (ToSqlWCharVector(wstring_var).data()) +#else +// Windows and macOS +# define GET_SQWCHAR_PTR(wstring_var) (wstring_var.c_str()) +#endif + namespace arrow::flight::sql::odbc { using config::Configuration; @@ -45,16 +52,31 @@ void PostArrowUtilError(arrow::Status error_status) { void PostLastInstallerError() { #define BUFFER_SIZE (1024) DWORD code; - wchar_t msg[BUFFER_SIZE]; - SQLInstallerError(1, &code, msg, BUFFER_SIZE, NULL); + std::vector msg(BUFFER_SIZE); + SQLInstallerError(1, &code, msg.data(), BUFFER_SIZE, NULL); std::wstringstream buf; - buf << L"Message: \"" << msg << L"\", Code: " << code; +#ifdef __linux__ + buf << L"Message: \""; + for (SQLWCHAR wch : msg) { + buf << static_cast(wch); + } + buf << L"\", Code: " << code; +#else + // Windows and macOS + buf << L"Message: \"" << msg.data() << L"\", Code: " << code; +#endif // __linux__ std::wstring error_msg = buf.str(); PostError(code, const_cast(error_msg.c_str())); } +std::vector ToSqlWCharVector(const std::wstring& ws) { + std::vector buf; + buf.assign(ws.begin(), ws.end()); + return buf; +} + /** * Unregister specified DSN. * @@ -62,7 +84,7 @@ void PostLastInstallerError() { * @return True on success and false on fail. */ bool UnregisterDsn(const std::wstring& dsn) { - if (SQLRemoveDSNFromIni(dsn.c_str())) { + if (SQLRemoveDSNFromIni(GET_SQWCHAR_PTR(dsn))) { return true; } @@ -86,7 +108,7 @@ bool RegisterDsn(const Configuration& config, LPCWSTR driver) { } std::wstring wdsn = wdsn_result.ValueOrDie(); - if (!SQLWriteDSNToIni(wdsn.c_str(), driver)) { + if (!SQLWriteDSNToIni(GET_SQWCHAR_PTR(wdsn), driver)) { PostLastInstallerError(); return false; } @@ -113,8 +135,9 @@ bool RegisterDsn(const Configuration& config, LPCWSTR driver) { } std::wstring wvalue = wvalue_result.ValueOrDie(); - if (!SQLWritePrivateProfileString(wdsn.c_str(), wkey.c_str(), wvalue.c_str(), - L"ODBC.INI")) { + if (!SQLWritePrivateProfileString(GET_SQWCHAR_PTR(wdsn), GET_SQWCHAR_PTR(wkey), + GET_SQWCHAR_PTR(wvalue), + reinterpret_cast(L"ODBC.INI"))) { PostLastInstallerError(); return false; } diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.cc index c23933287fb1..6792f7f7f52a 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.cc @@ -137,6 +137,7 @@ SqlDataType GetDataTypeFromArrowFieldV3(const std::shared_ptr& field, case Type::LARGE_LIST: case Type::MAX_ID: case Type::NA: + default: break; } @@ -804,6 +805,8 @@ std::shared_ptr GetDefaultDataTypeForTypeId(Type::type type_id) { return arrow::time64(TimeUnit::MICRO); case Type::TIMESTAMP: return arrow::timestamp(TimeUnit::SECOND); + default: + break; } throw DriverException(std::string("Invalid type id: ") + std::to_string(type_id)); From 0e114ba00b3a091c747054ac57795dff1ad32873 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Fri, 6 Mar 2026 12:15:46 -0800 Subject: [PATCH 02/12] Reuse existing docker image (no install of unixodbc) --- .github/workflows/cpp_extra.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index dd758058d9e9..54cc5dee1863 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -383,7 +383,14 @@ jobs: # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes sudo sysctl -w vm.mmap_rnd_bits=28 source ci/scripts/util_enable_core_dumps.sh - archery docker run ubuntu-cpp-odbc + archery docker run \ + -e ARROW_DEPENDENCY_SOURCE=BUNDLED \ + -e ARROW_DEPENDENCY_USE_SHARED=OFF \ + -e ARROW_FLIGHT_SQL_ODBC=ON \ + ubuntu-cpp + + # ubuntu-cpp \ + # bash -c "apt-get update -y -q && apt-get install -y -q --no-install-recommends unixodbc unixodbc-dev" - name: Docker Push if: >- success() && @@ -394,7 +401,7 @@ jobs: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} continue-on-error: true - run: archery docker push ubuntu-cpp-odbc + run: archery docker push ubuntu-cpp odbc-macos: needs: check-labels From f157cac752c57ef54bc48ac931c3510dd11d53ec Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Fri, 6 Mar 2026 15:22:15 -0800 Subject: [PATCH 03/12] Install unixodbc-dev inside docker --- .github/workflows/cpp_extra.yml | 3 --- ci/docker/ubuntu-24.04-cpp.dockerfile | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 54cc5dee1863..94eb2911b1fb 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -388,9 +388,6 @@ jobs: -e ARROW_DEPENDENCY_USE_SHARED=OFF \ -e ARROW_FLIGHT_SQL_ODBC=ON \ ubuntu-cpp - - # ubuntu-cpp \ - # bash -c "apt-get update -y -q && apt-get install -y -q --no-install-recommends unixodbc unixodbc-dev" - name: Docker Push if: >- success() && diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 7703046c75cd..5953c49ff9a0 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -121,6 +121,7 @@ RUN apt-get update -y -q && \ rsync \ tzdata \ tzdata-legacy \ + unixodbc-dev \ uuid-runtime \ unzip \ wget && \ From 68c1d62af669f67bcecc96549e5e99df4d6b0c00 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Mon, 9 Mar 2026 16:33:58 -0700 Subject: [PATCH 04/12] Clean up Linux changes --- .../flight/sql/odbc/odbc_impl/CMakeLists.txt | 23 +++++++++---------- .../odbc/odbc_impl/config/configuration.cc | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt b/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt index f8bbe5c5141e..7d879e95c3a2 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt @@ -136,25 +136,24 @@ if(WIN32) target_link_libraries(arrow_odbc_spi_impl PUBLIC arrow_flight_sql_shared arrow_compute_shared Boost::locale ${ODBCINST}) -elseif(APPLE) - target_include_directories(arrow_odbc_spi_impl SYSTEM BEFORE PUBLIC ${ODBC_INCLUDE_DIR}) - target_link_libraries(arrow_odbc_spi_impl - PUBLIC arrow_flight_sql_static - arrow_compute_static - Boost::locale - Boost::headers - RapidJSON) else() - # -AL- Linux build - message(status "-AL- ODBC_INCLUDE_DIR on Linux: ${ODBC_INCLUDE_DIR}") + # -AL- Linux build. share with apple + if(NOT APPLE) + message(status "-AL- ODBC_INCLUDE_DIR on Linux: ${ODBC_INCLUDE_DIR}") + endif() + target_include_directories(arrow_odbc_spi_impl SYSTEM BEFORE PUBLIC ${ODBC_INCLUDE_DIR}) target_link_libraries(arrow_odbc_spi_impl PUBLIC arrow_flight_sql_static arrow_compute_static Boost::locale Boost::headers - RapidJSON - ${ODBCINST}) # -AL- adding ${ODBCINST} as attempt to fix CI remotely + RapidJSON) + + if(NOT APPLE) + # Explicitly link to unix-odbc on Linux + target_link_libraries(arrow_odbc_spi_impl PUBLIC ${ODBCINST}) + endif() endif() # CLI diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc index e0c7baa3a5f6..b2d8552e1c31 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc @@ -50,7 +50,7 @@ std::string ReadDsnString(const std::string& dsn, std::string_view key, CONVERT_WIDE_STR(const std::wstring wdflt, dflt); // -AL- found workaround for `cannot convert 'const wchar_t*' to 'LPCWSTR' {aka - // 'const short unsigned int*'}` on Linux. + // 'const short unsigned int*'}` on Linux. Notes in this file for reference only. // Via CONVERT_WIDE_STR, Arrow correctly converts to UFT-32 on Unix systems, // so the conversion from wchar_t to short unsigned int* will work on Linux. From 489d57e8062b21e6a01c5d9bfa7ba27c59b2590c Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Tue, 10 Mar 2026 10:54:13 -0700 Subject: [PATCH 05/12] Comments for next TODO --- compose.yaml | 3 ++- cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compose.yaml b/compose.yaml index 3f3b1509a7c5..b782b55184cf 100644 --- a/compose.yaml +++ b/compose.yaml @@ -367,10 +367,11 @@ services: volumes: &debian-volumes - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated - command: &cpp-command > + command: &cpp-command > # -AL- &cpp-command is defined here and calls the build + test sh scripts /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/cpp_test.sh /arrow /build" +# -AL- so I can define a new one for odbc-cpp too. ubuntu-cpp: # Usage: diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc index ae97bf875676..5c31b5e5b043 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc @@ -26,7 +26,8 @@ #ifdef __linux__ # define GET_SQWCHAR_PTR(wstring_var) (ToSqlWCharVector(wstring_var).data()) #else -// Windows and macOS +// Windows and macOS // -AL- TODO raise GitHub issues for Linux functions +// Can't test it right now anyways without the tests. # define GET_SQWCHAR_PTR(wstring_var) (wstring_var.c_str()) #endif @@ -73,7 +74,8 @@ void PostLastInstallerError() { std::vector ToSqlWCharVector(const std::wstring& ws) { std::vector buf; - buf.assign(ws.begin(), ws.end()); + // buf.assign(ws.begin(), ws.end()); + // -AL- GitHub issue to implement. Also need to move this function else where like encoding_utils.h? return buf; } From 330f68a3a5fcc002fd6188701bd8a99c41f90069 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Tue, 10 Mar 2026 13:46:33 -0700 Subject: [PATCH 06/12] Use `ubuntu-cpp-odbc` compose command and change to release mode * remove ODBC dockerfile as it is not needed. * in-progress of register ODBC * move unix odbc registration script to common place --- .github/workflows/cpp_extra.yml | 8 +- ci/docker/ubuntu-24.04-cpp-odbc.dockerfile | 216 ------------------ compose.yaml | 31 ++- cpp/src/arrow/flight/sql/odbc/README | 6 +- .../sql/odbc/install/mac/install_odbc_ini.sh | 2 +- .../install/{mac => unix}/install_odbc.sh | 13 +- 6 files changed, 40 insertions(+), 236 deletions(-) delete mode 100644 ci/docker/ubuntu-24.04-cpp-odbc.dockerfile rename cpp/src/arrow/flight/sql/odbc/install/{mac => unix}/install_odbc.sh (91%) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 94eb2911b1fb..ab1706e88ef2 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -383,11 +383,7 @@ jobs: # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes sudo sysctl -w vm.mmap_rnd_bits=28 source ci/scripts/util_enable_core_dumps.sh - archery docker run \ - -e ARROW_DEPENDENCY_SOURCE=BUNDLED \ - -e ARROW_DEPENDENCY_USE_SHARED=OFF \ - -e ARROW_FLIGHT_SQL_ODBC=ON \ - ubuntu-cpp + archery docker run ubuntu-cpp-odbc - name: Docker Push if: >- success() && @@ -499,7 +495,7 @@ jobs: "$(pwd)/build/cpp/${{ matrix.build-type }}/libarrow_flight_sql_odbc.dylib" - name: Register Flight SQL ODBC Driver run: | - sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh $(pwd)/build/cpp/${{ matrix.build-type }}/libarrow_flight_sql_odbc.dylib + sudo cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh $(pwd)/build/cpp/${{ matrix.build-type }}/libarrow_flight_sql_odbc.dylib - name: Test shell: bash run: | diff --git a/ci/docker/ubuntu-24.04-cpp-odbc.dockerfile b/ci/docker/ubuntu-24.04-cpp-odbc.dockerfile deleted file mode 100644 index 1e5d7bd0c91b..000000000000 --- a/ci/docker/ubuntu-24.04-cpp-odbc.dockerfile +++ /dev/null @@ -1,216 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:24.04 -FROM ${base} - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN latest_system_llvm=18 && \ - if [ ${llvm} -gt ${latest_system_llvm} -o \ - ${clang_tools} -gt ${latest_system_llvm} ]; then \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - lsb-release \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - code_name=$(lsb_release --codename --short) && \ - if [ ${llvm} -gt 10 ]; then \ - echo "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list; \ - fi && \ - if [ ${clang_tools} -ne ${llvm} -a \ - ${clang_tools} -gt ${latest_system_llvm} ]; then \ - echo "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi; \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - libclang-rt-${llvm}-dev \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - curl \ - gdb \ - git \ - libbenchmark-dev \ - libbrotli-dev \ - libbz2-dev \ - libc-ares-dev \ - libcurl4-openssl-dev \ - libgflags-dev \ - libgmock-dev \ - libgoogle-glog-dev \ - libgrpc++-dev \ - libidn2-dev \ - libkrb5-dev \ - libldap-dev \ - liblz4-dev \ - libnghttp2-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libpsl-dev \ - libradospp-dev \ - libre2-dev \ - librtmp-dev \ - libsnappy-dev \ - libsqlite3-dev \ - libssh-dev \ - libssh2-1-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libxml2-dev \ - libzstd-dev \ - make \ - mold \ - ninja-build \ - nlohmann-json3-dev \ - npm \ - patch \ - pkg-config \ - protobuf-compiler \ - protobuf-compiler-grpc \ - python3-dev \ - python3-pip \ - python3-rados \ - python3-venv \ - rados-objclass-dev \ - rapidjson-dev \ - rsync \ - tzdata \ - tzdata-legacy \ - unixodbc-dev \ - uuid-runtime \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -ARG gcc="" -RUN if [ "${gcc}" = "" ]; then \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - g++ \ - gcc; \ - else \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - g++-${gcc} \ - gcc-${gcc} && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \ - update-alternatives --install \ - /usr/bin/$(uname --machine)-linux-gnu-gcc \ - $(uname --machine)-linux-gnu-gcc \ - /usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc} 100 && \ - update-alternatives --install \ - /usr/bin/$(uname --machine)-linux-gnu-g++ \ - $(uname --machine)-linux-gnu-g++ \ - /usr/bin/$(uname --machine)-linux-gnu-g++-${gcc} 100 && \ - update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \ - update-alternatives --set cc /usr/bin/gcc && \ - update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \ - update-alternatives --set c++ /usr/bin/g++; \ - fi - -COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh latest /usr/local - -COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_gcs_testbench.sh default - -COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_azurite.sh - -COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_ceph.sh - -COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - -# Prioritize system packages and local installation. -# -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - opentelemetry-cpp-dev is not packaged -ENV ARROW_ACERO=ON \ - ARROW_AZURE=ON \ - ARROW_BUILD_STATIC=ON \ - ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=BUNDLED \ - ARROW_DEPENDENCY_USE_SHARED=OFF \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=ON \ - ARROW_FLIGHT_SQL=ON \ - ARROW_FLIGHT_SQL_ODBC=ON \ - ARROW_GANDIVA=ON \ - ARROW_GCS=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_JEMALLOC=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_S3=ON \ - ARROW_SUBSTRAIT=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_MOLD=ON \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_OPENTELEMETRY=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \ - AWSSDK_SOURCE=BUNDLED \ - Azure_SOURCE=BUNDLED \ - google_cloud_cpp_storage_SOURCE=BUNDLED \ - opentelemetry_cpp_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - PYTHON=python3 \ - xsimd_SOURCE=BUNDLED diff --git a/compose.yaml b/compose.yaml index b782b55184cf..e7f468d56785 100644 --- a/compose.yaml +++ b/compose.yaml @@ -499,28 +499,41 @@ services: command: *cpp-command ubuntu-cpp-odbc: - # -AL- todo check this later # Arrow Flight SQL ODBC build with BUNDLED dependencies with downloaded dependencies. - image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-odbc + image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp build: context: . - dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-odbc.dockerfile + dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile cache_from: - - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-odbc + - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp args: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" + clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE} + gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size + cap_add: + - SYS_ADMIN + devices: + - "/dev/fuse:/dev/fuse" + security_opt: + - apparmor:unconfined ulimits: *ulimits environment: - <<: [*common, *ccache, *cpp] + <<: [*common, *ccache, *sccache, *cpp] + ARROW_BUILD_TYPE: RELEASE ARROW_DEPENDENCY_SOURCE: BUNDLED - # Apache ORC always uses external orc-format. - ARROW_ORC: OFF - CMAKE_GENERATOR: "Unix Makefiles" + ARROW_DEPENDENCY_USE_SHARED: OFF + ARROW_FLIGHT_SQL_ODBC: ON volumes: *ubuntu-volumes - command: *cpp-command + # Register ODBC before running tests + command: > + /bin/bash -c " + /arrow/ci/scripts/cpp_build.sh /arrow /build && + sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && + /arrow/ci/scripts/cpp_test.sh /arrow /build" ubuntu-cpp-minimal: # Arrow build with minimal components/dependencies diff --git a/cpp/src/arrow/flight/sql/odbc/README b/cpp/src/arrow/flight/sql/odbc/README index 120a92950f9b..f119efc82790 100644 --- a/cpp/src/arrow/flight/sql/odbc/README +++ b/cpp/src/arrow/flight/sql/odbc/README @@ -57,11 +57,11 @@ After the build succeeds, the ODBC DYLIB will be located in i. `cd to repo.` ii. `cd ` iii. Give script permission to execute - `chmod +x cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh` + `chmod +x cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh` iv. Run script with `sudo` to register your ODBC DYLIB as Apache Arrow Flight SQL ODBC Driver - `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh /cpp/build/< release | debug >/libarrow_flight_sql_odbc.dylib` + `sudo cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /cpp/build/< release | debug >/libarrow_flight_sql_odbc.dylib` Example command for reference: - `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh /path/to/arrow/cpp/build/release/libarrow_flight_sql_odbc.dylib` + `sudo cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /path/to/arrow/cpp/build/release/libarrow_flight_sql_odbc.dylib` If the registration is successful, then Apache Arrow Flight SQL ODBC Driver should be shown at `~/Library/ODBC/odbcinst.ini` diff --git a/cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc_ini.sh b/cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc_ini.sh index 652034f49460..84de8e0e1464 100755 --- a/cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc_ini.sh +++ b/cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc_ini.sh @@ -31,7 +31,7 @@ fi source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -odbc_install_script="${source_dir}/install_odbc.sh" +odbc_install_script="${source_dir}/../unix/install_odbc.sh" "$odbc_install_script" /Library/ODBC/arrow-odbc/libarrow_flight_sql_odbc.dylib diff --git a/cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh b/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh similarity index 91% rename from cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh rename to cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh index 069c534c2973..4e4513a84a22 100755 --- a/cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh +++ b/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh @@ -40,7 +40,18 @@ if [ ! -f "$ODBC_64BIT" ]; then exit 1 fi -USER_ODBCINST_FILE="$HOME/Library/ODBC/odbcinst.ini" +case "$(uname)" in + Linux) + USER_ODBCINST_FILE="/etc/odbcinst.ini" + ;; + *) + # macOS + USER_ODBCINST_FILE="$HOME/Library/ODBC/odbcinst.ini" + ;; +esac + +echo "-AL- USER_ODBCINST_FILE: $USER_ODBCINST_FILE" + DRIVER_NAME="Apache Arrow Flight SQL ODBC Driver" mkdir -p "$HOME"/Library/ODBC From de27e4209250e6f80b4f230f513b19c937174d06 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Wed, 11 Mar 2026 14:51:22 -0700 Subject: [PATCH 07/12] Export ODBC APIs on Linux * resolve the issue of ODBC APIs not exported on Linux --- cpp/cmake_modules/BuildUtils.cmake | 1 - cpp/src/arrow/symbols.map | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 9efa2009dd8e..d9e06b1657bc 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -722,7 +722,6 @@ function(ADD_TEST_CASE REL_TEST_NAME) set(TEST_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_NAME}") add_executable(${TEST_NAME} ${SOURCES}) - message(STATUS "-AL- TEST_NAME: ${TEST_NAME}") # With OSX and conda, we need to set the correct RPATH so that dependencies # are found. The installed libraries with conda have an RPATH that matches diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map index d3c38c22c907..397bd80060a9 100644 --- a/cpp/src/arrow/symbols.map +++ b/cpp/src/arrow/symbols.map @@ -35,6 +35,8 @@ Arrow*; # ARROW-14771: export Protobuf symbol table descriptor_table_Flight*_2eproto; + # Export Arrow Flight SQL ODBC symbols + SQL*; # Symbols marked as 'local' are not exported by the DSO and thus may not # be used by client applications. Everything except the above falls here. From 49e53df6e434b90d26cf42c65e61ad63d6b231c8 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Wed, 11 Mar 2026 15:13:31 -0700 Subject: [PATCH 08/12] Clean up ODBC code --- .github/workflows/cpp_extra.yml | 533 +++++++++--------- compose.yaml | 3 +- .../sql/odbc/install/unix/install_odbc.sh | 2 - .../flight/sql/odbc/odbc_impl/CMakeLists.txt | 6 +- .../odbc/odbc_impl/config/configuration.cc | 16 +- .../sql/odbc/odbc_impl/encoding_utils.h | 8 + .../sql/odbc/odbc_impl/get_info_cache.cc | 10 +- .../sql/odbc/odbc_impl/odbc_statement.cc | 3 +- .../flight/sql/odbc/odbc_impl/system_dsn.cc | 14 +- 9 files changed, 301 insertions(+), 294 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index ab1706e88ef2..21b780e4e359 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -91,250 +91,250 @@ jobs: with: parent-workflow: cpp_extra - # docker: - # needs: check-labels - # name: ${{ matrix.title }} - # runs-on: ${{ matrix.runs-on }} - # if: >- - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - # timeout-minutes: 75 - # strategy: - # fail-fast: false - # matrix: - # include: - # - image: alpine-linux-cpp - # runs-on: ubuntu-latest - # title: AMD64 Alpine Linux - # - image: conda-cpp - # run-options: >- - # -e ARROW_USE_MESON=ON - # runs-on: ubuntu-latest - # title: AMD64 Ubuntu Meson - # # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved - # - continue-on-error: true - # envs: - # - DEBIAN=13 - # image: debian-cpp - # run-options: >- - # -e CMAKE_CXX_STANDARD=23 - # runs-on: ubuntu-latest - # title: AMD64 Debian C++23 - # env: - # ARCHERY_DEBUG: 1 - # ARROW_ENABLE_TIMING_TESTS: OFF - # DOCKER_VOLUME_PREFIX: ".docker/" - # steps: - # - name: Checkout Arrow - # uses: actions/checkout@v6 - # with: - # fetch-depth: 0 - # submodules: recursive - # - name: Cache Docker Volumes - # uses: actions/cache@v5 - # with: - # path: .docker - # key: extra-ubuntu-cpp-odbc-${{ hashFiles('cpp/**') }} - # restore-keys: extra-ubuntu-cpp-odbc- - # - name: Setup Python - # uses: actions/setup-python@v6 - # with: - # python-version: 3 - # - name: Setup Archery - # run: python3 -m pip install -e dev/archery[docker] - # - name: Execute Docker Build - # continue-on-error: ${{ matrix.continue-on-error || false }} - # env: - # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - # ENVS: ${{ toJSON(matrix.envs) }} - # run: | - # # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - # sudo sysctl -w vm.mmap_rnd_bits=28 - # source ci/scripts/util_enable_core_dumps.sh - # if [ "${ENVS}" != "null" ]; then - # echo "${ENVS}" | jq -r '.[]' | while read env; do - # echo "${env}" >> .env - # done - # fi - # archery docker run ${{ matrix.run-options || '' }} ubuntu-cpp-odbc - # - name: Docker Push - # if: >- - # success() && - # github.event_name == 'push' && - # github.repository == 'apache/arrow' && - # github.ref_name == 'main' - # env: - # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - # continue-on-error: true - # run: archery docker push ubuntu-cpp-odbc + docker: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: >- + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - image: alpine-linux-cpp + runs-on: ubuntu-latest + title: AMD64 Alpine Linux + - image: conda-cpp + run-options: >- + -e ARROW_USE_MESON=ON + runs-on: ubuntu-latest + title: AMD64 Ubuntu Meson + # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved + - continue-on-error: true + envs: + - DEBIAN=13 + image: debian-cpp + run-options: >- + -e CMAKE_CXX_STANDARD=23 + runs-on: ubuntu-latest + title: AMD64 Debian C++23 + env: + ARCHERY_DEBUG: 1 + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker + key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.image }}- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3 + - name: Setup Archery + run: python3 -m pip install -e dev/archery[docker] + - name: Execute Docker Build + continue-on-error: ${{ matrix.continue-on-error || false }} + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + ENVS: ${{ toJSON(matrix.envs) }} + run: | + # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes + sudo sysctl -w vm.mmap_rnd_bits=28 + source ci/scripts/util_enable_core_dumps.sh + if [ "${ENVS}" != "null" ]; then + echo "${ENVS}" | jq -r '.[]' | while read env; do + echo "${env}" >> .env + done + fi + archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} + - name: Docker Push + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push ${{ matrix.image }} - # msvc-arm64: - # needs: check-labels - # if: >- - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - # name: ARM64 Windows 11 MSVC - # uses: ./.github/workflows/cpp_windows.yml - # with: - # arch: arm64 - # os: windows-11-arm - # simd-level: NONE + msvc-arm64: + needs: check-labels + if: >- + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + name: ARM64 Windows 11 MSVC + uses: ./.github/workflows/cpp_windows.yml + with: + arch: arm64 + os: windows-11-arm + simd-level: NONE - # jni-linux: - # needs: check-labels - # name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} - # runs-on: ${{ matrix.platform.runs-on }} - # if: >- - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - # timeout-minutes: 240 - # permissions: - # # This is for using GitHub Packages for vcpkg cache - # packages: write - # strategy: - # fail-fast: false - # matrix: - # platform: - # - arch: "amd64" - # runs-on: ubuntu-latest - # - arch: "arm64v8" - # runs-on: ubuntu-24.04-arm - # env: - # ARCH: ${{ matrix.platform.arch }} - # REPO: ghcr.io/${{ github.repository }}-dev - # steps: - # - name: Checkout Arrow - # uses: actions/checkout@v6 - # with: - # fetch-depth: 0 - # submodules: recursive - # - name: Free up disk space - # run: | - # ci/scripts/util_free_space.sh - # - name: Cache Docker Volumes - # uses: actions/cache@v5 - # with: - # path: .docker - # key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} - # restore-keys: jni-${{ matrix.platform.runs-on }}- - # - name: Setup Python - # uses: actions/setup-python@v6 - # with: - # python-version: 3 - # - name: Setup Archery - # run: python3 -m pip install -e dev/archery[docker] - # - name: Execute Docker Build - # env: - # ARCHERY_DOCKER_USER: ${{ github.actor }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" - # run: | - # source ci/scripts/util_enable_core_dumps.sh - # archery docker run cpp-jni - # - name: Docker Push - # if: >- - # success() && - # github.event_name == 'push' && - # github.ref_name == 'main' - # env: - # ARCHERY_DOCKER_USER: ${{ github.actor }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - # continue-on-error: true - # run: archery docker push cpp-jni + jni-linux: + needs: check-labels + name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs-on }} + if: >- + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + timeout-minutes: 240 + permissions: + # This is for using GitHub Packages for vcpkg cache + packages: write + strategy: + fail-fast: false + matrix: + platform: + - arch: "amd64" + runs-on: ubuntu-latest + - arch: "arm64v8" + runs-on: ubuntu-24.04-arm + env: + ARCH: ${{ matrix.platform.arch }} + REPO: ghcr.io/${{ github.repository }}-dev + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker + key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} + restore-keys: jni-${{ matrix.platform.runs-on }}- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3 + - name: Setup Archery + run: python3 -m pip install -e dev/archery[docker] + - name: Execute Docker Build + env: + ARCHERY_DOCKER_USER: ${{ github.actor }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" + run: | + source ci/scripts/util_enable_core_dumps.sh + archery docker run cpp-jni + - name: Docker Push + if: >- + success() && + github.event_name == 'push' && + github.ref_name == 'main' + env: + ARCHERY_DOCKER_USER: ${{ github.actor }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + run: archery docker push cpp-jni - # jni-macos: - # needs: check-labels - # name: JNI macOS - # runs-on: macos-14 - # if: >- - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - # timeout-minutes: 45 - # env: - # MACOSX_DEPLOYMENT_TARGET: "14.0" - # steps: - # - name: Checkout Arrow - # uses: actions/checkout@v6 - # with: - # fetch-depth: 0 - # submodules: recursive - # - name: Install dependencies - # run: | - # brew bundle --file=cpp/Brewfile - # # We want to link aws-sdk-cpp statically but Homebrew's - # # aws-sdk-cpp provides only shared library. If we have - # # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - # brew uninstall aws-sdk-cpp - # # We want to use bundled RE2 for static linking. If - # # Homebrew's RE2 is installed, its header file may be used. - # # We uninstall Homebrew's RE2 to ensure using bundled RE2. - # brew uninstall grpc || : # gRPC depends on RE2 - # brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - # brew uninstall re2 - # # We want to use bundled Protobuf for static linking. If - # # Homebrew's Protobuf is installed, its library file may be - # # used on test We uninstall Homebrew's Protobuf to ensure using - # # bundled Protobuf. - # brew uninstall protobuf - # - name: Prepare ccache - # run: | - # echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - # - name: Cache ccache - # uses: actions/cache@v5 - # with: - # path: ccache - # key: jni-macos-${{ hashFiles('cpp/**') }} - # restore-keys: jni-macos- - # - name: CMake - # run: | - # cmake \ - # -S cpp \ - # -B cpp.build \ - # --preset=ninja-release-jni-macos \ - # -DARROW_BUILD_TESTS=ON \ - # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - # - name: Build - # run: | - # cmake --build cpp.build - # - name: Install - # run: | - # cmake --install cpp.build - # - name: Test - # env: - # ARROW_TEST_DATA: ${{ github.workspace }}/testing/data - # PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data - # run: | - # # MinIO is required - # exclude_tests="arrow-s3fs-test" - # # unstable - # exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - # exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - # ctest \ - # --exclude-regex "${exclude_tests}" \ - # --label-regex unittest \ - # --output-on-failure \ - # --parallel "$(sysctl -n hw.ncpu)" \ - # --test-dir "cpp.build" \ - # --timeout 300 - # - name: Build example - # run: | - # cmake \ - # -S cpp/examples/minimal_build/ \ - # -B cpp/examples/minimal_build.build \ - # -GNinja \ - # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - # cmake --build cpp/examples/minimal_build.build - # cd cpp/examples/minimal_build - # ../minimal_build.build/arrow-example + jni-macos: + needs: check-labels + name: JNI macOS + runs-on: macos-14 + if: >- + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') + timeout-minutes: 45 + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Install dependencies + run: | + brew bundle --file=cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp + # We want to use bundled RE2 for static linking. If + # Homebrew's RE2 is installed, its header file may be used. + # We uninstall Homebrew's RE2 to ensure using bundled RE2. + brew uninstall grpc || : # gRPC depends on RE2 + brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + brew uninstall re2 + # We want to use bundled Protobuf for static linking. If + # Homebrew's Protobuf is installed, its library file may be + # used on test We uninstall Homebrew's Protobuf to ensure using + # bundled Protobuf. + brew uninstall protobuf + - name: Prepare ccache + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@v5 + with: + path: ccache + key: jni-macos-${{ hashFiles('cpp/**') }} + restore-keys: jni-macos- + - name: CMake + run: | + cmake \ + -S cpp \ + -B cpp.build \ + --preset=ninja-release-jni-macos \ + -DARROW_BUILD_TESTS=ON \ + -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + - name: Build + run: | + cmake --build cpp.build + - name: Install + run: | + cmake --install cpp.build + - name: Test + env: + ARROW_TEST_DATA: ${{ github.workspace }}/testing/data + PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data + run: | + # MinIO is required + exclude_tests="arrow-s3fs-test" + # unstable + exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" + exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" + ctest \ + --exclude-regex "${exclude_tests}" \ + --label-regex unittest \ + --output-on-failure \ + --parallel "$(sysctl -n hw.ncpu)" \ + --test-dir "cpp.build" \ + --timeout 300 + - name: Build example + run: | + cmake \ + -S cpp/examples/minimal_build/ \ + -B cpp/examples/minimal_build.build \ + -GNinja \ + -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + cmake --build cpp/examples/minimal_build.build + cd cpp/examples/minimal_build + ../minimal_build.build/arrow-example odbc-linux: needs: check-labels @@ -349,14 +349,12 @@ jobs: fail-fast: false env: ARCH: amd64 - CLANG_TOOLS: 18 - LLVM: 18 - UBUNTU: 24.04 - - # -AL- todo make the list alphabetical ARCHERY_DEBUG: 1 ARROW_ENABLE_TIMING_TESTS: OFF + CLANG_TOOLS: 18 DOCKER_VOLUME_PREFIX: ".docker/" + LLVM: 18 + UBUNTU: 24.04 steps: - name: Checkout Arrow uses: actions/checkout@v6 @@ -394,7 +392,7 @@ jobs: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} continue-on-error: true - run: archery docker push ubuntu-cpp + run: archery docker push ubuntu-cpp-odbc odbc-macos: needs: check-labels @@ -687,6 +685,37 @@ jobs: mkdir odbc-installer mv *.msi odbc-installer/ + # Add `dev-yyyy-mm-dd` to ODBC MSI before `win64.msi`: + # Apache Arrow Flight SQL ODBC-24.0.0-win64.msi -> + # Apache Arrow Flight SQL ODBC-24.0.0-dev-2026-02-06-win64.msi + cd odbc-installer + msi_name=$(ls *.msi) + dev_msi_name=$(echo ${msi_name} | sed -e "s/win64\.msi$/dev-$(date +%Y-%m-%d)-win64.msi/") + mv "${msi_name}" "${dev_msi_name}" + cd .. + + tree odbc-installer + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 1 + path: arrow + repository: apache/arrow + ref: main + submodules: recursive + - name: Sync to Remote + uses: ./arrow/.github/actions/sync-nightlies + with: + upload: true + switches: -avzh --update --delete --progress + local_path: odbc-installer + remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/odbc + remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} + remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} + remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} + remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} + remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} + odbc-release: needs: odbc-msvc name: ODBC release @@ -723,13 +752,13 @@ jobs: report-extra-cpp: if: github.event_name == 'schedule' && always() needs: - # - docker - # - jni-linux - # - jni-macos - # - msvc-arm64 + - docker + - jni-linux + - jni-macos + - msvc-arm64 - odbc-linux - odbc-macos - odbc-msvc - # - odbc-nightly + - odbc-nightly uses: ./.github/workflows/report_ci.yml secrets: inherit diff --git a/compose.yaml b/compose.yaml index e7f468d56785..407253b7b77c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -367,11 +367,10 @@ services: volumes: &debian-volumes - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated - command: &cpp-command > # -AL- &cpp-command is defined here and calls the build + test sh scripts + command: &cpp-command > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/cpp_test.sh /arrow /build" -# -AL- so I can define a new one for odbc-cpp too. ubuntu-cpp: # Usage: diff --git a/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh b/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh index 4e4513a84a22..a86f71caae51 100755 --- a/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh +++ b/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh @@ -50,8 +50,6 @@ case "$(uname)" in ;; esac -echo "-AL- USER_ODBCINST_FILE: $USER_ODBCINST_FILE" - DRIVER_NAME="Apache Arrow Flight SQL ODBC Driver" mkdir -p "$HOME"/Library/ODBC diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt b/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt index 7d879e95c3a2..5a16c0361f32 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/CMakeLists.txt @@ -137,11 +137,7 @@ if(WIN32) PUBLIC arrow_flight_sql_shared arrow_compute_shared Boost::locale ${ODBCINST}) else() - # -AL- Linux build. share with apple - if(NOT APPLE) - message(status "-AL- ODBC_INCLUDE_DIR on Linux: ${ODBC_INCLUDE_DIR}") - endif() - + # Unix target_include_directories(arrow_odbc_spi_impl SYSTEM BEFORE PUBLIC ${ODBC_INCLUDE_DIR}) target_link_libraries(arrow_odbc_spi_impl PUBLIC arrow_flight_sql_static diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc index b2d8552e1c31..51191950ff37 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc @@ -31,8 +31,6 @@ #include #include -#include "arrow/util/logging.h" // -AL- TEMP - using ODBC::SetAttributeSQLWCHAR; namespace arrow::flight::sql::odbc { @@ -49,14 +47,7 @@ std::string ReadDsnString(const std::string& dsn, std::string_view key, CONVERT_WIDE_STR(const std::wstring wkey, key); CONVERT_WIDE_STR(const std::wstring wdflt, dflt); - // -AL- found workaround for `cannot convert 'const wchar_t*' to 'LPCWSTR' {aka - // 'const short unsigned int*'}` on Linux. Notes in this file for reference only. - - // Via CONVERT_WIDE_STR, Arrow correctly converts to UFT-32 on Unix systems, - // so the conversion from wchar_t to short unsigned int* will work on Linux. - - // -AL- I just need to wrap `reinterpret_cast()` on all string args for - // SQLGetPrivateProfileString. + // TODO: implement proper Linux unicode support in separate PR #define BUFFER_SIZE (1024) std::vector buf(BUFFER_SIZE); @@ -75,11 +66,7 @@ std::string ReadDsnString(const std::string& dsn, std::string_view key, } std::string result(""); - ARROW_LOG(DEBUG) << "-AL- ReadDsnString key: " << key; - ARROW_LOG(DEBUG) << "-AL- ReadDsnString result before: " << result; SetAttributeSQLWCHAR(buf.data(), ret * GetSqlWCharSize(), result); - ARROW_LOG(DEBUG) << "-AL- ReadDsnString result: " << result; - ARROW_LOG(DEBUG) << "-AL- ReadDsnString ret: " << ret; return result; } @@ -127,7 +114,6 @@ std::vector ReadAllKeys(const std::string& dsn) { SQLINTEGER key_len = cur - begin; SetAttributeSQLWCHAR(begin, key_len * GetSqlWCharSize(), key); keys.emplace_back(key); - ARROW_LOG(DEBUG) << "-AL- ReadAllKeys key: " << key; begin = ++cur; } return keys; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h index 5e3a4ecbdae3..09489ddc06b8 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h @@ -117,4 +117,12 @@ inline std::string SqlStringToString(const unsigned char* sql_str, return res; } + +inline std::vector ToSqlWCharVector(const std::wstring& ws) { + std::vector buf; + // buf.assign(ws.begin(), ws.end()); + // TODO implement in separate PR + return buf; +} + } // namespace ODBC diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc index 55e8d72c9b7c..6923d7fafbe4 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/get_info_cache.cc @@ -395,9 +395,8 @@ bool GetInfoCache::LoadInfoFromServer() { // Unused by ODBC. break; case SqlInfoOptions::SQL_DDL_SCHEMA: { - // bool supports_schema_ddl = - // reinterpret_cast(scalar->child_value().get())->value; - // -AL- todo raise GitHub issues for finishing this work. + // GH-49500 TODO: use scalar bool to determine `SQL_CREATE_SCHEMA` and + // `SQL_DROP_SCHEMA` values // Note: this is a bitmask and we can't describe cascade or restrict // flags. @@ -409,9 +408,8 @@ bool GetInfoCache::LoadInfoFromServer() { break; } case SqlInfoOptions::SQL_DDL_TABLE: { - // bool supports_table_ddl = - // reinterpret_cast(scalar->child_value().get())->value; - // -AL- todo raise GitHub issues for finishing this work. + // GH-49500 TODO: use scalar bool to determine `SQL_CREATE_TABLE` and + // `SQL_DROP_TABLE` values // This is a bitmask and we cannot describe all clauses. info_[SQL_CREATE_TABLE] = static_cast(SQL_CT_CREATE_TABLE); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc index f7701ddc36c7..02579aaf5f74 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -// platform.h platform.h includes windows.h so it needs to be included first -// -AL- this is an attempt to fix the windows build +// platform.h platform.h includes windows.h so it needs to be included first #include "arrow/flight/sql/odbc/odbc_impl/platform.h" #include "arrow/type.h" diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc index 5c31b5e5b043..06d60df7a3eb 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc @@ -23,11 +23,12 @@ #include #include +#include "arrow/flight/sql/odbc/odbc_impl/encoding_utils.h" + #ifdef __linux__ -# define GET_SQWCHAR_PTR(wstring_var) (ToSqlWCharVector(wstring_var).data()) +# define GET_SQWCHAR_PTR(wstring_var) (ODBC::ToSqlWCharVector(wstring_var).data()) #else -// Windows and macOS // -AL- TODO raise GitHub issues for Linux functions -// Can't test it right now anyways without the tests. +// Windows and macOS # define GET_SQWCHAR_PTR(wstring_var) (wstring_var.c_str()) #endif @@ -72,13 +73,6 @@ void PostLastInstallerError() { PostError(code, const_cast(error_msg.c_str())); } -std::vector ToSqlWCharVector(const std::wstring& ws) { - std::vector buf; - // buf.assign(ws.begin(), ws.end()); - // -AL- GitHub issue to implement. Also need to move this function else where like encoding_utils.h? - return buf; -} - /** * Unregister specified DSN. * From 126bbe7b3c38cd180f37415d97c276a5d555ccf4 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Fri, 13 Mar 2026 11:47:11 -0700 Subject: [PATCH 09/12] Use `reinterpret_cast` as Linux wchar_t is char16 --- cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc index 06d60df7a3eb..790da11e3725 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc @@ -48,7 +48,7 @@ void PostArrowUtilError(arrow::Status error_status) { std::wstring werror_msg = arrow::util::UTF8ToWideString(error_msg).ValueOr( L"Error during utf8 to wide string conversion"); - PostError(ODBC_ERROR_GENERAL_ERR, const_cast(werror_msg.c_str())); + PostError(ODBC_ERROR_GENERAL_ERR, const_cast(GET_SQWCHAR_PTR(werror_msg))); } void PostLastInstallerError() { @@ -70,7 +70,7 @@ void PostLastInstallerError() { #endif // __linux__ std::wstring error_msg = buf.str(); - PostError(code, const_cast(error_msg.c_str())); + PostError(code, const_cast(GET_SQWCHAR_PTR(error_msg))); } /** From 64d8ada5661dfb032aa8b7f3607c341657d6fbcf Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Thu, 12 Mar 2026 15:36:54 -0700 Subject: [PATCH 10/12] Use const & string_view Trying to figure out why `Set(FlightSqlConnection::DISABLE_CERTIFICATE_VERIFICATION` just cannot pass default values. Somehow Add configuration.cc logs In-progress implement unicode support conversion [In-progress] One change that (somehow) enables DBTLabs Flight SQL connection - using `ENVID` as `static constexpr` enabled the environmentid variable to be read properly. Then other custom keys like `abc` and `Description` also started to work. Fixes the issue of strings too --- cpp/src/arrow/flight/sql/odbc/CMakeLists.txt | 2 +- .../odbc/odbc_impl/config/configuration.cc | 42 +++++------ .../sql/odbc/odbc_impl/encoding_utils.h | 38 ++++++++-- .../flight/sql/odbc/odbc_impl/system_dsn.cc | 71 +++++++++---------- .../arrow/flight/sql/odbc/odbc_impl/util.h | 16 +++++ .../flight/sql/odbc/tests/errors_test.cc | 1 - 6 files changed, 108 insertions(+), 62 deletions(-) diff --git a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt index 2a7a31733969..aaa36bc400f9 100644 --- a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt +++ b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt @@ -29,7 +29,7 @@ endif() add_subdirectory(odbc_impl) if(WIN32 OR APPLE) - # TODO: Enable Linux test build in separate PR + # GH-49552 TODO: Enable Linux test build add_subdirectory(tests) endif() diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc index 51191950ff37..bfbfe0cc7c07 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc @@ -41,28 +41,26 @@ static const char DEFAULT_USE_CERT_STORE[] = TRUE_STR; static const char DEFAULT_DISABLE_CERT_VERIFICATION[] = FALSE_STR; namespace { -std::string ReadDsnString(const std::string& dsn, std::string_view key, +std::string ReadDsnString(const std::string& dsn, const std::string_view& key, const std::string& dflt = "") { - CONVERT_WIDE_STR(const std::wstring wdsn, dsn); - CONVERT_WIDE_STR(const std::wstring wkey, key); - CONVERT_WIDE_STR(const std::wstring wdflt, dflt); - - // TODO: implement proper Linux unicode support in separate PR + CONVERT_SQLWCHAR_STR(wdsn, dsn); + CONVERT_SQLWCHAR_STR(wkey, key); + CONVERT_SQLWCHAR_STR(wdflt, dflt); #define BUFFER_SIZE (1024) std::vector buf(BUFFER_SIZE); int ret = SQLGetPrivateProfileString( reinterpret_cast(wdsn.c_str()), reinterpret_cast(wkey.c_str()), reinterpret_cast(wdflt.c_str()), buf.data(), static_cast(buf.size()), - reinterpret_cast(L"ODBC.INI")); + ODBC_INI); if (ret > BUFFER_SIZE) { // If there wasn't enough space, try again with the right size buffer. buf.resize(ret + 1); - ret = SQLGetPrivateProfileString( - reinterpret_cast(wdsn.c_str()), reinterpret_cast(wkey.c_str()), - reinterpret_cast(wdflt.c_str()), buf.data(), - static_cast(buf.size()), reinterpret_cast(L"ODBC.INI")); + ret = SQLGetPrivateProfileString(reinterpret_cast(wdsn.c_str()), + reinterpret_cast(wkey.c_str()), + reinterpret_cast(wdflt.c_str()), buf.data(), + static_cast(buf.size()), ODBC_INI); } std::string result(""); @@ -84,20 +82,20 @@ void RemoveAllKnownKeys(std::vector& keys) { } std::vector ReadAllKeys(const std::string& dsn) { - CONVERT_WIDE_STR(const std::wstring wdsn, dsn); + CONVERT_SQLWCHAR_STR(wdsn, dsn); std::vector buf(BUFFER_SIZE); - int ret = SQLGetPrivateProfileString( - reinterpret_cast(wdsn.c_str()), NULL, reinterpret_cast(L""), - buf.data(), static_cast(buf.size()), reinterpret_cast(L"ODBC.INI")); + int ret = SQLGetPrivateProfileString(reinterpret_cast(wdsn.c_str()), NULL, + reinterpret_cast(L""), buf.data(), + static_cast(buf.size()), ODBC_INI); if (ret > BUFFER_SIZE) { // If there wasn't enough space, try again with the right size buffer. buf.resize(ret + 1); - ret = SQLGetPrivateProfileString( - reinterpret_cast(wdsn.c_str()), NULL, reinterpret_cast(L""), - buf.data(), static_cast(buf.size()), reinterpret_cast(L"ODBC.INI")); + ret = SQLGetPrivateProfileString(reinterpret_cast(wdsn.c_str()), NULL, + reinterpret_cast(L""), buf.data(), + static_cast(buf.size()), ODBC_INI); } // When you pass NULL to SQLGetPrivateProfileString it gives back a \0 delimited list of @@ -137,6 +135,10 @@ void Configuration::LoadDefaults() { } void Configuration::LoadDsn(const std::string& dsn) { + // Read keys before reading DSN. This is a workaround to unixodbc driver manager + // unable to read DSN keys + auto customKeys = ReadAllKeys(dsn); + Set(FlightSqlConnection::DSN, dsn); Set(FlightSqlConnection::HOST, ReadDsnString(dsn, FlightSqlConnection::HOST)); Set(FlightSqlConnection::PORT, ReadDsnString(dsn, FlightSqlConnection::PORT)); @@ -145,6 +147,7 @@ void Configuration::LoadDsn(const std::string& dsn) { Set(FlightSqlConnection::PWD, ReadDsnString(dsn, FlightSqlConnection::PWD)); Set(FlightSqlConnection::TRUSTED_CERTS, ReadDsnString(dsn, FlightSqlConnection::TRUSTED_CERTS)); + #ifdef __APPLE__ // macOS iODBC treats non-empty defaults as the real values when reading from system // DSN, so we don't pass defaults on macOS. @@ -165,9 +168,8 @@ void Configuration::LoadDsn(const std::string& dsn) { Set(FlightSqlConnection::DISABLE_CERTIFICATE_VERIFICATION, ReadDsnString(dsn, FlightSqlConnection::DISABLE_CERTIFICATE_VERIFICATION, DEFAULT_DISABLE_CERT_VERIFICATION)); -#endif +#endif // __APPLE__ - auto customKeys = ReadAllKeys(dsn); RemoveAllKnownKeys(customKeys); for (auto key : customKeys) { std::string_view key_sv(key); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h index 09489ddc06b8..496a1b2dfaa1 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h @@ -28,8 +28,26 @@ #include #include +// Workaround for ODBC `BOOL` def conflict on Linux +#ifdef __linux__ +# ifdef BOOL +# undef BOOL +# endif // BOOL +#endif // __linux__ +// Include fwd.h headers after ODBC headers +#include "arrow/flight/sql/odbc/odbc_impl/util.h" + #define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING +#ifdef __linux__ +# define GET_SQWCHAR_PTR(wstring_var) (ODBC::ToSqlWCharVector(wstring_var).data()) +#else +// Windows and macOS +# define GET_SQWCHAR_PTR(wstring_var) (wstring_var.c_str()) +#endif + +#define ODBC_INI reinterpret_cast(GET_SQWCHAR_PTR(std::wstring(L"ODBC.INI"))) + namespace ODBC { using arrow::flight::sql::odbc::DriverException; using arrow::flight::sql::odbc::GetSqlWCharSize; @@ -119,10 +137,22 @@ inline std::string SqlStringToString(const unsigned char* sql_str, } inline std::vector ToSqlWCharVector(const std::wstring& ws) { - std::vector buf; - // buf.assign(ws.begin(), ws.end()); - // TODO implement in separate PR - return buf; + switch (GetSqlWCharSize()) { + case sizeof(wchar_t): { + return std::vector(ws.begin(), ws.end()); + } + case sizeof(char16_t): { + // Linux ODBC driver manager uses char16_t as SQLWCHAR + CONVERT_UTF8_STR(const std::string utf8s, ws); + CONVERT_UTF16_STR(const std::u16string utf16s, utf8s); + return std::vector(utf16s.begin(), utf16s.end()); + } + default: { + assert(false); + throw DriverException("Encoding is unsupported, SQLWCHAR size: " + + std::to_string(GetSqlWCharSize())); + } + } } } // namespace ODBC diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc index 790da11e3725..97c3a52d6c9f 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/system_dsn.cc @@ -26,11 +26,25 @@ #include "arrow/flight/sql/odbc/odbc_impl/encoding_utils.h" #ifdef __linux__ -# define GET_SQWCHAR_PTR(wstring_var) (ODBC::ToSqlWCharVector(wstring_var).data()) +// Linux driver manager uses utf16string +# define CONVERT_UTF8_TO_SQLWCHAR_OR_RETURN(wvar, var) \ + auto wvar##_result = arrow::util::UTF8StringToUTF16(var); \ + if (!wvar##_result.status().ok()) { \ + PostArrowUtilError(wvar##_result.status()); \ + return false; \ + } \ + std::u16string wvar = wvar##_result.ValueOrDie(); + #else // Windows and macOS -# define GET_SQWCHAR_PTR(wstring_var) (wstring_var.c_str()) -#endif +# define CONVERT_UTF8_TO_SQLWCHAR_OR_RETURN(wvar, var) \ + auto wvar##_result = arrow::util::UTF8ToWideString(var); \ + if (!wvar##_result.status().ok()) { \ + PostArrowUtilError(wvar##_result.status()); \ + return false; \ + } \ + std::wstring wvar = wvar##_result.ValueOrDie(); +#endif // __linux__ namespace arrow::flight::sql::odbc { @@ -57,20 +71,20 @@ void PostLastInstallerError() { std::vector msg(BUFFER_SIZE); SQLInstallerError(1, &code, msg.data(), BUFFER_SIZE, NULL); - std::wstringstream buf; #ifdef __linux__ - buf << L"Message: \""; - for (SQLWCHAR wch : msg) { - buf << static_cast(wch); - } - buf << L"\", Code: " << code; + std::string code_str = std::to_string(code); + std::u16string code_u16 = arrow::util::UTF8StringToUTF16(code_str).ValueOr( + u"unknown code. Error during utf8 to utf16 conversion"); + std::u16string error_msg = u"Message: \"" + + std::u16string(reinterpret_cast(msg.data())) + + u"\", Code: " + code_u16; #else - // Windows and macOS - buf << L"Message: \"" << msg.data() << L"\", Code: " << code; + // Windows/macOS + std::wstring error_msg = + L"Message: \"" + std::wstring(msg.data()) + L"\", Code: " + std::to_wstring(code); #endif // __linux__ - std::wstring error_msg = buf.str(); - PostError(code, const_cast(GET_SQWCHAR_PTR(error_msg))); + PostError(code, const_cast(reinterpret_cast(error_msg.c_str()))); } /** @@ -97,14 +111,9 @@ bool UnregisterDsn(const std::wstring& dsn) { */ bool RegisterDsn(const Configuration& config, LPCWSTR driver) { const std::string& dsn = config.Get(FlightSqlConnection::DSN); - auto wdsn_result = arrow::util::UTF8ToWideString(dsn); - if (!wdsn_result.status().ok()) { - PostArrowUtilError(wdsn_result.status()); - return false; - } - std::wstring wdsn = wdsn_result.ValueOrDie(); + CONVERT_UTF8_TO_SQLWCHAR_OR_RETURN(wdsn, dsn); - if (!SQLWriteDSNToIni(GET_SQWCHAR_PTR(wdsn), driver)) { + if (!SQLWriteDSNToIni(reinterpret_cast(wdsn.c_str()), driver)) { PostLastInstallerError(); return false; } @@ -117,23 +126,13 @@ bool RegisterDsn(const Configuration& config, LPCWSTR driver) { continue; } - auto wkey_result = arrow::util::UTF8ToWideString(key); - if (!wkey_result.status().ok()) { - PostArrowUtilError(wkey_result.status()); - return false; - } - std::wstring wkey = wkey_result.ValueOrDie(); - - auto wvalue_result = arrow::util::UTF8ToWideString(it->second); - if (!wvalue_result.status().ok()) { - PostArrowUtilError(wvalue_result.status()); - return false; - } - std::wstring wvalue = wvalue_result.ValueOrDie(); + CONVERT_UTF8_TO_SQLWCHAR_OR_RETURN(wkey, key); + CONVERT_UTF8_TO_SQLWCHAR_OR_RETURN(wvalue, it->second); - if (!SQLWritePrivateProfileString(GET_SQWCHAR_PTR(wdsn), GET_SQWCHAR_PTR(wkey), - GET_SQWCHAR_PTR(wvalue), - reinterpret_cast(L"ODBC.INI"))) { + if (!SQLWritePrivateProfileString(reinterpret_cast(wdsn.c_str()), + reinterpret_cast(wkey.c_str()), + reinterpret_cast(wvalue.c_str()), + ODBC_INI)) { PostLastInstallerError(); return false; } diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.h index 3bd11d60113b..1e1735756132 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/util.h @@ -28,6 +28,15 @@ #include "arrow/flight/types.h" #include "arrow/util/utf8.h" +#ifdef __linux__ +// Linux driver manager uses utf16string +# define CONVERT_SQLWCHAR_STR(wvar, var) \ + CONVERT_UTF16_STR(const std::u16string wvar, var) +#else +// Windows and macOS uses wstring +# define CONVERT_SQLWCHAR_STR(wvar, var) CONVERT_WIDE_STR(const std::wstring wvar, var) +#endif // __linux__ + #define CONVERT_WIDE_STR(wstring_var, utf8_target) \ wstring_var = [&] { \ arrow::Result res = arrow::util::UTF8ToWideString(utf8_target); \ @@ -42,6 +51,13 @@ return res.ValueOrDie(); \ }() +#define CONVERT_UTF16_STR(utf16string_var, utf8_target) \ + utf16string_var = [&] { \ + arrow::Result res = arrow::util::UTF8StringToUTF16(utf8_target); \ + arrow::flight::sql::odbc::util::ThrowIfNotOK(res.status()); \ + return res.ValueOrDie(); \ + }() + namespace arrow::flight::sql::odbc { namespace util { diff --git a/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc b/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc index 307b4812517a..640022429798 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/tests/errors_test.cc @@ -494,7 +494,6 @@ TYPED_TEST(ErrorsOdbcV2Test, TestSQLErrorEnvErrorFromDriverManager) { EXPECT_FALSE(std::wstring(message).empty()); } -// TODO: verify that `SQLGetConnectOption` is not required by Excel. #ifndef __APPLE__ TYPED_TEST(ErrorsOdbcV2Test, TestSQLErrorConnError) { // Test ODBC 2.0 API SQLError with ODBC ver 2. From 16c29c74144776347edf1cd26f3c6f0101b89265 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Thu, 19 Mar 2026 11:54:28 -0700 Subject: [PATCH 11/12] Address my own PR comments --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/flight/sql/odbc/CMakeLists.txt | 8 +++++--- .../flight/sql/odbc/odbc_impl/config/configuration.cc | 4 ++-- .../flight/sql/odbc/odbc_impl/flight_sql_result_set.cc | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index d18be5766824..eee63b11ca1c 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -259,6 +259,7 @@ function(ADD_ARROW_TEST REL_TEST_NAME) else() set(LABELS "arrow-tests") endif() + add_test_case(${REL_TEST_NAME} PREFIX ${PREFIX} diff --git a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt index aaa36bc400f9..4227873706ff 100644 --- a/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt +++ b/cpp/src/arrow/flight/sql/odbc/CMakeLists.txt @@ -28,9 +28,11 @@ else() endif() add_subdirectory(odbc_impl) -if(WIN32 OR APPLE) - # GH-49552 TODO: Enable Linux test build - add_subdirectory(tests) +if(ARROW_BUILD_TESTS) + if(WIN32 OR APPLE) + # GH-49552 TODO: Enable Linux test build + add_subdirectory(tests) + endif() endif() arrow_install_all_headers("arrow/flight/sql/odbc") diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc index bfbfe0cc7c07..f9780e173215 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc @@ -135,8 +135,8 @@ void Configuration::LoadDefaults() { } void Configuration::LoadDsn(const std::string& dsn) { - // Read keys before reading DSN. This is a workaround to unixodbc driver manager - // unable to read DSN keys + // Read keys before reading DSN to minimized unexpected behavior from ODBC driver + // managers. auto customKeys = ReadAllKeys(dsn); Set(FlightSqlConnection::DSN, dsn); diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc index 6a366ac5d001..fb743d1c1ec0 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/flight_sql_result_set.cc @@ -81,7 +81,7 @@ size_t FlightSqlResultSet::Move(size_t rows, size_t bind_offset, size_t bind_typ } // Reset GetData value offsets. - if ((size_t)num_binding_ != get_data_offsets_.size() && reset_get_data_) { + if (static_cast(num_binding_) != get_data_offsets_.size() && reset_get_data_) { std::fill(get_data_offsets_.begin(), get_data_offsets_.end(), 0); } From 308adde92569934315eaac4cc473f051fe999c2a Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Thu, 19 Mar 2026 14:01:49 -0700 Subject: [PATCH 12/12] Add patch for Windows build --- cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc | 2 +- cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc index f9780e173215..1a6f0725c22b 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/config/configuration.cc @@ -109,7 +109,7 @@ std::vector ReadAllKeys(const std::string& dsn) { } std::string key(""); - SQLINTEGER key_len = cur - begin; + SQLINTEGER key_len = static_cast(cur - begin); SetAttributeSQLWCHAR(begin, key_len * GetSqlWCharSize(), key); keys.emplace_back(key); begin = ++cur; diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h index 496a1b2dfaa1..f243ba9f6c09 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/encoding_utils.h @@ -141,12 +141,14 @@ inline std::vector ToSqlWCharVector(const std::wstring& ws) { case sizeof(wchar_t): { return std::vector(ws.begin(), ws.end()); } +#ifdef __linux__ case sizeof(char16_t): { // Linux ODBC driver manager uses char16_t as SQLWCHAR CONVERT_UTF8_STR(const std::string utf8s, ws); CONVERT_UTF16_STR(const std::u16string utf16s, utf8s); return std::vector(utf16s.begin(), utf16s.end()); } +#endif // __linux__ default: { assert(false); throw DriverException("Encoding is unsupported, SQLWCHAR size: " +