Skip to content

investigate

investigate #255

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: C++ Extra
on:
push:
branches:
- '**'
- '!dependabot/**'
paths:
- '.dockerignore'
- '.github/workflows/check_labels.yml'
- '.github/workflows/cpp_extra.yml'
- '.github/workflows/cpp_windows.yml'
- '.github/workflows/report_ci.yml'
- 'ci/conda_env_*'
- 'ci/docker/**'
- 'ci/scripts/ccache_setup.sh'
- 'ci/scripts/cpp_*'
- 'ci/scripts/install_azurite.sh'
- 'ci/scripts/install_gcs_testbench.sh'
- 'ci/scripts/install_minio.sh'
- 'ci/scripts/msys2_*'
- 'ci/scripts/util_*'
- 'cpp/**'
- 'compose.yaml'
- 'dev/archery/archery/**'
- 'format/Flight.proto'
- 'testing'
tags:
- '**'
pull_request:
paths:
- '.dockerignore'
- '.github/workflows/check_labels.yml'
- '.github/workflows/cpp_extra.yml'
- '.github/workflows/cpp_windows.yml'
- '.github/workflows/report_ci.yml'
- 'ci/conda_env_*'
- 'ci/docker/**'
- 'ci/scripts/ccache_setup.sh'
- 'ci/scripts/cpp_*'
- 'ci/scripts/install_azurite.sh'
- 'ci/scripts/install_gcs_testbench.sh'
- 'ci/scripts/install_minio.sh'
- 'ci/scripts/msys2_*'
- 'ci/scripts/util_*'
- 'cpp/**'
- 'compose.yaml'
- 'dev/archery/archery/**'
- 'format/Flight.proto'
- 'testing'
types:
- labeled
- opened
- reopened
- synchronize
schedule:
- cron: |
0 0 * * *
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
permissions:
actions: read
contents: read
pull-requests: read
jobs:
check-labels:
if: github.event_name != 'schedule' || github.repository == 'apache/arrow'
uses: ./.github/workflows/check_labels.yml
secrets: inherit
with:
parent-workflow: cpp_extra
docker:
needs: check-labels
name: ${{ matrix.title }}
runs-on: ${{ matrix.runs-on }}
if: >-
needs.check-labels.outputs.force == 'true' ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
timeout-minutes: 75
strategy:
fail-fast: false
matrix:
include:
- image: alpine-linux-cpp
runs-on: ubuntu-latest
title: AMD64 Alpine Linux
- image: conda-cpp
run-options: >-
-e ARROW_USE_MESON=ON
runs-on: ubuntu-latest
title: AMD64 Ubuntu Meson
# TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved
- continue-on-error: true
envs:
- DEBIAN=13
image: debian-cpp
run-options: >-
-e CMAKE_CXX_STANDARD=23
runs-on: ubuntu-latest
title: AMD64 Debian C++23
env:
ARCHERY_DEBUG: 1
ARROW_ENABLE_TIMING_TESTS: OFF
DOCKER_VOLUME_PREFIX: ".docker/"
steps:
- name: Checkout Arrow
uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: recursive
- name: Cache Docker Volumes
uses: actions/cache@v5
with:
path: .docker
key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }}
restore-keys: extra-${{ matrix.image }}-
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: 3
- name: Setup Archery
run: python3 -m pip install -e dev/archery[docker]
- name: Execute Docker Build
continue-on-error: ${{ matrix.continue-on-error || false }}
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
ENVS: ${{ toJSON(matrix.envs) }}
run: |
# GH-40558: reduce ASLR to avoid ASAN/LSAN crashes
sudo sysctl -w vm.mmap_rnd_bits=28
source ci/scripts/util_enable_core_dumps.sh
if [ "${ENVS}" != "null" ]; then
echo "${ENVS}" | jq -r '.[]' | while read env; do
echo "${env}" >> .env
done
fi
archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }}
- name: Docker Push
if: >-
success() &&
github.event_name == 'push' &&
github.repository == 'apache/arrow' &&
github.ref_name == 'main'
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
continue-on-error: true
run: archery docker push ${{ matrix.image }}
msvc-arm64:
needs: check-labels
if: >-
needs.check-labels.outputs.force == 'true' ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
name: ARM64 Windows 11 MSVC
uses: ./.github/workflows/cpp_windows.yml
with:
arch: arm64
os: windows-11-arm
simd-level: NONE
jni-linux:
needs: check-labels
name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }}
runs-on: ${{ matrix.platform.runs-on }}
if: >-
needs.check-labels.outputs.force == 'true' ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
timeout-minutes: 240
permissions:
# This is for using GitHub Packages for vcpkg cache
packages: write
strategy:
fail-fast: false
matrix:
platform:
- arch: "amd64"
runs-on: ubuntu-latest
- arch: "arm64v8"
runs-on: ubuntu-24.04-arm
env:
ARCH: ${{ matrix.platform.arch }}
REPO: ghcr.io/${{ github.repository }}-dev
steps:
- name: Checkout Arrow
uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: recursive
- name: Free up disk space
run: |
ci/scripts/util_free_space.sh
- name: Cache Docker Volumes
uses: actions/cache@v5
with:
path: .docker
key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }}
restore-keys: jni-${{ matrix.platform.runs-on }}-
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: 3
- name: Setup Archery
run: python3 -m pip install -e dev/archery[docker]
- name: Execute Docker Build
env:
ARCHERY_DOCKER_USER: ${{ github.actor }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
run: |
source ci/scripts/util_enable_core_dumps.sh
archery docker run cpp-jni
- name: Docker Push
if: >-
success() &&
github.event_name == 'push' &&
github.ref_name == 'main'
env:
ARCHERY_DOCKER_USER: ${{ github.actor }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
continue-on-error: true
run: archery docker push cpp-jni
jni-macos:
needs: check-labels
name: JNI macOS
runs-on: macos-14
if: >-
needs.check-labels.outputs.force == 'true' ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
timeout-minutes: 45
env:
MACOSX_DEPLOYMENT_TARGET: "14.0"
steps:
- name: Checkout Arrow
uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: recursive
- name: Install dependencies
run: |
brew bundle --file=cpp/Brewfile
# We want to link aws-sdk-cpp statically but Homebrew's
# aws-sdk-cpp provides only shared library. If we have
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
brew uninstall aws-sdk-cpp
# We want to use bundled RE2 for static linking. If
# Homebrew's RE2 is installed, its header file may be used.
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
brew uninstall grpc || : # gRPC depends on RE2
brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
brew uninstall re2
# We want to use bundled Protobuf for static linking. If
# Homebrew's Protobuf is installed, its library file may be
# used on test We uninstall Homebrew's Protobuf to ensure using
# bundled Protobuf.
brew uninstall protobuf
- name: Prepare ccache
run: |
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
- name: Cache ccache
uses: actions/cache@v5
with:
path: ccache
key: jni-macos-${{ hashFiles('cpp/**') }}
restore-keys: jni-macos-
- name: CMake
run: |
cmake \
-S cpp \
-B cpp.build \
--preset=ninja-release-jni-macos \
-DARROW_BUILD_TESTS=ON \
-DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
- name: Build
run: |
cmake --build cpp.build
- name: Install
run: |
cmake --install cpp.build
- name: Test
env:
ARROW_TEST_DATA: ${{ github.workspace }}/testing/data
PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data
run: |
# MinIO is required
exclude_tests="arrow-s3fs-test"
# unstable
exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"
ctest \
--exclude-regex "${exclude_tests}" \
--label-regex unittest \
--output-on-failure \
--parallel "$(sysctl -n hw.ncpu)" \
--test-dir "cpp.build" \
--timeout 300
- name: Build example
run: |
cmake \
-S cpp/examples/minimal_build/ \
-B cpp/examples/minimal_build.build \
-GNinja \
-DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
cmake --build cpp/examples/minimal_build.build
cd cpp/examples/minimal_build
../minimal_build.build/arrow-example
odbc-macos:
needs: check-labels
name: ODBC ${{ matrix.architecture }} macOS ${{ matrix.macos-version }}
runs-on: macos-${{ matrix.macos-version }}
if: >-
needs.check-labels.outputs.force == 'true' ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
timeout-minutes: 75
strategy:
fail-fast: false
matrix:
include:
- architecture: AMD64
macos-version: "15-intel"
- architecture: ARM64
macos-version: "14"
env:
ARROW_BUILD_TESTS: ON
ARROW_FLIGHT_SQL_ODBC: ON
ARROW_HOME: /tmp/local
steps:
- name: Checkout Arrow
uses: actions/checkout@v6.0.0
with:
fetch-depth: 0
submodules: recursive
- name: Install Dependencies
run: |
brew bundle --file=cpp/Brewfile
- name: Setup ccache
run: |
ci/scripts/ccache_setup.sh
- name: ccache info
id: ccache-info
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v4
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-ccache-macos-${{ matrix.macos-version }}-${{ hashFiles('cpp/**') }}
restore-keys: cpp-ccache-macos-${{ matrix.macos-version }}-
- name: Build
run: |
# Homebrew uses /usr/local as prefix. So packages
# installed by Homebrew also use /usr/local/include. We
# want to include headers for packages installed by
# Homebrew as system headers to ignore warnings in them.
# But "-isystem /usr/local/include" isn't used by CMake
# because /usr/local/include is marked as the default
# include path. So we disable -Werror to avoid build error
# by warnings from packages installed by Homebrew.
export BUILD_WARNING_LEVEL=PRODUCTION
LIBIODBC_DIR="$(brew --cellar libiodbc)/$(brew list --versions libiodbc | awk '{print $2}')"
export ODBC_INCLUDE_DIR=$LIBIODBC_DIR/include
export CXXFLAGS="$CXXFLAGS -I$ODBC_INCLUDE_DIR"
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- name: Register Arrow Flight SQL ODBC Driver
run: |
chmod +x cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh
sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh $(pwd)/build/cpp/debug/libarrow_flight_sql_odbc.dylib
- name: Test
shell: bash
run: |
sudo sysctl -w kern.coredump=1
sudo sysctl -w kern.corefile=/tmp/core.%N.%P
ulimit -c unlimited # must enable within the same shell
ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
# odbc-msvc:
# needs: check-labels
# name: ODBC Windows
# runs-on: windows-2022
# if: >-
# needs.check-labels.outputs.force == 'true' ||
# contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
# contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') ||
# contains(join(github.event.pull_request.changed_files, ' '), 'cpp/src/arrow/flight/sql/odbc/')
# timeout-minutes: 240
# permissions:
# packages: write
# env:
# ARROW_BUILD_SHARED: ON
# ARROW_BUILD_STATIC: OFF
# ARROW_BUILD_TESTS: ON
# ARROW_BUILD_TYPE: release
# ARROW_DEPENDENCY_SOURCE: VCPKG
# ARROW_FLIGHT_SQL_ODBC: ON
# ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON
# ARROW_HOME: /usr
# CMAKE_GENERATOR: Ninja
# CMAKE_INSTALL_PREFIX: /usr
# VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite'
# VCPKG_DEFAULT_TRIPLET: x64-windows
# steps:
# - name: Disable Crash Dialogs
# run: |
# reg add `
# "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
# /v DontShowUI `
# /t REG_DWORD `
# /d 1 `
# /f
# - name: Checkout Arrow
# uses: actions/checkout@v6
# with:
# fetch-depth: 0
# submodules: recursive
# - name: Download Timezone Database
# shell: bash
# run: ci/scripts/download_tz_database.sh
# - name: Install ccache
# shell: bash
# run: |
# ci/scripts/install_ccache.sh 4.12.1 /usr
# - name: Setup ccache
# shell: bash
# run: |
# ci/scripts/ccache_setup.sh
# - name: ccache info
# id: ccache-info
# shell: bash
# run: |
# echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
# - name: Cache ccache
# uses: actions/cache@v5
# with:
# path: ${{ steps.ccache-info.outputs.cache-dir }}
# key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }}
# restore-keys: cpp-odbc-ccache-windows-x64-
# - name: Checkout vcpkg
# uses: actions/checkout@v6
# with:
# fetch-depth: 0
# path: vcpkg
# repository: microsoft/vcpkg
# - name: Bootstrap vcpkg
# run: |
# vcpkg\bootstrap-vcpkg.bat
# $VCPKG_ROOT = $(Resolve-Path -LiteralPath "vcpkg").ToString()
# Write-Output ${VCPKG_ROOT} | `
# Out-File -FilePath ${Env:GITHUB_PATH} -Encoding utf8 -Append
# Write-Output "VCPKG_ROOT=${VCPKG_ROOT}" | `
# Out-File -FilePath ${Env:GITHUB_ENV} -Encoding utf8 -Append
# - name: Setup NuGet credentials for vcpkg caching
# shell: bash
# run: |
# $(vcpkg fetch nuget | tail -n 1) \
# sources add \
# -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \
# -storepasswordincleartext \
# -name "GitHub" \
# -username "$GITHUB_REPOSITORY_OWNER" \
# -password "${{ secrets.GITHUB_TOKEN }}"
# $(vcpkg fetch nuget | tail -n 1) \
# setapikey "${{ secrets.GITHUB_TOKEN }}" \
# -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json"
# - name: Build
# shell: cmd
# run: |
# set VCPKG_ROOT_KEEP=%VCPKG_ROOT%
# call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
# set VCPKG_ROOT=%VCPKG_ROOT_KEEP%
# bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
# - name: Register Arrow Flight SQL ODBC Driver
# shell: cmd
# run: |
# call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\arrow_flight_sql_odbc.dll
# # GH-48270 TODO: Resolve segementation fault during Arrow library unload, segementation fault is caught with `ci/scripts/cpp_test.sh`
# # GH-48269 TODO: Enable Flight & Flight SQL testing in MSVC CI and run tests with `ci/scripts/cpp_test.sh` instead
# # GH-48547 TODO: enable ODBC tests after GH-48270 and GH-48269 are resolved.
# - name: Run ODBC Unit Tests
# shell: cmd
# run: |
# build\cpp\%ARROW_BUILD_TYPE%\arrow-odbc-spi-impl-test.exe
# - name: Run ODBC Driver Tests
# shell: cmd
# run: |
# build\cpp\%ARROW_BUILD_TYPE%\arrow-flight-sql-odbc-test.exe
# - name: Install WiX Toolset
# shell: pwsh
# run: |
# Invoke-WebRequest -Uri https://github.com/wixtoolset/wix/releases/download/v6.0.0/wix-cli-x64.msi -OutFile wix-cli-x64.msi
# Start-Process -FilePath wix-cli-x64.msi -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
# echo "C:\Program Files\WiX Toolset v6.0\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
# - name: Build MSI ODBC installer
# shell: pwsh
# run: |
# # Verify WiX version
# wix --version
# cd build/cpp
# cpack
# - name: Upload the artifacts to the job
# uses: actions/upload-artifact@v6
# with:
# name: flight-sql-odbc-msi-installer
# path: build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi
# if-no-files-found: error
# # Upload ODBC installer as nightly release in scheduled runs
# - name: Prepare ODBC installer for sync
# if: github.event_name == 'schedule'
# run: |
# mkdir odbc-installer
# Move-Item "build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi" odbc-installer/
# tree odbc-installer /f
# - name: Sync to Remote
# if: github.event_name == 'schedule'
# uses: ./.github/actions/sync-nightlies
# with:
# upload: true
# switches: -avzh --update --delete --progress
# local_path: odbc-installer
# remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/odbc
# remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
# remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
# remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
# remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
# remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
report-extra-cpp:
if: github.event_name == 'schedule' && always()
needs:
- docker
- jni-linux
- jni-macos
- msvc-arm64
- odbc-macos
# - odbc-msvc
uses: ./.github/workflows/report_ci.yml
secrets: inherit