Skip to content

GH-49566: [Python] Skip header files when installing compiled Cython files and other Python release verification fixes #49571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
raulcd merged 12 commits into from
Mar 25, 2026
Merged

GH-49566: [Python] Skip header files when installing compiled Cython files and other Python release verification fixes #49571

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
41026d4
Skip header files when installing compiled Cython files
AlenkaF Mar 20, 2026
efc1ae9
Fix old build include location test
AlenkaF Mar 20, 2026
3f0f110
Use the virtualenv / conda env created instead of trying to create a …
raulcd Mar 24, 2026
f7d1f9b
The virtualenv must contain the requirements for building
raulcd Mar 24, 2026
b865cd5
Trying to diagnose crash importing flight on macOS intel
raulcd Mar 24, 2026
612c41b
Verification jobs shouldn't need to require editable builds
raulcd Mar 24, 2026
3ec70f6
Avoid testing on the python folder to use the installed pyarrow on th…
raulcd Mar 24, 2026
8918219
Add a bunch more debugging for macOS intel
raulcd Mar 24, 2026
9ef21fd
Append _arrow_lib_dir to RPATH so _flight.so (python) can find libarr…
raulcd Mar 24, 2026
5f8c18c
More debugging
raulcd Mar 24, 2026
a5ff56d
Fix protobug 33 and 34 on the same process
raulcd Mar 24, 2026
f9ca17b
Remove debugging
raulcd Mar 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 30 additions & 7 deletions dev/release/verify-release-candidate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ test_python() {
show_header "Build and test Python libraries"

# Build and test Python
maybe_setup_virtualenv
maybe_setup_virtualenv -r python/requirements-build.txt
maybe_setup_conda --file ci/conda_env_python.txt

if [ "${USE_CONDA}" -gt 0 ]; then
Expand Down Expand Up @@ -570,7 +570,9 @@ test_python() {
pushd python

# Build pyarrow
python -m pip install -e .
python -m pip install --no-build-isolation .

@raulcd raulcd Mar 25, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the release verification of Python. I am unsure why we were testing with build isolation. We are setting up a virtualenv or a conda env and we should use that, that's why I am adding the --no-build-isolation. Also I am unsure why we were testing with editable builds. That doesn't make much sense for verification.

@AlenkaF AlenkaF Mar 25, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with you, looks OK to me now.

@kou kou Mar 25, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1


popd

# Check mandatory and optional imports
python -c "
Expand All @@ -587,7 +589,30 @@ import pyarrow.parquet
python -c "import pyarrow.cuda"
fi
if [ "${ARROW_FLIGHT}" == "ON" ]; then
python -c "import pyarrow.flight"
python -c "
Comment thread
raulcd marked this conversation as resolved.
Outdated
import pyarrow, os, sysconfig, subprocess
print('pyarrow location:', pyarrow.__file__)
d = os.path.dirname(pyarrow.__file__)
ext = sysconfig.get_config_var('EXT_SUFFIX')
flight_so = os.path.join(d, f'_flight{ext}')
print('_flight.so exists:', os.path.exists(flight_so))
if os.path.exists(flight_so):
subprocess.run(['otool', '-L', flight_so])
# Show RPATH entries
result = subprocess.run(['otool', '-l', flight_so], capture_output=True, text=True)
lines = result.stdout.splitlines()
for i, line in enumerate(lines):
if 'RPATH' in line or 'rpath' in line:
print(lines[max(0,i-1):i+3])
flight_dylib = os.path.join(d, 'libarrow_flight.2400.dylib')
print('libarrow_flight.dylib exists:', os.path.exists(flight_dylib))
print('ARROW_HOME:', os.environ.get('ARROW_HOME', 'NOT SET'))
arrow_lib = os.path.join(os.environ.get('ARROW_HOME', ''), 'lib')
if os.path.isdir(arrow_lib):
print('Arrow libs:', [f for f in os.listdir(arrow_lib) if 'flight' in f])
print('DYLD_LIBRARY_PATH:', os.environ.get('DYLD_LIBRARY_PATH', 'NOT SET'))
"
python -X faulthandler -c "import pyarrow.flight"
fi
if [ "${ARROW_GANDIVA}" == "ON" ]; then
python -c "import pyarrow.gandiva"
Expand All @@ -601,12 +626,10 @@ import pyarrow.parquet


# Install test dependencies
pip install -r requirements-test.txt
pip install -r python/requirements-test.txt

# Execute pyarrow unittests
pytest pyarrow -v

popd
pytest --pyargs pyarrow -v
}

test_glib() {
Expand Down
4 changes: 4 additions & 0 deletions dev/tasks/verify-rc/github.macos.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ jobs:
brew uninstall pkg-config@0.29.2 || :
{% endif %}

# Workaround for https://github.com/grpc/grpc/issues/41755
# Remove once the runner ships a newer Homebrew.
brew update

brew bundle --file=arrow/cpp/Brewfile
brew bundle --file=arrow/c_glib/Brewfile

Expand Down
12 changes: 12 additions & 0 deletions python/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,15 @@ message(STATUS "Found Cython version: ${CYTHON_VERSION}")
include(GNUInstallDirs)
find_package(Arrow REQUIRED)

# When not bundling Arrow C++ libraries on macOS, add the Arrow library
# directory to the RPATH so that the extensions can find libarrow at runtime.
if(APPLE
AND NOT PYARROW_BUNDLE_ARROW_CPP
AND ARROW_SHARED_LIB)
get_filename_component(_arrow_lib_dir "${ARROW_SHARED_LIB}" DIRECTORY)
list(APPEND CMAKE_INSTALL_RPATH "${_arrow_lib_dir}")
endif()

macro(define_option name description arrow_option)
set("PYARROW_${name}"
"AUTO"
Expand Down Expand Up @@ -968,6 +977,9 @@ foreach(module ${CYTHON_EXTENSIONS})
continue()
endif()
endif()
if(output MATCHES "\\.h$")
continue()
endif()
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${output} DESTINATION ".")
endforeach()
endforeach()
Expand Down
5 changes: 2 additions & 3 deletions python/pyarrow/tests/test_cpp_internals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,8 @@ def test_pyarrow_include():
# created. Either with PyArrow C++ header files or with
# Arrow C++ and PyArrow C++ header files together

source = os.path.dirname(os.path.abspath(__file__))
pyarrow_dir = pjoin(source, '..')
pyarrow_include = pjoin(pyarrow_dir, 'include')
import pyarrow
pyarrow_include = pyarrow.get_include()
pyarrow_cpp_include = pjoin(pyarrow_include, 'arrow', 'python')

assert os.path.exists(pyarrow_include)
Expand Down
Loading