GH-49566: [Python] Skip header files when installing compiled Cython files and other Python release verification fixes (#49571)

### Rationale for this change

Local import is broken when doing editable install with scikit-build-core, also nightly verification jobs are failing for the same reason.

### What changes are included in this PR?

- `lib.h` and `lib_api.h` are already installed separately so we skip them when installing Cython extensions into the output destination (`side-packages/pyarrow`).
- Arrow library directory is added to the `RPATH`, in the case of not bundling ARROW_CPP, to fix linking issues on macos
- `test_pyarrow_include` is updated due to compiled files now being moved to the site-packages
- updates to the release verification script when building pyarrow
- update of `brew`  on the "Install System Dependencies" step in the rc verification job to fix an issue with protobuf mixed versions

### Are these changes tested?

Yes, locally and via the extended verification builds.

### Are there any user-facing changes?

No.

* GitHub Issue: #49566

Lead-authored-by: Raúl Cumplido <raulcumplido@gmail.com>
Co-authored-by: AlenkaF <frim.alenka@gmail.com>
Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>

Author: AlenkaF

dev/release/verify-release-candidate.sh

@@ -538,7 +538,7 @@ test_python() {
   show_header "Build and test Python libraries"
 
   # Build and test Python
-  maybe_setup_virtualenv
+  maybe_setup_virtualenv -r python/requirements-build.txt
   maybe_setup_conda --file ci/conda_env_python.txt
 
   if [ "${USE_CONDA}" -gt 0 ]; then
@@ -570,7 +570,9 @@ test_python() {
   pushd python
 
   # Build pyarrow
-  python -m pip install -e .
+  python -m pip install --no-build-isolation .
+
+  popd
 
   # Check mandatory and optional imports
   python -c "
@@ -601,12 +603,10 @@ import pyarrow.parquet
 
 
   # Install test dependencies
-  pip install -r requirements-test.txt
+  pip install -r python/requirements-test.txt
 
   # Execute pyarrow unittests
-  pytest pyarrow -v
-
-  popd
+  pytest --pyargs pyarrow -v
 }
 
 test_glib() {

dev/tasks/verify-rc/github.macos.yml

@@ -50,6 +50,10 @@ jobs:
           brew uninstall pkg-config@0.29.2 || :
           {% endif %}
 
+          # Workaround for https://github.com/grpc/grpc/issues/41755
+          # Remove once the runner ships a newer Homebrew.
+          brew update
+
           brew bundle --file=arrow/cpp/Brewfile
           brew bundle --file=arrow/c_glib/Brewfile
 

python/CMakeLists.txt

@@ -293,6 +293,15 @@ message(STATUS "Found Cython version: ${CYTHON_VERSION}")
 include(GNUInstallDirs)
 find_package(Arrow REQUIRED)
 
+# When not bundling Arrow C++ libraries on macOS, add the Arrow library
+# directory to the RPATH so that the extensions can find libarrow at runtime.
+if(APPLE
+   AND NOT PYARROW_BUNDLE_ARROW_CPP
+   AND ARROW_SHARED_LIB)
+  get_filename_component(_arrow_lib_dir "${ARROW_SHARED_LIB}" DIRECTORY)
+  list(APPEND CMAKE_INSTALL_RPATH "${_arrow_lib_dir}")
+endif()
+
 macro(define_option name description arrow_option)
   set("PYARROW_${name}"
       "AUTO"
@@ -968,6 +977,9 @@ foreach(module ${CYTHON_EXTENSIONS})
         continue()
       endif()
     endif()
+    if(output MATCHES "\\.h$")
+      continue()
+    endif()
     install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${output} DESTINATION ".")
   endforeach()
 endforeach()

python/pyarrow/tests/test_cpp_internals.py

@@ -49,9 +49,8 @@ def test_pyarrow_include():
     # created. Either with PyArrow C++ header files or with
     # Arrow C++ and PyArrow C++ header files together
 
-    source = os.path.dirname(os.path.abspath(__file__))
-    pyarrow_dir = pjoin(source, '..')
-    pyarrow_include = pjoin(pyarrow_dir, 'include')
+    import pyarrow
+    pyarrow_include = pyarrow.get_include()
     pyarrow_cpp_include = pjoin(pyarrow_include, 'arrow', 'python')
 
     assert os.path.exists(pyarrow_include)