GH-49831: [Python] Withhold annotations from Python wheel until they are complete (#50168)

### Rationale for this change

Since 24.0, pyarrow ships annotations and `py.typed` marker while the type stubs are still incomplete. This makes type checkers trust the partial stubs and report false errors in downstream code (e.g. Module has no attribute "all" for `pyarrow.compute`). Some type checkers also consume bundled `.pyi` files even without `py.typed`, so the stubs need to be withheld from wheels for now. See #49831.

### What changes are included in this PR?

This temporarily omits both `pyarrow/py.typed` and the bundled `pyarrow-stubs` / `.pyi` files from built wheels until the stubs are complete. Wheel-content validation now asserts that neither `py.typed` nor `.pyi` files are present, and wheel build scripts no longer request stub docstring injection while stubs are not installed.

### Are these changes tested?

Wheel-content validation has been updated to check the intended absence of `py.typed` and `.pyi` files.

### Are there any user-facing changes?

Type checkers no longer pick up pyarrow's incomplete stubs from wheels.
* GitHub Issue: #49831

Authored-by: Rok Mihevc <rok@mihevc.org>
Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>

Author: rok

ci/scripts/python_wheel_macos_build.sh

@@ -148,7 +148,8 @@ popd
 
 echo "=== (${PYTHON_VERSION}) Building wheel ==="
 export PYARROW_BUNDLE_ARROW_CPP=ON
-export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
+# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
+# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
 export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}

ci/scripts/python_wheel_validate_contents.py

@@ -37,6 +37,7 @@ def _count_docstrings(source):
     return count
 
 
+# TODO(GH-48970): Check stubs ARE present once annotations are complete
 def validate_wheel(path):
     p = Path(path)
     wheels = list(p.glob('*.whl'))
@@ -54,9 +55,9 @@ def validate_wheel(path):
                 info.filename.split("/")[-1] == filename for info in wheel_zip.filelist
             ), f"{filename} is missing from the wheel."
 
-        assert any(
+        assert not any(
             info.filename == "pyarrow/py.typed" for info in wheel_zip.filelist
-        ), "pyarrow/py.typed is missing from the wheel."
+        ), "pyarrow/py.typed is present in the wheel."
 
         source_root = Path(__file__).resolve().parents[2]
         stubs_dir = source_root / "python" / "pyarrow-stubs" / "pyarrow"
@@ -73,19 +74,22 @@ def validate_wheel(path):
             if info.filename.startswith("pyarrow/") and info.filename.endswith(".pyi")
         }
 
-        assert wheel_stub_files == expected_stub_files, (
-            "Wheel .pyi files differ from python/pyarrow-stubs/pyarrow.\n"
+        assert not (wheel_stub_files == expected_stub_files), (
+            "Wheel .pyi files do not differ from python/pyarrow-stubs/pyarrow.\n"
             f"Missing in wheel: {sorted(expected_stub_files - wheel_stub_files)}\n"
             f"Unexpected in wheel: {sorted(wheel_stub_files - expected_stub_files)}"
         )
+        assert not wheel_stub_files, (
+            f"Wheel contains unexpected .pyi files: {sorted(wheel_stub_files)}"
+        )
 
         wheel_docstring_count = sum(
             _count_docstrings(wheel_zip.read(wsf).decode("utf-8"))
             for wsf in wheel_stub_files
         )
 
         print(f"Found {wheel_docstring_count} docstring(s) in wheel stubs.")
-        assert wheel_docstring_count, "No docstrings found in wheel stub files."
+        assert wheel_docstring_count == 0, "Docstrings found in wheel stub files."
 
     print(f"The wheel: {wheels[0]} seems valid.")
 

ci/scripts/python_wheel_windows_build.bat

@@ -116,7 +116,8 @@ popd
 
 echo "=== (%PYTHON%) Building wheel ==="
 set PYARROW_BUNDLE_ARROW_CPP=ON
-set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
+rem TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
+rem set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
 set PYARROW_WITH_ACERO=%ARROW_ACERO%
 set PYARROW_WITH_AZURE=%ARROW_AZURE%
 set PYARROW_WITH_DATASET=%ARROW_DATASET%

ci/scripts/python_wheel_xlinux_build.sh

@@ -157,7 +157,8 @@ check_arrow_visibility
 
 echo "=== (${PYTHON_VERSION}) Building wheel ==="
 export PYARROW_BUNDLE_ARROW_CPP=ON
-export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
+# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
+# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
 export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}

python/CMakeLists.txt

@@ -1041,32 +1041,40 @@ endif()
 #
 # Type stubs with docstring injection
 #
+# TODO(GH-32609): Reintroduce type stubs into PyArrow wheels.
+# GH-49831: Temporarily do not install pyarrow-stubs into wheels.
 # Stubs live in pyarrow-stubs/pyarrow/ during development but are installed
 # alongside the package so type checkers can find them (PEP 561).
-set(PYARROW_STUBS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs/pyarrow")
-if(EXISTS "${PYARROW_STUBS_SOURCE_DIR}")
-  install(DIRECTORY "${PYARROW_STUBS_SOURCE_DIR}/"
-          DESTINATION "."
-          FILES_MATCHING
-          PATTERN "*.pyi")
-
-  if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
-    install(CODE "
-      execute_process(
-        COMMAND \"${Python3_EXECUTABLE}\"
-                \"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
-                \"${CMAKE_INSTALL_PREFIX}\"
-                \"${CMAKE_CURRENT_SOURCE_DIR}\"
-        RESULT_VARIABLE _pyarrow_stub_docstrings_result
-      )
-      if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
-        message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
-      endif()
-    ")
-  endif()
-else()
-  if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
-    message(FATAL_ERROR "PyArrow stub source directory not found at ${PYARROW_STUBS_SOURCE_DIR}; "
-                        "cannot build wheel without .pyi files.")
-  endif()
+# The stubs are currently incomplete, and some type checkers consume .pyi files
+# even without the py.typed marker. Re-enable this when the stubs are complete.
+if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
+  message(FATAL_ERROR "PYARROW_REQUIRE_STUB_DOCSTRINGS cannot be used while "
+                      "pyarrow-stubs are omitted from wheels (GH-49831).")
 endif()
+# set(PYARROW_STUBS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs/pyarrow")
+# if(EXISTS "${PYARROW_STUBS_SOURCE_DIR}")
+#   install(DIRECTORY "${PYARROW_STUBS_SOURCE_DIR}/"
+#           DESTINATION "."
+#           FILES_MATCHING
+#           PATTERN "*.pyi")
+#
+#   if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
+#     install(CODE "
+#       execute_process(
+#         COMMAND \"${Python3_EXECUTABLE}\"
+#                 \"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
+#                 \"${CMAKE_INSTALL_PREFIX}\"
+#                 \"${CMAKE_CURRENT_SOURCE_DIR}\"
+#         RESULT_VARIABLE _pyarrow_stub_docstrings_result
+#       )
+#       if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
+#         message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
+#       endif()
+#     ")
+#   endif()
+# else()
+#   if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
+#     message(FATAL_ERROR "PyArrow stub source directory not found at ${PYARROW_STUBS_SOURCE_DIR}; "
+#                         "cannot build wheel without .pyi files.")
+#   endif()
+# endif()

python/pyproject.toml

@@ -88,6 +88,12 @@ metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
 sdist.include = ["pyarrow/_generated_version.py", "cmake_modules/", "pyarrow-stubs/"]
 wheel.packages = ["pyarrow"]
 wheel.install-dir = "pyarrow"
+# TODO(GH-32609): Remove this when stubfiles are complete
+# Withhold the PEP 561 marker until the type stubs are complete. The .pyi files
+# are also temporarily omitted from wheels, so type checkers don't rely on the
+# incomplete stubs and break downstream users (GH-49831). py.typed is kept
+# in-tree for CI type-checking.
+wheel.exclude = ["pyarrow/py.typed"]
 
 [tool.scikit-build.cmake.define]
 PYARROW_BUNDLE_ARROW_CPP = {env = "PYARROW_BUNDLE_ARROW_CPP", default = "OFF"}