GH-50326: [Python] Convert arrays to Python objects without per-element Scalars in to_pylist

python/pyarrow/array.pxi

@@ -1864,7 +1864,19 @@ cdef class Array(_PandasConvertible):
         lst : list
         """
         self._assert_cpu()
-        return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self]
+        cdef int64_t i, n = self.length()
+        if maps_as_pydicts is not None:
+            # Converting maps to dicts has per-entry semantics (duplicate-key
+            # detection); use the Scalar-based conversion for exact behavior.
+            return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self]
+        return [self._getitem_py(i) for i in range(n)]
+
+    cdef object _getitem_py(self, int64_t i):
+        # Return self[i] as a Python object, without creating a Python Scalar
+        # (nor, for nested types, per-row Array wrappers) where a subclass
+        # provides a specialization; this base implementation goes through
+        # Scalar.as_py and thus preserves its semantics exactly (see GH-50326).
+        return self.getitem(i).as_py()
 
     def tolist(self):
         """
@@ -2444,6 +2456,11 @@ cdef class BooleanArray(Array):
     """
     Concrete class for Arrow arrays of boolean data type.
     """
+
+    cdef object _getitem_py(self, int64_t i):
+        if self.ap.IsNull(i):
+            return None
+        return (<CBooleanArray*> self.ap).Value(i)
     @property
     def false_count(self):
         return (<CBooleanArray*> self.ap).false_count()
@@ -2458,6 +2475,34 @@ cdef class NumericArray(Array):
     A base class for Arrow numeric arrays.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        cdef Type tid = self.ap.type_id()
+        if self.ap.IsNull(i):
+            return None
+        if tid == _Type_INT64:
+            return (<CInt64Array*> self.ap).Value(i)
+        elif tid == _Type_INT32:
+            return (<CInt32Array*> self.ap).Value(i)
+        elif tid == _Type_DOUBLE:
+            return (<CDoubleArray*> self.ap).Value(i)
+        elif tid == _Type_FLOAT:
+            return (<CFloatArray*> self.ap).Value(i)
+        elif tid == _Type_INT16:
+            return (<CInt16Array*> self.ap).Value(i)
+        elif tid == _Type_INT8:
+            return (<CInt8Array*> self.ap).Value(i)
+        elif tid == _Type_UINT64:
+            return (<CUInt64Array*> self.ap).Value(i)
+        elif tid == _Type_UINT32:
+            return (<CUInt32Array*> self.ap).Value(i)
+        elif tid == _Type_UINT16:
+            return (<CUInt16Array*> self.ap).Value(i)
+        elif tid == _Type_UINT8:
+            return (<CUInt8Array*> self.ap).Value(i)
+        # Subclasses whose as_py returns non-primitive objects (dates, times,
+        # timestamps, durations, half floats, ...) use the exact Scalar path.
+        return Array._getitem_py(self, i)
+
 
 cdef class IntegerArray(NumericArray):
     """
@@ -2776,6 +2821,16 @@ cdef class ListArray(BaseListArray):
     Concrete class for Arrow arrays of a list data type.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        cdef CListArray* arr = <CListArray*> self.ap
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = pyarrow_wrap_array(arr.values())
+        cdef Array values = <Array> self._children_cache
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        return [values._getitem_py(j) for j in range(start, end)]
+
     @staticmethod
     def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None):
         """
@@ -2961,6 +3016,16 @@ cdef class LargeListArray(BaseListArray):
     Identical to ListArray, but 64-bit offsets.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        cdef CLargeListArray* arr = <CLargeListArray*> self.ap
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = pyarrow_wrap_array(arr.values())
+        cdef Array values = <Array> self._children_cache
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        return [values._getitem_py(j) for j in range(start, end)]
+
     @staticmethod
     def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None):
         """
@@ -3551,6 +3616,19 @@ cdef class MapArray(ListArray):
     Concrete class for Arrow arrays of a map data type.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        cdef CListArray* arr = <CListArray*> self.ap
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = (self.keys, self.items)
+        cdef Array keys = <Array> (<tuple> self._children_cache)[0]
+        cdef Array items = <Array> (<tuple> self._children_cache)[1]
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        # Matches MapScalar.as_py with the default maps_as_pydicts=None:
+        # an association list of (key, value) tuples.
+        return [(keys._getitem_py(j), items._getitem_py(j)) for j in range(start, end)]
+
     @staticmethod
     def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None):
         """
@@ -3688,6 +3766,16 @@ cdef class FixedSizeListArray(BaseListArray):
     Concrete class for Arrow arrays of a fixed size list data type.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        cdef CFixedSizeListArray* arr = <CFixedSizeListArray*> self.ap
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = pyarrow_wrap_array(arr.values())
+        cdef Array values = <Array> self._children_cache
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        return [values._getitem_py(j) for j in range(start, end)]
+
     @staticmethod
     def from_arrays(values, list_size=None, DataType type=None, mask=None):
         """
@@ -3974,6 +4062,16 @@ cdef class StringArray(Array):
     Concrete class for Arrow arrays of string (or utf8) data type.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        cdef:
+            int32_t length
+            const uint8_t* data
+        if self.ap.IsNull(i):
+            return None
+        data = (<CStringArray*> self.ap).GetValue(i, &length)
+        # Matches StringScalar.as_py, which is str(buf, 'utf8').
+        return cp.PyUnicode_DecodeUTF8(<const char*> data, length, NULL)
+
     @staticmethod
     def from_buffers(int length, Buffer value_offsets, Buffer data,
                      Buffer null_bitmap=None, int null_count=-1,
@@ -4006,6 +4104,15 @@ cdef class LargeStringArray(Array):
     Concrete class for Arrow arrays of large string (or utf8) data type.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        cdef:
+            int64_t length
+            const uint8_t* data
+        if self.ap.IsNull(i):
+            return None
+        data = (<CLargeStringArray*> self.ap).GetValue(i, &length)
+        return cp.PyUnicode_DecodeUTF8(<const char*> data, length, NULL)
+
     @staticmethod
     def from_buffers(int length, Buffer value_offsets, Buffer data,
                      Buffer null_bitmap=None, int null_count=-1,
@@ -4043,6 +4150,16 @@ cdef class BinaryArray(Array):
     """
     Concrete class for Arrow arrays of variable-sized binary data type.
     """
+
+    cdef object _getitem_py(self, int64_t i):
+        cdef:
+            int32_t length
+            const uint8_t* data
+        if self.ap.IsNull(i):
+            return None
+        data = (<CBinaryArray*> self.ap).GetValue(i, &length)
+        return cp.PyBytes_FromStringAndSize(<const char*> data, length)
+
     @property
     def total_values_length(self):
         """
@@ -4056,6 +4173,16 @@ cdef class LargeBinaryArray(Array):
     """
     Concrete class for Arrow arrays of large variable-sized binary data type.
     """
+
+    cdef object _getitem_py(self, int64_t i):
+        cdef:
+            int64_t length
+            const uint8_t* data
+        if self.ap.IsNull(i):
+            return None
+        data = (<CLargeBinaryArray*> self.ap).GetValue(i, &length)
+        return cp.PyBytes_FromStringAndSize(<const char*> data, length)
+
     @property
     def total_values_length(self):
         """
@@ -4229,6 +4356,30 @@ cdef class StructArray(Array):
     Concrete class for Arrow arrays of a struct data type.
     """
 
+    cdef object _getitem_py(self, int64_t i):
+        if self.ap.IsNull(i):
+            return None
+        cdef int64_t k, num_fields = self.type.num_fields
+        if self._children_cache is None:
+            names = [self.type.field(k).name for k in range(num_fields)]
+            if len(set(names)) != len(names):
+                # StructScalar.as_py raises ValueError on duplicate field
+                # names; mark the cache so we take the Scalar path below.
+                self._children_cache = (None, None)
+            else:
+                self._children_cache = (
+                    names, [self.field(k) for k in range(num_fields)])
+        names = (<tuple> self._children_cache)[0]
+        if names is None:
+            return Array._getitem_py(self, i)
+        fields = (<tuple> self._children_cache)[1]
+        cdef Array field_arr
+        result = {}
+        for k in range(num_fields):
+            field_arr = <Array> fields[k]
+            result[names[k]] = field_arr._getitem_py(i)
+        return result
+
     def field(self, index):
         """
         Retrieves the child array belonging to field.

python/pyarrow/lib.pxd

@@ -282,6 +282,8 @@ cdef class Array(_PandasConvertible):
     cdef:
         shared_ptr[CArray] sp_array
         CArray* ap
+        # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326)
+        object _children_cache
 
     cdef readonly:
         DataType type
@@ -290,6 +292,7 @@ cdef class Array(_PandasConvertible):
 
     cdef void init(self, const shared_ptr[CArray]& sp_array) except *
     cdef getitem(self, int64_t i)
+    cdef object _getitem_py(self, int64_t i)
     cdef int64_t length(self)
     cdef void _assert_cpu(self) except *
 

python/pyarrow/tests/test_array.py

@@ -465,6 +465,38 @@ def test_array_getitem_numpy_scalars():
         assert arr[np.int32(idx)].as_py() == lst[idx]
 
 
+def test_to_pylist_bulk_paths():
+    # GH-50326: list-like and string arrays convert to Python objects in
+    # bulk instead of going through one Scalar per element; the result must
+    # match the per-scalar conversion exactly.
+    arrays = [
+        pa.array([[1, None, 3], None, [], [4]], type=pa.list_(pa.int32())),
+        pa.array([["a", None], None, [], ["bcd", ""]],
+                 type=pa.list_(pa.string())),
+        pa.array([["a", None], None, [], ["bcd", ""]],
+                 type=pa.large_list(pa.large_string())),
+        pa.array([[1, None], None, [3, 4]], type=pa.list_(pa.int32(), 2)),
+        pa.array([[[1], [2, None]], None, [None, [3]]],
+                 type=pa.list_(pa.list_(pa.int32()))),
+        pa.array([[("k1", 1), ("k2", None)], None, []],
+                 type=pa.map_(pa.string(), pa.int32())),
+        pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"],
+                 type=pa.string()),
+        pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"],
+                 type=pa.large_string()),
+        pa.array([], type=pa.list_(pa.int32())),
+        pa.array([None, None], type=pa.list_(pa.string())),
+    ]
+    for arr in arrays:
+        for view in (arr, arr.slice(1), arr.slice(0, 2), arr.slice(2)):
+            assert view.to_pylist() == [x.as_py() for x in view]
+
+    # Values inside numeric lists must stay Python ints/None, never floats
+    result = pa.array([[1, None, 3]], type=pa.list_(pa.int32())).to_pylist()
+    assert result == [[1, None, 3]]
+    assert [type(x) for x in result[0]] == [int, type(None), int]
+
+
 def test_array_slice():
     arr = pa.array(range(10))