GH-50429: [Python] Convert maps to dicts without per-element Scalars in to_pylist

python/pyarrow/array.pxi

@@ -1864,7 +1864,17 @@ cdef class Array(_PandasConvertible):
         lst : list
         """
         self._assert_cpu()
-        return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self]
+        cdef int64_t i, n = self.length()
+        # TODO(GH-50448): convert per range instead of per element to cut
+        # the per-element call overhead further.
+        return [self._getitem_py(i, maps_as_pydicts) for i in range(n)]
+
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        # Return self[i] as a Python object, without creating a Python Scalar
+        # (nor, for nested types, per-row Array wrappers) where a subclass
+        # provides a specialization; this base implementation goes through
+        # Scalar.as_py and thus preserves its semantics exactly (see GH-50326).
+        return self.getitem(i).as_py(maps_as_pydicts=maps_as_pydicts)
 
     def tolist(self):
         """
@@ -2444,6 +2454,12 @@ cdef class BooleanArray(Array):
     """
     Concrete class for Arrow arrays of boolean data type.
     """
+
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        return (<CBooleanArray*> self.ap).Value(i)
+
     @property
     def false_count(self):
         return (<CBooleanArray*> self.ap).false_count()
@@ -2458,6 +2474,34 @@ cdef class NumericArray(Array):
     A base class for Arrow numeric arrays.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        cdef Type tid = self.ap.type_id()
+        if self.ap.IsNull(i):
+            return None
+        if tid == _Type_INT8:
+            return (<CInt8Array*> self.ap).Value(i)
+        elif tid == _Type_INT16:
+            return (<CInt16Array*> self.ap).Value(i)
+        elif tid == _Type_INT32:
+            return (<CInt32Array*> self.ap).Value(i)
+        elif tid == _Type_INT64:
+            return (<CInt64Array*> self.ap).Value(i)
+        elif tid == _Type_UINT8:
+            return (<CUInt8Array*> self.ap).Value(i)
+        elif tid == _Type_UINT16:
+            return (<CUInt16Array*> self.ap).Value(i)
+        elif tid == _Type_UINT32:
+            return (<CUInt32Array*> self.ap).Value(i)
+        elif tid == _Type_UINT64:
+            return (<CUInt64Array*> self.ap).Value(i)
+        elif tid == _Type_FLOAT:
+            return (<CFloatArray*> self.ap).Value(i)
+        elif tid == _Type_DOUBLE:
+            return (<CDoubleArray*> self.ap).Value(i)
+        # Subclasses whose as_py returns non-primitive objects (dates, times,
+        # timestamps, durations, half floats, ...) use the exact Scalar path.
+        return Array._getitem_py(self, i, maps_as_pydicts)
+
 
 cdef class IntegerArray(NumericArray):
     """
@@ -2776,6 +2820,16 @@ cdef class ListArray(BaseListArray):
     Concrete class for Arrow arrays of a list data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        cdef CListArray* arr = <CListArray*> self.ap
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = pyarrow_wrap_array(arr.values())
+        cdef Array values = <Array> self._children_cache
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)]
+
     @staticmethod
     def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None):
         """
@@ -2961,6 +3015,16 @@ cdef class LargeListArray(BaseListArray):
     Identical to ListArray, but 64-bit offsets.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        cdef CLargeListArray* arr = <CLargeListArray*> self.ap
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = pyarrow_wrap_array(arr.values())
+        cdef Array values = <Array> self._children_cache
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)]
+
     @staticmethod
     def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None):
         """
@@ -3551,6 +3615,52 @@ cdef class MapArray(ListArray):
     Concrete class for Arrow arrays of a map data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        cdef CListArray* arr = <CListArray*> self.ap
+        cdef bint as_dicts = maps_as_pydicts is not None
+        if as_dicts and maps_as_pydicts != "lossy" and maps_as_pydicts != "strict":
+            # Matches MapScalar.as_py, which validates before the null check.
+            raise ValueError(
+                "Invalid value for 'maps_as_pydicts': "
+                + "valid values are 'lossy', 'strict' or `None` (default). "
+                + f"Received {maps_as_pydicts!r}."
+            )
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = (self.keys, self.items)
+        cdef Array keys = <Array> (<tuple> self._children_cache)[0]
+        cdef Array items = <Array> (<tuple> self._children_cache)[1]
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        if not as_dicts:
+            # Matches MapScalar.as_py with the default maps_as_pydicts=None:
+            # an association list of (key, value) tuples.
+            return [
+                (keys._getitem_py(j, None), items._getitem_py(j, maps_as_pydicts))
+                for j in range(start, end)
+            ]
+        cdef dict result = {}
+        for j in range(start, end):
+            result[keys._getitem_py(j, None)] = items._getitem_py(j, maps_as_pydicts)
+        if len(result) == end - start:
+            return result
+        # Duplicate keys: redo the row with the per-key loop so the 'lossy'
+        # warnings and 'strict' KeyError match MapScalar.as_py exactly.
+        result = {}
+        for j in range(start, end):
+            key = keys._getitem_py(j, None)
+            if key in result:
+                if maps_as_pydicts == "strict":
+                    raise KeyError(
+                        "Converting to Python dictionary is not supported in strict mode "
+                        f"when duplicate keys are present (duplicate key was '{key}')."
+                    )
+                else:
+                    warnings.warn(
+                        f"Encountered key '{key}' which was already encountered.")
+            result[key] = items._getitem_py(j, maps_as_pydicts)
+        return result
+
     @staticmethod
     def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None):
         """
@@ -3688,6 +3798,16 @@ cdef class FixedSizeListArray(BaseListArray):
     Concrete class for Arrow arrays of a fixed size list data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        cdef CFixedSizeListArray* arr = <CFixedSizeListArray*> self.ap
+        if arr.IsNull(i):
+            return None
+        if self._children_cache is None:
+            self._children_cache = pyarrow_wrap_array(arr.values())
+        cdef Array values = <Array> self._children_cache
+        cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
+        return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)]
+
     @staticmethod
     def from_arrays(values, list_size=None, DataType type=None, mask=None):
         """
@@ -3974,6 +4094,13 @@ cdef class StringArray(Array):
     Concrete class for Arrow arrays of string (or utf8) data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        cdef cpp_string_view view = (<CBinaryArray*> self.ap).GetView(i)
+        # Matches StringScalar.as_py, which is str(buf, 'utf8').
+        return cp.PyUnicode_DecodeUTF8(view.data(), <Py_ssize_t> view.size(), NULL)
+
     @staticmethod
     def from_buffers(int length, Buffer value_offsets, Buffer data,
                      Buffer null_bitmap=None, int null_count=-1,
@@ -4006,6 +4133,12 @@ cdef class LargeStringArray(Array):
     Concrete class for Arrow arrays of large string (or utf8) data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        cdef cpp_string_view view = (<CLargeBinaryArray*> self.ap).GetView(i)
+        return cp.PyUnicode_DecodeUTF8(view.data(), <Py_ssize_t> view.size(), NULL)
+
     @staticmethod
     def from_buffers(int length, Buffer value_offsets, Buffer data,
                      Buffer null_bitmap=None, int null_count=-1,
@@ -4038,11 +4171,24 @@ cdef class StringViewArray(Array):
     Concrete class for Arrow arrays of string (or utf8) view data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        cdef cpp_string_view view = (<CBinaryViewArray*> self.ap).GetView(i)
+        return cp.PyUnicode_DecodeUTF8(view.data(), <Py_ssize_t> view.size(), NULL)
+
 
 cdef class BinaryArray(Array):
     """
     Concrete class for Arrow arrays of variable-sized binary data type.
     """
+
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        cdef cpp_string_view view = (<CBinaryArray*> self.ap).GetView(i)
+        return cp.PyBytes_FromStringAndSize(view.data(), <Py_ssize_t> view.size())
+
     @property
     def total_values_length(self):
         """
@@ -4056,6 +4202,13 @@ cdef class LargeBinaryArray(Array):
     """
     Concrete class for Arrow arrays of large variable-sized binary data type.
     """
+
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        cdef cpp_string_view view = (<CLargeBinaryArray*> self.ap).GetView(i)
+        return cp.PyBytes_FromStringAndSize(view.data(), <Py_ssize_t> view.size())
+
     @property
     def total_values_length(self):
         """
@@ -4070,6 +4223,12 @@ cdef class BinaryViewArray(Array):
     Concrete class for Arrow arrays of variable-sized binary view data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        cdef cpp_string_view view = (<CBinaryViewArray*> self.ap).GetView(i)
+        return cp.PyBytes_FromStringAndSize(view.data(), <Py_ssize_t> view.size())
+
 
 cdef class DictionaryArray(Array):
     """
@@ -4229,6 +4388,28 @@ cdef class StructArray(Array):
     Concrete class for Arrow arrays of a struct data type.
     """
 
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
+        if self.ap.IsNull(i):
+            return None
+        cdef int64_t k, num_fields = self.type.num_fields
+        if self._children_cache is None:
+            names = [self.type.field(k).name for k in range(num_fields)]
+            if len(set(names)) != len(names):
+                # Matches StructScalar.as_py
+                raise ValueError(
+                    "Converting to Python dictionary is not supported when "
+                    "duplicate field names are present")
+            self._children_cache = (
+                names, [self.field(k) for k in range(num_fields)])
+        names = (<tuple> self._children_cache)[0]
+        fields = (<tuple> self._children_cache)[1]
+        cdef Array field_arr
+        result = {}
+        for k in range(num_fields):
+            field_arr = <Array> fields[k]
+            result[names[k]] = field_arr._getitem_py(i, maps_as_pydicts)
+        return result
+
     def field(self, index):
         """
         Retrieves the child array belonging to field.

python/pyarrow/includes/libarrow.pxd

@@ -946,13 +946,15 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
     cdef cppclass CBinaryArray" arrow::BinaryArray"(CArray):
         const uint8_t* GetValue(int i, int32_t* length)
+        cpp_string_view GetView(int64_t i)
         shared_ptr[CBuffer] value_data()
         int32_t value_offset(int64_t i)
         int32_t value_length(int64_t i)
         int32_t total_values_length()
 
     cdef cppclass CLargeBinaryArray" arrow::LargeBinaryArray"(CArray):
         const uint8_t* GetValue(int i, int64_t* length)
+        cpp_string_view GetView(int64_t i)
         shared_ptr[CBuffer] value_data()
         int64_t value_offset(int64_t i)
         int64_t value_length(int64_t i)
@@ -977,6 +979,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
 
         c_string GetString(int i)
 
+    cdef cppclass CBinaryViewArray" arrow::BinaryViewArray"(CArray):
+        cpp_string_view GetView(int64_t i)
+
+    cdef cppclass CStringViewArray" arrow::StringViewArray"(CBinaryViewArray):
+        pass
+
     cdef cppclass CStructArray" arrow::StructArray"(CArray):
         CStructArray(shared_ptr[CDataType]& type, int64_t length,
                      vector[shared_ptr[CArray]]& children,

python/pyarrow/lib.pxd

@@ -288,8 +288,15 @@ cdef class Array(_PandasConvertible):
         # To allow Table to propagate metadata to pandas.Series
         object _name
 
+    cdef:
+        # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326).
+        # Appended after the pre-existing attributes to keep their offsets
+        # stable for extensions compiled against an older pyarrow.
+        object _children_cache
+
     cdef void init(self, const shared_ptr[CArray]& sp_array) except *
     cdef getitem(self, int64_t i)
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts)
     cdef int64_t length(self)
     cdef void _assert_cpu(self) except *