GH-50429: [Python] Convert maps to dicts without per-element Scalars in to_pylist

Thread maps_as_pydicts through the scalar-free _getitem_py mechanism
instead of routing it to the Scalar-based path. MapArray builds the
dict directly from the flattened keys/items children; when the dict
size reveals duplicate keys, the row is redone with the per-key loop
so the 'lossy' warnings and 'strict' KeyError match MapScalar.as_py
exactly. Invalid values still raise the same ValueError when a map
value is converted (including null rows), non-map arrays still ignore
the option, and the option still propagates through nested types.
Unspecialized types keep the exact Scalar fallback, which now receives
the option.

Benchmark (M4 Max, 1M rows of 2-entry maps, 10% nulls):
to_pylist(maps_as_pydicts='lossy') 2.20s -> 0.10s (~21x). The dict
form is also faster than the default association-list form (0.78s),
which allocates a 2-tuple per entry.

Co-authored-by: Isaac

Author: viirya

python/pyarrow/array.pxi

@@ -1865,18 +1865,14 @@ cdef class Array(_PandasConvertible):
         """
         self._assert_cpu()
         cdef int64_t i, n = self.length()
-        if maps_as_pydicts is not None:
-            # Converting maps to dicts has per-entry semantics (duplicate-key
-            # detection); use the Scalar-based conversion for exact behavior.
-            return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self]
-        return [self._getitem_py(i) for i in range(n)]
+        return [self._getitem_py(i, maps_as_pydicts) for i in range(n)]
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         # Return self[i] as a Python object, without creating a Python Scalar
         # (nor, for nested types, per-row Array wrappers) where a subclass
         # provides a specialization; this base implementation goes through
         # Scalar.as_py and thus preserves its semantics exactly (see GH-50326).
-        return self.getitem(i).as_py()
+        return self.getitem(i).as_py(maps_as_pydicts=maps_as_pydicts)
 
     def tolist(self):
         """
@@ -2457,7 +2453,7 @@ cdef class BooleanArray(Array):
     Concrete class for Arrow arrays of boolean data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         if self.ap.IsNull(i):
             return None
         return (<CBooleanArray*> self.ap).Value(i)
@@ -2476,7 +2472,7 @@ cdef class NumericArray(Array):
     A base class for Arrow numeric arrays.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef Type tid = self.ap.type_id()
         if self.ap.IsNull(i):
             return None
@@ -2502,7 +2498,7 @@ cdef class NumericArray(Array):
             return (<CUInt8Array*> self.ap).Value(i)
         # Subclasses whose as_py returns non-primitive objects (dates, times,
         # timestamps, durations, half floats, ...) use the exact Scalar path.
-        return Array._getitem_py(self, i)
+        return Array._getitem_py(self, i, maps_as_pydicts)
 
 
 cdef class IntegerArray(NumericArray):
@@ -2822,15 +2818,15 @@ cdef class ListArray(BaseListArray):
     Concrete class for Arrow arrays of a list data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef CListArray* arr = <CListArray*> self.ap
         if arr.IsNull(i):
             return None
         if self._children_cache is None:
             self._children_cache = pyarrow_wrap_array(arr.values())
         cdef Array values = <Array> self._children_cache
         cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
-        return [values._getitem_py(j) for j in range(start, end)]
+        return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)]
 
     @staticmethod
     def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None):
@@ -3017,15 +3013,15 @@ cdef class LargeListArray(BaseListArray):
     Identical to ListArray, but 64-bit offsets.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef CLargeListArray* arr = <CLargeListArray*> self.ap
         if arr.IsNull(i):
             return None
         if self._children_cache is None:
             self._children_cache = pyarrow_wrap_array(arr.values())
         cdef Array values = <Array> self._children_cache
         cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
-        return [values._getitem_py(j) for j in range(start, end)]
+        return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)]
 
     @staticmethod
     def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None):
@@ -3617,18 +3613,51 @@ cdef class MapArray(ListArray):
     Concrete class for Arrow arrays of a map data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef CListArray* arr = <CListArray*> self.ap
+        cdef bint as_dicts = maps_as_pydicts is not None
+        if as_dicts and maps_as_pydicts != "lossy" and maps_as_pydicts != "strict":
+            # Matches MapScalar.as_py, which validates before the null check.
+            raise ValueError(
+                "Invalid value for 'maps_as_pydicts': "
+                + "valid values are 'lossy', 'strict' or `None` (default). "
+                + f"Received {maps_as_pydicts!r}."
+            )
         if arr.IsNull(i):
             return None
         if self._children_cache is None:
             self._children_cache = (self.keys, self.items)
         cdef Array keys = <Array> (<tuple> self._children_cache)[0]
         cdef Array items = <Array> (<tuple> self._children_cache)[1]
         cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
-        # Matches MapScalar.as_py with the default maps_as_pydicts=None:
-        # an association list of (key, value) tuples.
-        return [(keys._getitem_py(j), items._getitem_py(j)) for j in range(start, end)]
+        if not as_dicts:
+            # Matches MapScalar.as_py with the default maps_as_pydicts=None:
+            # an association list of (key, value) tuples.
+            return [
+                (keys._getitem_py(j, None), items._getitem_py(j, maps_as_pydicts))
+                for j in range(start, end)
+            ]
+        cdef dict result = {}
+        for j in range(start, end):
+            result[keys._getitem_py(j, None)] = items._getitem_py(j, maps_as_pydicts)
+        if len(result) == end - start:
+            return result
+        # Duplicate keys: redo the row with the per-key loop so the 'lossy'
+        # warnings and 'strict' KeyError match MapScalar.as_py exactly.
+        result = {}
+        for j in range(start, end):
+            key = keys._getitem_py(j, None)
+            if key in result:
+                if maps_as_pydicts == "strict":
+                    raise KeyError(
+                        "Converting to Python dictionary is not supported in strict mode "
+                        f"when duplicate keys are present (duplicate key was '{key}')."
+                    )
+                else:
+                    warnings.warn(
+                        f"Encountered key '{key}' which was already encountered.")
+            result[key] = items._getitem_py(j, maps_as_pydicts)
+        return result
 
     @staticmethod
     def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None):
@@ -3767,15 +3796,15 @@ cdef class FixedSizeListArray(BaseListArray):
     Concrete class for Arrow arrays of a fixed size list data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef CFixedSizeListArray* arr = <CFixedSizeListArray*> self.ap
         if arr.IsNull(i):
             return None
         if self._children_cache is None:
             self._children_cache = pyarrow_wrap_array(arr.values())
         cdef Array values = <Array> self._children_cache
         cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1)
-        return [values._getitem_py(j) for j in range(start, end)]
+        return [values._getitem_py(j, maps_as_pydicts) for j in range(start, end)]
 
     @staticmethod
     def from_arrays(values, list_size=None, DataType type=None, mask=None):
@@ -4063,7 +4092,7 @@ cdef class StringArray(Array):
     Concrete class for Arrow arrays of string (or utf8) data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef:
             int32_t length
             const uint8_t* data
@@ -4105,7 +4134,7 @@ cdef class LargeStringArray(Array):
     Concrete class for Arrow arrays of large string (or utf8) data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef:
             int64_t length
             const uint8_t* data
@@ -4152,7 +4181,7 @@ cdef class BinaryArray(Array):
     Concrete class for Arrow arrays of variable-sized binary data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef:
             int32_t length
             const uint8_t* data
@@ -4175,7 +4204,7 @@ cdef class LargeBinaryArray(Array):
     Concrete class for Arrow arrays of large variable-sized binary data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         cdef:
             int64_t length
             const uint8_t* data
@@ -4357,7 +4386,7 @@ cdef class StructArray(Array):
     Concrete class for Arrow arrays of a struct data type.
     """
 
-    cdef object _getitem_py(self, int64_t i):
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts):
         if self.ap.IsNull(i):
             return None
         cdef int64_t k, num_fields = self.type.num_fields
@@ -4372,13 +4401,13 @@ cdef class StructArray(Array):
                     names, [self.field(k) for k in range(num_fields)])
         names = (<tuple> self._children_cache)[0]
         if names is None:
-            return Array._getitem_py(self, i)
+            return Array._getitem_py(self, i, maps_as_pydicts)
         fields = (<tuple> self._children_cache)[1]
         cdef Array field_arr
         result = {}
         for k in range(num_fields):
             field_arr = <Array> fields[k]
-            result[names[k]] = field_arr._getitem_py(i)
+            result[names[k]] = field_arr._getitem_py(i, maps_as_pydicts)
         return result
 
     def field(self, index):

python/pyarrow/lib.pxd

@@ -296,7 +296,7 @@ cdef class Array(_PandasConvertible):
 
     cdef void init(self, const shared_ptr[CArray]& sp_array) except *
     cdef getitem(self, int64_t i)
-    cdef object _getitem_py(self, int64_t i)
+    cdef object _getitem_py(self, int64_t i, object maps_as_pydicts)
     cdef int64_t length(self)
     cdef void _assert_cpu(self) except *
 

python/pyarrow/tests/test_array.py

@@ -515,6 +515,36 @@ def test_to_pylist_bulk_paths():
         dup.to_pylist()
 
 
+def test_to_pylist_maps_as_pydicts():
+    # GH-50429: maps_as_pydicts converts through the scalar-free path; the
+    # semantics must match MapScalar.as_py exactly.
+    map_type = pa.map_(pa.string(), pa.int32())
+    arrays = [
+        pa.array([[("k1", 1), ("k2", None)], None, []], type=map_type),
+        pa.array([[[("k", 1)], None], None], type=pa.list_(map_type)),
+        pa.array([[("o", [("i", 5)])]],
+                 type=pa.map_(pa.string(), map_type)),
+        pa.array([{"m": [("k", 1)]}, None],
+                 type=pa.struct([("m", map_type)])),
+    ]
+    for arr in arrays:
+        for mode in ("lossy", "strict"):
+            for view in (arr, arr.slice(1)):
+                expected = [x.as_py(maps_as_pydicts=mode) for x in view]
+                assert view.to_pylist(maps_as_pydicts=mode) == expected
+
+    dup = pa.array([[("k", 1), ("k", 2)]], type=map_type)
+    with pytest.warns(UserWarning, match="already encountered"):
+        assert dup.to_pylist(maps_as_pydicts="lossy") == [{"k": 2}]
+    with pytest.raises(KeyError, match="strict mode"):
+        dup.to_pylist(maps_as_pydicts="strict")
+    with pytest.raises(ValueError, match="Invalid value for 'maps_as_pydicts'"):
+        dup.to_pylist(maps_as_pydicts="bogus")
+    # invalid values are only rejected when a map value is converted,
+    # matching the Scalar-based behavior
+    assert pa.array([1, 2]).to_pylist(maps_as_pydicts="bogus") == [1, 2]
+
+
 def test_array_slice():
     arr = pa.array(range(10))