From 92966b45c9eb695bbc8cb2577f094da486b2f8d3 Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Wed, 11 Feb 2026 20:59:32 +0100 Subject: [PATCH 1/4] Initial commit --- python/pyarrow/array.pxi | 7 +++++++ python/pyarrow/tests/test_pandas.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 2c1597e21586..50504c7b9b42 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2308,6 +2308,13 @@ cdef _array_like_to_pandas(obj, options, types_mapper): dtype = "object" elif types_mapper: dtype = types_mapper(original_type) + elif _pandas_api.uses_string_dtype() and ( + original_type.id == _Type_STRING or + original_type.id == _Type_LARGE_STRING or + original_type.id == _Type_STRING_VIEW + ): + # for pandas 3.0+, use pandas' new default string dtype + dtype = _pandas_api.pd.StringDtype(na_value=np.nan) else: dtype = None diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 5fde980dd8d3..214f781ed2a3 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4652,6 +4652,29 @@ def test_chunked_array_to_pandas_types_mapper(): assert result.dtype == np.dtype("int64") +@pytest.mark.parametrize( + "string_type", [pa.string(), pa.large_string(), pa.string_view()] +) +@pytest.mark.parametrize("data", [[], [None]]) +def test_array_to_pandas_string_dtype(string_type, data): + # GH-49002 + if Version(pd.__version__) < Version("3.0.0"): + pytest.skip("PyArrow backed string dtype missing") + + arr = pa.array(data, type=string_type) + result = arr.to_pandas() + assert result.dtype == pd.StringDtype(na_value=np.nan) + + arr = pa.chunked_array([data], type=string_type) + result = arr.to_pandas() + assert result.dtype == pd.StringDtype(na_value=np.nan) + + # Test types_mapper takes precedence + types_mapper = {string_type: None}.get + result = arr.to_pandas(types_mapper=types_mapper) + assert result.dtype == np.dtype("object") + + # ---------------------------------------------------------------------- # Legacy metadata compatibility tests From fec9077a6ee0a06eed0ab32b0be9b092106f5cb7 Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Wed, 25 Mar 2026 15:35:22 +0100 Subject: [PATCH 2/4] Add suggestions, fix typo --- python/pyarrow/array.pxi | 14 +++++++------- python/pyarrow/tests/test_pandas.py | 25 ++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 50504c7b9b42..374511185322 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2276,6 +2276,13 @@ cdef _array_like_to_pandas(obj, options, types_mapper): dtype = original_type.to_pandas_dtype() except NotImplementedError: pass + elif pandas_api.uses_string_dtype() and not options["strings_to_categorical"] and ( + original_type.id == _Type_STRING or + original_type.id == _Type_LARGE_STRING or + original_type.id == _Type_STRING_VIEW + ): + # for pandas 3.0+, use pandas' new default string dtyp + dtype = pandas_api.pd.StringDtype(na_value=np.nan) # Only call __from_arrow__ for Arrow extension types or when explicitly # overridden via types_mapper @@ -2308,13 +2315,6 @@ cdef _array_like_to_pandas(obj, options, types_mapper): dtype = "object" elif types_mapper: dtype = types_mapper(original_type) - elif _pandas_api.uses_string_dtype() and ( - original_type.id == _Type_STRING or - original_type.id == _Type_LARGE_STRING or - original_type.id == _Type_STRING_VIEW - ): - # for pandas 3.0+, use pandas' new default string dtype - dtype = _pandas_api.pd.StringDtype(na_value=np.nan) else: dtype = None diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 214f781ed2a3..660f684dbedf 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -2975,7 +2975,9 @@ def check_zero_copy_failure(self, arr): arr.to_pandas(zero_copy_only=True) def test_zero_copy_failure_on_object_types(self): - self.check_zero_copy_failure(pa.array(['A', 'B', 'C'])) + if Version(pd.__version__) < Version("3.0.0"): + # pandas 3.0 includes default string dtype support + self.check_zero_copy_failure(pa.array(['A', 'B', 'C'])) def test_zero_copy_failure_with_int_when_nulls(self): self.check_zero_copy_failure(pa.array([0, 1, None])) @@ -3047,6 +3049,10 @@ def test_all_none_category(self): def test_empty_arrays(self): for dtype_str, pa_type in self.type_pairs: + if (Version(pd.__version__) >= Version("3.0.0") and + pa_type == pa.string()): + # PyArrow backed string dtype are set by default + dtype_str = 'str' arr = np.array([], dtype=np.dtype(dtype_str)) _check_array_roundtrip(arr, type=pa_type) @@ -3231,13 +3237,19 @@ def test_convert_empty_table(self): empty_objects = pd.Series(np.array([], dtype=object)) tm.assert_series_equal(arr.to_pandas(), pd.Series(np.array([], dtype=np.int64))) - arr = pa.array([], type=pa.string()) - tm.assert_series_equal(arr.to_pandas(), empty_objects) arr = pa.array([], type=pa.list_(pa.int64())) tm.assert_series_equal(arr.to_pandas(), empty_objects) arr = pa.array([], type=pa.struct([pa.field('a', pa.int64())])) tm.assert_series_equal(arr.to_pandas(), empty_objects) + arr = pa.array([], type=pa.string()) + if Version(pd.__version__) >= Version("3.0.0"): + # PyArrow backed string dtype are set by default + empty_str = pd.Series(np.array([], dtype=str)) + tm.assert_series_equal(arr.to_pandas(), empty_str) + else: + tm.assert_series_equal(arr.to_pandas(), empty_objects) + def test_non_natural_stride(self): """ ARROW-2172: converting from a Numpy array with a stride that's @@ -4674,6 +4686,13 @@ def test_array_to_pandas_string_dtype(string_type, data): result = arr.to_pandas(types_mapper=types_mapper) assert result.dtype == np.dtype("object") + # Test strings_to_categorical + result = arr.to_pandas(strings_to_categorical=False) + assert result.dtype == pd.StringDtype(na_value=np.nan) + result = arr.to_pandas(strings_to_categorical=True) + assert result.dtype == pd.CategoricalDtype(categories=[], + ordered=False) + # ---------------------------------------------------------------------- # Legacy metadata compatibility tests From 600c4e48ca9c31b36867ee6599c918849b463075 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 1 Apr 2026 10:21:20 +0200 Subject: [PATCH 3/4] Update python/pyarrow/array.pxi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Raúl Cumplido --- python/pyarrow/array.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 374511185322..58af9b77eaa1 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2281,7 +2281,7 @@ cdef _array_like_to_pandas(obj, options, types_mapper): original_type.id == _Type_LARGE_STRING or original_type.id == _Type_STRING_VIEW ): - # for pandas 3.0+, use pandas' new default string dtyp + # for pandas 3.0+, use pandas' new default string dtype dtype = pandas_api.pd.StringDtype(na_value=np.nan) # Only call __from_arrow__ for Arrow extension types or when explicitly From 3f6af35f901aef34992e964476df174e95f949f4 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 1 Apr 2026 11:10:02 +0200 Subject: [PATCH 4/4] Apply suggestion from @jorisvandenbossche Co-authored-by: Joris Van den Bossche --- python/pyarrow/tests/test_pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 660f684dbedf..0339975f4571 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -3245,7 +3245,7 @@ def test_convert_empty_table(self): arr = pa.array([], type=pa.string()) if Version(pd.__version__) >= Version("3.0.0"): # PyArrow backed string dtype are set by default - empty_str = pd.Series(np.array([], dtype=str)) + empty_str = pd.Series([], dtype=str) tm.assert_series_equal(arr.to_pandas(), empty_str) else: tm.assert_series_equal(arr.to_pandas(), empty_objects)