Merge branch 'main' into api-nan-vs-na

jbrockmendel · jbrockmendel · commit 54ae640ecee1 · 2025-09-08T12:57:49.000-07:00
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -32,7 +32,7 @@ repos:
         exclude: ^pandas/tests
         args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
     -   id: ruff-format
-        exclude: ^scripts|^pandas/tests/frame/test_query_eval.py
+        exclude: ^pandas/tests/frame/test_query_eval.py
 -   repo: https://github.com/jendrikseipp/vulture
     rev: v2.14
     hooks:
diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst
@@ -708,6 +708,7 @@ Other Deprecations
 - Deprecated strings ``w``, ``d``, ``MIN``, ``MS``, ``US`` and ``NS`` denoting units in :class:`Timedelta` in favour of ``W``, ``D``, ``min``, ``ms``, ``us`` and ``ns`` (:issue:`59051`)
 - Deprecated the ``arg`` parameter of ``Series.map``; pass the added ``func`` argument instead. (:issue:`61260`)
 - Deprecated using ``epoch`` date format in :meth:`DataFrame.to_json` and :meth:`Series.to_json`, use ``iso`` instead. (:issue:`57063`)
+- Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.unstack` and :meth:`DataFrame.unstack` (:issue:`12189`, :issue:`53868`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.prior_deprecations:
diff --git a/pandas/core/arrays/string_.py b/pandas/core/arrays/string_.py
@@ -344,10 +344,10 @@ def __from_arrow__(
         if self.storage == "pyarrow":
             from pandas.core.arrays.string_arrow import (
                 ArrowStringArray,
-                _chk_pyarrow_available,
+                _check_pyarrow_available,
             )
 
-            _chk_pyarrow_available()
+            _check_pyarrow_available()
 
             if not pa.types.is_large_string(array.type):
                 array = pc.cast(array, pa.large_string())
diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py
@@ -13,7 +13,10 @@
 from pandas._config.config import get_option
 
 import pandas._libs.reshape as libreshape
-from pandas.errors import PerformanceWarning
+from pandas.errors import (
+    Pandas4Warning,
+    PerformanceWarning,
+)
 from pandas.util._decorators import cache_readonly
 from pandas.util._exceptions import find_stack_level
 
@@ -28,7 +31,10 @@
     needs_i8_conversion,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
-from pandas.core.dtypes.missing import notna
+from pandas.core.dtypes.missing import (
+    isna,
+    notna,
+)
 
 import pandas.core.algorithms as algos
 from pandas.core.algorithms import (
@@ -298,7 +304,25 @@ def get_new_values(self, values, fill_value=None):
                 new_values[:] = fill_value
         else:
             if not mask_all:
+                old_dtype = dtype
                 dtype, fill_value = maybe_promote(dtype, fill_value)
+                if old_dtype != dtype:
+                    if old_dtype.kind not in "iub":
+                        warnings.warn(
+                            # GH#12189, GH#53868
+                            "Using a fill_value that cannot be held in the existing "
+                            "dtype is deprecated and will raise in a future version.",
+                            Pandas4Warning,
+                            stacklevel=find_stack_level(),
+                        )
+                    elif not isna(fill_value):
+                        warnings.warn(
+                            # GH#12189, GH#53868
+                            "Using a fill_value that cannot be held in the existing "
+                            "dtype is deprecated and will raise in a future version.",
+                            Pandas4Warning,
+                            stacklevel=find_stack_level(),
+                        )
             new_values = np.empty(result_shape, dtype=dtype)
             if not mask_all:
                 new_values.fill(fill_value)
diff --git a/pandas/tests/frame/test_stack_unstack.py b/pandas/tests/frame/test_stack_unstack.py
@@ -105,8 +105,12 @@ def test_unstack_fill(self, future_stack):
         )
         tm.assert_frame_equal(result, expected)
 
-        # From a series with incorrect data type for fill_value
-        result = data.unstack(fill_value=0.5)
+        msg = (
+            "Using a fill_value that cannot be held in the existing dtype is deprecated"
+        )
+        with tm.assert_produces_warning(Pandas4Warning, match=msg):
+            # From a series with incorrect data type for fill_value
+            result = data.unstack(fill_value=0.5)
         expected = DataFrame(
             {"a": [1, 0.5, 5], "b": [2, 4, 0.5]}, index=["x", "y", "z"], dtype=float
         )
@@ -161,8 +165,12 @@ def test_unstack_fill_frame(self):
         expected["B"] = expected["B"].astype(np.float64)
         tm.assert_frame_equal(result, expected)
 
-        # From a dataframe with incorrect data type for fill_value
-        result = df.unstack(fill_value=0.5)
+        msg = (
+            "Using a fill_value that cannot be held in the existing dtype is deprecated"
+        )
+        with tm.assert_produces_warning(Pandas4Warning, match=msg):
+            # From a dataframe with incorrect data type for fill_value
+            result = df.unstack(fill_value=0.5)
 
         rows = [[1, 3, 2, 4], [0.5, 5, 0.5, 6], [7, 0.5, 8, 0.5]]
         expected = DataFrame(rows, index=list("xyz"), dtype=float)
diff --git a/scripts/check_for_inconsistent_pandas_namespace.py b/scripts/check_for_inconsistent_pandas_namespace.py
@@ -30,8 +30,7 @@
 from typing import NamedTuple
 
 ERROR_MESSAGE = (
-    "{path}:{lineno}:{col_offset}: "
-    "Found both '{prefix}.{name}' and '{name}' in {path}"
+    "{path}:{lineno}:{col_offset}: Found both '{prefix}.{name}' and '{name}' in {path}"
 )
 
 
diff --git a/scripts/check_test_naming.py b/scripts/check_test_naming.py
@@ -8,6 +8,7 @@
 NOTE: if this finds a false positive, you can add the comment `# not a test` to the
 class or function definition. Though hopefully that shouldn't be necessary.
 """
+
 from __future__ import annotations
 
 import argparse
diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py
@@ -12,6 +12,7 @@
     generated with this script:
     $ python scripts/generate_pip_deps_from_conda.py --compare
 """
+
 import argparse
 import pathlib
 import re
diff --git a/scripts/pandas_errors_documented.py b/scripts/pandas_errors_documented.py
@@ -6,6 +6,7 @@
 
     pre-commit run pandas-errors-documented --all-files
 """
+
 from __future__ import annotations
 
 import argparse
diff --git a/scripts/sort_whatsnew_note.py b/scripts/sort_whatsnew_note.py
@@ -23,6 +23,7 @@
 
     pre-commit run sort-whatsnew-items --all-files
 """
+
 from __future__ import annotations
 
 import argparse
diff --git a/scripts/tests/test_check_test_naming.py b/scripts/tests/test_check_test_naming.py
@@ -24,10 +24,7 @@
             0,
         ),
         (
-            "class Foo:  # not a test\n"
-            "    pass\n"
-            "def test_foo():\n"
-            "    Class.foo()\n",
+            "class Foo:  # not a test\n    pass\ndef test_foo():\n    Class.foo()\n",
             "",
             0,
         ),
diff --git a/scripts/tests/test_inconsistent_namespace_check.py b/scripts/tests/test_inconsistent_namespace_check.py
@@ -5,14 +5,10 @@
 )
 
 BAD_FILE_0 = (
-    "from pandas import Categorical\n"
-    "cat_0 = Categorical()\n"
-    "cat_1 = pd.Categorical()"
+    "from pandas import Categorical\ncat_0 = Categorical()\ncat_1 = pd.Categorical()"
 )
 BAD_FILE_1 = (
-    "from pandas import Categorical\n"
-    "cat_0 = pd.Categorical()\n"
-    "cat_1 = Categorical()"
+    "from pandas import Categorical\ncat_0 = pd.Categorical()\ncat_1 = Categorical()"
 )
 BAD_FILE_2 = (
     "from pandas import Categorical\n"
diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py
@@ -6,6 +6,7 @@
 from scripts import validate_docstrings
 
 
+# fmt: off
 class BadDocstrings:
     """Everything here has a bad docstring"""
 
@@ -88,6 +89,7 @@ def leftover_files(self) -> None:
         >>> import pathlib
         >>> pathlib.Path("foo.txt").touch()
         """
+# fmt: on
 
 
 class TestValidator:
@@ -227,13 +229,13 @@ def test_validate_all_ignore_errors(self, monkeypatch):
                 "errors": [
                     ("ER01", "err desc"),
                     ("ER02", "err desc"),
-                    ("ER03", "err desc")
+                    ("ER03", "err desc"),
                 ],
                 "warnings": [],
                 "examples_errors": "",
                 "deprecated": True,
                 "file": "file1",
-                "file_line": "file_line1"
+                "file_line": "file_line1",
             },
         )
         monkeypatch.setattr(
@@ -272,14 +274,13 @@ def test_validate_all_ignore_errors(self, monkeypatch):
                 None: {"ER03"},
                 "pandas.DataFrame.align": {"ER01"},
                 # ignoring an error that is not requested should be of no effect
-                "pandas.Index.all": {"ER03"}
-            }
+                "pandas.Index.all": {"ER03"},
+            },
         )
         # two functions * two not global ignored errors - one function ignored error
         assert exit_status == 2 * 2 - 1
 
 
-
 class TestApiItems:
     @property
     def api_doc(self):
diff --git a/scripts/tests/test_validate_unwanted_patterns.py b/scripts/tests/test_validate_unwanted_patterns.py
@@ -307,7 +307,7 @@ def test_nodefault_used_not_only_for_typing_raises(self, data, expected) -> None
         "DeprecationWarning",
         "PendingDeprecationWarning",
         "Pandas4Warning",
-        "RuntimeWarning"
+        "RuntimeWarning",
     ],
 )
 @pytest.mark.parametrize("pdlint_ignore", [True, False])
diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py
@@ -13,6 +13,7 @@
     $ ./validate_docstrings.py
     $ ./validate_docstrings.py pandas.DataFrame.head
 """
+
 from __future__ import annotations
 
 import argparse
@@ -380,13 +381,13 @@ def print_validate_all_results(
         )
         for err_code in actual_failures - expected_failures:
             sys.stdout.write(
-                f'{prefix}{res["file"]}:{res["file_line"]}:'
+                f"{prefix}{res['file']}:{res['file_line']}:"
                 f"{err_code}:{func_name}:{error_messages[err_code]}\n"
             )
             exit_status += 1
         for err_code in ignore_errors.get(func_name, set()) - actual_failures:
             sys.stdout.write(
-                f'{prefix}{res["file"]}:{res["file_line"]}:'
+                f"{prefix}{res['file']}:{res['file_line']}:"
                 f"{err_code}:{func_name}:"
                 "EXPECTED TO FAIL, BUT NOT FAILING\n"
             )
@@ -419,7 +420,7 @@ def header(title, width=80, char="#") -> str:
 
     sys.stderr.write(header("Validation"))
     if result["errors"]:
-        sys.stderr.write(f'{len(result["errors"])} Errors found for `{func_name}`:\n')
+        sys.stderr.write(f"{len(result['errors'])} Errors found for `{func_name}`:\n")
         for err_code, err_desc in result["errors"]:
             sys.stderr.write(f"\t{err_code}\t{err_desc}\n")
     else:
diff --git a/scripts/validate_exception_location.py b/scripts/validate_exception_location.py
@@ -18,6 +18,7 @@
 As a pre-commit hook:
     pre-commit run validate-errors-locations --all-files
 """
+
 from __future__ import annotations
 
 import argparse
diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py
@@ -12,6 +12,7 @@
 
     pre-commit run validate-min-versions-in-sync --all-files
 """
+
 from __future__ import annotations
 
 import pathlib
@@ -100,14 +101,11 @@ def get_operator_from(dependency: str) -> str | None:
 
 
 def get_yaml_map_from(
-    yaml_dic: list[str | dict[str, list[str]]]
+    yaml_dic: list[str | dict[str, list[str]]],
 ) -> dict[str, list[str] | None]:
     yaml_map: dict[str, list[str] | None] = {}
     for dependency in yaml_dic:
-        if (
-            isinstance(dependency, dict)
-            or dependency in yaml_map
-        ):
+        if isinstance(dependency, dict) or dependency in yaml_map:
             continue
         search_text = str(dependency)
         operator = get_operator_from(search_text)
diff --git a/scripts/validate_rst_title_capitalization.py b/scripts/validate_rst_title_capitalization.py
@@ -11,6 +11,7 @@
 From the command-line:
     python scripts/validate_rst_title_capitalization.py <rst file>
 """
+
 from __future__ import annotations
 
 import argparse
@@ -271,7 +272,8 @@ def main(source_paths: list[str]) -> int:
             if title != correct_title_capitalization(title):
                 print(
                     f"""{filename}:{line_number}:{err_msg} "{title}" to "{
-                    correct_title_capitalization(title)}" """
+                        correct_title_capitalization(title)
+                    }" """
                 )
                 number_of_errors += 1
 
diff --git a/scripts/validate_unwanted_patterns.py b/scripts/validate_unwanted_patterns.py

Original file line number	Diff line number	Diff line change
`@@ -30,8 +30,7 @@`
`30`	`30`	`from typing import NamedTuple`
`31`	`31`
`32`	`32`	`ERROR_MESSAGE = (`
`33`		`- "{path}:{lineno}:{col_offset}: "`
`34`		`- "Found both '{prefix}.{name}' and '{name}' in {path}"`
	`33`	`+ "{path}:{lineno}:{col_offset}: Found both '{prefix}.{name}' and '{name}' in {path}"`
`35`	`34`	`)`
`36`	`35`
`37`	`36`