ffi: define compat for Py_NewRef and Py_XNewRef (#4445)

* ffi: define compat for `Py_NewRef` and `Py_XNewRef`

* add missing inline hint

Co-authored-by: Nathan Goldbaum <[email protected]>

* don't use std::ffi::c_int (requires MSRV 1.64)

* add test to guard against ambiguity

* fix `Py_NewRef` cfg on PyPy

---------

Co-authored-by: Nathan Goldbaum <[email protected]>

Author: davidhewitt

newsfragments/4445.added.md

@@ -0,0 +1 @@
+Add FFI definitions `compat::Py_NewRef` and `compat::Py_XNewRef`.

newsfragments/4445.removed.md

@@ -0,0 +1 @@
+Remove private FFI definitions `_Py_NewRef` and `_Py_XNewRef`.

pyo3-ffi/Cargo.toml

@@ -37,6 +37,9 @@ abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"]
 # Automatically generates `python3.dll` import libraries for Windows targets.
 generate-import-lib = ["pyo3-build-config/python3-dll-a"]
 
+[dev-dependencies]
+paste = "1"
+
 [build-dependencies]
 pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.0-dev", features = ["resolve-config"] }
 

pyo3-ffi/src/compat.rs

@@ -0,0 +1,57 @@
+//! C API Compatibility Shims
+//!
+//! Some CPython C API functions added in recent versions of Python are
+//! inherently safer to use than older C API constructs. This module
+//! exposes functions available on all Python versions that wrap the
+//! old C API on old Python versions and wrap the function directly
+//! on newer Python versions.
+
+// Unless otherwise noted, the compatibility shims are adapted from
+// the pythoncapi-compat project: https://github.com/python/pythoncapi-compat
+
+/// Internal helper macro which defines compatibility shims for C API functions, deferring to a
+/// re-export when that's available.
+macro_rules! compat_function {
+    (
+        originally_defined_for($cfg:meta);
+
+        $(#[$attrs:meta])*
+        pub unsafe fn $name:ident($($arg_names:ident: $arg_types:ty),* $(,)?) -> $ret:ty $body:block
+    ) => {
+        // Define as a standalone function under docsrs cfg so that this shows as a unique function in the docs,
+        // not a re-export (the re-export has the wrong visibility)
+        #[cfg(any(docsrs, not($cfg)))]
+        #[cfg_attr(docsrs, doc(cfg(all())))]
+        $(#[$attrs])*
+        pub unsafe fn $name(
+            $($arg_names: $arg_types,)*
+        ) -> $ret $body
+
+        #[cfg(all($cfg, not(docsrs)))]
+        pub use $crate::$name;
+
+        #[cfg(test)]
+        paste::paste! {
+            // Test that the compat function does not overlap with the original function. If the
+            // cfgs line up, then the the two glob imports will resolve to the same item via the
+            // re-export. If the cfgs mismatch, then the use of $name will be ambiguous in cases
+            // where the function is defined twice, and the test will fail to compile.
+            #[allow(unused_imports)]
+            mod [<test_ $name _export>] {
+                use $crate::*;
+                use $crate::compat::*;
+
+                #[test]
+                fn test_export() {
+                    let _ = $name;
+                }
+            }
+        }
+    };
+}
+
+mod py_3_10;
+mod py_3_13;
+
+pub use self::py_3_10::*;
+pub use self::py_3_13::*;

pyo3-ffi/src/compat/mod.rs

@@ -0,0 +1,19 @@
+compat_function!(
+    originally_defined_for(Py_3_10);
+
+    #[inline]
+    pub unsafe fn Py_NewRef(obj: *mut crate::PyObject) -> *mut crate::PyObject {
+        crate::Py_INCREF(obj);
+        obj
+    }
+);
+
+compat_function!(
+    originally_defined_for(Py_3_10);
+
+    #[inline]
+    pub unsafe fn Py_XNewRef(obj: *mut crate::PyObject) -> *mut crate::PyObject {
+        crate::Py_XINCREF(obj);
+        obj
+    }
+);

pyo3-ffi/src/compat/py_3_10.rs

@@ -0,0 +1,39 @@
+compat_function!(
+    originally_defined_for(Py_3_13);
+
+    #[inline]
+    pub unsafe fn PyDict_GetItemRef(
+        dp: *mut crate::PyObject,
+        key: *mut crate::PyObject,
+        result: *mut *mut crate::PyObject,
+    ) -> std::os::raw::c_int {
+        use crate::{compat::Py_NewRef, PyDict_GetItemWithError, PyErr_Occurred};
+
+        let item = PyDict_GetItemWithError(dp, key);
+        if !item.is_null() {
+            *result = Py_NewRef(item);
+            return 1; // found
+        }
+        *result = std::ptr::null_mut();
+        if PyErr_Occurred().is_null() {
+            return 0; // not found
+        }
+        -1
+    }
+);
+
+compat_function!(
+    originally_defined_for(Py_3_13);
+
+    #[inline]
+    pub unsafe fn PyList_GetItemRef(
+        arg1: *mut crate::PyObject,
+        arg2: crate::Py_ssize_t,
+    ) -> *mut crate::PyObject {
+        use crate::{PyList_GetItem, Py_XINCREF};
+
+        let item = PyList_GetItem(arg1, arg2);
+        Py_XINCREF(item);
+        item
+    }
+);

pyo3-ffi/src/compat/py_3_13.rs

@@ -713,40 +713,33 @@ pub unsafe fn Py_XDECREF(op: *mut PyObject) {
 }
 
 extern "C" {
-    #[cfg(all(Py_3_10, Py_LIMITED_API))]
+    #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))]
+    #[cfg_attr(docsrs, doc(cfg(Py_3_10)))]
     pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject;
-    #[cfg(all(Py_3_10, Py_LIMITED_API))]
+    #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))]
+    #[cfg_attr(docsrs, doc(cfg(Py_3_10)))]
     pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject;
 }
 
-// Technically these macros are only available in the C header from 3.10 and up, however their
-// implementation works on all supported Python versions so we define these macros on all
-// versions for simplicity.
+// macro _Py_NewRef not public; reimplemented directly inside Py_NewRef here
+// macro _Py_XNewRef not public; reimplemented directly inside Py_XNewRef here
 
+#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))]
+#[cfg_attr(docsrs, doc(cfg(Py_3_10)))]
 #[inline]
-pub unsafe fn _Py_NewRef(obj: *mut PyObject) -> *mut PyObject {
+pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject {
     Py_INCREF(obj);
     obj
 }
 
+#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))]
+#[cfg_attr(docsrs, doc(cfg(Py_3_10)))]
 #[inline]
-pub unsafe fn _Py_XNewRef(obj: *mut PyObject) -> *mut PyObject {
+pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject {
     Py_XINCREF(obj);
     obj
 }
 
-#[cfg(all(Py_3_10, not(Py_LIMITED_API)))]
-#[inline]
-pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject {
-    _Py_NewRef(obj)
-}
-
-#[cfg(all(Py_3_10, not(Py_LIMITED_API)))]
-#[inline]
-pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject {
-    _Py_XNewRef(obj)
-}
-
 #[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {
     #[cfg(not(GraalPy))]

pyo3-ffi/src/object.rs

@@ -250,7 +250,7 @@ impl PyTypeBuilder {
                             if (*dict_ptr).is_null() {
                                 std::ptr::write(dict_ptr, ffi::PyDict_New());
                             }
-                            Ok(ffi::_Py_XNewRef(*dict_ptr))
+                            Ok(ffi::compat::Py_XNewRef(*dict_ptr))
                         })
                     }
                 }