From 87b0b3cf345ad3ebf644c045c07f2289049e1cad Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 11 Apr 2025 09:01:27 +0100 Subject: [PATCH 1/9] remove private types from `pyo3-ffi` --- pyo3-ffi/src/bytearrayobject.rs | 2 +- pyo3-ffi/src/code.rs | 2 +- pyo3-ffi/src/cpython/bytesobject.rs | 2 +- pyo3-ffi/src/cpython/code.rs | 89 +++++++----------------- pyo3-ffi/src/cpython/compile.rs | 2 +- pyo3-ffi/src/cpython/critical_section.rs | 4 +- pyo3-ffi/src/cpython/dictobject.rs | 4 +- pyo3-ffi/src/cpython/frameobject.rs | 2 +- pyo3-ffi/src/cpython/genobject.rs | 5 +- pyo3-ffi/src/cpython/pyframe.rs | 3 +- pyo3-ffi/src/cpython/pystate.rs | 11 +-- pyo3-ffi/src/cpython/weakrefobject.rs | 1 + pyo3-ffi/src/dictobject.rs | 2 +- pyo3-ffi/src/floatobject.rs | 2 +- pyo3-ffi/src/lib.rs | 4 +- pyo3-ffi/src/longobject.rs | 2 +- pyo3-ffi/src/object.rs | 2 +- pyo3-ffi/src/pyarena.rs | 2 +- pyo3-ffi/src/pyframe.rs | 2 +- pyo3-ffi/src/pystate.rs | 4 +- pyo3-ffi/src/pythonrun.rs | 6 +- pyo3-ffi/src/weakrefobject.rs | 2 +- 22 files changed, 60 insertions(+), 95 deletions(-) diff --git a/pyo3-ffi/src/bytearrayobject.rs b/pyo3-ffi/src/bytearrayobject.rs index 24a97bcc31b..d27dfa8b0ec 100644 --- a/pyo3-ffi/src/bytearrayobject.rs +++ b/pyo3-ffi/src/bytearrayobject.rs @@ -17,7 +17,7 @@ pub struct PyByteArrayObject { } #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] -opaque_struct!(PyByteArrayObject); +opaque_struct!(pub PyByteArrayObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/code.rs b/pyo3-ffi/src/code.rs index d28f68cded7..296b17f6aa4 100644 --- a/pyo3-ffi/src/code.rs +++ b/pyo3-ffi/src/code.rs @@ -1,4 +1,4 @@ // This header doesn't exist in CPython, but Include/cpython/code.h does. We add // this here so that PyCodeObject has a definition under the limited API. -opaque_struct!(PyCodeObject); +opaque_struct!(pub PyCodeObject); diff --git a/pyo3-ffi/src/cpython/bytesobject.rs b/pyo3-ffi/src/cpython/bytesobject.rs index 306702de25e..dd78e646a12 100644 --- a/pyo3-ffi/src/cpython/bytesobject.rs +++ b/pyo3-ffi/src/cpython/bytesobject.rs @@ -17,7 +17,7 @@ pub struct PyBytesObject { } #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] -opaque_struct!(PyBytesObject); +opaque_struct!(pub PyBytesObject); extern "C" { #[cfg_attr(PyPy, link_name = "_PyPyBytes_Resize")] diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 60989901052..f365fddb57d 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -9,73 +9,32 @@ use std::ptr::addr_of_mut; #[cfg(all(Py_3_8, not(any(PyPy, GraalPy)), not(Py_3_11)))] opaque_struct!(_PyOpcache); -#[cfg(Py_3_12)] -pub const _PY_MONITORING_LOCAL_EVENTS: usize = 10; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 15; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_EVENTS: usize = 17; +// skipped private _PY_MONITORING_LOCAL_EVENTS +// skipped private _PY_MONITORING_UNGROUPED_EVENTS +// skipped private _PY_MONITORING_EVENTS -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_LocalMonitors { - pub tools: [u8; if cfg!(Py_3_13) { - _PY_MONITORING_LOCAL_EVENTS - } else { - _PY_MONITORING_UNGROUPED_EVENTS - }], -} +// skipped private _PyLocalMonitors +// skipped private _Py_GlobalMonitors -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_GlobalMonitors { - pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], -} +// skipped private _Py_CODEUNIT -// skipped _Py_CODEUNIT +// skipped private _Py_OPCODE +// skipped private _Py_OPARG -// skipped _Py_OPCODE -// skipped _Py_OPARG +// skipped private _py_make_codeunit -// skipped _py_make_codeunit +// skipped private _py_set_opcode -// skipped _py_set_opcode - -// skipped _Py_MAKE_CODEUNIT -// skipped _Py_SET_OPCODE +// skipped private _Py_MAKE_CODEUNIT +// skipped private _Py_SET_OPCODE #[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoCached { - pub _co_code: *mut PyObject, - pub _co_varnames: *mut PyObject, - pub _co_cellvars: *mut PyObject, - pub _co_freevars: *mut PyObject, -} +opaque_struct!(_PyCoCached); -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoLineInstrumentationData { - pub original_opcode: u8, - pub line_delta: i8, -} +// skipped private _PyCoLineInstrumentationData #[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoMonitoringData { - pub local_monitors: _Py_LocalMonitors, - pub active_monitors: _Py_LocalMonitors, - pub tools: *mut u8, - pub lines: *mut _PyCoLineInstrumentationData, - pub line_tools: *mut u8, - pub per_instruction_opcodes: *mut u8, - pub per_instruction_tools: *mut u8, -} +opaque_struct!(_PyCoMonitoringData); #[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_7)))] opaque_struct!(PyCodeObject); @@ -177,20 +136,24 @@ pub struct PyCodeObject { pub co_linetable: *mut PyObject, pub co_weakreflist: *mut PyObject, #[cfg(not(Py_3_12))] - pub _co_code: *mut PyObject, + _co_code: *mut PyObject, #[cfg(not(Py_3_12))] - pub _co_linearray: *mut c_char, + _co_linearray: *mut c_char, #[cfg(Py_3_13)] + #[allow( + private_interfaces, + reason = "field is public, but the type is opaque and private" + )] pub co_executors: *mut _PyExecutorArray, #[cfg(Py_3_12)] - pub _co_cached: *mut _PyCoCached, + _co_cached: *mut _PyCoCached, #[cfg(all(Py_3_12, not(Py_3_13)))] - pub _co_instrumentation_version: u64, + _co_instrumentation_version: u64, #[cfg(Py_3_13)] - pub _co_instrumentation_version: libc::uintptr_t, + _co_instrumentation_version: libc::uintptr_t, #[cfg(Py_3_12)] - pub _co_monitoring: *mut _PyCoMonitoringData, - pub _co_firsttraceable: c_int, + _co_monitoring: *mut _PyCoMonitoringData, + _co_firsttraceable: c_int, pub co_extra: *mut c_void, pub co_code_adaptive: [c_char; 1], } diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 79f06c92003..608e50858bc 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -33,7 +33,7 @@ pub struct PyCompilerFlags { #[cfg(all(Py_3_12, not(any(Py_3_13, PyPy, GraalPy))))] #[repr(C)] #[derive(Copy, Clone)] -pub struct _PyCompilerSrcLocation { +struct _PyCompilerSrcLocation { pub lineno: c_int, pub end_lineno: c_int, pub col_offset: c_int, diff --git a/pyo3-ffi/src/cpython/critical_section.rs b/pyo3-ffi/src/cpython/critical_section.rs index 97b2f5e0559..808dba870c6 100644 --- a/pyo3-ffi/src/cpython/critical_section.rs +++ b/pyo3-ffi/src/cpython/critical_section.rs @@ -17,10 +17,10 @@ pub struct PyCriticalSection2 { } #[cfg(not(Py_GIL_DISABLED))] -opaque_struct!(PyCriticalSection); +opaque_struct!(pub PyCriticalSection); #[cfg(not(Py_GIL_DISABLED))] -opaque_struct!(PyCriticalSection2); +opaque_struct!(pub PyCriticalSection2); extern "C" { pub fn PyCriticalSection_Begin(c: *mut PyCriticalSection, op: *mut PyObject); diff --git a/pyo3-ffi/src/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs index 79dcbfdb62e..93ce560c573 100644 --- a/pyo3-ffi/src/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -2,10 +2,10 @@ use crate::object::*; use crate::pyport::Py_ssize_t; use std::os::raw::c_int; -opaque_struct!(PyDictKeysObject); +opaque_struct!(pub PyDictKeysObject); #[cfg(Py_3_11)] -opaque_struct!(PyDictValues); +opaque_struct!(pub PyDictValues); #[cfg(not(GraalPy))] #[repr(C)] diff --git a/pyo3-ffi/src/cpython/frameobject.rs b/pyo3-ffi/src/cpython/frameobject.rs index e9b9c183f37..993e93c838b 100644 --- a/pyo3-ffi/src/cpython/frameobject.rs +++ b/pyo3-ffi/src/cpython/frameobject.rs @@ -53,7 +53,7 @@ pub struct PyFrameObject { } #[cfg(any(PyPy, GraalPy, Py_3_11))] -opaque_struct!(PyFrameObject); +opaque_struct!(pub PyFrameObject); // skipped _PyFrame_IsRunnable // skipped _PyFrame_IsExecuting diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index c9d419e3782..b3de063d768 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -1,7 +1,5 @@ use crate::object::*; use crate::PyFrameObject; -#[cfg(not(any(PyPy, GraalPy)))] -use crate::_PyErr_StackItem; #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] use std::os::raw::c_char; use std::os::raw::c_int; @@ -20,7 +18,8 @@ pub struct PyGenObject { pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, pub gi_qualname: *mut PyObject, - pub gi_exc_state: _PyErr_StackItem, + #[allow(private_interfaces, reason = "field is public but type is private")] + pub gi_exc_state: crate::cpython::pystate::_PyErr_StackItem, #[cfg(Py_3_11)] pub gi_origin_or_finalizer: *mut PyObject, #[cfg(Py_3_11)] diff --git a/pyo3-ffi/src/cpython/pyframe.rs b/pyo3-ffi/src/cpython/pyframe.rs index 5e1e16a7d08..f0c38be47be 100644 --- a/pyo3-ffi/src/cpython/pyframe.rs +++ b/pyo3-ffi/src/cpython/pyframe.rs @@ -1,2 +1,3 @@ +// NB used in `_PyEval_EvalFrameDefault`, maybe we remove this too. #[cfg(all(Py_3_11, not(PyPy)))] -opaque_struct!(_PyInterpreterFrame); +opaque_struct!(pub _PyInterpreterFrame); diff --git a/pyo3-ffi/src/cpython/pystate.rs b/pyo3-ffi/src/cpython/pystate.rs index 650cd6a1f7f..707d4945f49 100644 --- a/pyo3-ffi/src/cpython/pystate.rs +++ b/pyo3-ffi/src/cpython/pystate.rs @@ -27,16 +27,17 @@ pub const PyTrace_OPCODE: c_int = 7; // skipped PyTraceInfo // skipped CFrame +/// Private structure used inline in `PyGenObject` #[cfg(not(PyPy))] #[repr(C)] #[derive(Clone, Copy)] -pub struct _PyErr_StackItem { +pub(crate) struct _PyErr_StackItem { #[cfg(not(Py_3_11))] - pub exc_type: *mut PyObject, - pub exc_value: *mut PyObject, + exc_type: *mut PyObject, + exc_value: *mut PyObject, #[cfg(not(Py_3_11))] - pub exc_traceback: *mut PyObject, - pub previous_item: *mut _PyErr_StackItem, + exc_traceback: *mut PyObject, + previous_item: *mut _PyErr_StackItem, } // skipped _PyStackChunk diff --git a/pyo3-ffi/src/cpython/weakrefobject.rs b/pyo3-ffi/src/cpython/weakrefobject.rs index 88bb501bcc5..1c50c7a759f 100644 --- a/pyo3-ffi/src/cpython/weakrefobject.rs +++ b/pyo3-ffi/src/cpython/weakrefobject.rs @@ -1,3 +1,4 @@ +// NB publicly re-exported in `src/weakrefobject.rs` #[cfg(not(any(PyPy, GraalPy)))] pub struct _PyWeakReference { pub ob_base: crate::PyObject, diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index 710be80243f..e609352ac1b 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -118,4 +118,4 @@ extern "C" { #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyDictObject); +opaque_struct!(pub PyDictObject); diff --git a/pyo3-ffi/src/floatobject.rs b/pyo3-ffi/src/floatobject.rs index 65fc1d4c316..4e1d6476f58 100644 --- a/pyo3-ffi/src/floatobject.rs +++ b/pyo3-ffi/src/floatobject.rs @@ -4,7 +4,7 @@ use std::ptr::addr_of_mut; #[cfg(Py_LIMITED_API)] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyFloatObject); +opaque_struct!(pub PyFloatObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index aba8866b266..f6d5b14fab7 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -341,9 +341,9 @@ // model opaque types: // https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs macro_rules! opaque_struct { - ($name:ident) => { + ($pub:vis $name:ident) => { #[repr(C)] - pub struct $name([u8; 0]); + $pub struct $name([u8; 0]); }; } diff --git a/pyo3-ffi/src/longobject.rs b/pyo3-ffi/src/longobject.rs index 68b4ecba540..eca0af3d0a5 100644 --- a/pyo3-ffi/src/longobject.rs +++ b/pyo3-ffi/src/longobject.rs @@ -4,7 +4,7 @@ use libc::size_t; use std::os::raw::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void}; use std::ptr::addr_of_mut; -opaque_struct!(PyLongObject); +opaque_struct!(pub PyLongObject); #[inline] pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int { diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 087cd32920c..38f79ba111a 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -10,7 +10,7 @@ use std::ptr; use std::sync::atomic::{AtomicIsize, AtomicU32, AtomicU8, Ordering::Relaxed}; #[cfg(Py_LIMITED_API)] -opaque_struct!(PyTypeObject); +opaque_struct!(pub PyTypeObject); #[cfg(not(Py_LIMITED_API))] pub use crate::cpython::object::PyTypeObject; diff --git a/pyo3-ffi/src/pyarena.rs b/pyo3-ffi/src/pyarena.rs index 87d5f28a7a5..1200de3df48 100644 --- a/pyo3-ffi/src/pyarena.rs +++ b/pyo3-ffi/src/pyarena.rs @@ -1 +1 @@ -opaque_struct!(PyArena); +opaque_struct!(pub PyArena); diff --git a/pyo3-ffi/src/pyframe.rs b/pyo3-ffi/src/pyframe.rs index 4dd3d2b31a5..1693b20b0af 100644 --- a/pyo3-ffi/src/pyframe.rs +++ b/pyo3-ffi/src/pyframe.rs @@ -6,7 +6,7 @@ use crate::PyFrameObject; use std::os::raw::c_int; #[cfg(Py_LIMITED_API)] -opaque_struct!(PyFrameObject); +opaque_struct!(pub PyFrameObject); extern "C" { pub fn PyFrame_GetLineNumber(f: *mut PyFrameObject) -> c_int; diff --git a/pyo3-ffi/src/pystate.rs b/pyo3-ffi/src/pystate.rs index a6caf421ff6..e97cca94ed5 100644 --- a/pyo3-ffi/src/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -9,8 +9,8 @@ use std::os::raw::c_long; pub const MAX_CO_EXTRA_USERS: c_int = 255; -opaque_struct!(PyThreadState); -opaque_struct!(PyInterpreterState); +opaque_struct!(pub PyThreadState); +opaque_struct!(pub PyInterpreterState); extern "C" { #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index e7ea2d2efd0..80209b5875a 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -49,12 +49,12 @@ pub const PYOS_STACK_MARGIN: c_int = 2048; // skipped PyOS_CheckStack under Microsoft C #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] -opaque_struct!(_mod); +opaque_struct!(pub _mod); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(symtable); +opaque_struct!(pub symtable); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(_node); +opaque_struct!(pub _node); #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] diff --git a/pyo3-ffi/src/weakrefobject.rs b/pyo3-ffi/src/weakrefobject.rs index 305dc290fa8..0a1afe1a619 100644 --- a/pyo3-ffi/src/weakrefobject.rs +++ b/pyo3-ffi/src/weakrefobject.rs @@ -4,7 +4,7 @@ use std::os::raw::c_int; use std::ptr::addr_of_mut; #[cfg(all(not(PyPy), Py_LIMITED_API, not(GraalPy)))] -opaque_struct!(PyWeakReference); +opaque_struct!(pub PyWeakReference); #[cfg(all(not(PyPy), not(Py_LIMITED_API), not(GraalPy)))] pub use crate::_PyWeakReference as PyWeakReference; From 442efbad9ccc3e412e14ceb47b2391e0aa98b6fc Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 11 Apr 2025 09:22:39 +0100 Subject: [PATCH 2/9] newsfragments --- newsfragments/5064.changed.md | 1 + newsfragments/5064.removed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 newsfragments/5064.changed.md create mode 100644 newsfragments/5064.removed.md diff --git a/newsfragments/5064.changed.md b/newsfragments/5064.changed.md new file mode 100644 index 00000000000..f391cd9384c --- /dev/null +++ b/newsfragments/5064.changed.md @@ -0,0 +1 @@ +The following underscore-prefixed fields of FFI definition `PyCodeObject` are now private: `_co_code`, `_co_linearray`, `_co_cached`, `_co_instrumentation_version`, `_co_monitoring` and `_co_firsttraceable`. diff --git a/newsfragments/5064.removed.md b/newsfragments/5064.removed.md new file mode 100644 index 00000000000..e0bee8f11bd --- /dev/null +++ b/newsfragments/5064.removed.md @@ -0,0 +1 @@ +Remove private types from `pyo3-ffi` (i.e. starting with `_Py`) which are not referenced by public APIs: `_PyLocalMonitors`, `_Py_GlobalMonitors`, `_PyCoCached`, `_PyCoLineInstrumentationData`, `_PyCoMonitoringData`, `_PyCompilerSrcLocation`, `_PyErr_StackItem`. From d18a3120947013df771297ab5c8b75af58a313f1 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 11 Apr 2025 11:44:14 +0100 Subject: [PATCH 3/9] make `PyCodeObject` opaque on all versions --- newsfragments/5064.changed.md | 2 +- pyo3-ffi/src/cpython/code.rs | 153 +++------------------------------- pyo3-ffi/src/lib.rs | 3 +- 3 files changed, 15 insertions(+), 143 deletions(-) diff --git a/newsfragments/5064.changed.md b/newsfragments/5064.changed.md index f391cd9384c..233614bccdf 100644 --- a/newsfragments/5064.changed.md +++ b/newsfragments/5064.changed.md @@ -1 +1 @@ -The following underscore-prefixed fields of FFI definition `PyCodeObject` are now private: `_co_code`, `_co_linearray`, `_co_cached`, `_co_instrumentation_version`, `_co_monitoring` and `_co_firsttraceable`. +The FFI definition `PyCodeObject` is now an opaque struct on all Python versions. diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index f365fddb57d..8177ddde912 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -1,14 +1,12 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -#[allow(unused_imports)] -use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; +#[cfg(not(GraalPy))] +use std::os::raw::c_char; +use std::os::raw::{c_int, c_void}; #[cfg(not(any(PyPy, GraalPy)))] use std::ptr::addr_of_mut; -#[cfg(all(Py_3_8, not(any(PyPy, GraalPy)), not(Py_3_11)))] -opaque_struct!(_PyOpcache); - // skipped private _PY_MONITORING_LOCAL_EVENTS // skipped private _PY_MONITORING_UNGROUPED_EVENTS // skipped private _PY_MONITORING_EVENTS @@ -28,145 +26,18 @@ opaque_struct!(_PyOpcache); // skipped private _Py_MAKE_CODEUNIT // skipped private _Py_SET_OPCODE -#[cfg(Py_3_12)] -opaque_struct!(_PyCoCached); - +// skipped private _PyCoCached // skipped private _PyCoLineInstrumentationData +// skipped private _PyCoMontoringData -#[cfg(Py_3_12)] -opaque_struct!(_PyCoMonitoringData); - -#[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_7)))] -opaque_struct!(PyCodeObject); - -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_7, not(Py_3_8)))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_lnotab: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, -} - -#[cfg(Py_3_13)] -opaque_struct!(_PyExecutorArray); - -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_8, not(Py_3_11)))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - #[cfg(not(Py_3_10))] - pub co_lnotab: *mut PyObject, - #[cfg(Py_3_10)] - pub co_linetable: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, - pub co_opcache_map: *mut c_uchar, - pub co_opcache: *mut _PyOpcache, - pub co_opcache_flag: c_int, - pub co_opcache_size: c_uchar, -} - -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_11))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyVarObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_exceptiontable: *mut PyObject, - pub co_flags: c_int, - #[cfg(not(Py_3_12))] - pub co_warmup: c_int, - - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_stacksize: c_int, - pub co_firstlineno: c_int, - - pub co_nlocalsplus: c_int, - #[cfg(Py_3_12)] - pub co_framesize: c_int, - pub co_nlocals: c_int, - #[cfg(not(Py_3_12))] - pub co_nplaincellvars: c_int, - pub co_ncellvars: c_int, - pub co_nfreevars: c_int, - #[cfg(Py_3_12)] - pub co_version: u32, +// skipped private _PyExecutorArray - pub co_localsplusnames: *mut PyObject, - pub co_localspluskinds: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_qualname: *mut PyObject, - pub co_linetable: *mut PyObject, - pub co_weakreflist: *mut PyObject, - #[cfg(not(Py_3_12))] - _co_code: *mut PyObject, - #[cfg(not(Py_3_12))] - _co_linearray: *mut c_char, - #[cfg(Py_3_13)] - #[allow( - private_interfaces, - reason = "field is public, but the type is opaque and private" - )] - pub co_executors: *mut _PyExecutorArray, - #[cfg(Py_3_12)] - _co_cached: *mut _PyCoCached, - #[cfg(all(Py_3_12, not(Py_3_13)))] - _co_instrumentation_version: u64, - #[cfg(Py_3_13)] - _co_instrumentation_version: libc::uintptr_t, - #[cfg(Py_3_12)] - _co_monitoring: *mut _PyCoMonitoringData, - _co_firsttraceable: c_int, - pub co_extra: *mut c_void, - pub co_code_adaptive: [c_char; 1], -} - -#[cfg(PyPy)] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_name: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_argcount: c_int, - pub co_flags: c_int, -} +opaque_struct!( + #[doc = "A Python code object.\n"] + #[doc = "\n"] + #[doc = "`pyo3-ffi` does not expose the contents of this struct, as it has no stability guarantees."] + pub PyCodeObject +); /* Masks for co_flags */ pub const CO_OPTIMIZED: c_int = 0x0001; diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index f6d5b14fab7..9b98d08c3c5 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -341,7 +341,8 @@ // model opaque types: // https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs macro_rules! opaque_struct { - ($pub:vis $name:ident) => { + ($(#[$attrs:meta])* $pub:vis $name:ident) => { + $(#[$attrs])* #[repr(C)] $pub struct $name([u8; 0]); }; From 495b5f5825f130dcef9c18e281774c98a447a6d5 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 11 Apr 2025 11:51:33 +0100 Subject: [PATCH 4/9] fix msrv --- pyo3-ffi/src/cpython/genobject.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index b3de063d768..fc9217c8dca 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -18,7 +18,8 @@ pub struct PyGenObject { pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, pub gi_qualname: *mut PyObject, - #[allow(private_interfaces, reason = "field is public but type is private")] + // field is public but type is private, private_interfaces lint requires MSRV 1.74 + #[allow(unknown_lints, private_interfaces)] pub gi_exc_state: crate::cpython::pystate::_PyErr_StackItem, #[cfg(Py_3_11)] pub gi_origin_or_finalizer: *mut PyObject, From ec3c6a4cf38b9d7db2d196a2bbf384d3dc47447d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 11 Apr 2025 11:55:34 +0100 Subject: [PATCH 5/9] fix msrv, try 2 --- pyo3-ffi/src/cpython/genobject.rs | 2 -- pyo3-ffi/src/cpython/pystate.rs | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index fc9217c8dca..51c80f1a212 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -18,8 +18,6 @@ pub struct PyGenObject { pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, pub gi_qualname: *mut PyObject, - // field is public but type is private, private_interfaces lint requires MSRV 1.74 - #[allow(unknown_lints, private_interfaces)] pub gi_exc_state: crate::cpython::pystate::_PyErr_StackItem, #[cfg(Py_3_11)] pub gi_origin_or_finalizer: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/pystate.rs b/pyo3-ffi/src/cpython/pystate.rs index 707d4945f49..b8f6fd667b7 100644 --- a/pyo3-ffi/src/cpython/pystate.rs +++ b/pyo3-ffi/src/cpython/pystate.rs @@ -31,7 +31,8 @@ pub const PyTrace_OPCODE: c_int = 7; #[cfg(not(PyPy))] #[repr(C)] #[derive(Clone, Copy)] -pub(crate) struct _PyErr_StackItem { +#[doc(hidden)] // TODO should be able to make pub(crate) after MSRV 1.74 +pub struct _PyErr_StackItem { #[cfg(not(Py_3_11))] exc_type: *mut PyObject, exc_value: *mut PyObject, From b0563f1b8ebe2bf9390d0843edade221975bea4d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 12 Apr 2025 08:07:47 +0100 Subject: [PATCH 6/9] remove unused constant --- pyo3-ffi-check/macro/src/lib.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index 217fec42c8d..41092b9020e 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -9,11 +9,6 @@ const PY_3_12: PythonVersion = PythonVersion { minor: 12, }; -const PY_3_13: PythonVersion = PythonVersion { - major: 3, - minor: 13, -}; - /// Macro which expands to multiple macro calls, one per pyo3-ffi struct. #[proc_macro] pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream { From 6c6cbff6797d92dd159f3fe1e27e63ba33443f47 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 21 Apr 2025 10:04:48 +0100 Subject: [PATCH 7/9] sync `cpython/compile.rs` with header at 3.13 --- newsfragments/5064.added.md | 1 + newsfragments/5064.changed.md | 4 ++- pyo3-ffi/src/cpython/compile.rs | 63 ++++++++++++++------------------- pyo3-ffi/src/pyport.rs | 2 ++ 4 files changed, 32 insertions(+), 38 deletions(-) create mode 100644 newsfragments/5064.added.md diff --git a/newsfragments/5064.added.md b/newsfragments/5064.added.md new file mode 100644 index 00000000000..21e2cce6430 --- /dev/null +++ b/newsfragments/5064.added.md @@ -0,0 +1 @@ +Add FFI definition `PY_INVALID_STACK_EFFECT`. diff --git a/newsfragments/5064.changed.md b/newsfragments/5064.changed.md index 233614bccdf..0ca3ad7d542 100644 --- a/newsfragments/5064.changed.md +++ b/newsfragments/5064.changed.md @@ -1 +1,3 @@ -The FFI definition `PyCodeObject` is now an opaque struct on all Python versions. +Reduce visibility of some CPython implementation details: +- The FFI definition `PyCodeObject` is now an opaque struct on all Python versions. +- The FFI definition `PyFutureFeatures` is now only defined up until Python 3.10 (it was present in CPython headers but unused in 3.11 and 3.12). diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 608e50858bc..078b5e0d675 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -6,19 +6,21 @@ use crate::pyarena::*; use crate::pythonrun::*; #[cfg(not(any(PyPy, Py_3_10)))] use crate::PyCodeObject; +use crate::INT_MAX; #[cfg(not(any(PyPy, Py_3_10)))] use std::os::raw::c_char; use std::os::raw::c_int; -// skipped non-limited PyCF_MASK -// skipped non-limited PyCF_MASK_OBSOLETE -// skipped non-limited PyCF_SOURCE_IS_UTF8 -// skipped non-limited PyCF_DONT_IMPLY_DEDENT -// skipped non-limited PyCF_ONLY_AST -// skipped non-limited PyCF_IGNORE_COOKIE -// skipped non-limited PyCF_TYPE_COMMENTS -// skipped non-limited PyCF_ALLOW_TOP_LEVEL_AWAIT -// skipped non-limited PyCF_COMPILE_MASK +// skipped PyCF_MASK +// skipped PyCF_MASK_OBSOLETE +// skipped PyCF_SOURCE_IS_UTF8 +// skipped PyCF_DONT_IMPLY_DEDENT +// skipped PyCF_ONLY_AST +// skipped PyCF_IGNORE_COOKIE +// skipped PyCF_TYPE_COMMENTS +// skipped PyCF_ALLOW_TOP_LEVEL_AWAIT +// skipped PyCF_OPTIMIZED_AST +// skipped PyCF_COMPILE_MASK #[repr(C)] #[derive(Copy, Clone)] @@ -28,31 +30,23 @@ pub struct PyCompilerFlags { pub cf_feature_version: c_int, } -// skipped non-limited _PyCompilerFlags_INIT +// skipped _PyCompilerFlags_INIT -#[cfg(all(Py_3_12, not(any(Py_3_13, PyPy, GraalPy))))] -#[repr(C)] -#[derive(Copy, Clone)] -struct _PyCompilerSrcLocation { - pub lineno: c_int, - pub end_lineno: c_int, - pub col_offset: c_int, - pub end_col_offset: c_int, -} - -// skipped SRC_LOCATION_FROM_AST - -#[cfg(not(any(PyPy, GraalPy, Py_3_13)))] +// NB this type technically existed in the header until 3.13, when it was +// moved to the internal CPython headers. +// +// We choose not to expose it in the public API past 3.10, as it is +// not used in the public API past that point. +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { pub ff_features: c_int, - #[cfg(not(Py_3_12))] pub ff_lineno: c_int, - #[cfg(Py_3_12)] - pub ff_location: _PyCompilerSrcLocation, } +// FIXME: these constants should probably be &CStr, if they are used at all + pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes"; pub const FUTURE_GENERATORS: &str = "generators"; pub const FUTURE_DIVISION: &str = "division"; @@ -62,13 +56,12 @@ pub const FUTURE_PRINT_FUNCTION: &str = "print_function"; pub const FUTURE_UNICODE_LITERALS: &str = "unicode_literals"; pub const FUTURE_BARRY_AS_BDFL: &str = "barry_as_FLUFL"; pub const FUTURE_GENERATOR_STOP: &str = "generator_stop"; -// skipped non-limited FUTURE_ANNOTATIONS +pub const FUTURE_ANNOTATIONS: &str = "annotations"; +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] extern "C" { - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyNode_Compile(arg1: *mut _node, arg2: *const c_char) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileEx( _mod: *mut _mod, filename: *const c_char, @@ -77,7 +70,6 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileObject( _mod: *mut _mod, filename: *mut PyObject, @@ -86,23 +78,20 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromAST(_mod: *mut _mod, filename: *const c_char) -> *mut PyFutureFeatures; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromASTObject( _mod: *mut _mod, filename: *mut PyObject, ) -> *mut PyFutureFeatures; +} - // skipped non-limited _Py_Mangle - // skipped non-limited PY_INVALID_STACK_EFFECT +pub const PY_INVALID_STACK_EFFECT: c_int = INT_MAX; + +extern "C" { pub fn PyCompile_OpcodeStackEffect(opcode: c_int, oparg: c_int) -> c_int; #[cfg(Py_3_8)] pub fn PyCompile_OpcodeStackEffectWithJump(opcode: c_int, oparg: c_int, jump: c_int) -> c_int; - - // skipped non-limited _PyASTOptimizeState - // skipped non-limited _PyAST_Optimize } diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index a144c67fb1b..b432c0de8f1 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -1,3 +1,5 @@ +pub const INT_MAX: std::os::raw::c_int = libc::INT_MAX; + pub type PY_UINT32_T = u32; pub type PY_UINT64_T = u64; From 54755afa682b5decd8e82a57c54813601abaffe4 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 21 Apr 2025 10:36:43 +0100 Subject: [PATCH 8/9] fix size of INT_MAX --- pyo3-ffi/src/pyport.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index b432c0de8f1..e524831d80d 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -1,4 +1,7 @@ -pub const INT_MAX: std::os::raw::c_int = libc::INT_MAX; +// NB libc does not define this constant on all platforms, so we hard code it +// like CPython does. +// https://github.com/python/cpython/blob/d8b9011702443bb57579f8834f3effe58e290dfc/Include/pyport.h#L372 +pub const INT_MAX: std::os::raw::c_int = 2147483647; pub type PY_UINT32_T = u32; pub type PY_UINT64_T = u64; @@ -13,8 +16,8 @@ pub type Py_ssize_t = ::libc::ssize_t; pub type Py_hash_t = Py_ssize_t; pub type Py_uhash_t = ::libc::size_t; -pub const PY_SSIZE_T_MIN: Py_ssize_t = isize::MIN as Py_ssize_t; -pub const PY_SSIZE_T_MAX: Py_ssize_t = isize::MAX as Py_ssize_t; +pub const PY_SSIZE_T_MIN: Py_ssize_t = Py_ssize_t::MIN; +pub const PY_SSIZE_T_MAX: Py_ssize_t = Py_ssize_t::MAX; #[cfg(target_endian = "big")] pub const PY_BIG_ENDIAN: usize = 1; From 59ca90fab5695d15e84f4198bbcefa4734552f99 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 23 Apr 2025 22:01:51 +0100 Subject: [PATCH 9/9] remove `PyCode_GetNumFree` --- newsfragments/5064.removed.md | 1 + pyo3-ffi/src/cpython/code.rs | 18 ++---------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/newsfragments/5064.removed.md b/newsfragments/5064.removed.md index e0bee8f11bd..f1a04a78aa7 100644 --- a/newsfragments/5064.removed.md +++ b/newsfragments/5064.removed.md @@ -1 +1,2 @@ Remove private types from `pyo3-ffi` (i.e. starting with `_Py`) which are not referenced by public APIs: `_PyLocalMonitors`, `_Py_GlobalMonitors`, `_PyCoCached`, `_PyCoLineInstrumentationData`, `_PyCoMonitoringData`, `_PyCompilerSrcLocation`, `_PyErr_StackItem`. +Remove FFI definition `PyCode_GetNumFree` (PyO3 cannot support it due to knowledge of the code object). diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 8177ddde912..3d47a1bc8c3 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -83,28 +83,14 @@ pub unsafe fn PyCode_Check(op: *mut PyObject) -> c_int { (Py_TYPE(op) == addr_of_mut!(PyCode_Type)) as c_int } -#[inline] -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_10, not(Py_3_11)))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t { - crate::PyTuple_GET_SIZE((*op).co_freevars) -} - -#[inline] -#[cfg(all(not(Py_3_10), Py_3_11, not(any(PyPy, GraalPy))))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> c_int { - (*op).co_nfreevars -} - extern "C" { #[cfg(PyPy)] #[link_name = "PyPyCode_Check"] pub fn PyCode_Check(op: *mut PyObject) -> c_int; - - #[cfg(PyPy)] - #[link_name = "PyPyCode_GetNumFree"] - pub fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t; } +// skipped PyCode_GetNumFree (requires knowledge of code object layout) + extern "C" { #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyCode_New")]