auto-initialize: new feature to control initializing Python

Author: davidhewitt

.github/workflows/ci.yml

@@ -85,27 +85,28 @@ jobs:
         run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV
 
       - name: Build docs
-        run: cargo doc --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
+        run: cargo doc --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
 
-      - name: Build without default features
+      - name: Build (no features)
         run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }}
 
-      - name: Build with default features
-        run: cargo build --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
+      - name: Build (all additive features)
+        run: cargo build --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
 
       # Run tests (except on PyPy, because no embedding API).
       - if: matrix.python-version != 'pypy-3.6'
         name: Test
-        run: cargo test --features "num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features "macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
+
       # Run tests again, but in abi3 mode
       - if: matrix.python-version != 'pypy-3.6'
         name: Test (abi3)
-        run: cargo test --no-default-features --features "abi3,macros" --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features "abi3 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
 
       # Run tests again, for abi3-py36 (the minimal Python version)
       - if: (matrix.python-version != 'pypy-3.6') && (matrix.python-version != '3.6')
         name: Test (abi3-py36)
-        run: cargo test --no-default-features --features "abi3-py36,macros" --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features "abi3-py36 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
 
       - name: Test proc-macro code
         run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml --target ${{ matrix.platform.rust-target }}
@@ -125,6 +126,9 @@ jobs:
     env:
       RUST_BACKTRACE: 1
       RUSTFLAGS: "-D warnings"
+      # TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy
+      # Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
+      PYO3_CI: 1
 
   coverage:
     needs: [fmt]

Cargo.toml

@@ -33,26 +33,36 @@ assert_approx_eq = "1.1.0"
 trybuild = "1.0.23"
 rustversion = "1.0"
 proptest = { version = "0.10.1", default-features = false, features = ["std"] }
+# features needed to run the PyO3 test suite
+pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] }
 
 [features]
-default = ["macros"]
-macros = ["ctor", "indoc", "inventory", "paste", "pyo3-macros", "unindent"]
+default = ["macros", "auto-initialize"]
+
+# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
+macros = ["pyo3-macros", "ctor", "indoc", "inventory", "paste", "unindent"]
+
+# Use this feature when building an extension module.
+# It tells the linker to keep the python symbols unresolved,
+# so that the module can also be used with statically linked python interpreters.
+extension-module = []
+
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
 abi3 = []
+
 # With abi3, we can manually set the minimum Python version.
 abi3-py36 = ["abi3-py37"]
 abi3-py37 = ["abi3-py38"]
 abi3-py38 = ["abi3-py39"]
 abi3-py39 = ["abi3"]
 
+# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
+# Python interpreter if needed.
+auto-initialize = []
+
 # Optimizes PyObject to Vec conversion and so on.
 nightly = []
 
-# Use this feature when building an extension module.
-# It tells the linker to keep the python symbols unresolved,
-# so that the module can also be used with statically linked python interpreters.
-extension-module = []
-
 [workspace]
 members = [
     "pyo3-macros",

README.md

@@ -104,8 +104,9 @@ If you want your Rust application to create a Python interpreter internally and
 use it to run Python code, add `pyo3` to your `Cargo.toml` like this:
 
 ```toml
-[dependencies]
-pyo3 = "0.13.0"
+[dependencies.pyo3]
+version = "0.13.0"
+features = ["auto-initialize"]
 ```
 
 Example program displaying the value of `sys.version` and the current user name:

build.rs

@@ -749,6 +749,10 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result<()> {
         }
     }
 
+    if interpreter_config.shared {
+        println!("cargo:rustc-cfg=Py_SHARED");
+    }
+
     if interpreter_config.version.implementation == PythonInterpreterKind::PyPy {
         println!("cargo:rustc-cfg=PyPy");
     };
@@ -883,5 +887,11 @@ fn main() -> Result<()> {
         }
     }
 
+    // TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy
+    // Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
+    if env::var_os("PYO3_CI").is_some() {
+        println!("cargo:rustc-cfg=__pyo3_ci");
+    }
+
     Ok(())
 }

src/gil.rs

@@ -3,11 +3,11 @@
 //! Interaction with python's global interpreter lock
 
 use crate::{ffi, internal_tricks::Unsendable, Python};
-use parking_lot::{const_mutex, Mutex};
+use parking_lot::{const_mutex, Mutex, Once};
 use std::cell::{Cell, RefCell};
-use std::{mem::ManuallyDrop, ptr::NonNull, sync};
+use std::{mem::ManuallyDrop, ptr::NonNull};
 
-static START: sync::Once = sync::Once::new();
+static START: Once = Once::new();
 
 thread_local! {
     /// This is a internal counter in pyo3 monitoring whether this thread has the GIL.
@@ -45,16 +45,20 @@ pub(crate) fn gil_is_acquired() -> bool {
 /// If both the Python interpreter and Python threading are already initialized,
 /// this function has no effect.
 ///
+/// # Availability
+///
+/// This function is only available when linking against Python distributions that contain a
+/// shared library.
+///
+/// This function is not available on PyPy.
+///
 /// # Panic
 /// If the Python interpreter is initialized but Python threading is not,
 /// a panic occurs.
 /// It is not possible to safely access the Python runtime unless the main
 /// thread (the thread which originally initialized Python) also initializes
 /// threading.
-///
-/// When writing an extension module, the `#[pymodule]` macro
-/// will ensure that Python threading is initialized.
-///
+#[cfg(all(Py_SHARED, not(PyPy)))]
 pub fn prepare_freethreaded_python() {
     // Protect against race conditions when Python is not yet initialized
     // and multiple threads concurrently call 'prepare_freethreaded_python()'.
@@ -72,34 +76,29 @@ pub fn prepare_freethreaded_python() {
             // Note that the 'main thread' notion in Python isn't documented properly;
             // and running Python without one is not officially supported.
 
-            // PyPy does not support the embedding API
-            #[cfg(not(PyPy))]
-            {
-                ffi::Py_InitializeEx(0);
-
-                // Make sure Py_Finalize will be called before exiting.
-                extern "C" fn finalize() {
-                    unsafe {
-                        if ffi::Py_IsInitialized() != 0 {
-                            ffi::PyGILState_Ensure();
-                            ffi::Py_Finalize();
-                        }
+            ffi::Py_InitializeEx(0);
+
+            // Make sure Py_Finalize will be called before exiting.
+            extern "C" fn finalize() {
+                unsafe {
+                    if ffi::Py_IsInitialized() != 0 {
+                        ffi::PyGILState_Ensure();
+                        ffi::Py_Finalize();
                     }
                 }
-                libc::atexit(finalize);
             }
+            libc::atexit(finalize);
 
             // > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
             // > to call it yourself anymore.
             #[cfg(not(Py_3_7))]
             if ffi::PyEval_ThreadsInitialized() == 0 {
                 ffi::PyEval_InitThreads();
             }
-            // PyEval_InitThreads() will acquire the GIL,
-            // but we don't want to hold it at this point
+
+            // Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
             // (it's not acquired in the other code paths)
             // So immediately release the GIL:
-            #[cfg(not(PyPy))]
             let _thread_state = ffi::PyEval_SaveThread();
             // Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
             // and will be restored by PyGILState_Ensure.
@@ -137,7 +136,51 @@ impl GILGuard {
     /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this
     /// new `GILGuard` will also contain a `GILPool`.
     pub(crate) fn acquire() -> GILGuard {
-        prepare_freethreaded_python();
+        // Maybe auto-initialize the GIL:
+        //  - If auto-initialize feature set and supported, try to initalize the interpreter.
+        //  - If the auto-initialize feature is set but unsupported, emit hard errors only when
+        //    the extension-module feature is not activated - extension modules don't care about
+        //    auto-initialize so this avoids breaking existing builds.
+        //  - Otherwise, just check the GIL is initialized.
+        cfg_if::cfg_if! {
+            if #[cfg(all(feature = "auto-initialize", Py_SHARED, not(PyPy)))] {
+                prepare_freethreaded_python();
+            } else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), not(Py_SHARED), not(__pyo3_ci)))] {
+                compile_error!(concat!(
+                    "The `auto-initialize` feature is not supported when linking Python ",
+                    "statically instead of with a shared library.\n\n",
+                    "Please disable the `auto-initialize` feature, for example by entering the following ",
+                    "in your cargo.toml:\n\n",
+                    "    pyo3 = { version = \"0.13.0\", default-features = false }\n\n",
+                    "Alternatively, compile PyO3 using a Python distribution which contains a shared ",
+                    "libary."
+                ));
+            } else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), PyPy, not(__pyo3_ci)))] {
+                compile_error!(concat!(
+                    "The `auto-initialize` feature is not supported by PyPy.\n\n",
+                    "Please disable the `auto-initialize` feature, for example by entering the following ",
+                    "in your cargo.toml:\n\n",
+                    "    pyo3 = { version = \"0.13.0\", default-features = false }",
+                ));
+            } else {
+                // extension module feature enabled and PyPy or static linking
+                // OR auto-initialize feature not enabled
+                START.call_once_force(|_| unsafe {
+                    // Use call_once_force because if there is a panic because the interpreter is not
+                    // initialized, it's fine for the user to initialize the interpreter and retry.
+                    assert_ne!(
+                        ffi::Py_IsInitialized(),
+                        0,
+                        "The Python interpreter is not initalized and the `auto-initialize` feature is not enabled."
+                    );
+                    assert_ne!(
+                        ffi::PyEval_ThreadsInitialized(),
+                        0,
+                        "Python threading is not initalized and the `auto-initialize` feature is not enabled."
+                    );
+                });
+            }
+        }
 
         let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL
 

src/lib.rs

@@ -114,8 +114,9 @@
 //! Add `pyo3` to your `Cargo.toml`:
 //!
 //! ```toml
-//! [dependencies]
-//! pyo3 = "0.13.0"
+//! [dependencies.pyo3]
+//! version = "0.13.0"
+//! features = ["auto-initialize"]
 //! ```
 //!
 //! Example program displaying the value of `sys.version`:
@@ -145,12 +146,14 @@ pub use crate::conversion::{
     ToBorrowedObject, ToPyObject,
 };
 pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
+#[cfg(all(Py_SHARED, not(PyPy)))]
+pub use crate::gil::prepare_freethreaded_python;
 pub use crate::gil::{GILGuard, GILPool};
 pub use crate::instance::{Py, PyNativeType, PyObject};
 pub use crate::pycell::{PyCell, PyRef, PyRefMut};
 pub use crate::pyclass::PyClass;
 pub use crate::pyclass_init::PyClassInitializer;
-pub use crate::python::{prepare_freethreaded_python, Python, PythonVersionInfo};
+pub use crate::python::{Python, PythonVersionInfo};
 pub use crate::type_object::{type_flags, PyTypeInfo};
 // Since PyAny is as important as PyObject, we expose it to the top level.
 pub use crate::types::PyAny;

src/python.rs

@@ -11,8 +11,6 @@ use std::ffi::{CStr, CString};
 use std::marker::PhantomData;
 use std::os::raw::{c_char, c_int};
 
-pub use gil::prepare_freethreaded_python;
-
 /// Represents the major, minor, and patch (if any) versions of this interpreter.
 ///
 /// See [Python::version].
@@ -134,8 +132,13 @@ impl Python<'_> {
     /// Acquires the global interpreter lock, which allows access to the Python runtime. The
     /// provided closure F will be executed with the acquired `Python` marker token.
     ///
-    /// If the Python runtime is not already initialized, this function will initialize it.
-    /// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
+    /// If the `auto-initialize` feature is enabled and the Python runtime is not already
+    /// initialized, this function will initialize it. See
+    /// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
+    ///
+    /// # Panics
+    /// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
+    ///   initialized.
     ///
     /// # Example
     /// ```
@@ -158,8 +161,9 @@ impl Python<'_> {
 impl<'p> Python<'p> {
     /// Acquires the global interpreter lock, which allows access to the Python runtime.
     ///
-    /// If the Python runtime is not already initialized, this function will initialize it.
-    /// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
+    /// If the `auto-initialize` feature is enabled and the Python runtime is not already
+    /// initialized, this function will initialize it. See
+    /// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
     ///
     /// Most users should not need to use this API directly, and should prefer one of two options:
     /// 1. When implementing `#[pymethods]` or `#[pyfunction]` add a function argument
@@ -172,6 +176,10 @@ impl<'p> Python<'p> {
     /// allowed, and will not deadlock. However, `GILGuard`s must be dropped in the reverse order
     /// to acquisition. If PyO3 detects this order is not maintained, it may be forced to begin
     /// an irrecoverable panic.
+    ///
+    /// # Panics
+    /// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
+    ///   initialized.
     #[inline]
     pub fn acquire_gil() -> GILGuard {
         GILGuard::acquire()