embedding: new feature to enable embedding Python

Author: davidhewitt

Cargo.toml

@@ -33,26 +33,39 @@ assert_approx_eq = "1.1.0"
 trybuild = "1.0.23"
 rustversion = "1.0"
 proptest = { version = "0.10.1", default-features = false, features = ["std"] }
+# features needed to run the PyO3 test suite
+pyo3 = { path = ".", default-features = false, features = ["macros", "embedding", "auto-initialize"] }
 
 [features]
-default = ["macros"]
-macros = ["ctor", "indoc", "inventory", "paste", "pyo3-macros", "unindent"]
+default = ["macros", "embedding", "auto-initialize"]
+
+# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
+macros = ["pyo3-macros", "ctor", "indoc", "inventory", "paste", "unindent"]
+
+# Use this feature when building an extension module.
+# It tells the linker to keep the python symbols unresolved,
+# so that the module can also be used with statically linked python interpreters.
+extension-module = []
+
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
 abi3 = []
+
 # With abi3, we can manually set the minimum Python version.
 abi3-py36 = ["abi3-py37"]
 abi3-py37 = ["abi3-py38"]
 abi3-py38 = ["abi3-py39"]
 abi3-py39 = ["abi3"]
 
+# Enables embedding a Python interpreter inside Rust programs.
+embedding = []
+
+# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
+# Python interpreter if needed.
+auto-initialize = ["embedding"]
+
 # Optimizes PyObject to Vec conversion and so on.
 nightly = []
 
-# Use this feature when building an extension module.
-# It tells the linker to keep the python symbols unresolved,
-# so that the module can also be used with statically linked python interpreters.
-extension-module = []
-
 [workspace]
 members = [
     "pyo3-macros",

README.md

@@ -104,8 +104,9 @@ If you want your Rust application to create a Python interpreter internally and
 use it to run Python code, add `pyo3` to your `Cargo.toml` like this:
 
 ```toml
-[dependencies]
-pyo3 = "0.13.0"
+[dependencies.pyo3]
+version = "0.13.0"
+features = ["embedding", "auto-initialize"]
 ```
 
 Example program displaying the value of `sys.version` and the current user name:

build.rs

@@ -764,6 +764,11 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result<String> {
     }
 
     check_target_architecture(interpreter_config)?;
+    // TODO just let this be an error on PyO3 0.14
+    if let Err(e) = check_embedding_feature(interpreter_config) {
+        println!("cargo:warning={}", e)
+    }
+
     let target_os = env::var_os("CARGO_CFG_TARGET_OS").unwrap();
 
     let is_extension_module = env::var_os("CARGO_FEATURE_EXTENSION_MODULE").is_some();
@@ -854,6 +859,38 @@ fn check_target_architecture(interpreter_config: &InterpreterConfig) -> Result<(
     Ok(())
 }
 
+fn check_embedding_feature(interpreter_config: &InterpreterConfig) -> Result<()> {
+    if env::var_os("CARGO_FEATURE_EMBEDDING").is_some() {
+        if !interpreter_config.shared {
+            bail!(concat!(
+                "The `embedding` feature is not supported when linking Python statically instead ",
+                "of with a shared library.\n",
+                "\n",
+                "Please disable the `embedding` feature, for example by entering the following ",
+                "in your cargo.toml:\n",
+                "\n",
+                "    pyo3 = { version = \"0.13.0\", default-features = false }\n",
+                "\n",
+                "Alternatively, compile PyO3 using a Python distribution which contains a shared ",
+                "libary."
+            ));
+        }
+
+        if interpreter_config.version.implementation == PythonInterpreterKind::PyPy {
+            bail!(concat!(
+                "The `embedding` feature is not supported by PyPy.\n",
+                "\n",
+                "Please disable the `embedding` feature, for example by entering the following ",
+                "in your cargo.toml:\n",
+                "\n",
+                "    pyo3 = { version = \"0.13.0\", default-features = false }\n",
+            ));
+        }
+    }
+
+    Ok(())
+}
+
 fn abi3_without_interpreter() -> Result<()> {
     println!("cargo:rustc-cfg=Py_LIMITED_API");
     let mut flags = "FLAG_WITH_THREAD=1".to_string();

src/gil.rs

@@ -3,11 +3,11 @@
 //! Interaction with python's global interpreter lock
 
 use crate::{ffi, internal_tricks::Unsendable, Python};
-use parking_lot::{const_mutex, Mutex};
+use parking_lot::{const_mutex, Mutex, Once};
 use std::cell::{Cell, RefCell};
-use std::{mem::ManuallyDrop, ptr::NonNull, sync};
+use std::{mem::ManuallyDrop, ptr::NonNull};
 
-static START: sync::Once = sync::Once::new();
+static START: Once = Once::new();
 
 thread_local! {
     /// This is a internal counter in pyo3 monitoring whether this thread has the GIL.
@@ -45,16 +45,17 @@ pub(crate) fn gil_is_acquired() -> bool {
 /// If both the Python interpreter and Python threading are already initialized,
 /// this function has no effect.
 ///
+/// # Features
+///
+/// This function is only available with the `embedding` feature.
+///
 /// # Panic
 /// If the Python interpreter is initialized but Python threading is not,
 /// a panic occurs.
 /// It is not possible to safely access the Python runtime unless the main
 /// thread (the thread which originally initialized Python) also initializes
 /// threading.
-///
-/// When writing an extension module, the `#[pymodule]` macro
-/// will ensure that Python threading is initialized.
-///
+#[cfg(feature = "embedding")]
 pub fn prepare_freethreaded_python() {
     // Protect against race conditions when Python is not yet initialized
     // and multiple threads concurrently call 'prepare_freethreaded_python()'.
@@ -66,15 +67,15 @@ pub fn prepare_freethreaded_python() {
             // as we can't make the existing Python main thread acquire the GIL.
             assert_ne!(ffi::PyEval_ThreadsInitialized(), 0);
         } else {
-            // Initialize Python.
-            // We use Py_InitializeEx() with initsigs=0 to disable Python signal handling.
-            // Signal handling depends on the notion of a 'main thread', which doesn't exist in this case.
-            // Note that the 'main thread' notion in Python isn't documented properly;
-            // and running Python without one is not officially supported.
-
-            // PyPy does not support the embedding API
+            // TODO remove this cfg once build.rs rejects embedding feature misuse.
             #[cfg(not(PyPy))]
             {
+                // Initialize Python.
+                // We use Py_InitializeEx() with initsigs=0 to disable Python signal handling.
+                // Signal handling depends on the notion of a 'main thread', which doesn't exist in this case.
+                // Note that the 'main thread' notion in Python isn't documented properly;
+                // and running Python without one is not officially supported.
+
                 ffi::Py_InitializeEx(0);
 
                 // Make sure Py_Finalize will be called before exiting.
@@ -87,22 +88,21 @@ pub fn prepare_freethreaded_python() {
                     }
                 }
                 libc::atexit(finalize);
-            }
 
-            // > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
-            // > to call it yourself anymore.
-            #[cfg(not(Py_3_7))]
-            if ffi::PyEval_ThreadsInitialized() == 0 {
-                ffi::PyEval_InitThreads();
+                // > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
+                // > to call it yourself anymore.
+                #[cfg(not(Py_3_7))]
+                if ffi::PyEval_ThreadsInitialized() == 0 {
+                    ffi::PyEval_InitThreads();
+                }
+
+                // Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
+                // (it's not acquired in the other code paths)
+                // So immediately release the GIL:
+                let _thread_state = ffi::PyEval_SaveThread();
+                // Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
+                // and will be restored by PyGILState_Ensure.
             }
-            // PyEval_InitThreads() will acquire the GIL,
-            // but we don't want to hold it at this point
-            // (it's not acquired in the other code paths)
-            // So immediately release the GIL:
-            #[cfg(not(PyPy))]
-            let _thread_state = ffi::PyEval_SaveThread();
-            // Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
-            // and will be restored by PyGILState_Ensure.
         }
     });
 }
@@ -137,8 +137,25 @@ impl GILGuard {
     /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this
     /// new `GILGuard` will also contain a `GILPool`.
     pub(crate) fn acquire() -> GILGuard {
+        #[cfg(feature = "auto-initialize")]
         prepare_freethreaded_python();
 
+        #[cfg(not(feature = "auto-initialize"))]
+        START.call_once_force(|_| unsafe {
+            // Use call_once_force because if there is a panic because the interpreter is not
+            // initialized, it's fine for the user to initialize the interpreter and retry.
+            assert_ne!(
+                ffi::Py_IsInitialized(),
+                0,
+                "The Python interpreter is not initalized and the `auto-initialize` feature is not enabled."
+            );
+            assert_ne!(
+                ffi::PyEval_ThreadsInitialized(),
+                0,
+                "Python threading is not initalized and the `auto-initialize` feature is not enabled."
+            );
+        });
+
         let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL
 
         // If there's already a GILPool, we should not create another or this could lead to

src/lib.rs

@@ -114,8 +114,9 @@
 //! Add `pyo3` to your `Cargo.toml`:
 //!
 //! ```toml
-//! [dependencies]
-//! pyo3 = "0.13.0"
+//! [dependencies.pyo3]
+//! version = "0.13.0"
+//! features = ["embedding", "auto-initialize"]
 //! ```
 //!
 //! Example program displaying the value of `sys.version`:
@@ -145,12 +146,14 @@ pub use crate::conversion::{
     ToBorrowedObject, ToPyObject,
 };
 pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
+#[cfg(feature = "embedding")]
+pub use crate::gil::prepare_freethreaded_python;
 pub use crate::gil::{GILGuard, GILPool};
 pub use crate::instance::{Py, PyNativeType, PyObject};
 pub use crate::pycell::{PyCell, PyRef, PyRefMut};
 pub use crate::pyclass::PyClass;
 pub use crate::pyclass_init::PyClassInitializer;
-pub use crate::python::{prepare_freethreaded_python, Python, PythonVersionInfo};
+pub use crate::python::{Python, PythonVersionInfo};
 pub use crate::type_object::{type_flags, PyTypeInfo};
 // Since PyAny is as important as PyObject, we expose it to the top level.
 pub use crate::types::PyAny;

src/python.rs

@@ -11,8 +11,6 @@ use std::ffi::{CStr, CString};
 use std::marker::PhantomData;
 use std::os::raw::{c_char, c_int};
 
-pub use gil::prepare_freethreaded_python;
-
 /// Represents the major, minor, and patch (if any) versions of this interpreter.
 ///
 /// See [Python::version].
@@ -134,8 +132,13 @@ impl Python<'_> {
     /// Acquires the global interpreter lock, which allows access to the Python runtime. The
     /// provided closure F will be executed with the acquired `Python` marker token.
     ///
-    /// If the Python runtime is not already initialized, this function will initialize it.
-    /// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
+    /// If the `auto-initialize` feature is enabled and the Python runtime is not already
+    /// initialized, this function will initialize it. See
+    /// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
+    ///
+    /// # Panics
+    /// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
+    ///   initialized.
     ///
     /// # Example
     /// ```
@@ -158,20 +161,24 @@ impl Python<'_> {
 impl<'p> Python<'p> {
     /// Acquires the global interpreter lock, which allows access to the Python runtime.
     ///
-    /// If the Python runtime is not already initialized, this function will initialize it.
-    /// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
+    /// If the `auto-initialize` feature is enabled and the Python runtime is not already
+    /// initialized, this function will initialize it. See
+    /// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
     ///
     /// Most users should not need to use this API directly, and should prefer one of two options:
     /// 1. When implementing `#[pymethods]` or `#[pyfunction]` add a function argument
     ///    `py: Python` to receive access to the GIL context in which the function is running.
     /// 2. Use [`Python::with_gil`](#method.with_gil) to run a closure with the GIL, acquiring
     ///    only if needed.
-    ///
     /// **Note:** This return type from this function, `GILGuard`, is implemented as a RAII guard
     /// around the C-API Python_EnsureGIL. This means that multiple `acquire_gil()` calls are
     /// allowed, and will not deadlock. However, `GILGuard`s must be dropped in the reverse order
     /// to acquisition. If PyO3 detects this order is not maintained, it may be forced to begin
     /// an irrecoverable panic.
+    ///
+    /// # Panics
+    /// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
+    ///   initialized.
     #[inline]
     pub fn acquire_gil() -> GILGuard {
         GILGuard::acquire()