add second lifetime to FromPyObject

guide/src/class.md

@@ -1410,6 +1410,7 @@ impl pyo3::PyClass for MyClass {
 impl<'a, 'holder, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'holder, 'py, false> for &'holder MyClass
 {
     type Holder = ::std::option::Option<pyo3::PyClassGuard<'a, MyClass>>;
+    type Error = pyo3::PyErr;
     #[cfg(feature = "experimental-inspect")]
     const INPUT_TYPE: &'static str = "MyClass";
 
@@ -1422,6 +1423,7 @@ impl<'a, 'holder, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'ho
 impl<'a, 'holder, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'holder, 'py, false> for &'holder mut MyClass
 {
     type Holder = ::std::option::Option<pyo3::PyClassGuardMut<'a, MyClass>>;
+    type Error = pyo3::PyErr;
     #[cfg(feature = "experimental-inspect")]
     const INPUT_TYPE: &'static str = "MyClass";
 

guide/src/migration.md

@@ -3,6 +3,136 @@
 This guide can help you upgrade code through breaking changes from one PyO3 version to the next.
 For a detailed list of all changes, see the [CHANGELOG](changelog.md).
 
+## from 0.26.* to 0.27
+### `FromPyObject` reworked for flexibility and efficiency
+<details open>
+<summary><small>Click to expand</small></summary>
+
+With the removal of the `gil-ref` API in PyO3 0.23 it is now possible to fully split the Python lifetime
+`'py` and the input lifetime `'a`. This allows borrowing from the input data without extending the
+lifetime of being attached to the interpreter.
+
+`FromPyObject` now takes an additional lifetime `'a` describing the input lifetime. The argument
+type of the `extract` method changed from `&Bound<'py, PyAny>` to `Borrowed<'a, 'py, PyAny>`. This was
+done because `&'a Bound<'py, PyAny>` would have an implicit restriction `'py: 'a` due to the reference type.
+
+This new form was partly implemented already in 0.22 using the internal `FromPyObjectBound` trait and
+is now extended to all types.
+
+Most implementations can just add an elided lifetime to migrate.
+
+Additionally `FromPyObject` gained an associated type `Error`. This is the error type that can be used
+in case of a conversion error. During migration using `PyErr` is a good default, later a custom error
+type can be introduced to prevent unneccessary creation of Python exception objects and improved type safety.
+
+Before:
+```rust,ignore
+impl<'py> FromPyObject<'py> for IpAddr {
+    fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult<Self> {
+        ...
+    }
+}
+```
+
+After
+```rust,ignore
+impl<'py> FromPyObject<'_, 'py> for IpAddr {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result<Self, Self::Error> {
+        ...
+        // since `Borrowed` derefs to `&Bound`, the body often
+        // needs no changes, or adding an occasional `&`
+    }
+}
+```
+
+Occasionally, more steps are necessary. For generic types, the bounds need to be adjusted. The
+correct bound depends on how the type is used.
+
+For simple wrapper types usually it's possible to just forward the bound.
+
+Before:
+```rust,ignore
+struct MyWrapper<T>(T);
+
+impl<'py, T> FromPyObject<'py> for MyWrapper<T>
+where 
+    T: FromPyObject<'py>
+{
+    fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult<Self> {
+        ob.extract().map(MyWrapper)
+    }
+}
+```
+
+After: 
+```rust
+# use pyo3::prelude::*;
+# #[allow(dead_code)]
+# pub struct MyWrapper<T>(T);
+impl<'a, 'py, T> FromPyObject<'a, 'py> for MyWrapper<T>
+where
+    T: FromPyObject<'a, 'py>
+{
+    type Error = T::Error;
+
+    fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result<Self, Self::Error> {
+        obj.extract().map(MyWrapper)
+    }
+}
+```
+
+Container types that need to create temporary Python references during extraction, for example
+extracing from a `PyList`, requires a stronger bound. For these the `FromPyObjectOwned` trait was
+introduced. It is automatically implemented for any type that implements `FromPyObject` and does not
+borrow from the input. It is intended to be used as a trait bound in these situations.
+
+Before:
+```rust,ignore
+struct MyVec<T>(Vec<T>);
+impl<'py, T> FromPyObject<'py> for Vec<T>
+where
+    T: FromPyObject<'py>,
+{
+    fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult<Self> {
+        let mut v = MyVec(Vec::new());
+        for item in obj.try_iter()? {
+            v.0.push(item?.extract::<T>()?);
+        }
+        Ok(v)
+    }
+}
+```
+
+After:
+```rust
+# use pyo3::prelude::*;
+# #[allow(dead_code)]
+# pub struct MyVec<T>(Vec<T>);
+impl<'py, T> FromPyObject<'_, 'py> for MyVec<T>
+where
+    T: FromPyObjectOwned<'py> // 👈 can only extract owned values, because each `item` below
+                              //    is a temporary short lived owned reference
+{
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result<Self, Self::Error> {
+        let mut v = MyVec(Vec::new());
+        for item in obj.try_iter()? {
+            v.0.push(item?.extract::<T>().map_err(Into::into)?); // `map_err` is needed because `?` uses `From`, not `Into` 🙁
+        }
+        Ok(v)
+    }
+}
+```
+
+This is very similar to `serde`s [`Deserialize`] and [`DeserializeOwned`] traits, see [here](https://serde.rs/lifetimes.html).
+
+[`Deserialize`]: https://docs.rs/serde/latest/serde/trait.Deserialize.html
+[`DeserializeOwned`]: https://docs.rs/serde/latest/serde/de/trait.DeserializeOwned.html
+</details>
+
 ## from 0.25.* to 0.26
 ### Rename of `Python::with_gil`, `Python::allow_threads`, and `pyo3::prepare_freethreaded_python`
 <details open>

newsfragments/4390.added.md

@@ -0,0 +1,2 @@
+added `FromPyObjectOwned` as more convenient trait bound
+added `Borrowed::extract`, same as `PyAnyMethods::extract`, but does not restrict the lifetime by deref

newsfragments/4390.changed.md

@@ -0,0 +1,2 @@
+added second lifetime to `FromPyObject`
+reintroduced `extract` method

newsfragments/4390.removed.md

@@ -0,0 +1 @@
+removed `FromPyObjectBound`

pyo3-benches/benches/bench_dict.rs

@@ -62,8 +62,9 @@ fn extract_hashmap(b: &mut Bencher<'_>) {
         let dict = (0..LEN as u64)
             .map(|i| (i, i * 2))
             .into_py_dict(py)
-            .unwrap();
-        b.iter(|| HashMap::<u64, u64>::extract_bound(&dict));
+            .unwrap()
+            .into_any();
+        b.iter(|| HashMap::<u64, u64>::extract(dict.as_borrowed()));
     });
 }
 
@@ -73,8 +74,9 @@ fn extract_btreemap(b: &mut Bencher<'_>) {
         let dict = (0..LEN as u64)
             .map(|i| (i, i * 2))
             .into_py_dict(py)
-            .unwrap();
-        b.iter(|| BTreeMap::<u64, u64>::extract_bound(&dict));
+            .unwrap()
+            .into_any();
+        b.iter(|| BTreeMap::<u64, u64>::extract(dict.as_borrowed()));
     });
 }
 
@@ -84,8 +86,9 @@ fn extract_hashbrown_map(b: &mut Bencher<'_>) {
         let dict = (0..LEN as u64)
             .map(|i| (i, i * 2))
             .into_py_dict(py)
-            .unwrap();
-        b.iter(|| hashbrown::HashMap::<u64, u64>::extract_bound(&dict));
+            .unwrap()
+            .into_any();
+        b.iter(|| hashbrown::HashMap::<u64, u64>::extract(dict.as_borrowed()));
     });
 }
 

pyo3-macros-backend/src/frompyobject.rs

@@ -499,7 +499,7 @@ impl<'a> Container<'a> {
             let ty = ty.clone().elide_lifetimes();
             let pyo3_crate_path = &ctx.pyo3_path;
             builder.push_tokens(
-                quote! { <#ty as #pyo3_crate_path::FromPyObject<'_>>::INPUT_TYPE.as_bytes() },
+                quote! { <#ty as #pyo3_crate_path::FromPyObject<'_, '_>>::INPUT_TYPE.as_bytes() },
             )
         }
     }
@@ -543,7 +543,7 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result<TokenStream> {
         let gen_ident = &param.ident;
         where_clause
             .predicates
-            .push(parse_quote!(#gen_ident: #pyo3_path::FromPyObject<'py>))
+            .push(parse_quote!(#gen_ident: #pyo3_path::conversion::FromPyObjectOwned<#lt_param>))
     }
 
     let derives = match &tokens.data {
@@ -605,8 +605,10 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result<TokenStream> {
     let ident = &tokens.ident;
     Ok(quote!(
         #[automatically_derived]
-        impl #impl_generics #pyo3_path::FromPyObject<#lt_param> for #ident #ty_generics #where_clause {
-            fn extract_bound(obj: &#pyo3_path::Bound<#lt_param, #pyo3_path::PyAny>) -> #pyo3_path::PyResult<Self>  {
+        impl #impl_generics #pyo3_path::FromPyObject<'_, #lt_param> for #ident #ty_generics #where_clause {
+            type Error = #pyo3_path::PyErr;
+            fn extract(obj: #pyo3_path::Borrowed<'_, #lt_param, #pyo3_path::PyAny>) -> ::std::result::Result<Self, Self::Error> {
+                let obj: &#pyo3_path::Bound<'_, _> = &*obj;
                 #derives
             }
             #input_type

pyo3-macros-backend/src/pyclass.rs

@@ -2327,6 +2327,7 @@ impl<'a> PyClassImplsBuilder<'a> {
                 impl<'a, 'holder, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'holder, 'py, false> for &'holder #cls
                 {
                     type Holder = ::std::option::Option<#pyo3_path::PyClassGuard<'a, #cls>>;
+                    type Error = #pyo3_path::PyErr;
 
                     #input_type
 
@@ -2341,6 +2342,7 @@ impl<'a> PyClassImplsBuilder<'a> {
                 impl<'a, 'holder, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'holder, 'py, false> for &'holder #cls
                 {
                     type Holder = ::std::option::Option<#pyo3_path::PyClassGuard<'a, #cls>>;
+                    type Error = #pyo3_path::PyErr;
 
                     #input_type
 
@@ -2353,6 +2355,7 @@ impl<'a> PyClassImplsBuilder<'a> {
                 impl<'a, 'holder, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'holder, 'py, false> for &'holder mut #cls
                 {
                     type Holder = ::std::option::Option<#pyo3_path::PyClassGuardMut<'a, #cls>>;
+                    type Error =#pyo3_path::PyErr;
 
                     #input_type
 

src/buffer.rs

@@ -18,8 +18,8 @@
 // DEALINGS IN THE SOFTWARE.
 
 //! `PyBuffer` implementation
-use crate::Bound;
 use crate::{err, exceptions::PyBufferError, ffi, FromPyObject, PyAny, PyResult, Python};
+use crate::{Borrowed, Bound, PyErr};
 use std::ffi::{
     c_char, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong,
     c_ushort, c_void,
@@ -185,9 +185,11 @@ pub unsafe trait Element: Copy {
     fn is_compatible_format(format: &CStr) -> bool;
 }
 
-impl<T: Element> FromPyObject<'_> for PyBuffer<T> {
-    fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult<PyBuffer<T>> {
-        Self::get(obj)
+impl<T: Element> FromPyObject<'_, '_> for PyBuffer<T> {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'_, '_, PyAny>) -> Result<PyBuffer<T>, Self::Error> {
+        Self::get(&obj)
     }
 }