reintroduce vectorcall optimization with new PyCallArgs trait

guide/src/performance.md

@@ -97,6 +97,19 @@ impl PartialEq<Foo> for FooBound<'_> {
 }
 ```
 
+## Calling Python callables (`__call__`)
+CPython support multiple calling protocols: [`tp_call`] and [`vectorcall`]. [`vectorcall`] is a more efficient protocol unlocking faster calls.
+PyO3 will try to dispatch Python `call`s using the [`vectorcall`] calling convention to archive maximum performance if possible and falling back to [`tp_call`] otherwise.
+This is implemented using the (internal) `PyCallArgs` trait. It defines how Rust types can be used as Python `call` arguments. This trait is currently implemented for
+- Rust tuples, where each member implements `IntoPyObject`,
+- `Bound<'_, PyTuple>`
+- `Py<PyTuple>`
+Rust tuples may make use of [`vectorcall`] where as `Bound<'_, PyTuple>` and `Py<PyTuple>` can only use [`tp_call`]. For maximum performance prefer using Rust tuples as arguments.
+
+
+[`tp_call`]: https://docs.python.org/3/c-api/call.html#the-tp-call-protocol
+[`vectorcall`]: https://docs.python.org/3/c-api/call.html#the-vectorcall-protocol
+
 ## Disable the global reference pool
 
 PyO3 uses global mutable state to keep track of deferred reference count updates implied by `impl<T> Drop for Py<T>` being called without the GIL being held. The necessary synchronization to obtain and apply these reference count updates when PyO3-based code next acquires the GIL is somewhat expensive and can become a significant part of the cost of crossing the Python-Rust boundary.

newsfragments/4768.added.md

@@ -0,0 +1 @@
+Added `PyCallArgs` trait for arguments into the Python calling protocol. This enabled using a faster calling convention for certain types, improving performance.

newsfragments/4768.changed.md

@@ -0,0 +1 @@
+`PyAnyMethods::call` an friends now require `PyCallArgs` for their positional arguments.

src/call.rs

@@ -0,0 +1,150 @@
+//! Defines how Python calls are dispatched, see [`PyCallArgs`].for more information.
+
+use crate::ffi_ptr_ext::FfiPtrExt as _;
+use crate::types::{PyAnyMethods as _, PyDict, PyString, PyTuple};
+use crate::{ffi, Borrowed, Bound, IntoPyObjectExt as _, Py, PyAny, PyResult};
+
+pub(crate) mod private {
+    use super::*;
+
+    pub trait Sealed {}
+
+    impl Sealed for () {}
+    impl Sealed for Bound<'_, PyTuple> {}
+    impl Sealed for Py<PyTuple> {}
+
+    pub struct Token;
+}
+
+/// This trait marks types that can be used as arguments to Python function
+/// calls.
+///
+/// This trait is currently implemented for Rust tuple (up to a size of 12),
+/// [`Bound<'py, PyTuple>`] and [`Py<PyTuple>`]. Custom types that are
+/// convertable to `PyTuple` via `IntoPyObject` need to do so before passing it
+/// to `call`.
+///
+/// This trait is not intended to used by downstream crates directly. As such it
+/// has no publicly available methods and cannot be implemented ouside of
+/// `pyo3`. The corresponding public API is available through [`call`]
+/// ([`call0`], [`call1`] and friends) on [`PyAnyMethods`].
+///
+/// # What is `PyCallArgs` used for?
+/// `PyCallArgs` is used internally in `pyo3` to dispatch the Python calls in
+/// the most optimal way for the current build configuration. Certain types,
+/// such as Rust tuples, do allow the usage of a faster calling convention of
+/// the Python interpreter (if available). More types that may take advantage
+/// from this may be added in the future.
+///
+/// [`call0`]: crate::types::PyAnyMethods::call0
+/// [`call1`]: crate::types::PyAnyMethods::call1
+/// [`call`]: crate::types::PyAnyMethods::call
+/// [`PyAnyMethods`]: crate::types::PyAnyMethods
+#[cfg_attr(
+    diagnostic_namespace,
+    diagnostic::on_unimplemented(
+        message = "`{Self}` cannot used as a Python `call` argument",
+        note = "`PyCallArgs` is implemented for Rust tuples, `Bound<'py, PyTuple>` and `Py<PyTuple>`",
+        note = "if your type is convertable to `PyTuple` via `IntoPyObject`, call `<arg>.into_pyobject(py)` manually",
+        note = "if you meant to pass the type as a single argument, wrap it in a 1-tuple, `(<arg>,)`"
+    )
+)]
+pub trait PyCallArgs<'py>: Sized + private::Sealed {
+    #[doc(hidden)]
+    fn call(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        kwargs: Borrowed<'_, 'py, PyDict>,
+        token: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>>;
+
+    #[doc(hidden)]
+    fn call_positional(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        token: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>>;
+
+    #[doc(hidden)]
+    fn call_method_positional(
+        self,
+        object: Borrowed<'_, 'py, PyAny>,
+        method_name: Borrowed<'_, 'py, PyString>,
+        _: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        object
+            .getattr(method_name)
+            .and_then(|method| method.call1(self))
+    }
+}
+
+impl<'py> PyCallArgs<'py> for () {
+    fn call(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        kwargs: Borrowed<'_, 'py, PyDict>,
+        token: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let args = self.into_pyobject_or_pyerr(function.py())?;
+        args.call(function, kwargs, token)
+    }
+
+    fn call_positional(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        token: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let args = self.into_pyobject_or_pyerr(function.py())?;
+        args.call_positional(function, token)
+    }
+}
+
+impl<'py> PyCallArgs<'py> for Bound<'py, PyTuple> {
+    fn call(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        kwargs: Borrowed<'_, '_, PyDict>,
+        _: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        unsafe {
+            ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), kwargs.as_ptr())
+                .assume_owned_or_err(function.py())
+        }
+    }
+
+    fn call_positional(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        _: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        unsafe {
+            ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), std::ptr::null_mut())
+                .assume_owned_or_err(function.py())
+        }
+    }
+}
+
+impl<'py> PyCallArgs<'py> for Py<PyTuple> {
+    fn call(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        kwargs: Borrowed<'_, '_, PyDict>,
+        _: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        unsafe {
+            ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), kwargs.as_ptr())
+                .assume_owned_or_err(function.py())
+        }
+    }
+
+    fn call_positional(
+        self,
+        function: Borrowed<'_, 'py, PyAny>,
+        _: private::Token,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        unsafe {
+            ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), std::ptr::null_mut())
+                .assume_owned_or_err(function.py())
+        }
+    }
+}

src/lib.rs

@@ -428,6 +428,7 @@ mod internal_tricks;
 mod internal;
 
 pub mod buffer;
+pub mod call;
 pub mod conversion;
 mod conversions;
 #[cfg(feature = "experimental-async")]

src/types/any.rs

@@ -1,3 +1,4 @@
+use crate::call::PyCallArgs;
 use crate::class::basic::CompareOp;
 use crate::conversion::{AsPyPointer, FromPyObjectBound, IntoPyObject};
 use crate::err::{DowncastError, DowncastIntoError, PyErr, PyResult};
@@ -10,7 +11,7 @@ use crate::py_result_ext::PyResultExt;
 use crate::type_object::{PyTypeCheck, PyTypeInfo};
 #[cfg(not(any(PyPy, GraalPy)))]
 use crate::types::PySuper;
-use crate::types::{PyDict, PyIterator, PyList, PyString, PyTuple, PyType};
+use crate::types::{PyDict, PyIterator, PyList, PyString, PyType};
 use crate::{err, ffi, Borrowed, BoundObject, IntoPyObjectExt, Python};
 use std::cell::UnsafeCell;
 use std::cmp::Ordering;
@@ -436,7 +437,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed {
     /// ```
     fn call<A>(&self, args: A, kwargs: Option<&Bound<'py, PyDict>>) -> PyResult<Bound<'py, PyAny>>
     where
-        A: IntoPyObject<'py, Target = PyTuple>;
+        A: PyCallArgs<'py>;
 
     /// Calls the object without arguments.
     ///
@@ -491,7 +492,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed {
     /// ```
     fn call1<A>(&self, args: A) -> PyResult<Bound<'py, PyAny>>
     where
-        A: IntoPyObject<'py, Target = PyTuple>;
+        A: PyCallArgs<'py>;
 
     /// Calls a method on the object.
     ///
@@ -538,7 +539,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed {
     ) -> PyResult<Bound<'py, PyAny>>
     where
         N: IntoPyObject<'py, Target = PyString>,
-        A: IntoPyObject<'py, Target = PyTuple>;
+        A: PyCallArgs<'py>;
 
     /// Calls a method on the object without arguments.
     ///
@@ -614,7 +615,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed {
     fn call_method1<N, A>(&self, name: N, args: A) -> PyResult<Bound<'py, PyAny>>
     where
         N: IntoPyObject<'py, Target = PyString>,
-        A: IntoPyObject<'py, Target = PyTuple>;
+        A: PyCallArgs<'py>;
 
     /// Returns whether the object is considered to be true.
     ///
@@ -1209,25 +1210,17 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> {
 
     fn call<A>(&self, args: A, kwargs: Option<&Bound<'py, PyDict>>) -> PyResult<Bound<'py, PyAny>>
     where
-        A: IntoPyObject<'py, Target = PyTuple>,
+        A: PyCallArgs<'py>,
     {
-        fn inner<'py>(
-            any: &Bound<'py, PyAny>,
-            args: Borrowed<'_, 'py, PyTuple>,
-            kwargs: Option<&Bound<'py, PyDict>>,
-        ) -> PyResult<Bound<'py, PyAny>> {
-            unsafe {
-                ffi::PyObject_Call(
-                    any.as_ptr(),
-                    args.as_ptr(),
-                    kwargs.map_or(std::ptr::null_mut(), |dict| dict.as_ptr()),
-                )
-                .assume_owned_or_err(any.py())
-            }
+        if let Some(kwargs) = kwargs {
+            args.call(
+                self.as_borrowed(),
+                kwargs.as_borrowed(),
+                crate::call::private::Token,
+            )
+        } else {
+            args.call_positional(self.as_borrowed(), crate::call::private::Token)
         }
-
-        let py = self.py();
-        inner(self, args.into_pyobject_or_pyerr(py)?.as_borrowed(), kwargs)
     }
 
     #[inline]
@@ -1237,9 +1230,9 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> {
 
     fn call1<A>(&self, args: A) -> PyResult<Bound<'py, PyAny>>
     where
-        A: IntoPyObject<'py, Target = PyTuple>,
+        A: PyCallArgs<'py>,
     {
-        self.call(args, None)
+        args.call_positional(self.as_borrowed(), crate::call::private::Token)
     }
 
     #[inline]
@@ -1251,10 +1244,14 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> {
     ) -> PyResult<Bound<'py, PyAny>>
     where
         N: IntoPyObject<'py, Target = PyString>,
-        A: IntoPyObject<'py, Target = PyTuple>,
+        A: PyCallArgs<'py>,
     {
-        self.getattr(name)
-            .and_then(|method| method.call(args, kwargs))
+        if kwargs.is_none() {
+            self.call_method1(name, args)
+        } else {
+            self.getattr(name)
+                .and_then(|method| method.call(args, kwargs))
+        }
     }
 
     #[inline]
@@ -1273,9 +1270,14 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> {
     fn call_method1<N, A>(&self, name: N, args: A) -> PyResult<Bound<'py, PyAny>>
     where
         N: IntoPyObject<'py, Target = PyString>,
-        A: IntoPyObject<'py, Target = PyTuple>,
+        A: PyCallArgs<'py>,
     {
-        self.call_method(name, args, None)
+        let name = name.into_pyobject_or_pyerr(self.py())?;
+        args.call_method_positional(
+            self.as_borrowed(),
+            name.as_borrowed(),
+            crate::call::private::Token,
+        )
     }
 
     fn is_truthy(&self) -> PyResult<bool> {