diff --git a/README/ReleaseNotes/v632/index.md b/README/ReleaseNotes/v632/index.md index 7d88a0f4fbf72..f7acf1583a7c4 100644 --- a/README/ReleaseNotes/v632/index.md +++ b/README/ReleaseNotes/v632/index.md @@ -224,6 +224,48 @@ Please use the higher-level functions `RooAbsPdf::createNLL()` and `RooAbsPdf::c ## PROOF Libraries +## PyROOT + + +### Rebase of PyROOT on the current cppyy + +PyROOT was rebased on the latest version of the [cppyy library](https://cppyy.readthedocs.io/en/latest/). +This means PyROOT benefits from many upstream improvements and fixes, for example related to the conversion of NumPy arrays to vectors, implicit conversion from nested Python tuples to nested initializer lists, and improved overload resolution. + +Related to this cppyy upgrade, there is one change in PyROOT behavior. +A static size character buffer of type `char[n]` is not converted to a Python string anymore. +The reason for this: since it was previously assumed the string was +null-terminated, there was no way to get the bytes after a `null`, even if you +wanted to. + +``` +import ROOT + +ROOT.gInterpreter.Declare(""" +struct Struct { char char_buffer[5] {}; }; // struct with char[n] +void fill_char_buffer(Struct & st) { + std::string foo{"foo"}; + std::memcpy(st.char_buffer, foo.data(), foo.size()); +} +""") + +struct = ROOT.Struct() +ROOT.fill_char_buffer(struct) +char_buffer = struct.char_buffer + +# With thew new cppyy, you get access to the lower level buffer instead of a +# Python string: +print("struct.char_buffer : ", char_buffer) + +# However, you can turn the buffer into a string very easily with as_string(): +print("struct.char_buffer.as_string(): ", char_buffer.as_string()) +``` +The output of this script with ROOT 6.32: +``` +struct.char_buffer : +struct.char_buffer.as_string(): foo +``` + ## Language Bindings diff --git a/bindings/pyroot/cppyy/CPyCppyy/.gitignore b/bindings/pyroot/cppyy/CPyCppyy/.gitignore index 0de22a4662245..6562f33442ca3 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/.gitignore +++ b/bindings/pyroot/cppyy/CPyCppyy/.gitignore @@ -10,5 +10,8 @@ # build products dist *.egg-info +__pycache__ .cache +.pytest_cache +.vscode build diff --git a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt index caee0b4714524..42acd581d056c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt +++ b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt @@ -5,12 +5,12 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. set(headers - inc/CPyCppyy/API.h - inc/CPyCppyy/Reflex.h - inc/CPyCppyy/PyResult.h - inc/CPyCppyy/CommonDefs.h - inc/CPyCppyy/PyException.h - inc/CPyCppyy/DispatchPtr.h + include/CPyCppyy/API.h + include/CPyCppyy/Reflex.h + include/CPyCppyy/PyResult.h + include/CPyCppyy/CommonDefs.h + include/CPyCppyy/PyException.h + include/CPyCppyy/DispatchPtr.h ) set(sources @@ -20,10 +20,12 @@ set(sources src/CPPClassMethod.cxx src/CPPConstructor.cxx src/CPPDataMember.cxx + src/CPPEnum.cxx src/CPPExcInstance.cxx src/CPPFunction.cxx src/CPPInstance.cxx src/CPPMethod.cxx + src/CPPOperator.cxx src/CPPOverload.cxx src/CPPScope.cxx src/CPPGetSetItem.cxx @@ -64,8 +66,7 @@ else() endif() if(NOT MSVC) - target_compile_options(${libname} PRIVATE - -Wno-shadow -Wno-strict-aliasing) + target_compile_options(${libname} PRIVATE -Wno-strict-aliasing) endif() if(NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" AND NOT MSVC) target_compile_options(${libname} PRIVATE @@ -77,11 +78,16 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND ${CMAKE_CXX_COMPILER_VERSION} V target_compile_options(${libname} PRIVATE -Wno-cast-function-type) endif() -# Disables warnings due to new field tp_vectorcall in Python 3.8 -if(NOT MSVC AND PYTHON_VERSION_STRING VERSION_GREATER_EQUAL 3.8) +# Disables warnings in Python 3.8 caused by the temporary extra filed for tp_print compatibility +# (see https://github.com/python/cpython/blob/3.8/Include/cpython/object.h#L260). +# Note that Python 3.8 is the lowers Python version that is still supported by +# ROOT, so this compile option can be completely removed soon. +if(NOT MSVC AND PYTHON_VERSION_STRING VERSION_LESS 3.9) target_compile_options(${libname} PRIVATE -Wno-missing-field-initializers) endif() +target_compile_definitions(${libname} PRIVATE NO_CPPYY_LEGACY_NAMESPACE) + target_include_directories(${libname} SYSTEM PUBLIC ${PYTHON_INCLUDE_DIRS}) @@ -89,7 +95,7 @@ target_include_directories(${libname} PRIVATE ${CMAKE_BINARY_DIR}/ginclude PUBLIC - $ + $ ) set_property(GLOBAL APPEND PROPERTY ROOT_EXPORTED_TARGETS ${libname}) diff --git a/bindings/pyroot/cppyy/CPyCppyy/LICENSE.txt b/bindings/pyroot/cppyy/CPyCppyy/LICENSE.txt index 19d00e1fad28b..383f5d3644d7e 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/LICENSE.txt +++ b/bindings/pyroot/cppyy/CPyCppyy/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2002-2019, The Regents of the University of California, +Copyright (c) 2002-2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or @@ -48,5 +48,11 @@ the same conditions (except for some compatible licenses as retained in the source code): CERN - Toby StClere-Smithe + Lucio Asnaghi + Torok Attila + Simone Bacchio + Niko Fink + Aaron Jomy Mac Kolin + Baidyanath Kundu + Toby StClere-Smithe diff --git a/bindings/pyroot/cppyy/CPyCppyy/README.rst b/bindings/pyroot/cppyy/CPyCppyy/README.rst index fa6b43bbc7a47..2b931e7d3518c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/README.rst +++ b/bindings/pyroot/cppyy/CPyCppyy/README.rst @@ -28,4 +28,4 @@ Change log: https://cppyy.readthedocs.io/en/latest/changelog.html Bug reports/feedback: - https://bitbucket.org/wlav/cppyy/issues?status=new&status=open + https://github.com/wlav/cppyy/issues diff --git a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/Reflex.h b/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/Reflex.h deleted file mode 100644 index 7b988d7fb9ee9..0000000000000 --- a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/Reflex.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef CPYCPPYY_REFLEX_H -#define CPYCPPYY_REFLEX_H - -// -// Access to the C++ reflection information -// - -namespace Cppyy { - -namespace Reflex { - -typedef int RequestId_t; - -const RequestId_t IS_NAMESPACE = 1; - -const RequestId_t OFFSET = 2; -const RequestId_t RETURN_TYPE = 3; -const RequestId_t TYPE = 4; - -typedef int FormatId_t; -const FormatId_t OPTIMAL = 1; -const FormatId_t AS_TYPE = 2; -const FormatId_t AS_STRING = 3; - -} // namespace Reflex - -} // namespace Cppyy - -#endif // !CPYCPPYY_REFLEX_H diff --git a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/API.h b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/API.h similarity index 87% rename from bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/API.h rename to bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/API.h index 35f15602950e8..c2ccb4da31e0e 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/API.h +++ b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/API.h @@ -1,13 +1,9 @@ -#ifndef CPYCPPYY_TPYTHON -#define CPYCPPYY_TPYTHON +#ifndef CPYCPPYY_API_H +#define CPYCPPYY_API_H -////////////////////////////////////////////////////////////////////////////// -// // -// TPython // -// // -// Access to the python interpreter and API onto CPyCppyy. // -// // -////////////////////////////////////////////////////////////////////////////// +// +// Access to the python interpreter and API onto CPyCppyy. +// // Python #ifdef _WIN32 @@ -29,6 +25,8 @@ #endif #include "Python.h" +#define CPYCPPYY_VERSION_HEX 0x010c10 + // Cppyy types namespace Cppyy { typedef size_t TCppScope_t; @@ -86,6 +84,25 @@ struct Parameter { // CallContext is not currently exposed struct CallContext; +// Dimensions class not currently exposed +#ifndef CPYCPPYY_DIMENSIONS_H +#define CPYCPPYY_DIMENSIONS_H +typedef Py_ssize_t dim_t; + +class Dimensions { // Windows note: NOT exported/imported + dim_t* fDims; + +public: + Dimensions(dim_t /*ndim*/ = 0, dim_t* /*dims*/ = nullptr) : fDims(nullptr) {} + ~Dimensions() { delete [] fDims; } + +public: + operator bool() const { return (bool)fDims; } +}; + +typedef Dimensions dims_t; +typedef const dims_t& cdims_t; +#endif // !CPYCPPYY_DIMENSIONS_H // type converter base class class CPYCPPYY_CLASS_EXTERN Converter { @@ -106,13 +123,13 @@ class CPYCPPYY_CLASS_EXTERN Converter { }; // create a converter based on its full type name and dimensions -CPYCPPYY_EXTERN Converter* CreateConverter(const std::string& name, Py_ssize_t* dims = nullptr); +CPYCPPYY_EXTERN Converter* CreateConverter(const std::string& name, cdims_t = 0); // delete a previously created converter CPYCPPYY_EXTERN void DestroyConverter(Converter* p); // register a custom converter -typedef Converter* (*ConverterFactory_t)(Py_ssize_t* dims); +typedef Converter* (*ConverterFactory_t)(cdims_t); CPYCPPYY_EXTERN bool RegisterConverter(const std::string& name, ConverterFactory_t); // remove a custom converter @@ -133,13 +150,13 @@ class CPYCPPYY_CLASS_EXTERN Executor { }; // create an executor based on its full type name -CPYCPPYY_EXTERN Executor* CreateExecutor(const std::string& name); +CPYCPPYY_EXTERN Executor* CreateExecutor(const std::string& name, cdims_t = 0); // delete a previously created executor CPYCPPYY_EXTERN void DestroyConverter(Converter* p); // register a custom executor -typedef Executor* (*ExecutorFactory_t)(); +typedef Executor* (*ExecutorFactory_t)(cdims_t); CPYCPPYY_EXTERN bool RegisterExecutor(const std::string& name, ExecutorFactory_t); // remove a custom executor diff --git a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/CommonDefs.h b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/CommonDefs.h similarity index 100% rename from bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/CommonDefs.h rename to bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/CommonDefs.h diff --git a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/DispatchPtr.h b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/DispatchPtr.h similarity index 95% rename from bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/DispatchPtr.h rename to bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/DispatchPtr.h index 010d485e98c0c..760443c17380b 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/DispatchPtr.h +++ b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/DispatchPtr.h @@ -31,7 +31,7 @@ class CPYCPPYY_CLASS_EXTERN DispatchPtr { // Conversion constructor: called with C++ object construction when the PyObject // is known (eg. when instantiating from Python), with pyobj the Python-side // representation of the C++ object. - explicit DispatchPtr(PyObject* pyobj); + explicit DispatchPtr(PyObject* pyobj, bool strong = false); // Copy constructor: only ever called from C++. The Python object needs to be // copied, in case it has added state, and rebound to the new C++ instance. @@ -47,10 +47,7 @@ class CPYCPPYY_CLASS_EXTERN DispatchPtr { DispatchPtr& operator=(const DispatchPtr& other) = delete; // lifetime is directly bound to the lifetime of the dispatcher object - ~DispatchPtr() { - Py_XDECREF(fPyWeakRef); - Py_XDECREF(fPyHardRef); - } + ~DispatchPtr(); // either C++ owns the Python object through a reference count (on fPyHardRef) or // Python owns the C++ object and we only have a weak reference (through fPyWeakRef) diff --git a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/PyException.h b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/PyException.h similarity index 94% rename from bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/PyException.h rename to bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/PyException.h index 3fa1850a0123d..cd31cc0b1f95f 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/PyException.h +++ b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/PyException.h @@ -28,6 +28,7 @@ // Standard #include +#include // Bindings #include "CPyCppyy/CommonDefs.h" @@ -42,6 +43,12 @@ class CPYCPPYY_CLASS_EXTERN PyException : public std::exception { // give reason for raised exception virtual const char* what() const noexcept; + +// clear Python error, to allow full error handling C++ side + void clear() const noexcept; + +private: + std::string fMsg; }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/PyResult.h b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/PyResult.h similarity index 100% rename from bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/PyResult.h rename to bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/PyResult.h diff --git a/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/Reflex.h b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/Reflex.h new file mode 100644 index 0000000000000..b37b9eed9621e --- /dev/null +++ b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/Reflex.h @@ -0,0 +1,30 @@ +#ifndef CPYCPPYY_REFLEX_H +#define CPYCPPYY_REFLEX_H + +// +// Access to the C++ reflection information +// + +namespace Cppyy { + +namespace Reflex { + +typedef int RequestId_t; + +const RequestId_t IS_NAMESPACE = 1; +const RequestId_t IS_AGGREGATE = 2; + +const RequestId_t OFFSET = 16; +const RequestId_t RETURN_TYPE = 17; +const RequestId_t TYPE = 18; + +typedef int FormatId_t; +const FormatId_t OPTIMAL = 1; +const FormatId_t AS_TYPE = 2; +const FormatId_t AS_STRING = 3; + +} // namespace Reflex + +} // namespace Cppyy + +#endif // !CPYCPPYY_REFLEX_H diff --git a/bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/TPyArg.h b/bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/TPyArg.h similarity index 100% rename from bindings/pyroot/cppyy/CPyCppyy/inc/CPyCppyy/TPyArg.h rename to bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/TPyArg.h diff --git a/bindings/pyroot/cppyy/CPyCppyy/pyproject.toml b/bindings/pyroot/cppyy/CPyCppyy/pyproject.toml new file mode 100644 index 0000000000000..76e57228dba35 --- /dev/null +++ b/bindings/pyroot/cppyy/CPyCppyy/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["cppyy-cling==6.30.0", "cppyy-backend==1.15.2", "setuptools", "wheel"] diff --git a/bindings/pyroot/cppyy/CPyCppyy/setup.cfg b/bindings/pyroot/cppyy/CPyCppyy/setup.cfg index 8debd01371e4f..9ae562903000d 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/setup.cfg +++ b/bindings/pyroot/cppyy/CPyCppyy/setup.cfg @@ -2,4 +2,4 @@ universal=0 [metadata] -license_file = LICENSE.txt +license_files = LICENSE.txt diff --git a/bindings/pyroot/cppyy/CPyCppyy/setup.py b/bindings/pyroot/cppyy/CPyCppyy/setup.py index 46b37188b3e42..320d3ca9122df 100755 --- a/bindings/pyroot/cppyy/CPyCppyy/setup.py +++ b/bindings/pyroot/cppyy/CPyCppyy/setup.py @@ -2,7 +2,6 @@ from setuptools import setup, find_packages, Extension from distutils import log -from setuptools.dist import Distribution from distutils.command.build_ext import build_ext as _build_ext try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel @@ -10,12 +9,7 @@ except ImportError: has_wheel = False -force_bdist = False -if '--force-bdist' in sys.argv: - force_bdist = True - sys.argv.remove('--force-bdist') - -requirements = ['cppyy-cling==6.18.2.*', 'cppyy-backend==1.10.*'] +requirements = ['cppyy-cling==6.30.0', 'cppyy-backend==1.15.2'] setup_requirements = ['wheel'] if 'build' in sys.argv or 'install' in sys.argv: setup_requirements += requirements @@ -28,18 +22,9 @@ # # platform-dependent helpers # -def is_manylinux(): - try: - for line in open('/etc/redhat-release').readlines(): - if 'CentOS release 6.10 (Final)' in line: - return True - except (OSError, IOError): - pass - return False - def _get_link_libraries(): if 'win32' in sys.platform: - return ['libcppyy_backend', 'libCore'] + return ['libcppyy_backend', 'libCoreLegacy'] return [] def _get_link_dirs(): @@ -93,32 +78,9 @@ def build_extension(self, ext): 'build_ext': my_build_extension } -# -# customized distribition to disable binaries -# -class MyDistribution(Distribution): - def run_commands(self): - # pip does not resolve dependencies before building binaries, so unless - # packages are installed one-by-one, on old install is used or the build - # will simply fail hard. The following is not completely quiet, but at - # least a lot less conspicuous. - if not is_manylinux() and not force_bdist: - disabled = set(( - 'bdist_wheel', 'bdist_egg', 'bdist_wininst', 'bdist_rpm')) - for cmd in self.commands: - if not cmd in disabled: - self.run_command(cmd) - else: - log.info('Command "%s" is disabled', cmd) - cmd_obj = self.get_command_obj(cmd) - cmd_obj.get_outputs = lambda: None - else: - return Distribution.run_commands(self) - - setup( name='CPyCppyy', - version='1.10.2', + version='1.12.16', description='Cling-based Python-C++ bindings for CPython', long_description=long_description, @@ -140,11 +102,12 @@ def run_commands(self): 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: C', 'Programming Language :: C++', @@ -165,7 +128,6 @@ def run_commands(self): headers=glob.glob(os.path.join('include', 'CPyCppyy', '*.h')), cmdclass=cmdclass, - distclass=MyDistribution, zip_safe=False, ) diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/API.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/API.cxx index 43897a808c79b..ae286b572ef63 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/API.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/API.cxx @@ -59,14 +59,14 @@ static bool Initialize() #endif #if PY_VERSION_HEX >= 0x03020000 #if PY_VERSION_HEX < 0x03090000 - PyEval_InitThreads(); + PyEval_InitThreads(); #endif #endif // try again to see if the interpreter is initialized if (!Py_IsInitialized()) { // give up ... - std::cerr << "Error: python has not been intialized; returning." << std::endl; + std::cerr << "Error: python has not been initialized; returning." << std::endl; return false; } @@ -79,6 +79,7 @@ static bool Initialize() #if PY_VERSION_HEX < 0x03080000 PySys_SetArgv(sizeof(argv)/sizeof(argv[0]), argv); #endif + // force loading of the cppyy module PyRun_SimpleString(const_cast("import cppyy")); } @@ -183,7 +184,7 @@ bool CPyCppyy::Instance_CheckExact(PyObject* pyobject) //----------------------------------------------------------------------------- bool CPyCppyy::Instance_IsLively(PyObject* pyobject) { -// Test whether the given instance can safely return to C++, or whether +// Test whether the given instance can safely return to C++ if (!CPPInstance_Check(pyobject)) return true; // simply don't know @@ -314,7 +315,7 @@ void CPyCppyy::ExecScript(const std::string& name, const std::vector= 0x03080000 + self +#endif + , CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt) { // preliminary check in case keywords are accidently used (they are ignored otherwise) - if (kwds && PyDict_Size(kwds)) { + if (kwds && ((PyDict_Check(kwds) && PyDict_Size(kwds)) || + (PyTuple_CheckExact(kwds) && PyTuple_GET_SIZE(kwds)))) { PyErr_SetString(PyExc_TypeError, "keyword arguments are not yet supported"); return nullptr; } @@ -18,7 +23,26 @@ PyObject* CPyCppyy::CPPClassMethod::Call( return nullptr; // translate the arguments - if (!this->ConvertAndSetArgs(args, ctxt)) +#if PY_VERSION_HEX >= 0x03080000 +// TODO: The following is not robust and should be revisited e.g. by making CPPOverloads +// that have only CPPClassMethods be true Python classmethods? Note that the original +// implementation wasn't 100% correct either (e.g. static size() mapped to len()). +// +// As-is, if no suitable `self` is given (normal case), but the type of the first argument +// matches the enclosing scope of the class method and it isn't needed for the call, then +// assume that the method was (incorrectly) bound and so drop that instance from args. + int nargs = (int)CPyCppyy_PyArgs_GET_SIZE(args, nargsf); + if ((!self || (PyObject*)self == Py_None) && nargs) { + PyObject* arg0 = CPyCppyy_PyArgs_GET_ITEM(args, 0); + if ((CPPInstance_Check(arg0) && ((CPPInstance*)arg0)->ObjectIsA() == GetScope()) && \ + (fArgsRequired <= nargs-1) && (GetMaxArgs() < nargs)) { + args += 1; // drops first argument + nargsf -= 1; + } + } +#endif + + if (!this->ConvertAndSetArgs(args, nargsf, ctxt)) return nullptr; // execute function diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPClassMethod.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPClassMethod.h index 4cbfdc055fae9..33ecdf8222e9e 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPClassMethod.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPClassMethod.h @@ -12,8 +12,8 @@ class CPPClassMethod : public CPPMethod { using CPPMethod::CPPMethod; virtual PyCallable* Clone() { return new CPPClassMethod(*this); } - virtual PyObject* Call( - CPPInstance*&, PyObject* args, PyObject* kwds, CallContext* ctxt = nullptr); + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.cxx index e4bc1266953a7..37d37aa1aa7f7 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.cxx @@ -7,12 +7,17 @@ #include "ProxyWrappers.h" #include "PyStrings.h" -#include "CPyCppyy/DispatchPtr.h" - // Standard +#include #include +//- data _____________________________________________________________________ +namespace CPyCppyy { + extern PyObject* gNullPtrObject; +} + + //- protected members -------------------------------------------------------- bool CPyCppyy::CPPConstructor::InitExecutor_(Executor*& executor, CallContext*) { @@ -48,37 +53,42 @@ PyObject* CPyCppyy::CPPConstructor::Reflex( } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPConstructor::Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt) +PyObject* CPyCppyy::CPPConstructor::Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt) { // setup as necessary if (fArgsRequired == -1 && !this->Initialize(ctxt)) return nullptr; // important: 0, not Py_None // fetch self, verify, and put the arguments in usable order - if (!(args = this->PreProcessArgs(self, args, kwds))) + PyCallArgs cargs{self, args, nargsf, kwds}; + if (!this->ProcessArgs(cargs)) return nullptr; // verify existence of self (i.e. tp_new called) if (!self) { - PyErr_Print(); PyErr_SetString(PyExc_ReferenceError, "no python object allocated"); return nullptr; } if (self->GetObject()) { - Py_DECREF(args); PyErr_SetString(PyExc_ReferenceError, "object already constructed; use __assign__ instead of __init__"); return nullptr; } +// self provides the python context for lifelines + if (!ctxt->fPyContext) + ctxt->fPyContext = (PyObject*)cargs.fSelf; // no Py_INCREF as no ownership + // perform the call, nullptr 'this' makes the other side allocate the memory Cppyy::TCppScope_t disp = self->ObjectIsA(false /* check_smart */); - ptrdiff_t address = 0; + intptr_t address = 0; if (GetScope() != disp) { - // happens for Python derived types, which have a dispatcher inserted that - // is not otherwise user-visible: call it instead + // happens for Python derived types (which have a dispatcher inserted that + // is not otherwise user-visible: call it instead) and C++ derived classes + // without public constructors + // first, check whether we at least had a proper meta class, or whether that // was also replaced user-side if (!GetScope() || !disp) { @@ -86,41 +96,43 @@ PyObject* CPyCppyy::CPPConstructor::Call( return nullptr; } - // get the dispatcher class + // get the dispatcher class and verify PyObject* dispproxy = CPyCppyy::GetScopeProxy(disp); if (!dispproxy) { PyErr_SetString(PyExc_TypeError, "dispatcher proxy was never created"); return nullptr; } - PyObject* pyobj = PyObject_Call(dispproxy, args, kwds); + if (!(((CPPClass*)dispproxy)->fFlags & CPPScope::kIsPython)) { + PyErr_SetString(PyExc_TypeError, const_cast(( + "constructor for " + Cppyy::GetScopedFinalName(disp) + " is not a dispatcher").c_str())); + return nullptr; + } + + PyObject* pyobj = CPyCppyy_PyObject_Call(dispproxy, cargs.fArgs, cargs.fNArgsf, kwds); if (!pyobj) return nullptr; - // retrieve the actual pointer, take over control, and set m_self - address = (ptrdiff_t)((CPPInstance*)pyobj)->GetObject(); + // retrieve the actual pointer, take over control, and set set _internal_self + address = (intptr_t)((CPPInstance*)pyobj)->GetObject(); if (address) { - ((CPPInstance*)pyobj)->CppOwns(); + ((CPPInstance*)pyobj)->CppOwns(); // b/c self will control the object on address PyObject* res = PyObject_CallMethodObjArgs( - dispproxy, PyStrings::gDispInit, pyobj, (PyObject*)self, nullptr); + dispproxy, PyStrings::gDispInit, pyobj, (PyObject*)self, nullptr); Py_XDECREF(res); } - Py_DECREF(pyobj); Py_DECREF(dispproxy); } else { // translate the arguments - if (!this->ConvertAndSetArgs(args, ctxt)) { - Py_DECREF(args); + if (((CPPClass*)Py_TYPE(self))->fFlags & CPPScope::kNoImplicit) + ctxt->fFlags |= CallContext::kNoImplicit; + if (!this->ConvertAndSetArgs(cargs.fArgs, cargs.fNArgsf, ctxt)) return nullptr; - } - address = (ptrdiff_t)this->Execute(nullptr, 0, ctxt); + address = (intptr_t)this->Execute(nullptr, 0, ctxt); } -// done with filtered args - Py_DECREF(args); - // return object if successful, lament if not if (address) { Py_INCREF(self); @@ -129,8 +141,10 @@ PyObject* CPyCppyy::CPPConstructor::Call( // decided by the method proxy (which carries a creator flag) upon return self->Set((void*)address); - // TODO: consistent up or down cast ... - MemoryRegulator::RegisterPyObject(self, (Cppyy::TCppObject_t)address); + // mark as actual to prevent needless auto-casting and register on its class + self->fFlags |= CPPInstance::kIsActual; + if (!(((CPPClass*)Py_TYPE(self))->fFlags & CPPScope::kIsSmart)) + MemoryRegulator::RegisterPyObject(self, (Cppyy::TCppObject_t)address); // handling smart types this way is deeply fugly, but if CPPInstance sets the proper // types in op_new first, then the wrong init is called @@ -160,13 +174,167 @@ PyObject* CPyCppyy::CPPConstructor::Call( //---------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPAbstractClassConstructor::Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt) +CPyCppyy::CPPMultiConstructor::CPPMultiConstructor(Cppyy::TCppScope_t scope, Cppyy::TCppMethod_t method) : + CPPConstructor(scope, method) +{ + fNumBases = Cppyy::GetNumBases(scope); +} + +//---------------------------------------------------------------------------- +CPyCppyy::CPPMultiConstructor::CPPMultiConstructor(const CPPMultiConstructor& s) : + CPPConstructor(s), fNumBases(s.fNumBases) +{ +} + +//---------------------------------------------------------------------------- +CPyCppyy::CPPMultiConstructor& CPyCppyy::CPPMultiConstructor::operator=(const CPPMultiConstructor& s) +{ + if (this != &s) { + CPPConstructor::operator=(s); + fNumBases = s.fNumBases; + } + return *this; +} + +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::CPPMultiConstructor::Call(CPPInstance*& self, + CPyCppyy_PyArgs_t argsin, size_t nargsf, PyObject* kwds, CallContext* ctxt) +{ +// By convention, initialization parameters of multiple base classes are grouped +// by target base class. Here, we disambiguate and put in "sentinel" parameters +// that allow the dispatcher to propagate them. + +// Three options supported: +// 0. empty args: default constructor call +// 1. fNumBases tuples, each handed to individual constructors +// 2. less than fNumBases, assuming (void) for the missing base constructors +// 3. normal arguments, going to the first base only + +// TODO: this way of forwarding is expensive as the loop is external to this call; +// it would be more efficient to have the argument handling happen beforehand + +#if PY_VERSION_HEX >= 0x03080000 +// fetch self, verify, and put the arguments in usable order (if self is not handled +// first, arguments can not be reordered with sentinels in place) + PyCallArgs cargs{self, argsin, nargsf, kwds}; + if (!this->ProcessArgs(cargs)) + return nullptr; + +// to re-use the argument handling, simply change the argument array into a tuple (the +// benefits of not allocating the tuple are relatively minor in this case) + Py_ssize_t nargs = CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf); + PyObject* args = PyTuple_New(nargs); + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(cargs.fArgs[i]); + PyTuple_SET_ITEM(args, i, cargs.fArgs[i]); + } + +// copy out self as it may have been updated + self = cargs.fSelf; + +#else + PyObject* args = argsin; + Py_INCREF(args); +#endif + + if (PyTuple_CheckExact(args) && PyTuple_GET_SIZE(args)) { // case 0. falls through + Py_ssize_t nArgs = PyTuple_GET_SIZE(args); + + bool isAllTuples = true; + Py_ssize_t nArgsTot = 0; + for (Py_ssize_t i = 0; i < nArgs; ++i) { + PyObject* argi = PyTuple_GET_ITEM(args, i); + if (!PyTuple_CheckExact(argi)) { + isAllTuples = false; + break; + } + nArgsTot += PyTuple_GET_SIZE(argi); + } + + if (isAllTuples) { + // copy over the arguments, while filling in the sentinels (case 1. & 2.), with + // just sentinels for the remaining (void) calls (case 2.) + PyObject* newArgs = PyTuple_New(nArgsTot + fNumBases - 1); + Py_ssize_t idx = 0; + for (Py_ssize_t i = 0; i < nArgs; ++i) { + if (i != 0) { + // add sentinel + Py_INCREF(gNullPtrObject); + PyTuple_SET_ITEM(newArgs, idx, gNullPtrObject); + idx += 1; + } + + PyObject* argi = PyTuple_GET_ITEM(args, i); + for (Py_ssize_t j = 0; j < PyTuple_GET_SIZE(argi); ++j) { + PyObject* item = PyTuple_GET_ITEM(argi, j); + Py_INCREF(item); + PyTuple_SET_ITEM(newArgs, idx, item); + idx += 1; + } + } + + // add final sentinels as needed + while (idx < (nArgsTot+fNumBases-1)) { + Py_INCREF(gNullPtrObject); + PyTuple_SET_ITEM(newArgs, idx, gNullPtrObject); + idx += 1; + } + + Py_DECREF(args); + args = newArgs; + } else { // case 3. add sentinels + // copy arguments as-is, then add sentinels at the end + PyObject* newArgs = PyTuple_New(PyTuple_GET_SIZE(args) + fNumBases - 1); + for (Py_ssize_t i = 0; i < nArgs; ++i) { + PyObject* item = PyTuple_GET_ITEM(args, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newArgs, i, item); + } + for (Py_ssize_t i = 0; i < fNumBases - 1; ++i) { + Py_INCREF(gNullPtrObject); + PyTuple_SET_ITEM(newArgs, i+nArgs, gNullPtrObject); + } + Py_DECREF(args); + args = newArgs; + } + } + +#if PY_VERSION_HEX < 0x03080000 + Py_ssize_t +#endif + nargs = PyTuple_GET_SIZE(args); + +#if PY_VERSION_HEX >= 0x03080000 +// now unroll the new args tuple into a vector of objects + auto argsu = std::unique_ptr{new PyObject*[nargs]}; + for (Py_ssize_t i = 0; i < nargs; ++i) + argsu[i] = PyTuple_GET_ITEM(args, i); + CPyCppyy_PyArgs_t _args = argsu.get(); +#else + CPyCppyy_PyArgs_t _args = args; +#endif + + PyObject* result = CPPConstructor::Call(self, _args, nargs, kwds, ctxt); + Py_DECREF(args); + + return result; +} + + +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::CPPAbstractClassConstructor::Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt) { // do not allow instantiation of abstract classes - if (self && GetScope() != self->ObjectIsA()) { + if ((self && GetScope() != self->ObjectIsA() +#if PY_VERSION_HEX >= 0x03080000 + ) || (!self && !(ctxt->fFlags & CallContext::kFromDescr) && \ + CPyCppyy_PyArgs_GET_SIZE(args, nargsf) && CPPInstance_Check(args[0]) && \ + GetScope() != ((CPPInstance*)args[0])->ObjectIsA() +#endif + )) { // happens if a dispatcher is inserted; allow constructor call - return CPPConstructor::Call(self, args, kwds, ctxt); + return CPPConstructor::Call(self, args, nargsf, kwds, ctxt); } PyErr_Format(PyExc_TypeError, "cannot instantiate abstract class \'%s\'" @@ -175,9 +343,10 @@ PyObject* CPyCppyy::CPPAbstractClassConstructor::Call( return nullptr; } + //---------------------------------------------------------------------------- PyObject* CPyCppyy::CPPNamespaceConstructor::Call( - CPPInstance*&, PyObject*, PyObject*, CallContext*) + CPPInstance*&, CPyCppyy_PyArgs_t, size_t, PyObject*, CallContext*) { // do not allow instantiation of namespaces PyErr_Format(PyExc_TypeError, "cannot instantiate namespace \'%s\'", @@ -185,12 +354,23 @@ PyObject* CPyCppyy::CPPNamespaceConstructor::Call( return nullptr; } + //---------------------------------------------------------------------------- PyObject* CPyCppyy::CPPIncompleteClassConstructor::Call( - CPPInstance*&, PyObject*, PyObject*, CallContext*) + CPPInstance*&, CPyCppyy_PyArgs_t, size_t, PyObject*, CallContext*) { // do not allow instantiation of incomplete (forward declared) classes) PyErr_Format(PyExc_TypeError, "cannot instantiate incomplete class \'%s\'", Cppyy::GetScopedFinalName(this->GetScope()).c_str()); return nullptr; } + +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::CPPAllPrivateClassConstructor::Call( + CPPInstance*&, CPyCppyy_PyArgs_t, size_t, PyObject*, CallContext*) +{ +// do not allow instantiation of classes with only private constructors + PyErr_Format(PyExc_TypeError, "cannot instantiate class \'%s\' that has no public constructors", + Cppyy::GetScopedFinalName(this->GetScope()).c_str()); + return nullptr; +} diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.h index 6bc20997dbb20..adb113ec6fb90 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPConstructor.h @@ -17,23 +17,40 @@ class CPPConstructor : public CPPMethod { Cppyy::Reflex::FormatId_t = Cppyy::Reflex::OPTIMAL); virtual PyCallable* Clone() { return new CPPConstructor(*this); } - -public: - virtual PyObject* Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt = nullptr); + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); protected: virtual bool InitExecutor_(Executor*&, CallContext* ctxt = nullptr); }; +// specialization for multiple inheritance disambiguation +class CPPMultiConstructor : public CPPConstructor { +public: + CPPMultiConstructor(Cppyy::TCppScope_t scope, Cppyy::TCppMethod_t method); + CPPMultiConstructor(const CPPMultiConstructor&); + CPPMultiConstructor& operator=(const CPPMultiConstructor&); + +public: + virtual PyCallable* Clone() { return new CPPMultiConstructor(*this); } + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); + +private: + Py_ssize_t fNumBases; +}; + + // specializations of prohibiting constructors class CPPAbstractClassConstructor : public CPPConstructor { public: using CPPConstructor::CPPConstructor; public: - virtual PyObject* Call(CPPInstance*&, PyObject*, PyObject*, CallContext* = nullptr); + virtual PyCallable* Clone() { return new CPPAbstractClassConstructor(*this); } + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); }; class CPPNamespaceConstructor : public CPPConstructor { @@ -41,7 +58,9 @@ class CPPNamespaceConstructor : public CPPConstructor { using CPPConstructor::CPPConstructor; public: - virtual PyObject* Call(CPPInstance*&, PyObject*, PyObject*, CallContext* = nullptr); + virtual PyCallable* Clone() { return new CPPNamespaceConstructor(*this); } + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); }; class CPPIncompleteClassConstructor : public CPPConstructor { @@ -49,7 +68,19 @@ class CPPIncompleteClassConstructor : public CPPConstructor { using CPPConstructor::CPPConstructor; public: - virtual PyObject* Call(CPPInstance*&, PyObject*, PyObject*, CallContext* = nullptr); + virtual PyCallable* Clone() { return new CPPIncompleteClassConstructor(*this); } + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); +}; + +class CPPAllPrivateClassConstructor : public CPPConstructor { +public: + using CPPConstructor::CPPConstructor; + +public: + virtual PyCallable* Clone() { return new CPPAllPrivateClassConstructor(*this); } + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.cxx index 153a1a3bf7bcd..bb250158bff37 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.cxx @@ -4,15 +4,18 @@ #include "PyStrings.h" #include "CPPDataMember.h" #include "CPPInstance.h" +#include "Dimensions.h" #include "LowLevelViews.h" +#include "ProxyWrappers.h" #include "PyStrings.h" +#include "TypeManip.h" #include "Utility.h" // Standard #include #include #include - +#include namespace CPyCppyy { @@ -21,17 +24,19 @@ enum ETypeDetails { kIsStaticData = 0x0001, kIsConstData = 0x0002, kIsArrayType = 0x0004, - kIsCachable = 0x0008 + kIsEnumPrep = 0x0008, + kIsEnumType = 0x0010, + kIsCachable = 0x0020 }; //= CPyCppyy data member as Python property behavior ========================= -static PyObject* pp_get(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* /* kls */) +static PyObject* dm_get(CPPDataMember* dm, CPPInstance* pyobj, PyObject* /* kls */) { // cache lookup for low level views - if (pyprop->fFlags & kIsCachable) { + if (pyobj && dm->fFlags & kIsCachable) { CPyCppyy::CI_DatamemberCache_t& cache = pyobj->GetDatamemberCache(); for (auto it = cache.begin(); it != cache.end(); ++it) { - if (it->first == pyprop->fOffset) { + if (it->first == dm->fOffset) { if (it->second) { Py_INCREF(it->second); return it->second; @@ -42,24 +47,51 @@ static PyObject* pp_get(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* /* } } -// normal getter access - void* address = pyprop->GetAddress(pyobj); +// non-initialized or public data accesses through class (e.g. by help()) + void* address = dm->GetAddress(pyobj); if (!address || (intptr_t)address == -1 /* Cling error */) return nullptr; -// for fixed size arrays - void* ptr = address; - if (pyprop->fFlags & kIsArrayType) - ptr = &address; + if (dm->fFlags & (kIsEnumPrep | kIsEnumType)) { + if (dm->fFlags & kIsEnumPrep) { + // still need to do lookup; only ever try this once, then fallback on converter + dm->fFlags &= ~kIsEnumPrep; + + // fDescription contains the full name of the actual enum value object + const std::string& lookup = CPyCppyy_PyText_AsString(dm->fDescription); + const std::string& enum_type = TypeManip::extract_namespace(lookup); + const std::string& enum_scope = TypeManip::extract_namespace(enum_type); + + PyObject* pyscope = nullptr; + if (enum_scope.empty()) pyscope = GetScopeProxy(Cppyy::gGlobalScope); + else pyscope = CreateScopeProxy(enum_scope); + if (pyscope) { + PyObject* pyEnumType = PyObject_GetAttrString(pyscope, + enum_type.substr(enum_scope.size() ? enum_scope.size()+2 : 0, std::string::npos).c_str()); + if (pyEnumType) { + PyObject* pyval = PyObject_GetAttrString(pyEnumType, + lookup.substr(enum_type.size()+2, std::string::npos).c_str()); + Py_DECREF(pyEnumType); + if (pyval) { + Py_DECREF(dm->fDescription); + dm->fDescription = pyval; + dm->fFlags |= kIsEnumType; + } + } + Py_DECREF(pyscope); + } + if (!(dm->fFlags & kIsEnumType)) + PyErr_Clear(); + } -// non-initialized or public data accesses through class (e.g. by help()) - if (!ptr || (intptr_t)ptr == -1 /* Cling error */) { - Py_INCREF(pyprop); - return (PyObject*)pyprop; + if (dm->fFlags & kIsEnumType) { + Py_INCREF(dm->fDescription); + return dm->fDescription; + } } - if (pyprop->fConverter != 0) { - PyObject* result = pyprop->fConverter->FromMemory(ptr); + if (dm->fConverter != 0) { + PyObject* result = dm->fConverter->FromMemory((dm->fFlags & kIsArrayType) ? &address : address); if (!result) return result; @@ -67,8 +99,8 @@ static PyObject* pp_get(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* /* bool isLLView = LowLevelView_CheckExact(result); if (isLLView && CPPInstance_Check(pyobj)) { Py_INCREF(result); - pyobj->GetDatamemberCache().push_back(std::make_pair(pyprop->fOffset, result)); - pyprop->fFlags |= kIsCachable; + pyobj->GetDatamemberCache().push_back(std::make_pair(dm->fOffset, result)); + dm->fFlags |= kIsCachable; } // ensure that the encapsulating class does not go away for the duration @@ -76,7 +108,7 @@ static PyObject* pp_get(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* /* // for builtin types, b/c those are copied over into python types and thus // end up being "stand-alone") // TODO: should be done for LLViews as well - else if (pyobj && CPPInstance_Check(result)) { + else if (pyobj && !(dm->fFlags & kIsStaticData) && CPPInstance_Check(result)) { if (PyObject_SetAttr(result, PyStrings::gLifeLine, (PyObject*)pyobj) == -1) PyErr_Clear(); // ignored } @@ -85,27 +117,42 @@ static PyObject* pp_get(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* /* } PyErr_Format(PyExc_NotImplementedError, - "no converter available for \"%s\"", pyprop->GetName().c_str()); + "no converter available for \"%s\"", dm->GetName().c_str()); return nullptr; } //----------------------------------------------------------------------------- -static int pp_set(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* value) +static int dm_set(CPPDataMember* dm, CPPInstance* pyobj, PyObject* value) { // Set the value of the C++ datum held. const int errret = -1; + if (!value) { + // we're being deleted; fine for namespaces (redo lookup next time), but makes + // no sense for classes/structs + if (!Cppyy::IsNamespace(dm->fEnclosingScope)) { + PyErr_SetString(PyExc_TypeError, "data member deletion is not supported"); + return errret; + } + + // deletion removes the proxy, with the idea that a fresh lookup can be made, + // to support Cling's shadowing of declarations (TODO: the use case here is + // redeclared variables, for which fDescription is indeed th ename; it might + // fail for enums). + return PyObject_DelAttr((PyObject*)Py_TYPE(pyobj), dm->fDescription); + } + // filter const objects to prevent changing their values - if (pyprop->fFlags & kIsConstData) { + if (dm->fFlags & kIsConstData) { PyErr_SetString(PyExc_TypeError, "assignment to const data not allowed"); return errret; } // remove cached low level view, if any (will be restored upon reaeding) - if (pyprop->fFlags & kIsCachable) { + if (dm->fFlags & kIsCachable) { CPyCppyy::CI_DatamemberCache_t& cache = pyobj->GetDatamemberCache(); for (auto it = cache.begin(); it != cache.end(); ++it) { - if (it->first == pyprop->fOffset) { + if (it->first == dm->fOffset) { Py_XDECREF(it->second); cache.erase(it); break; @@ -113,17 +160,17 @@ static int pp_set(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* value) } } - intptr_t address = (intptr_t)pyprop->GetAddress(pyobj); + intptr_t address = (intptr_t)dm->GetAddress(pyobj); if (!address || address == -1 /* Cling error */) return errret; // for fixed size arrays void* ptr = (void*)address; - if (pyprop->fFlags & kIsArrayType) + if (dm->fFlags & kIsArrayType) ptr = &address; // actual conversion; return on success - if (pyprop->fConverter && pyprop->fConverter->ToMemory(value, ptr, (PyObject*)pyobj)) + if (dm->fConverter && dm->fConverter->ToMemory(value, ptr, (PyObject*)pyobj)) return 0; // set a python error, if not already done @@ -135,35 +182,43 @@ static int pp_set(CPPDataMember* pyprop, CPPInstance* pyobj, PyObject* value) } //= CPyCppyy data member construction/destruction =========================== -static CPPDataMember* pp_new(PyTypeObject* pytype, PyObject*, PyObject*) +static CPPDataMember* dm_new(PyTypeObject* pytype, PyObject*, PyObject*) { // Create and initialize a new property descriptor. - CPPDataMember* pyprop = (CPPDataMember*)pytype->tp_alloc(pytype, 0); + CPPDataMember* dm = (CPPDataMember*)pytype->tp_alloc(pytype, 0); - pyprop->fOffset = 0; - pyprop->fFlags = 0; - pyprop->fConverter = nullptr; - pyprop->fEnclosingScope = 0; - pyprop->fName = nullptr; + dm->fOffset = 0; + dm->fFlags = 0; + dm->fConverter = nullptr; + dm->fEnclosingScope = 0; + dm->fDescription = nullptr; + dm->fDoc = nullptr; - new (&pyprop->fFullType) std::string{}; + new (&dm->fFullType) std::string{}; - return pyprop; + return dm; } //---------------------------------------------------------------------------- -static void pp_dealloc(CPPDataMember* pyprop) +static void dm_dealloc(CPPDataMember* dm) { // Deallocate memory held by this descriptor. - using std::string; - if (pyprop->fConverter && pyprop->fConverter->HasState()) delete pyprop->fConverter; - Py_XDECREF(pyprop->fName); // never exposed so no GC necessary + using namespace std; + if (dm->fConverter && dm->fConverter->HasState()) delete dm->fConverter; + Py_XDECREF(dm->fDescription); // never exposed so no GC necessary + Py_XDECREF(dm->fDoc); - pyprop->fFullType.~string(); + dm->fFullType.~string(); - Py_TYPE(pyprop)->tp_free((PyObject*)pyprop); + Py_TYPE(dm)->tp_free((PyObject*)dm); } +static PyMemberDef dm_members[] = { + {(char*)"__doc__", T_OBJECT, offsetof(CPPDataMember, fDoc), 0, + (char*)"writable documentation"}, + {NULL, 0, 0, 0, nullptr} /* Sentinel */ +}; + //= CPyCppyy datamember proxy access to internals ============================ static PyObject* dm_reflex(CPPDataMember* dm, PyObject* args) { @@ -192,17 +247,18 @@ static PyMethodDef dm_methods[] = { {(char*)nullptr, nullptr, 0, nullptr } }; + //= CPyCppyy data member type ================================================ PyTypeObject CPPDataMember_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) (char*)"cppyy.CPPDataMember", // tp_name sizeof(CPPDataMember), // tp_basicsize 0, // tp_itemsize - (destructor)pp_dealloc, // tp_dealloc - 0, // tp_print + (destructor)dm_dealloc, // tp_dealloc + 0, // tp_vectorcall_offset / tp_print 0, // tp_getattr 0, // tp_setattr - 0, // tp_compare + 0, // tp_as_async / tp_compare 0, // tp_repr 0, // tp_as_number 0, // tp_as_sequence @@ -222,16 +278,16 @@ PyTypeObject CPPDataMember_Type = { 0, // tp_iter 0, // tp_iternext dm_methods, // tp_methods - 0, // tp_members + dm_members, // tp_members 0, // tp_getset 0, // tp_base 0, // tp_dict - (descrgetfunc)pp_get, // tp_descr_get - (descrsetfunc)pp_set, // tp_descr_set + (descrgetfunc)dm_get, // tp_descr_get + (descrsetfunc)dm_set, // tp_descr_set 0, // tp_dictoffset 0, // tp_init 0, // tp_alloc - (newfunc)pp_new, // tp_new + (newfunc)dm_new, // tp_new 0, // tp_free 0, // tp_is_gc 0, // tp_bases @@ -248,6 +304,9 @@ PyTypeObject CPPDataMember_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; } // namespace CPyCppyy @@ -257,40 +316,60 @@ PyTypeObject CPPDataMember_Type = { void CPyCppyy::CPPDataMember::Set(Cppyy::TCppScope_t scope, Cppyy::TCppIndex_t idata) { fEnclosingScope = scope; - fName = CPyCppyy_PyText_FromString(Cppyy::GetDatamemberName(scope, idata).c_str()); fOffset = Cppyy::GetDatamemberOffset(scope, idata); // TODO: make lazy fFlags = Cppyy::IsStaticData(scope, idata) ? kIsStaticData : 0; std::vector dims; - int ndim = 0; dim_t size = 0; + int ndim = 0; Py_ssize_t size = 0; while (0 < (size = Cppyy::GetDimensionSize(scope, idata, ndim))) { ndim += 1; if (size == INT_MAX) // meaning: incomplete array type - size = -1; - if (ndim == 1) { dims.reserve(4); dims.push_back(0); } - dims.push_back(size); + size = UNKNOWN_SIZE; + if (ndim == 1) dims.reserve(4); + dims.push_back((dim_t)size); } - if (ndim) { - dims[0] = ndim; + if (!dims.empty()) fFlags |= kIsArrayType; - } + const std::string name = Cppyy::GetDatamemberName(scope, idata); fFullType = Cppyy::GetDatamemberType(scope, idata); if (Cppyy::IsEnumData(scope, idata)) { - fFullType = Cppyy::ResolveEnum(fFullType); // enum might be any type of int + if (fFullType.find("(anonymous)") == std::string::npos && + fFullType.find("(unnamed)") == std::string::npos) { + // repurpose fDescription for lazy lookup of the enum later + fDescription = CPyCppyy_PyText_FromString((fFullType + "::" + name).c_str()); + fFlags |= kIsEnumPrep; + } + fFullType = Cppyy::ResolveEnum(fFullType); fFlags |= kIsConstData; } else if (Cppyy::IsConstData(scope, idata)) { fFlags |= kIsConstData; } - fConverter = CreateConverter(fFullType, dims.empty() ? nullptr : dims.data()); +// if this data member is an array, the conversion needs to be pointer to object for instances, +// to prevent the need for copying in the conversion; furthermore, fixed arrays' full type for +// builtins are not declared as such if more than 1-dim (TODO: fix in clingwrapper) + if (!dims.empty() && fFullType.back() != '*') { + if (Cppyy::GetScope(fFullType)) fFullType += '*'; + else if (fFullType.back() != ']') { + for (auto d: dims) fFullType += d == UNKNOWN_SIZE ? "*" : "[]"; + } + } + + if (dims.empty()) + fConverter = CreateConverter(fFullType); + else + fConverter = CreateConverter(fFullType, {(dim_t)dims.size(), dims.data()}); + + if (!(fFlags & kIsEnumPrep)) + fDescription = CPyCppyy_PyText_FromString(name.c_str()); } //----------------------------------------------------------------------------- void CPyCppyy::CPPDataMember::Set(Cppyy::TCppScope_t scope, const std::string& name, void* address) { fEnclosingScope = scope; - fName = CPyCppyy_PyText_FromString(name.c_str()); + fDescription = CPyCppyy_PyText_FromString(name.c_str()); fOffset = (intptr_t)address; fFlags = kIsStaticData | kIsConstData; fConverter = CreateConverter("internal_enum_type_t"); @@ -326,8 +405,30 @@ void* CPyCppyy::CPPDataMember::GetAddress(CPPInstance* pyobj) // the proxy's internal offset is calculated from the enclosing class ptrdiff_t offset = 0; - if (pyobj->ObjectIsA() != fEnclosingScope) - offset = Cppyy::GetBaseOffset(pyobj->ObjectIsA(), fEnclosingScope, obj, 1 /* up-cast */); + Cppyy::TCppType_t oisa = pyobj->ObjectIsA(); + if (oisa != fEnclosingScope) + offset = Cppyy::GetBaseOffset(oisa, fEnclosingScope, obj, 1 /* up-cast */); return (void*)((intptr_t)obj + offset + fOffset); } + + +//----------------------------------------------------------------------------- +std::string CPyCppyy::CPPDataMember::GetName() +{ + if (fFlags & kIsEnumType) { + PyObject* repr = PyObject_Repr(fDescription); + if (repr) { + std::string res = CPyCppyy_PyText_AsString(repr); + Py_DECREF(repr); + return res; + } + PyErr_Clear(); + return ""; + } else if (fFlags & kIsEnumPrep) { + std::string fullName = CPyCppyy_PyText_AsString(fDescription); + return fullName.substr(fullName.rfind("::")+2, std::string::npos); + } + + return CPyCppyy_PyText_AsString(fDescription); +} diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.h index 8740fbf9e3473..9fe32cfd93e72 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPDataMember.h @@ -17,16 +17,20 @@ class CPPDataMember { void Set(Cppyy::TCppScope_t scope, Cppyy::TCppIndex_t idata); void Set(Cppyy::TCppScope_t scope, const std::string& name, void* address); - std::string GetName() { return CPyCppyy_PyText_AsString(fName); } + std::string GetName(); void* GetAddress(CPPInstance* pyobj /* owner */); public: // public, as the python C-API works with C structs PyObject_HEAD intptr_t fOffset; - Long_t fFlags; + long fFlags; Converter* fConverter; Cppyy::TCppScope_t fEnclosingScope; - PyObject* fName; + PyObject* fDescription; + PyObject* fDoc; + + // TODO: data members should have a unique identifier, just like methods, + // so that reflection information can be recovered post-initialization std::string fFullType; private: // private, as the python C-API will handle creation diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPEnum.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPEnum.cxx new file mode 100644 index 0000000000000..58400fbece2c2 --- /dev/null +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPEnum.cxx @@ -0,0 +1,207 @@ +// Bindings +#include "CPyCppyy.h" +#include "CPPEnum.h" +#include "PyStrings.h" +#include "Utility.h" + + +//- private helpers ---------------------------------------------------------- +static PyObject* pytype_from_enum_type(const std::string& enum_type) +{ + if (enum_type == "char") + return (PyObject*)&CPyCppyy_PyText_Type; + else if (enum_type == "bool") + return (PyObject*)&PyInt_Type; // can't use PyBool_Type as base + else if (strstr("long", enum_type.c_str())) + return (PyObject*)&PyLong_Type; + return (PyObject*)&PyInt_Type; // covers most cases +} + +//---------------------------------------------------------------------------- +static PyObject* pyval_from_enum(const std::string& enum_type, PyObject* pytype, + PyObject* btype, Cppyy::TCppEnum_t etype, Cppyy::TCppIndex_t idata) { + long long llval = Cppyy::GetEnumDataValue(etype, idata); + + if (enum_type == "bool") { + PyObject* result = (bool)llval ? Py_True : Py_False; + Py_INCREF(result); + return result; // <- immediate return; + } + + PyObject* bval; + if (enum_type == "char") { + char val = (char)llval; + bval = CPyCppyy_PyText_FromStringAndSize(&val, 1); + } else if (enum_type == "int" || enum_type == "unsigned int") + bval = PyInt_FromLong((long)llval); + else + bval = PyLong_FromLongLong(llval); + + PyObject* args = PyTuple_New(1); + PyTuple_SET_ITEM(args, 0, bval); + PyObject* result = ((PyTypeObject*)btype)->tp_new((PyTypeObject*)pytype, args, nullptr); + Py_DECREF(args); + return result; +} + + +//- enum methods ------------------------------------------------------------- +static int enum_setattro(PyObject* /* pyclass */, PyObject* /* pyname */, PyObject* /* pyval */) +{ +// Helper to make enums read-only. + PyErr_SetString(PyExc_TypeError, "enum values are read-only"); + return -1; +} + +//---------------------------------------------------------------------------- +static PyObject* enum_repr(PyObject* self) +{ + using namespace CPyCppyy; + + PyObject* kls_cppname = PyObject_GetAttr((PyObject*)Py_TYPE(self), PyStrings::gCppName); + if (!kls_cppname) PyErr_Clear(); + PyObject* obj_cppname = PyObject_GetAttr(self, PyStrings::gCppName); + if (!obj_cppname) PyErr_Clear(); + PyObject* obj_str = Py_TYPE(self)->tp_str(self); + + PyObject* repr = nullptr; + if (kls_cppname && obj_cppname && obj_str) { + const std::string resolved = Cppyy::ResolveEnum(CPyCppyy_PyText_AsString(kls_cppname)); + repr = CPyCppyy_PyText_FromFormat("(%s::%s) : (%s) %s", + CPyCppyy_PyText_AsString(kls_cppname), CPyCppyy_PyText_AsString(obj_cppname), + resolved.c_str(), CPyCppyy_PyText_AsString(obj_str)); + } + Py_XDECREF(obj_cppname); + Py_XDECREF(kls_cppname); + + if (repr) { + Py_DECREF(obj_str); + return repr; + } + + return obj_str; +} + + +//---------------------------------------------------------------------------- +// TODO: factor the following lookup with similar codes in Convertes and TemplateProxy.cxx + +static std::map gCTypesNames = { + {"bool", "c_bool"}, + {"char", "c_char"}, {"wchar_t", "c_wchar"}, + {"std::byte", "c_byte"}, {"int8_t", "c_byte"}, {"uint8_t", "c_ubyte"}, + {"short", "c_short"}, {"int16_t", "c_int16"}, {"unsigned short", "c_ushort"}, {"uint16_t", "c_uint16"}, + {"int", "c_int"}, {"unsigned int", "c_uint"}, + {"long", "c_long"}, {"unsigned long", "c_ulong"}, + {"long long", "c_longlong"}, {"unsigned long long", "c_ulonglong"}}; + +// Both GetCTypesType and GetCTypesPtrType, rely on the ctypes module itself +// caching the types (thus also making them unique), so no ref-count is needed. +// Further, by keeping a ref-count on the module, it won't be off-loaded until +// the 2nd cleanup cycle. +static PyTypeObject* GetCTypesType(const std::string& cppname) +{ + static PyObject* ctmod = PyImport_ImportModule("ctypes"); // ref-count kept + if (!ctmod) + return nullptr; + + auto nn = gCTypesNames.find(cppname); + if (nn == gCTypesNames.end()) { + PyErr_Format(PyExc_TypeError, "Can not find ctypes type for \"%s\"", cppname.c_str()); + return nullptr; + } + + return (PyTypeObject*)PyObject_GetAttrString(ctmod, nn->second.c_str()); +} + +static PyObject* enum_ctype(PyObject* cls, PyObject* args, PyObject* kwds) +{ + PyObject* pyres = PyObject_GetAttr(cls, CPyCppyy::PyStrings::gUnderlying); + if (!pyres) PyErr_Clear(); + + std::string underlying = pyres ? CPyCppyy_PyText_AsString(pyres) : "int"; + PyTypeObject* ct = GetCTypesType(underlying); + if (!ct) + return nullptr; + + return PyType_Type.tp_call((PyObject*)ct, args, kwds); +} + + +//- creation ----------------------------------------------------------------- +CPyCppyy::CPPEnum* CPyCppyy::CPPEnum_New(const std::string& name, Cppyy::TCppScope_t scope) +{ +// Create a new enum type based on the actual C++ type. Enum values are added to +// the type by may also live in the enclosing scope. + + CPPEnum* pyenum = nullptr; + + const std::string& ename = scope == Cppyy::gGlobalScope ? name : Cppyy::GetScopedFinalName(scope)+"::"+name; + Cppyy::TCppEnum_t etype = Cppyy::GetEnum(scope, name); + if (etype) { + // create new enum type with labeled values in place, with a meta-class + // to make sure the enum values are read-only + const std::string& resolved = Cppyy::ResolveEnum(ename); + PyObject* pyside_type = pytype_from_enum_type(resolved); + PyObject* pymetabases = PyTuple_New(1); + PyObject* btype = (PyObject*)Py_TYPE(pyside_type); + Py_INCREF(btype); + PyTuple_SET_ITEM(pymetabases, 0, btype); + + PyObject* args = Py_BuildValue((char*)"sO{}", (name+"_meta").c_str(), pymetabases); + Py_DECREF(pymetabases); + PyObject* pymeta = PyType_Type.tp_new(Py_TYPE(pyside_type), args, nullptr); + Py_DECREF(args); + + // prepare the base class + PyObject* pybases = PyTuple_New(1); + Py_INCREF(pyside_type); + PyTuple_SET_ITEM(pybases, 0, (PyObject*)pyside_type); + + // create the __cpp_name__ for templates + PyObject* dct = PyDict_New(); + PyObject* pycppname = CPyCppyy_PyText_FromString(ename.c_str()); + PyDict_SetItem(dct, PyStrings::gCppName, pycppname); + Py_DECREF(pycppname); + PyObject* pyresolved = CPyCppyy_PyText_FromString(resolved.c_str()); + PyDict_SetItem(dct, PyStrings::gUnderlying, pyresolved); + Py_DECREF(pyresolved); + + // create the actual enum class + args = Py_BuildValue((char*)"sOO", name.c_str(), pybases, dct); + Py_DECREF(pybases); + Py_DECREF(dct); + pyenum = ((PyTypeObject*)pymeta)->tp_new((PyTypeObject*)pymeta, args, nullptr); + + // add pythonizations + Utility::AddToClass( + (PyObject*)Py_TYPE(pyenum), "__ctype__", (PyCFunction)enum_ctype, METH_VARARGS | METH_KEYWORDS); + ((PyTypeObject*)pyenum)->tp_repr = enum_repr; + ((PyTypeObject*)pyenum)->tp_str = ((PyTypeObject*)pyside_type)->tp_repr; + + // collect the enum values + Cppyy::TCppIndex_t ndata = Cppyy::GetNumEnumData(etype); + for (Cppyy::TCppIndex_t idata = 0; idata < ndata; ++idata) { + PyObject* val = pyval_from_enum(resolved, pyenum, pyside_type, etype, idata); + PyObject* pydname = CPyCppyy_PyText_FromString(Cppyy::GetEnumDataName(etype, idata).c_str()); + PyObject_SetAttr(pyenum, pydname, val); + PyObject_SetAttr(val, PyStrings::gCppName, pydname); + Py_DECREF(pydname); + Py_DECREF(val); + } + + // disable writing onto enum values + ((PyTypeObject*)pymeta)->tp_setattro = enum_setattro; + + // final cleanup + Py_DECREF(args); + Py_DECREF(pymeta); + + } else { + // presumably not a class enum; simply pretend int + Py_INCREF(&PyInt_Type); + pyenum = (PyObject*)&PyInt_Type; + } + + return pyenum; +} diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPEnum.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPEnum.h new file mode 100644 index 0000000000000..730baf4ba75d9 --- /dev/null +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPEnum.h @@ -0,0 +1,16 @@ +#ifndef CPYCPPYY_CPPENUM_H +#define CPYCPPYY_CPPENUM_H + + +namespace CPyCppyy { + +// CPPEnum does not carry any additional C-side data for now, but can be of +// several types, based on the declared or compile-dependent types chosen. +typedef PyObject CPPEnum; + +//- creation ----------------------------------------------------------------- +CPPEnum* CPPEnum_New(const std::string& name, Cppyy::TCppScope_t scope); + +} // namespace CPyCppyy + +#endif // !CPYCPPYY_CPPENUM_H diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPExcInstance.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPExcInstance.cxx index e3a5663d099e0..68c366e191c79 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPExcInstance.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPExcInstance.cxx @@ -29,7 +29,7 @@ static PyObject* ep_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) PyObject* ulc = PyObject_GetAttr((PyObject*)subtype, PyStrings::gUnderlying); excobj->fCppInstance = PyType_Type.tp_call(ulc, args, kwds); if (!excobj->fCppInstance) { - // if this fails, then the contruction may have been attempted from a string + // if this fails, then the construction may have been attempted from a string // (e.g. from PyErr_Format); if so, drop the proxy and use fTopMessage instead PyErr_Clear(); if (PyTuple_GET_SIZE(args) == 1) { @@ -229,10 +229,10 @@ PyTypeObject CPPExcInstance_Type = { sizeof(CPPExcInstance), // tp_basicsize 0, // tp_itemsize (destructor)ep_dealloc, // tp_dealloc - 0, // tp_print + 0, // tp_vectorcall_offset / tp_print 0, // tp_getattr 0, // tp_setattr - 0, // tp_compare + 0, // tp_as_async / tp_compare (reprfunc)ep_repr, // tp_repr &ep_as_number, // tp_as_number 0, // tp_as_sequence @@ -247,7 +247,11 @@ PyTypeObject CPPExcInstance_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_CHECKTYPES, // tp_flags + Py_TPFLAGS_CHECKTYPES +#if PY_VERSION_HEX >= 0x03120000 + | Py_TPFLAGS_MANAGED_DICT +#endif + , // tp_flags (char*)"cppyy exception object proxy (internal)", // tp_doc (traverseproc)ep_traverse, // tp_traverse (inquiry)ep_clear, // tp_clear @@ -282,6 +286,9 @@ PyTypeObject CPPExcInstance_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.cxx index 37a527b6f4875..d5a4d45007d29 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.cxx @@ -3,31 +3,63 @@ #include "CPPFunction.h" #include "CPPInstance.h" +// Standard +#include -//- CPPFunction public members -------------------------------------------------- -PyObject* CPyCppyy::CPPFunction::PreProcessArgs( - CPPInstance*& self, PyObject* args, PyObject* kwds) -{ -// add self as part of the function arguments (means bound member) - if (kwds) return this->ProcessKeywords((PyObject*)self, args, kwds); - Py_ssize_t sz = PyTuple_GET_SIZE(args); - PyObject* newArgs = PyTuple_New(sz+1); +//- CFunction helpers ----------------------------------------------------------- +bool CPyCppyy::AdjustSelf(PyCallArgs& cargs) +{ +#if PY_VERSION_HEX >= 0x03080000 + if (cargs.fNArgsf & PY_VECTORCALL_ARGUMENTS_OFFSET) { // mutation allowed? + std::swap(((PyObject**)cargs.fArgs-1)[0], (PyObject*&)cargs.fSelf); + cargs.fFlags |= PyCallArgs::kSelfSwap; + cargs.fArgs -= 1; + cargs.fNArgsf &= ~PY_VECTORCALL_ARGUMENTS_OFFSET; + cargs.fNArgsf += 1; + } else { + Py_ssize_t nkwargs = cargs.fKwds ? PyTuple_GET_SIZE(cargs.fKwds) : 0; + Py_ssize_t totalargs = PyVectorcall_NARGS(cargs.fNArgsf)+nkwargs; + PyObject** newArgs = (PyObject**)PyMem_Malloc((totalargs+1) * sizeof(PyObject*)); + if (!newArgs) + return false; + + newArgs[0] = (PyObject*)cargs.fSelf; + if (0 < totalargs) + memcpy((void*)&newArgs[1], cargs.fArgs, totalargs * sizeof(PyObject*)); + cargs.fArgs = newArgs; + cargs.fFlags |= PyCallArgs::kDoFree; + cargs.fNArgsf += 1; + } +#else + Py_ssize_t sz = PyTuple_GET_SIZE(cargs.fArgs); + CPyCppyy_PyArgs_t newArgs = PyTuple_New(sz+1); for (int i = 0; i < sz; ++i) { - PyObject* item = PyTuple_GET_ITEM(args, i); + PyObject* item = PyTuple_GET_ITEM(cargs.fArgs, i); Py_INCREF(item); PyTuple_SET_ITEM(newArgs, i+1, item); } + Py_INCREF(cargs.fSelf); + PyTuple_SET_ITEM(newArgs, 0, (PyObject*)cargs.fSelf); + + cargs.fArgs = newArgs; + cargs.fFlags |= PyCallArgs::kDoDecref; + cargs.fNArgsf += 1; +#endif + return true; +} - Py_INCREF(self); - PyTuple_SET_ITEM(newArgs, 0, (PyObject*)self); - - return newArgs; +bool CPyCppyy::CPPFunction::ProcessArgs(PyCallArgs& cargs) +{ +// add self as part of the function arguments (means bound member) + if (cargs.fKwds) + return this->ProcessKwds((PyObject*)cargs.fSelf, cargs); + return AdjustSelf(cargs); } -//--------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPFunction::Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt) +//- CPPFunction public members -------------------------------------------------- +PyObject* CPyCppyy::CPPFunction::Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt) { // setup as necessary if (fArgsRequired == -1 && !this->Initialize(ctxt)) @@ -35,48 +67,68 @@ PyObject* CPyCppyy::CPPFunction::Call( // if function was attached to a class, self will be non-zero and should be // the first function argument, so reorder + PyCallArgs cargs{self, args, nargsf, kwds}; if (self || kwds) { - if (!(args = this-> PreProcessArgs(self, args, kwds))) + if (!this->ProcessArgs(cargs)) return nullptr; } -// translate the arguments as normal - bool bConvertOk = this->ConvertAndSetArgs(args, ctxt); - - if (self || kwds) Py_DECREF(args); +#if PY_VERSION_HEX >= 0x03080000 +// special case, if this method was inserted as a constructor, then self is nullptr +// and it will be the first argument and needs to be used as Python context + if (IsConstructor(ctxt->fFlags) && !ctxt->fPyContext && \ + CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf)) { + ctxt->fPyContext = cargs.fArgs[0]; + } +#endif - if (bConvertOk == false) +// translate the arguments as normal + if (!this->ConvertAndSetArgs(cargs.fArgs, cargs.fNArgsf, ctxt)) return nullptr; // execute function - return this->Execute(nullptr, 0, ctxt); + PyObject* result = this->Execute(nullptr, 0, ctxt); + +#if PY_VERSION_HEX >= 0x03080000 +// special case, if this method was inserted as a constructor, then if no self was +// provided, it will be the first argument and may have been updated + if (IsConstructor(ctxt->fFlags) && result && !cargs.fSelf && \ + CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf) && CPPInstance_Check(cargs.fArgs[0])) { + self = (CPPInstance*)cargs.fArgs[0]; + Py_INCREF(self); + } +#endif + + return result; } -//- CPPReverseBinary public members --------------------------------------------- -PyObject* CPyCppyy::CPPReverseBinary::PreProcessArgs( - CPPInstance*& self, PyObject* args, PyObject* kwds) +//- CPPReverseBinary private helper --------------------------------------------- +bool CPyCppyy::CPPReverseBinary::ProcessArgs(PyCallArgs& cargs) { - if (self || kwds) { + if (cargs.fSelf || cargs.fKwds) { // add self as part of the function arguments (means bound member) - if (!(args = this->CPPFunction::PreProcessArgs(self, args, kwds))) - return nullptr; + if (!this->CPPFunction::ProcessArgs(cargs)) + return false; } // swap the arguments - PyObject* tmp = PyTuple_GET_ITEM(args, 0); - PyTuple_SET_ITEM(args, 0, PyTuple_GET_ITEM(args, 1)); - PyTuple_SET_ITEM(args, 1, tmp); - - return args; +#if PY_VERSION_HEX >= 0x03080000 + std::swap(((PyObject**)cargs.fArgs)[0], ((PyObject**)cargs.fArgs)[1]); +#else + std::swap(PyTuple_GET_ITEM(cargs.fArgs, 0), PyTuple_GET_ITEM(cargs.fArgs, 1)); +#endif + cargs.fFlags |= PyCallArgs::kArgsSwap; + + return true; } -//--------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPReverseBinary::Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt) +//- CPPReverseBinary public members --------------------------------------------- +PyObject* CPyCppyy::CPPReverseBinary::Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt) { -// This Call() function is very similar to the one of CPPFunction: only -// difference is that PreProcessArgs() is always called. +// This Call() function is very similar to the one of CPPFunction: only difference is +// that ProcessArgs() is always called. // setup as necessary if (fArgsRequired == -1 && !this->Initialize(ctxt)) @@ -84,15 +136,12 @@ PyObject* CPyCppyy::CPPReverseBinary::Call( // if function was attached to a class, self will be non-zero and should be // the first function argument, further, the arguments needs swapping - if (!(args = this->PreProcessArgs(self, args, kwds))) + PyCallArgs cargs{self, args, nargsf, kwds}; + if (!this->ProcessArgs(cargs)) return nullptr; // translate the arguments as normal - bool bConvertOk = this->ConvertAndSetArgs(args, ctxt); - - if (self || kwds) Py_DECREF(args); - - if (bConvertOk == false) + if (!this->ConvertAndSetArgs(cargs.fArgs, cargs.fNArgsf, ctxt)) return nullptr; // execute function diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.h index 20d8d89e7fb45..2a770ce1f6a30 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPFunction.h @@ -13,12 +13,11 @@ class CPPFunction : public CPPMethod { using CPPMethod::CPPMethod; virtual PyCallable* Clone() { return new CPPFunction(*this); } - - virtual PyObject* Call( - CPPInstance*&, PyObject* args, PyObject* kwds, CallContext* ctx = nullptr); + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); protected: - virtual PyObject* PreProcessArgs(CPPInstance*& self, PyObject* args, PyObject* kwds); + virtual bool ProcessArgs(PyCallArgs& args); }; // Wrapper for global binary operators that swap arguments @@ -27,14 +26,16 @@ class CPPReverseBinary : public CPPFunction { using CPPFunction::CPPFunction; virtual PyCallable* Clone() { return new CPPFunction(*this); } - - virtual PyObject* Call( - CPPInstance*&, PyObject* args, PyObject* kwds, CallContext* ctx = nullptr); + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); protected: - virtual PyObject* PreProcessArgs(CPPInstance*& self, PyObject* args, PyObject* kwds); + virtual bool ProcessArgs(PyCallArgs& args); }; +// Helper to add self to the arguments tuple if rebound +bool AdjustSelf(PyCallArgs& cargs); + } // namespace CPyCppyy #endif // !CPYCPPYY_CPPFUNCTION_H diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.cxx index 01bed171ad9fe..2ed852ff9d02c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.cxx @@ -4,6 +4,26 @@ #include "Executors.h" +//- private helpers ----------------------------------------------------------- +static inline +void unroll(CPyCppyy_PyArgs_t packed_args, CPyCppyy_PyArgs_t unrolled, Py_ssize_t nArgs) +{ +// Unroll up to nArgs arguments from packed_args into unrolled. + for (int i = 0, iur = 0; i < nArgs; ++i, ++iur) { + PyObject* item = CPyCppyy_PyArgs_GET_ITEM(packed_args, i); + if (PyTuple_Check(item)) { + for (int j = 0; j < PyTuple_GET_SIZE(item); ++j, ++iur) { + PyObject* subitem = PyTuple_GET_ITEM(item, j); + Py_INCREF(subitem); + CPyCppyy_PyArgs_SET_ITEM(unrolled, iur, subitem); + } + } else { + Py_INCREF(item); + CPyCppyy_PyArgs_SET_ITEM(unrolled, iur, item); + } + } +} + //- protected members --------------------------------------------------------- bool CPyCppyy::CPPSetItem::InitExecutor_(Executor*& executor, CallContext*) { @@ -23,97 +43,77 @@ bool CPyCppyy::CPPSetItem::InitExecutor_(Executor*& executor, CallContext*) } //----------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPSetItem::PreProcessArgs( - CPPInstance*& self, PyObject* args, PyObject* kwds) +bool CPyCppyy::CPPSetItem::ProcessArgs(PyCallArgs& cargs) { // Prepare executor with a buffer for the return value. - Py_ssize_t nArgs = PyTuple_GET_SIZE(args); + Py_ssize_t nArgs = CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf); if (nArgs <= 1) { PyErr_SetString(PyExc_TypeError, "insufficient arguments to __setitem__"); - return nullptr; + return false; } -// strip the last element of args to be used on return - ((RefExecutor*)this->GetExecutor())->SetAssignable(PyTuple_GET_ITEM(args, nArgs-1)); - PyObject* subset = PyTuple_GetSlice(args, 0, nArgs-1); +// use the last element of args for assignment upon return, then slice it from +// the (unrolled) actual arguments + ((RefExecutor*)this->GetExecutor())->SetAssignable(CPyCppyy_PyArgs_GET_ITEM(cargs.fArgs, nArgs-1)); -// see whether any of the arguments is a tuple itself +// see whether any of the arguments is a tuple to be unrolled Py_ssize_t realsize = 0; - for (Py_ssize_t i = 0; i < nArgs - 1; ++i) { - PyObject* item = PyTuple_GET_ITEM(subset, i); + for (Py_ssize_t i = 0; i < nArgs-1; ++i) { + PyObject* item = CPyCppyy_PyArgs_GET_ITEM(cargs.fArgs, i); realsize += PyTuple_Check(item) ? PyTuple_GET_SIZE(item) : 1; } // unroll any tuples, if present in the arguments - PyObject* unrolled = 0; +#if PY_VERSION_HEX >= 0x03080000 if (realsize != nArgs-1) { - unrolled = PyTuple_New(realsize); - - int current = 0; - for (int i = 0; i < nArgs - 1; ++i, ++current) { - PyObject* item = PyTuple_GET_ITEM(subset, i); - if (PyTuple_Check(item)) { - for (int j = 0; j < PyTuple_GET_SIZE(item); ++j, ++current) { - PyObject* subitem = PyTuple_GET_ITEM(item, j); - Py_INCREF(subitem); - PyTuple_SET_ITEM(unrolled, current, subitem); - } - } else { - Py_INCREF(item); - PyTuple_SET_ITEM(unrolled, current, item); - } - } + CPyCppyy_PyArgs_t unrolled = (PyObject**)PyMem_Malloc(realsize * sizeof(PyObject*)); + unroll(cargs.fArgs, unrolled, nArgs-1); + cargs.fArgs = unrolled; + cargs.fFlags |= PyCallArgs::kDoFree; } +#else + if (realsize != nArgs-1) { + CPyCppyy_PyArgs_t unrolled = PyTuple_New(realsize); + unroll(cargs.fArgs, unrolled, nArgs-1); + cargs.fArgs = unrolled; + } else + cargs.fArgs = PyTuple_GetSlice(cargs.fArgs, 0, nArgs-1); + cargs.fFlags |= PyCallArgs::kDoDecref; +#endif + cargs.fNArgsf = realsize; // continue normal method processing - PyObject* result = CPPMethod::PreProcessArgs(self, unrolled ? unrolled : subset, kwds); - - Py_XDECREF(unrolled); - Py_DECREF(subset); - return result; + return CPPMethod::ProcessArgs(cargs); } //----------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPGetItem::PreProcessArgs( - CPPInstance*& self, PyObject* args, PyObject* kwds) +bool CPyCppyy::CPPGetItem::ProcessArgs(PyCallArgs& cargs) { -// Unroll tuples for call, otherwise just like CPPMethod (this is very similar -// to the code in CPPSetItem above, but subtly different in the details, hence -// not factored out). - Py_ssize_t nArgs = PyTuple_GET_SIZE(args); +// Unroll tuples for call, otherwise just like regular CPPMethod of __getitem__. + Py_ssize_t nArgs = CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf); -// see whether any of the arguments is a tuple itself +// see whether any of the arguments is a tuple to be unrolled Py_ssize_t realsize = 0; for (Py_ssize_t i = 0; i < nArgs; ++i) { - PyObject* item = PyTuple_GET_ITEM(args, i); + PyObject* item = CPyCppyy_PyArgs_GET_ITEM(cargs.fArgs, i); realsize += PyTuple_Check(item) ? PyTuple_GET_SIZE(item) : 1; } // unroll any tuples, if present in the arguments - PyObject* unrolled = 0; - if (realsize != nArgs-1) { - unrolled = PyTuple_New(realsize); - - int current = 0; - for (int i = 0; i < nArgs; ++i, ++current) { - PyObject* item = PyTuple_GET_ITEM(args, i); - if (PyTuple_Check(item)) { - for (int j = 0; j < PyTuple_GET_SIZE(item); ++j, ++current) { - PyObject* subitem = PyTuple_GET_ITEM(item, j); - Py_INCREF(subitem); - PyTuple_SET_ITEM(unrolled, current, subitem); - } - } else { - Py_INCREF(item); - PyTuple_SET_ITEM(unrolled, current, item); - } - } + if (realsize != nArgs) { + CPyCppyy_PyArgs_t packed_args = cargs.fArgs; +#if PY_VERSION_HEX >= 0x03080000 + cargs.fArgs = (PyObject**)PyMem_Malloc(realsize * sizeof(PyObject*)); + cargs.fFlags |= PyCallArgs::kDoFree; +#else + cargs.fArgs = PyTuple_New(realsize); + cargs.fFlags |= PyCallArgs::kDoDecref; +#endif + cargs.fNArgsf = realsize; + unroll(packed_args, cargs.fArgs, nArgs); } // continue normal method processing - PyObject* result = CPPMethod::PreProcessArgs(self, unrolled ? unrolled : args, kwds); - - Py_XDECREF(unrolled); - return result; + return CPPMethod::ProcessArgs(cargs); } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.h index d7244a7a9f581..76f0cceead7e1 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPGetSetItem.h @@ -15,7 +15,7 @@ class CPPSetItem : public CPPMethod { virtual PyCallable* Clone() { return new CPPSetItem(*this); } protected: - virtual PyObject* PreProcessArgs(CPPInstance*& self, PyObject* args, PyObject* kwds); + virtual bool ProcessArgs(PyCallArgs& args); virtual bool InitExecutor_(Executor*&, CallContext* ctxt = nullptr); }; @@ -27,7 +27,7 @@ class CPPGetItem : public CPPMethod { virtual PyCallable* Clone() { return new CPPGetItem(*this); } protected: - virtual PyObject* PreProcessArgs(CPPInstance*& self, PyObject* args, PyObject* kwds); + virtual bool ProcessArgs(PyCallArgs& args); }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx index f2eea396af1bc..9cc4f47d8aa52 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx @@ -16,6 +16,11 @@ #include +//- data _____________________________________________________________________ +namespace CPyCppyy { + extern PyObject* gNullPtrObject; +} + //______________________________________________________________________________ // Python-side proxy objects // ========================= @@ -45,7 +50,7 @@ namespace { // extended data can slot in place of fObject for those use cases. struct ExtendedData { - ExtendedData() : fObject(nullptr), fSmartClass(nullptr), fTypeSize(0), fLastState(nullptr), fDispatchPtr(nullptr) {} + ExtendedData() : fObject(nullptr), fSmartClass(nullptr), fDispatchPtr(nullptr), fArraySize(0) {} ~ExtendedData() { for (auto& pc : fDatamemberCache) Py_XDECREF(pc.second); @@ -56,25 +61,27 @@ struct ExtendedData { // in GetObjectRaw(), e.g. for ptr-ptr passing) void* fObject; -// for smart pointer types - CPyCppyy::CPPSmartClass* fSmartClass; - size_t fTypeSize; - void* fLastState; - // for caching expensive-to-create data member representations CPyCppyy::CI_DatamemberCache_t fDatamemberCache; +// for smart pointer types + CPyCppyy::CPPSmartClass* fSmartClass; + // for back-referencing from Python-derived instances CPyCppyy::DispatchPtr* fDispatchPtr; + +// for representing T* as a low-level array + Py_ssize_t fArraySize; }; } // unnamed namespace #define EXT_OBJECT(pyobj) ((ExtendedData*)((pyobj)->fObject))->fObject +#define DATA_CACHE(pyobj) ((ExtendedData*)((pyobj)->fObject))->fDatamemberCache #define SMART_CLS(pyobj) ((ExtendedData*)((pyobj)->fObject))->fSmartClass #define SMART_TYPE(pyobj) SMART_CLS(pyobj)->fCppType #define DISPATCHPTR(pyobj) ((ExtendedData*)((pyobj)->fObject))->fDispatchPtr -#define DATA_CACHE(pyobj) ((ExtendedData*)((pyobj)->fObject))->fDatamemberCache +#define ARRAY_SIZE(pyobj) ((ExtendedData*)((pyobj)->fObject))->fArraySize inline void CPyCppyy::CPPInstance::CreateExtension() { if (fFlags & kIsExtended) @@ -97,12 +104,12 @@ void* CPyCppyy::CPPInstance::GetExtendedObject() //- public methods ----------------------------------------------------------- -CPyCppyy::CPPInstance* CPyCppyy::CPPInstance::Copy(void* cppinst) +CPyCppyy::CPPInstance* CPyCppyy::CPPInstance::Copy(void* cppinst, PyTypeObject* target) { // create a fresh instance; args and kwds are not used by op_new (see below) PyObject* self = (PyObject*)this; - PyTypeObject* pytype = Py_TYPE(self); - PyObject* newinst = pytype->tp_new(pytype, nullptr, nullptr); + if (!target) target = Py_TYPE(self); + PyObject* newinst = target->tp_new(target, nullptr, nullptr); // set the C++ instance as given ((CPPInstance*)newinst)->fObject = cppinst; @@ -207,13 +214,12 @@ void CPyCppyy::op_dealloc_nofree(CPPInstance* pyobj) { if (pyobj->fFlags & CPPInstance::kIsRegulated) MemoryRegulator::UnregisterPyObject(pyobj, (PyObject*)Py_TYPE((PyObject*)pyobj)); - if (pyobj->fFlags & CPPInstance::kIsOwner) { - if (pyobj->fFlags & CPPInstance::kIsValue) { - Cppyy::CallDestructor(klass, cppobj); - Cppyy::Deallocate(klass, cppobj); - } else { - if (cppobj) Cppyy::Destruct(klass, cppobj); - } + if (cppobj && (pyobj->fFlags & CPPInstance::kIsOwner)) { + if (pyobj->fFlags & CPPInstance::kIsValue) { + Cppyy::CallDestructor(klass, cppobj); + Cppyy::Deallocate(klass, cppobj); + } else + Cppyy::Destruct(klass, cppobj); } cppobj = nullptr; @@ -300,14 +306,95 @@ static PyObject* op_get_smartptr(CPPInstance* self) return CPyCppyy::BindCppObjectNoCast(self->GetSmartObject(), SMART_TYPE(self), CPPInstance::kNoWrapConv); } +//= pointer-as-array support for legacy C code =============================== +void CPyCppyy::CPPInstance::CastToArray(Py_ssize_t sz) +{ + CreateExtension(); + fFlags |= kIsArray; + ARRAY_SIZE(this) = sz; +} + +Py_ssize_t CPyCppyy::CPPInstance::ArrayLength() { + if (!(fFlags & kIsArray)) + return -1; + return (Py_ssize_t)ARRAY_SIZE(this); +} + +static PyObject* op_reshape(CPPInstance* self, PyObject* shape) +{ +// Allow the user to fix up the actual (type-strided) size of the buffer. + if (!PyTuple_Check(shape) || PyTuple_GET_SIZE(shape) != 1) { + PyErr_SetString(PyExc_TypeError, "tuple object of size 1 expected"); + return nullptr; + } + + long sz = PyLong_AsLong(PyTuple_GET_ITEM(shape, 0)); + if (sz <= 0) { + PyErr_SetString(PyExc_ValueError, "array length must be positive"); + return nullptr; + } + + self->CastToArray(sz); + + Py_RETURN_NONE; +} + +static PyObject* op_getitem(CPPInstance* self, PyObject* pyidx) +{ +// In C, it is common to represent an array of structs as a pointer to the first +// object in the array. If the caller indexes a pointer to an object that does not +// define indexing, then highly likely such C-style indexing is the goal. Just +// like C, this is potentially unsafe, so caveat emptor. + + if (!(self->fFlags & (CPPInstance::kIsReference | CPPInstance::kIsArray))) { + PyErr_Format(PyExc_TypeError, "%s object does not support indexing", Py_TYPE(self)->tp_name); + return nullptr; + } + + Py_ssize_t idx = PyInt_AsSsize_t(pyidx); + if (idx == (Py_ssize_t)-1 && PyErr_Occurred()) + return nullptr; + + if (idx < 0) { + // this is debatable, and probably should not care, but the use case is pretty + // circumscribed anyway, so might as well keep the functionality simple + PyErr_SetString(PyExc_IndexError, "negative indices not supported for array of structs"); + return nullptr; + } + + if (self->fFlags & CPPInstance::kIsArray) { + Py_ssize_t maxidx = ARRAY_SIZE(self); + if (0 <= maxidx && maxidx <= idx) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return nullptr; + } + } + + unsigned flags = 0; size_t sz = sizeof(void*); + if (self->fFlags & CPPInstance::kIsPtrPtr) { + flags = CPPInstance::kIsReference; + } else { + sz = Cppyy::SizeOf(((CPPClass*)Py_TYPE(self))->fCppType); + } + + uintptr_t address = (uintptr_t)(flags ? self->GetObjectRaw() : self->GetObject()); + void* indexed_obj = (void*)(address+(uintptr_t)(idx*sz)); + + return BindCppObjectNoCast(indexed_obj, ((CPPClass*)Py_TYPE(self))->fCppType, flags); +} //---------------------------------------------------------------------------- static PyMethodDef op_methods[] = { - {(char*)"__destruct__", (PyCFunction)op_destruct, METH_NOARGS, nullptr}, + {(char*)"__destruct__", (PyCFunction)op_destruct, METH_NOARGS, + (char*)"call the C++ destructor"}, {(char*)"__dispatch__", (PyCFunction)op_dispatch, METH_VARARGS, (char*)"dispatch to selected overload"}, {(char*)"__smartptr__", (PyCFunction)op_get_smartptr, METH_NOARGS, (char*)"get associated smart pointer, if any"}, + {(char*)"__getitem__", (PyCFunction)op_getitem, METH_O, + (char*)"pointer dereferencing"}, + {(char*)"__reshape__", (PyCFunction)op_reshape, METH_O, + (char*)"cast pointer to 1D array type"}, {(char*)nullptr, nullptr, 0, nullptr} }; @@ -348,6 +435,21 @@ static inline PyObject* eqneq_binop(CPPClass* klass, PyObject* self, PyObject* o { using namespace Utility; +// special case for C++11 style nullptr + if (obj == gNullPtrObject) { + void* rawcpp = ((CPPInstance*)self)->GetObjectRaw(); + switch (op) { + case Py_EQ: + if (rawcpp == nullptr) Py_RETURN_TRUE; + Py_RETURN_FALSE; + case Py_NE: + if (rawcpp != nullptr) Py_RETURN_TRUE; + Py_RETURN_FALSE; + default: + return nullptr; // not implemented + } + } + if (!klass->fOperators) klass->fOperators = new PyOperators{}; @@ -397,37 +499,109 @@ static inline PyObject* eqneq_binop(CPPClass* klass, PyObject* self, PyObject* o Py_RETURN_TRUE; } +static inline void* cast_actual(void* obj) { + void* address = ((CPPInstance*)obj)->GetObject(); + if (((CPPInstance*)obj)->fFlags & CPPInstance::kIsActual) + return address; + + Cppyy::TCppType_t klass = ((CPPClass*)Py_TYPE((PyObject*)obj))->fCppType; + Cppyy::TCppType_t clActual = Cppyy::GetActualClass(klass, address); + if (clActual && clActual != klass) { + intptr_t offset = Cppyy::GetBaseOffset( + clActual, klass, address, -1 /* down-cast */, true /* report errors */); + if (offset != -1) address = (void*)((intptr_t)address + offset); + } + + return address; +} + + +#define CPYCPPYY_ORDERED_OPERATOR_STUB(op, ometh, label) \ + if (!ometh) { \ + PyCallable* pyfunc = Utility::FindBinaryOperator((PyObject*)self, other, #op);\ + if (pyfunc) \ + ometh = (PyObject*)CPPOverload_New(#label, pyfunc); \ + } \ + meth = ometh; + static PyObject* op_richcompare(CPPInstance* self, PyObject* other, int op) { -// Rich set of comparison objects; only equals and not-equals are defined. - if (op != Py_EQ && op != Py_NE) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } +// Rich set of comparison objects; currently supported: +// == : Py_EQ +// != : Py_NE +// +// < : Py_LT +// <= : Py_LE +// > : Py_GT +// >= : Py_GE +// + +// associative comparison operators + if (op == Py_EQ || op == Py_NE) { + // special case for None to compare True to a null-pointer + if ((PyObject*)other == Py_None && !self->fObject) { + if (op == Py_EQ) { Py_RETURN_TRUE; } + Py_RETURN_FALSE; + } + + // use C++-side operators if available + PyObject* result = eqneq_binop((CPPClass*)Py_TYPE(self), (PyObject*)self, other, op); + if (!result && CPPInstance_Check(other)) + result = eqneq_binop((CPPClass*)Py_TYPE(other), other, (PyObject*)self, op); + if (result) return result; + + // default behavior: type + held pointer value defines identity; if both are + // CPPInstance objects, perform an additional autocast if need be + bool bIsEq = false; + + if ((Py_TYPE(self) == Py_TYPE(other) && \ + self->GetObject() == ((CPPInstance*)other)->GetObject())) { + // direct match + bIsEq = true; + } else if (CPPInstance_Check(other)) { + // try auto-cast match + void* addr1 = cast_actual(self); + void* addr2 = cast_actual(other); + bIsEq = addr1 && addr2 && (addr1 == addr2); + } + + if ((op == Py_EQ && bIsEq) || (op == Py_NE && !bIsEq)) + Py_RETURN_TRUE; -// special case for None to compare True to a null-pointer - if ((PyObject*)other == Py_None && !self->fObject) { - if (op == Py_EQ) { Py_RETURN_TRUE; } Py_RETURN_FALSE; } -// use C++-side operators if available - PyObject* result = eqneq_binop((CPPClass*)Py_TYPE(self), (PyObject*)self, other, op); - if (!result && CPPInstance_Check(other)) - result = eqneq_binop((CPPClass*)Py_TYPE(other), other, (PyObject*)self, op); - if (result) return result; - -// default behavior: type + held pointer value defines identity (covers if -// other is not actually an CPPInstance, as ob_type will be unequal) - bool bIsEq = false; - if (Py_TYPE(self) == Py_TYPE(other) && \ - self->GetObject() == ((CPPInstance*)other)->GetObject()) - bIsEq = true; - - if ((op == Py_EQ && bIsEq) || (op == Py_NE && !bIsEq)) { - Py_RETURN_TRUE; +// ordered comparison operators + else if (op == Py_LT || op == Py_LE || op == Py_GT || op == Py_GE) { + CPPClass* klass = (CPPClass*)Py_TYPE(self); + if (!klass->fOperators) klass->fOperators = new Utility::PyOperators{}; + PyObject* meth = nullptr; + + switch (op) { + case Py_LT: + CPYCPPYY_ORDERED_OPERATOR_STUB(<, klass->fOperators->fLt, __lt__) + break; + case Py_LE: + CPYCPPYY_ORDERED_OPERATOR_STUB(<=, klass->fOperators->fLe, __ge__) + break; + case Py_GT: + CPYCPPYY_ORDERED_OPERATOR_STUB(>, klass->fOperators->fGt, __gt__) + break; + case Py_GE: + CPYCPPYY_ORDERED_OPERATOR_STUB(>=, klass->fOperators->fGe, __ge__) + break; + } + + if (!meth) { + PyErr_SetString(PyExc_NotImplementedError, ""); + return nullptr; + } + + return PyObject_CallFunctionObjArgs(meth, (PyObject*)self, other, nullptr); } - Py_RETURN_FALSE; + + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; } //---------------------------------------------------------------------------- @@ -436,11 +610,15 @@ static PyObject* op_repr(CPPInstance* self) // Build a representation string of the object proxy that shows the address // of the C++ object that is held, as well as its type. PyObject* pyclass = (PyObject*)Py_TYPE(self); + if (CPPScope_Check(pyclass) && (((CPPScope*)pyclass)->fFlags & CPPScope::kIsPython)) + return PyBaseObject_Type.tp_repr((PyObject*)self); PyObject* modname = PyObject_GetAttr(pyclass, PyStrings::gModule); Cppyy::TCppType_t klass = self->ObjectIsA(); std::string clName = klass ? Cppyy::GetFinalName(klass) : ""; - if (self->fFlags & CPPInstance::kIsReference) + if (self->fFlags & CPPInstance::kIsPtrPtr) + clName.append("**"); + else if (self->fFlags & CPPInstance::kIsReference) clName.append("*"); PyObject* repr = nullptr; @@ -517,54 +695,155 @@ static PyObject* op_str_internal(PyObject* pyobj, PyObject* lshift, bool isBound static Cppyy::TCppScope_t sOStringStreamID = Cppyy::GetScope("std::ostringstream"); std::ostringstream s; PyObject* pys = BindCppObjectNoCast(&s, sOStringStreamID); + Py_INCREF(pys); +#if PY_VERSION_HEX >= 0x03000000 +// for py3 and later, a ref-count of 2 is okay to consider the object temporary, but +// in this case, we can't lose our existing ostrinstring (otherwise, we'd have to peel +// it out of the return value, if moves are used + Py_INCREF(pys); +#endif + PyObject* res; if (isBound) res = PyObject_CallFunctionObjArgs(lshift, pys, NULL); else res = PyObject_CallFunctionObjArgs(lshift, pys, pyobj, NULL); + Py_DECREF(pys); - Py_DECREF(lshift); +#if PY_VERSION_HEX >= 0x03000000 + Py_DECREF(pys); +#endif + if (res) { Py_DECREF(res); return CPyCppyy_PyText_FromString(s.str().c_str()); } - PyErr_Clear(); + return nullptr; } + +static inline bool ScopeFlagCheck(CPPInstance* self, CPPScope::EFlags flag) { + return ((CPPScope*)Py_TYPE((PyObject*)self))->fFlags & flag; +} + +static inline void ScopeFlagSet(CPPInstance* self, CPPScope::EFlags flag) { + ((CPPScope*)Py_TYPE((PyObject*)self))->fFlags |= flag; +} + static PyObject* op_str(CPPInstance* self) { -#ifndef _WIN64 -// Forward to C++ insertion operator if available, otherwise forward to repr. - PyObject* result = nullptr; - PyObject* pyobj = (PyObject*)self; - PyObject* lshift = PyObject_GetAttr(pyobj, PyStrings::gLShift); - if (lshift) result = op_str_internal(pyobj, lshift, true); +// There are three possible options here: +// 1. Available operator<< to convert through an ostringstream +// 2. Cling's pretty printing +// 3. Generic printing as done in op_repr +// +// Additionally, there may be a mapped __str__ from the C++ type defining `operator char*` +// or `operator const char*`. Results are memoized for performance reasons. + +// 0. Protect against trying to print a typed nullptr object through an insertion operator + if (!self->GetObject()) + return op_repr(self); + +// 1. Available operator<< to convert through an ostringstream + if (!ScopeFlagCheck(self, CPPScope::kNoOSInsertion)) { + for (PyObject* pyname : {PyStrings::gLShift, PyStrings::gLShiftC, (PyObject*)0x01, (PyObject*)0x02}) { + if (pyname == PyStrings::gLShift && ScopeFlagCheck(self, CPPScope::kGblOSInsertion)) + continue; + + else if (pyname == (PyObject*)0x01) { + // normal lookup failed; attempt lazy install of global operator<<(ostream&, type&) + std::string rcname = Utility::ClassName((PyObject*)self); + Cppyy::TCppScope_t rnsID = Cppyy::GetScope(TypeManip::extract_namespace(rcname)); + PyCallable* pyfunc = Utility::FindBinaryOperator("std::ostream", rcname, "<<", rnsID); + if (!pyfunc) + continue; + + Utility::AddToClass((PyObject*)Py_TYPE((PyObject*)self), "__lshiftc__", pyfunc); + + pyname = PyStrings::gLShiftC; + ScopeFlagSet(self, CPPScope::kGblOSInsertion); + + } else if (pyname == (PyObject*)0x02) { + // TODO: the only reason this still exists, is b/c friend functions are otherwise not found + // TODO: ToString() still leaks ... + const std::string& pretty = Cppyy::ToString(self->ObjectIsA(), self->GetObject()); + if (!pretty.empty()) + return CPyCppyy_PyText_FromString(pretty.c_str()); + continue; + } + + PyObject* lshift = PyObject_GetAttr( + pyname == PyStrings::gLShift ? (PyObject*)self : (PyObject*)Py_TYPE((PyObject*)self), pyname); + + if (lshift) { + PyObject* result = op_str_internal((PyObject*)self, lshift, pyname == PyStrings::gLShift); + Py_DECREF(lshift); + if (result) + return result; + } - if (!result) { - PyErr_Clear(); - PyObject* pyclass = (PyObject*)Py_TYPE(pyobj); - lshift = PyObject_GetAttr(pyclass, PyStrings::gLShiftC); - if (!lshift) { PyErr_Clear(); - // attempt lazy install of global operator<<(ostream&) - std::string rcname = Utility::ClassName(pyobj); - Cppyy::TCppScope_t rnsID = Cppyy::GetScope(TypeManip::extract_namespace(rcname)); - PyCallable* pyfunc = Utility::FindBinaryOperator("std::ostream", rcname, "<<", rnsID); - if (pyfunc) { - Utility::AddToClass(pyclass, "__lshiftc__", pyfunc); - lshift = PyObject_GetAttr(pyclass, PyStrings::gLShiftC); - } else - PyType_Type.tp_setattro(pyclass, PyStrings::gLShiftC, Py_None); - } else if (lshift == Py_None) { - Py_DECREF(lshift); - lshift = nullptr; } - if (lshift) result = op_str_internal(pyobj, lshift, false); + + // failed ostream printing; don't try again + ScopeFlagSet(self, CPPScope::kNoOSInsertion); } - if (result) - return result; -#endif //!_WIN64 +// 2. Cling's pretty printing (not done through backend for performance reasons) + if (!ScopeFlagCheck(self, CPPScope::kNoPrettyPrint)) { + static PyObject* printValue = nullptr; + if (!printValue) { + PyObject* gbl = PyDict_GetItemString(PySys_GetObject((char*)"modules"), "cppyy.gbl"); + PyObject* cl = PyObject_GetAttrString(gbl, (char*)"cling"); + printValue = PyObject_GetAttrString(cl, (char*)"printValue"); + Py_DECREF(cl); + // gbl is borrowed + if (printValue) { + Py_DECREF(printValue); // make borrowed + if (!PyCallable_Check(printValue)) + printValue = nullptr; // unusable ... + } + if (!printValue) // unlikely + ScopeFlagSet(self, CPPScope::kNoPrettyPrint); + } + + if (printValue) { + // as printValue only works well for templates taking pointer arguments, we'll + // have to force the issue by working with a by-ptr object + Cppyy::TCppObject_t cppobj = self->GetObjectRaw(); + PyObject* byref = (PyObject*)self; + if (!(self->fFlags & CPPInstance::kIsReference)) { + byref = BindCppObjectNoCast((Cppyy::TCppObject_t)&cppobj, + self->ObjectIsA(), CPPInstance::kIsReference | CPPInstance::kNoMemReg); + } else { + Py_INCREF(byref); + } + // explicit template lookup + PyObject* clName = CPyCppyy_PyText_FromString(Utility::ClassName((PyObject*)self).c_str()); + PyObject* OL = PyObject_GetItem(printValue, clName); + Py_DECREF(clName); + + PyObject* pretty = OL ? PyObject_CallFunctionObjArgs(OL, byref, nullptr) : nullptr; + Py_XDECREF(OL); + Py_DECREF(byref); + + PyObject* result = nullptr; + if (pretty) { + const std::string& pv = *(std::string*)((CPPInstance*)pretty)->GetObject(); + if (!pv.empty() && pv.find("@0x") == std::string::npos) + result = CPyCppyy_PyText_FromString(pv.c_str()); + Py_DECREF(pretty); + if (result) return result; + } + + PyErr_Clear(); + } + + // if not available/specialized, don't try again + ScopeFlagSet(self, CPPScope::kNoPrettyPrint); + } + +// 3. Generic printing as done in op_repr return op_repr(self); } @@ -600,6 +879,7 @@ static PyGetSetDef op_getset[] = { //= CPyCppyy type number stubs to allow dynamic overrides ===================== #define CPYCPPYY_STUB_BODY(name, op) \ + bool previously_resolved_overload = (bool)meth; \ if (!meth) { \ PyErr_Clear(); \ PyCallable* pyfunc = Utility::FindBinaryOperator(left, right, #op); \ @@ -610,8 +890,8 @@ static PyGetSetDef op_getset[] = { } \ } \ PyObject* res = PyObject_CallFunctionObjArgs(meth, cppobj, other, nullptr);\ - if (!res) { \ - /* try again, in case there is a better overload out there */ \ + if (!res && previously_resolved_overload) { \ + /* try again, in case (left, right) are different types than before */ \ PyErr_Clear(); \ PyCallable* pyfunc = Utility::FindBinaryOperator(left, right, #op); \ if (pyfunc) ((CPPOverload*&)meth)->AdoptMethod(pyfunc); \ @@ -664,7 +944,7 @@ static PyObject* op_##name##_stub(PyObject* pyobj) \ /* placeholder to lazily install unary operators */ \ PyCallable* pyfunc = Utility::FindUnaryOperator((PyObject*)Py_TYPE(pyobj), #op);\ if (pyfunc && Utility::AddToClass((PyObject*)Py_TYPE(pyobj), #label, pyfunc))\ - return PyObject_CallMethod(pyobj, (char*)#label, nullptr); \ + return PyObject_CallMethod(pyobj, (char*)#label, nullptr); \ PyErr_SetString(PyExc_NotImplementedError, ""); \ return nullptr; \ } @@ -748,10 +1028,10 @@ PyTypeObject CPPInstance_Type = { sizeof(CPPInstance), // tp_basicsize 0, // tp_itemsize (destructor)op_dealloc, // tp_dealloc - 0, // tp_print + 0, // tp_vectorcall_offset / tp_print 0, // tp_getattr 0, // tp_setattr - 0, // tp_compare + 0, // tp_as_async / tp_compare (reprfunc)op_repr, // tp_repr &op_as_number, // tp_as_number 0, // tp_as_sequence @@ -764,7 +1044,11 @@ PyTypeObject CPPInstance_Type = { 0, // tp_as_buffer Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_CHECKTYPES, // tp_flags + Py_TPFLAGS_CHECKTYPES +#if PY_VERSION_HEX >= 0x03120000 + | Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_WEAKREF +#endif + , // tp_flags (char*)"cppyy object proxy (internal)", // tp_doc 0, // tp_traverse (inquiry)op_clear, // tp_clear @@ -799,6 +1083,9 @@ PyTypeObject CPPInstance_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h index 45dd10cd5f6b0..f61c041e2777d 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h @@ -38,21 +38,21 @@ class CPPInstance { kIsArray = 0x0100, kIsSmartPtr = 0x0200, kNoMemReg = 0x0400, - kHasLifeline = 0x0800, + kHasLifeLine = 0x0800, kIsRegulated = 0x1000, kIsActual = 0x2000 }; public: // public, as the python C-API works with C structs PyObject_HEAD void* fObject; - int fFlags; + uint32_t fFlags; public: // construction (never done directly) CPPInstance() = delete; void Set(void* address, EFlags flags = kDefault); - CPPInstance* Copy(void* cppinst); + CPPInstance* Copy(void* cppinst, PyTypeObject* target = nullptr); // state checking bool IsExtended() const { return fFlags & kIsExtended; } @@ -67,17 +67,21 @@ class CPPInstance { void PythonOwns(); void CppOwns(); +// data member cache + CI_DatamemberCache_t& GetDatamemberCache(); + // smart pointer management void SetSmart(PyObject* smart_type); void* GetSmartObject() { return GetObjectRaw(); } Cppyy::TCppType_t GetSmartIsA() const; -// data member cache - CI_DatamemberCache_t& GetDatamemberCache(); - -// cross-inheritence dispatch +// cross-inheritance dispatch void SetDispatchPtr(void*); +// redefine pointer to object as fixed-size array + void CastToArray(Py_ssize_t sz); + Py_ssize_t ArrayLength(); + private: void CreateExtension(); void* GetExtendedObject(); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx index d0f893246f0b4..097fb527cab19 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx @@ -14,9 +14,11 @@ #include "CPyCppyy/PyException.h" // Standard +#include #include #include #include +#include #include #include #include @@ -33,7 +35,56 @@ namespace CPyCppyy { } +//- public helper ------------------------------------------------------------ +CPyCppyy::PyCallArgs::~PyCallArgs() { + if (fFlags & kSelfSwap) // if self swap, fArgs has been offset by -1 + std::swap((PyObject*&)fSelf, ((PyObject**)fArgs)[0]); + +#if PY_VERSION_HEX >= 0x03080000 + if (fFlags & kIsOffset) fArgs -= 1; + + if (fFlags & kDoItemDecref) { + for (Py_ssize_t iarg = 0; iarg < CPyCppyy_PyArgs_GET_SIZE(fArgs, fNArgsf); ++iarg) + Py_DECREF(fArgs[iarg]); + } + + if (fFlags & kDoFree) + PyMem_Free((void*)fArgs); + else if (fFlags & kArgsSwap) { + // if self swap, fArgs has been offset by -1 + int offset = (fFlags & kSelfSwap) ? 1 : 0; + std::swap(((PyObject**)fArgs+offset)[0], ((PyObject**)fArgs+offset)[1]); + } +#else + if (fFlags & kDoDecref) + Py_DECREF((PyObject*)fArgs); + else if (fFlags & kArgsSwap) + std::swap(PyTuple_GET_ITEM(fArgs, 0), PyTuple_GET_ITEM(fArgs, 1)); +#endif +} + + //- private helpers ---------------------------------------------------------- +inline bool CPyCppyy::CPPMethod::VerifyArgCount_(Py_ssize_t actual) +{ +// actual number of arguments must be between required and max args + Py_ssize_t maxargs = (Py_ssize_t)fConverters.size(); + + if (maxargs != actual) { + if (actual < (Py_ssize_t)fArgsRequired) { + SetPyError_(CPyCppyy_PyText_FromFormat( + "takes at least %d arguments (%zd given)", fArgsRequired, actual)); + return false; + } else if (maxargs < actual) { + SetPyError_(CPyCppyy_PyText_FromFormat( + "takes at most %zd arguments (%zd given)", maxargs, actual)); + return false; + } + } + return true; +} + +//---------------------------------------------------------------------------- inline void CPyCppyy::CPPMethod::Copy_(const CPPMethod& /* other */) { // fScope and fMethod handled separately @@ -49,16 +100,14 @@ inline void CPyCppyy::CPPMethod::Destroy_() { // destroy executor and argument converters if (fExecutor && fExecutor->HasState()) delete fExecutor; + fExecutor = nullptr; for (auto p : fConverters) { if (p && p->HasState()) delete p; } - - delete fArgIndices; - - fExecutor = nullptr; - fArgIndices = nullptr; fConverters.clear(); + + delete fArgIndices; fArgIndices = nullptr; fArgsRequired = -1; } @@ -73,11 +122,14 @@ inline PyObject* CPyCppyy::CPPMethod::ExecuteFast( try { // C++ try block result = fExecutor->Execute(fMethod, (Cppyy::TCppObject_t)((intptr_t)self+offset), ctxt); } catch (PyException&) { + ctxt->fFlags |= CallContext::kPyException; result = nullptr; // error already set } catch (std::exception& e) { // attempt to set the exception to the actual type, to allow catching with the Python C++ type static Cppyy::TCppType_t exc_type = (Cppyy::TCppType_t)Cppyy::GetScope("std::exception"); + ctxt->fFlags |= CallContext::kCppException; + PyObject* pyexc_type = nullptr; PyObject* pyexc_obj = nullptr; @@ -120,6 +172,8 @@ inline PyObject* CPyCppyy::CPPMethod::ExecuteFast( result = nullptr; } catch (...) { + // don't set the kCppException flag here, as there is basically no useful + // extra information to be had and caller has to catch Exception either way PyErr_SetString(PyExc_Exception, "unhandled, unknown C++ exception"); result = nullptr; } @@ -145,10 +199,17 @@ inline PyObject* CPyCppyy::CPPMethod::ExecuteProtected( // fatal signal PyObject* result = 0; - TRY { // copy call environment to be able to jump back on signal + CLING_EXCEPTION_TRY { // copy call environment to be able to jump back on signal result = ExecuteFast(self, offset, ctxt); - } CATCH(excode) { - // Unfortunately, the excodes are not the ones from signal.h, but enums from TSysEvtHandler.h + } CLING_EXCEPTION_CATCH(excode) { + // report any outstanding Python exceptions first + if (PyErr_Occurred()) { + std::cerr << "Python exception outstanding during C++ longjmp:" << std::endl; + PyErr_Print(); + std::cerr << std::endl; + } + + // unfortunately, the excodes are not the ones from signal.h, but enums from TSysEvtHandler.h if (excode == 0) PyErr_SetString(gBusException, "bus error in C++; program state was reset"); else if (excode == 1) @@ -162,7 +223,7 @@ inline PyObject* CPyCppyy::CPPMethod::ExecuteProtected( else PyErr_SetString(PyExc_SystemError, "problem in C++; program state was reset"); result = 0; - } ENDTRY; + } CLING_EXCEPTION_ENDTRY; return result; } @@ -207,27 +268,7 @@ bool CPyCppyy::CPPMethod::InitExecutor_(Executor*& executor, CallContext* /* ctx std::string CPyCppyy::CPPMethod::GetSignatureString(bool fa) { // built a signature representation (used for doc strings) - std::stringstream sig; sig << "("; - int count = 0; - const size_t nArgs = Cppyy::GetMethodNumArgs(fMethod); - for (int iarg = 0; iarg < (int)nArgs; ++iarg) { - if (count) sig << (fa ? ", " : ","); - - sig << Cppyy::GetMethodArgType(fMethod, iarg); - - if (fa) { - const std::string& parname = Cppyy::GetMethodArgName(fMethod, iarg); - if (!parname.empty()) - sig << " " << parname; - - const std::string& defvalue = Cppyy::GetMethodArgDefault(fMethod, iarg); - if (!defvalue.empty()) - sig << " = " << defvalue; - } - count++; - } - sig << ")"; - return sig.str(); + return Cppyy::GetMethodSignature(fMethod, fa); } //---------------------------------------------------------------------------- @@ -272,7 +313,7 @@ void CPyCppyy::CPPMethod::SetPyError_(PyObject* msg) PyErr_Format(errtype, "%s =>\n %s: %s", CPyCppyy_PyText_AsString(doc), cname, details.c_str()); } - } else if (evalue) { + } else { Py_XDECREF(((CPPExcInstance*)evalue)->fTopMessage); if (msg) { ((CPPExcInstance*)evalue)->fTopMessage = CPyCppyy_PyText_FromFormat(\ @@ -365,6 +406,15 @@ PyObject* CPyCppyy::CPPMethod::GetPrototype(bool fa) Cppyy::GetMethodName(fMethod).c_str(), GetSignatureString(fa).c_str()); } + +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::CPPMethod::GetTypeName() +{ + PyObject* cppname = CPyCppyy_PyText_FromString((GetReturnTypeName() + " (*)").c_str()); + CPyCppyy_PyText_AppendAndDel(&cppname, GetSignature(false /* show_formalargs */)); + return cppname; +} + //---------------------------------------------------------------------------- PyObject* CPyCppyy::CPPMethod::Reflex(Cppyy::Reflex::RequestId_t request, Cppyy::Reflex::FormatId_t format) { @@ -418,6 +468,10 @@ int CPyCppyy::CPPMethod::GetPriority() const std::string aname = Cppyy::GetMethodArgType(fMethod, iarg); if (Cppyy::IsBuiltin(aname)) { + // complex type (note: double penalty: for complex and the template type) + if (strstr(aname.c_str(), "std::complex")) + priority -= 10; // prefer double, float, etc. over conversion + // integer types if (strstr(aname.c_str(), "bool")) priority += 1; // bool over int (does accept 1 and 0) @@ -468,7 +522,7 @@ int CPyCppyy::CPPMethod::GetPriority() priority += 150; // needed for proper implicit conversion rules } else if (aname.rfind("&&", aname.size()-2) != std::string::npos) { priority += 100; // prefer moves over other ref/ptr - } else if (!aname.empty() && !Cppyy::IsComplete(aname)) { + } else if (scope && !Cppyy::IsComplete(clean_name)) { // class is known, but no dictionary available, 2 more cases: * and & if (aname[aname.size() - 1] == '&') priority += -5000; @@ -539,33 +593,78 @@ PyObject* CPyCppyy::CPPMethod::GetCoVarNames() return co_varnames; } -PyObject* CPyCppyy::CPPMethod::GetArgDefault(int iarg) +PyObject* CPyCppyy::CPPMethod::GetArgDefault(int iarg, bool silent) { -// get the default value (if any) of argument iarg of this method +// get and evaluate the default value (if any) of argument iarg of this method if (iarg >= (int)GetMaxArgs()) return nullptr; - const std::string& defvalue = Cppyy::GetMethodArgDefault(fMethod, iarg); +// borrowed reference to cppyy.gbl module to use its dictionary to eval in + static PyObject* gbl = PyDict_GetItemString(PySys_GetObject((char*)"modules"), "cppyy.gbl"); + + std::string defvalue = Cppyy::GetMethodArgDefault(fMethod, iarg); if (!defvalue.empty()) { + PyObject** dctptr = _PyObject_GetDictPtr(gbl); + if (!(dctptr && *dctptr)) + return nullptr; - // attempt to evaluate the string representation (will work for all builtin types) - PyObject* pyval = (PyObject*)PyRun_String( - (char*)defvalue.c_str(), Py_eval_input, gThisModule, gThisModule); - if (!pyval && PyErr_Occurred()) { + PyObject* gdct = *dctptr; + PyObject* scope = nullptr; + + if (defvalue.find("::") != std::string::npos) { + // try to tickle scope creation, just in case + scope = CreateScopeProxy(defvalue.substr(0, defvalue.rfind('('))); + if (!scope) PyErr_Clear(); + + // rename '::' -> '.' + TypeManip::cppscope_to_pyscope(defvalue); + } + + if (!scope) { + // a couple of common cases that python doesn't like (technically, 'L' is okay with older + // pythons, but C long will always fit in Python int, so no need to bother) + char c = defvalue.back(); + if (c == 'F' || c == 'D' || c == 'L') { + int offset = 1; + if (2 < defvalue.size() && defvalue[defvalue.size()-2] == 'U') + offset = 2; + defvalue = defvalue.substr(0, defvalue.size()-offset); + } + } + + // attempt to evaluate the string representation (compilation is first to code to allow + // the error message to indicate where it's coming from) + PyObject* pyval = nullptr; + + PyObject* pycode = Py_CompileString((char*)defvalue.c_str(), "cppyy_default_compiler", Py_eval_input); + if (pycode) { + pyval = PyEval_EvalCode( +#if PY_VERSION_HEX < 0x03000000 + (PyCodeObject*) +#endif + pycode, gdct, gdct); + Py_DECREF(pycode); + } + + if (!pyval && PyErr_Occurred() && silent) { PyErr_Clear(); - return CPyCppyy_PyText_FromString(defvalue.c_str()); + pyval = CPyCppyy_PyText_FromString(defvalue.c_str()); // allows continuation, but is likely to fail } - return pyval; + Py_XDECREF(scope); + return pyval; // may be nullptr } + PyErr_Format(PyExc_TypeError, "Could not construct default value for: %s", Cppyy::GetMethodArgName(fMethod, iarg).c_str()); return nullptr; } + bool CPyCppyy::CPPMethod::IsConst() { return Cppyy::IsConstMethod(GetMethod()); } + //---------------------------------------------------------------------------- PyObject* CPyCppyy::CPPMethod::GetScopeProxy() { @@ -635,17 +734,24 @@ bool CPyCppyy::CPPMethod::Initialize(CallContext* ctxt) } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPMethod::ProcessKeywords(PyObject* self, PyObject* args, PyObject* kwds) +bool CPyCppyy::CPPMethod::ProcessKwds(PyObject* self_in, PyCallArgs& cargs) { - if (!PyDict_CheckExact(kwds)) { +#if PY_VERSION_HEX >= 0x03080000 + if (!PyTuple_CheckExact(cargs.fKwds)) { + SetPyError_(CPyCppyy_PyText_FromString("received unknown keyword names object")); + return false; + } + Py_ssize_t nKeys = PyTuple_GET_SIZE(cargs.fKwds); +#else + if (!PyDict_CheckExact(cargs.fKwds)) { SetPyError_(CPyCppyy_PyText_FromString("received unknown keyword arguments object")); - return nullptr; + return false; } + Py_ssize_t nKeys = PyDict_Size(cargs.fKwds); +#endif - if (PyDict_Size(kwds) == 0 && !self) { - Py_INCREF(args); - return args; - } + if (nKeys == 0 && !self_in) + return true; if (!fArgIndices) { fArgIndices = new std::map{}; @@ -653,102 +759,150 @@ PyObject* CPyCppyy::CPPMethod::ProcessKeywords(PyObject* self, PyObject* args, P (*fArgIndices)[Cppyy::GetMethodArgName(fMethod, iarg)] = iarg; } - Py_ssize_t nKeys = PyDict_Size(kwds); - Py_ssize_t nArgs = PyTuple_GET_SIZE(args) + (self ? 1 : 0); - if (nKeys+nArgs < fArgsRequired) { - SetPyError_(CPyCppyy_PyText_FromFormat( - "takes at least %d arguments (%zd given)", fArgsRequired, nKeys+nArgs)); - return nullptr; - } - - PyObject* newArgs = PyTuple_New(nArgs+nKeys); + Py_ssize_t nArgs = CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf) + (self_in ? 1 : 0); + if (!VerifyArgCount_(nArgs+nKeys)) + return false; -// set all values to zero to be able to check them later (this also guarantees normal -// cleanup by the tuple deallocation) - for (Py_ssize_t i = 0; i < nArgs+nKeys; ++i) - PyTuple_SET_ITEM(newArgs, i, static_cast(nullptr)); + std::vector vArgs{fConverters.size()}; // next, insert the keyword values PyObject *key, *value; + Py_ssize_t maxpos = -1; + +#if PY_VERSION_HEX >= 0x03080000 + Py_ssize_t npos_args = CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf); + for (Py_ssize_t ikey = 0; ikey < nKeys; ++ikey) { + key = PyTuple_GET_ITEM(cargs.fKwds, ikey); + value = cargs.fArgs[npos_args+ikey]; +#else Py_ssize_t pos = 0; - - while (PyDict_Next(kwds, &pos, &key, &value)) { + while (PyDict_Next(cargs.fKwds, &pos, &key, &value)) { +#endif const char* ckey = CPyCppyy_PyText_AsStringChecked(key); - if (!ckey) { - Py_DECREF(newArgs); - return nullptr; - } + if (!ckey) + return false; + auto p = fArgIndices->find(ckey); if (p == fArgIndices->end()) { SetPyError_(CPyCppyy_PyText_FromFormat("%s::%s got an unexpected keyword argument \'%s\'", Cppyy::GetFinalName(fScope).c_str(), Cppyy::GetMethodName(fMethod).c_str(), ckey)); - Py_DECREF(newArgs); - return nullptr; + return false; } - Py_INCREF(value); - PyTuple_SetItem(newArgs, (*fArgIndices)[ckey], value); + + maxpos = p->second > maxpos ? p->second : maxpos; + vArgs[p->second] = value; // no INCREF yet for simple cleanup in case of error } -// fill out the rest of the arguments +// if maxpos < nArgs, it will be detected & reported as a duplicate below + Py_ssize_t maxargs = maxpos + 1; + CPyCppyy_PyArgs_t newArgs = CPyCppyy_PyArgs_New(maxargs); + +// set all values to zero to be able to check them later (this also guarantees normal +// cleanup by the tuple deallocation) + for (Py_ssize_t i = 0; i < maxargs; ++i) + CPyCppyy_PyArgs_SET_ITEM(newArgs, i, nullptr); + +// fill out the positional arguments Py_ssize_t start = 0; - if (self) { - Py_INCREF(self); - PyTuple_SET_ITEM(newArgs, 0, self); + if (self_in) { + Py_INCREF(self_in); + CPyCppyy_PyArgs_SET_ITEM(newArgs, 0, self_in); start = 1; } for (Py_ssize_t i = start; i < nArgs; ++i) { - if (PyTuple_GET_ITEM(newArgs, i)) { + if (vArgs[i]) { SetPyError_(CPyCppyy_PyText_FromFormat("%s::%s got multiple values for argument %d", Cppyy::GetFinalName(fScope).c_str(), Cppyy::GetMethodName(fMethod).c_str(), (int)i+1)); - Py_DECREF(newArgs); - return nullptr; + CPyCppyy_PyArgs_DEL(newArgs); + return false; } - PyObject* item = PyTuple_GET_ITEM(args, i); + PyObject* item = CPyCppyy_PyArgs_GET_ITEM(cargs.fArgs, i); Py_INCREF(item); - PyTuple_SET_ITEM(newArgs, i, item); + CPyCppyy_PyArgs_SET_ITEM(newArgs, i, item); + } + +// fill out the keyword arguments + for (Py_ssize_t i = nArgs; i < maxargs; ++i) { + PyObject* item = vArgs[i]; + if (item) { + Py_INCREF(item); + CPyCppyy_PyArgs_SET_ITEM(newArgs, i, item); + } else { + // try retrieving the default + item = GetArgDefault((int)i, false /* i.e. not silent */); + if (!item) { + CPyCppyy_PyArgs_DEL(newArgs); + return false; + } + CPyCppyy_PyArgs_SET_ITEM(newArgs, i, item); + } + } + +#if PY_VERSION_HEX >= 0x03080000 + if (cargs.fFlags & PyCallArgs::kDoFree) { + if (cargs.fFlags & PyCallArgs::kIsOffset) + cargs.fArgs -= 1; +#else + if (cargs.fFlags & PyCallArgs::kDoDecref) { +#endif + CPyCppyy_PyArgs_DEL(cargs.fArgs); } - return newArgs; + cargs.fArgs = newArgs; + cargs.fNArgsf = maxargs; +#if PY_VERSION_HEX >= 0x03080000 + cargs.fFlags = PyCallArgs::kDoFree | PyCallArgs::kDoItemDecref; +#else + cargs.fFlags = PyCallArgs::kDoDecref; +#endif + + return true; } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPMethod::PreProcessArgs( - CPPInstance*& self, PyObject* args, PyObject* kwds) +bool CPyCppyy::CPPMethod::ProcessArgs(PyCallArgs& cargs) { // verify existence of self, return if ok - if (self) { - if (kwds) return ProcessKeywords(nullptr, args, kwds); - Py_INCREF(args); - return args; + if (cargs.fSelf) { + if (cargs.fKwds) { return ProcessKwds(nullptr, cargs); } + return true; } // otherwise, check for a suitable 'self' in args and update accordingly - if (PyTuple_GET_SIZE(args) != 0) { - CPPInstance* pyobj = (CPPInstance*)PyTuple_GET_ITEM(args, 0); + if (CPyCppyy_PyArgs_GET_SIZE(cargs.fArgs, cargs.fNArgsf) != 0) { + CPPInstance* pyobj = (CPPInstance*)CPyCppyy_PyArgs_GET_ITEM(cargs.fArgs, 0); // demand CPyCppyy object, and an argument that may match down the road - if (CPPInstance_Check(pyobj) && - (fScope == Cppyy::gGlobalScope || // free global - (pyobj->ObjectIsA() == 0) || // null pointer or ctor call - (Cppyy::IsSubtype(pyobj->ObjectIsA(), fScope)))) { // matching types - - // reset self - Py_INCREF(pyobj); // corresponding Py_DECREF is in CPPOverload - self = pyobj; - - // offset args by 1 (new ref) - PyObject* newArgs = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); - - // put the keywords, if any, in their places in the arguments array - if (kwds) { - args = ProcessKeywords(nullptr, newArgs, kwds); - Py_DECREF(newArgs); - newArgs = args; - } + if (CPPInstance_Check(pyobj)) { + Cppyy::TCppType_t oisa = pyobj->ObjectIsA(); + if (fScope == Cppyy::gGlobalScope || // free global + oisa == 0 || // null pointer or ctor call + oisa == fScope || // matching types + Cppyy::IsSubtype(oisa, fScope)) { // id. + + // reset self + Py_INCREF(pyobj); // corresponding Py_DECREF is in CPPOverload + cargs.fSelf = pyobj; + + // offset args by 1 +#if PY_VERSION_HEX >= 0x03080000 + cargs.fArgs += 1; + cargs.fFlags |= PyCallArgs::kIsOffset; +#else + if (cargs.fFlags & PyCallArgs::kDoDecref) + Py_DECREF((PyObject*)cargs.fArgs); + cargs.fArgs = PyTuple_GetSlice(cargs.fArgs, 1, PyTuple_GET_SIZE(cargs.fArgs)); + cargs.fFlags |= PyCallArgs::kDoDecref; +#endif + cargs.fNArgsf -= 1; - return newArgs; // may be nullptr if kwds insertion failed + // put the keywords, if any, in their places in the arguments array + if (cargs.fKwds) + return ProcessKwds(nullptr, cargs); + return true; + } } } @@ -757,39 +911,27 @@ PyObject* CPyCppyy::CPPMethod::PreProcessArgs( "unbound method %s::%s must be called with a %s instance as first argument", Cppyy::GetFinalName(fScope).c_str(), Cppyy::GetMethodName(fMethod).c_str(), Cppyy::GetFinalName(fScope).c_str())); - return nullptr; + return false; } //---------------------------------------------------------------------------- -bool CPyCppyy::CPPMethod::ConvertAndSetArgs(PyObject* args, CallContext* ctxt) +bool CPyCppyy::CPPMethod::ConvertAndSetArgs(CPyCppyy_PyArgs_t args, size_t nargsf, CallContext* ctxt) { - Py_ssize_t argc = PyTuple_GET_SIZE(args); - Py_ssize_t argMax = (Py_ssize_t)fConverters.size(); + Py_ssize_t argc = CPyCppyy_PyArgs_GET_SIZE(args, nargsf); + if (!VerifyArgCount_(argc)) + return false; - if (argMax != argc) { - // argc must be between min and max number of arguments - if (argc < (Py_ssize_t)fArgsRequired) { - SetPyError_(CPyCppyy_PyText_FromFormat( - "takes at least %d arguments (%zd given)", fArgsRequired, argc)); - return false; - } else if (argMax < argc) { - SetPyError_(CPyCppyy_PyText_FromFormat( - "takes at most %zd arguments (%zd given)", argMax, argc)); - return false; - } - } +// pass current scope for which the call is made + ctxt->fCurScope = fScope; if (argc == 0) return true; -// pass current scope for which the call is made - ctxt->fCurScope = fScope; - // convert the arguments to the method call array bool isOK = true; Parameter* cppArgs = ctxt->GetArgs(argc); for (int i = 0; i < (int)argc; ++i) { - if (!fConverters[i]->SetArg(PyTuple_GET_ITEM(args, i), cppArgs[i], ctxt)) { + if (!fConverters[i]->SetArg(CPyCppyy_PyArgs_GET_ITEM(args, i), cppArgs[i], ctxt)) { SetPyError_(CPyCppyy_PyText_FromFormat("could not convert argument %d", i+1)); isOK = false; break; @@ -828,23 +970,26 @@ PyObject* CPyCppyy::CPPMethod::Execute(void* self, ptrdiff_t offset, CallContext } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::CPPMethod::Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt) +PyObject* CPyCppyy::CPPMethod::Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt) { // setup as necessary if (fArgsRequired == -1 && !Initialize(ctxt)) return nullptr; // fetch self, verify, and put the arguments in usable order - if (!(args = PreProcessArgs(self, args, kwds))) + PyCallArgs cargs{self, args, nargsf, kwds}; + if (!ProcessArgs(cargs)) return nullptr; +// self provides the python context for lifelines + if (!ctxt->fPyContext) + ctxt->fPyContext = (PyObject*)cargs.fSelf; // no Py_INCREF as no ownership + // translate the arguments - if (fArgsRequired || PyTuple_GET_SIZE(args)) { - if (!ConvertAndSetArgs(args, ctxt)) { - Py_DECREF(args); + if (fArgsRequired || CPyCppyy_PyArgs_GET_SIZE(args, nargsf)) { + if (!ConvertAndSetArgs(cargs.fArgs, cargs.fNArgsf, ctxt)) return nullptr; - } } // get the C++ object that this object proxy is a handle for @@ -853,7 +998,6 @@ PyObject* CPyCppyy::CPPMethod::Call( // validity check that should not fail if (!object) { PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer"); - Py_DECREF(args); return nullptr; } @@ -867,8 +1011,6 @@ PyObject* CPyCppyy::CPPMethod::Call( // actual call; recycle self instead of returning new object for same address objects CPPInstance* pyobj = (CPPInstance*)Execute(object, offset, ctxt); - Py_DECREF(args); - if (CPPInstance_Check(pyobj) && derived && pyobj->ObjectIsA() == derived && pyobj->GetObject() == object) { diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.h index 0801efaa7fbe9..62ba6e1f0974c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.h @@ -15,6 +15,33 @@ namespace CPyCppyy { class Executor; class Converter; +class PyCallArgs { +public: + PyCallArgs(CPPInstance*& self, CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds) + : fSelf(self), fArgs(args), fNArgsf(nargsf), fKwds(kwds), fFlags(kNone) {} + ~PyCallArgs(); + + enum ECleanupFlags { + kNone = 0x0000, + kIsOffset = 0x0001, // args were offset by 1 to drop self + kSelfSwap = 0x0002, // args[-1] and self need swapping + kArgsSwap = 0x0004, // args[0] and args[1] need swapping +#if PY_VERSION_HEX >= 0x03080000 + kDoFree = 0x0008, // args need to be free'd (vector call only) + kDoItemDecref = 0x0010 // items in args need a decref (vector call only) +#else + kDoDecref = 0x0020 // args need a decref +#endif + }; + +public: + CPPInstance*& fSelf; + CPyCppyy_PyArgs_t fArgs; + size_t fNArgsf; + PyObject* fKwds; + int fFlags; +}; + class CPPMethod : public PyCallable { public: CPPMethod(Cppyy::TCppScope_t scope, Cppyy::TCppMethod_t method); @@ -25,14 +52,16 @@ class CPPMethod : public PyCallable { public: virtual PyObject* GetSignature(bool show_formalargs = true); virtual PyObject* GetPrototype(bool show_formalargs = true); + virtual PyObject* GetTypeName(); virtual PyObject* Reflex(Cppyy::Reflex::RequestId_t request, Cppyy::Reflex::FormatId_t = Cppyy::Reflex::OPTIMAL); + virtual int GetPriority(); - virtual bool IsGreedy(); + virtual bool IsGreedy(); virtual int GetMaxArgs(); virtual PyObject* GetCoVarNames(); - virtual PyObject* GetArgDefault(int iarg); + virtual PyObject* GetArgDefault(int iarg, bool silent=true); virtual bool IsConst(); virtual PyObject* GetScopeProxy(); @@ -43,21 +72,18 @@ class CPPMethod : public PyCallable { virtual int GetArgMatchScore(PyObject* args_tuple); public: - virtual PyObject* Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt = nullptr); + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); protected: - virtual PyObject* PreProcessArgs(CPPInstance*& self, PyObject* args, PyObject* kwds); + virtual bool ProcessArgs(PyCallArgs& args); - bool Initialize(CallContext* ctxt = nullptr); - PyObject* ProcessKeywords(PyObject* self, PyObject* args, PyObject* kwds); - bool ConvertAndSetArgs(PyObject* args, CallContext* ctxt = nullptr); + bool Initialize(CallContext* ctxt = nullptr); + bool ProcessKwds(PyObject* self_in, PyCallArgs& args); + bool ConvertAndSetArgs(CPyCppyy_PyArgs_t, size_t nargsf, CallContext* ctxt = nullptr); PyObject* Execute(void* self, ptrdiff_t offset, CallContext* ctxt = nullptr); Cppyy::TCppMethod_t GetMethod() { return fMethod; } -// TODO: the following is a special case to allow shimming of the -// constructor; there's probably a better way ... - void SetMethod(Cppyy::TCppMethod_t m) { fMethod = m; } Cppyy::TCppScope_t GetScope() { return fScope; } Executor* GetExecutor() { return fExecutor; } std::string GetSignatureString(bool show_formalargs = true); @@ -68,6 +94,7 @@ class CPPMethod : public PyCallable { private: void Copy_(const CPPMethod&); void Destroy_(); + bool VerifyArgCount_(Py_ssize_t); PyObject* ExecuteFast(void*, ptrdiff_t, CallContext*); PyObject* ExecuteProtected(void*, ptrdiff_t, CallContext*); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOperator.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOperator.cxx new file mode 100644 index 0000000000000..9aacc3dd50528 --- /dev/null +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOperator.cxx @@ -0,0 +1,66 @@ +// Bindings +#include "CPyCppyy.h" +#include "CPPOperator.h" +#include "CPPInstance.h" + + +//- constructor -------------------------------------------------------------- +CPyCppyy::CPPOperator::CPPOperator( + Cppyy::TCppScope_t scope, Cppyy::TCppMethod_t method, const std::string& name) + : CPPMethod(scope, method) +{ +// a bit silly but doing it this way allows decoupling the initialization order + if (name == "__mul__") + fStub = CPPInstance_Type.tp_as_number->nb_multiply; + else if (name == CPPYY__div__) +#if PY_VERSION_HEX < 0x03000000 + fStub = CPPInstance_Type.tp_as_number->nb_divide; +#else + fStub = CPPInstance_Type.tp_as_number->nb_true_divide; +#endif + else if (name == "__add__") + fStub = CPPInstance_Type.tp_as_number->nb_add; + else if (name == "__sub__") + fStub = CPPInstance_Type.tp_as_number->nb_subtract; + else + fStub = nullptr; +} + +//----------------------------------------------------------------------------- +PyObject* CPyCppyy::CPPOperator::Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt) +{ +// some operators can be a mix of global and class overloads; this method will +// first try class overloads (the existence of this method means that such were +// defined) and if failed, fall back on the global stubs +// TODO: the fact that this is a method and not an overload means that the global +// ones are tried for each method that fails during the overload resolution + PyObject* result = this->CPPMethod::Call(self, args, nargsf, kwds, ctxt); + if (result || !fStub || !self) + return result; + + Py_ssize_t idx_other = 0; + if (CPyCppyy_PyArgs_GET_SIZE(args, nargsf) != 1) { +#if PY_VERSION_HEX >= 0x03080000 + if ((CPyCppyy_PyArgs_GET_SIZE(args, nargsf) == 2 && CPyCppyy_PyArgs_GET_ITEM(args, 0) == (PyObject*)self)) + idx_other = 1; + else +#endif + return result; + } + + PyObject* pytype = 0, *pyvalue = 0, *pytrace = 0; + PyErr_Fetch(&pytype, &pyvalue, &pytrace); + + result = fStub((PyObject*)self, CPyCppyy_PyArgs_GET_ITEM(args, idx_other)); + + if (!result) + PyErr_Restore(pytype, pyvalue, pytrace); + else { + Py_XDECREF(pytype); + Py_XDECREF(pyvalue); + Py_XDECREF(pytrace); + } + + return result; +} diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOperator.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOperator.h new file mode 100644 index 0000000000000..b680b895e50b0 --- /dev/null +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOperator.h @@ -0,0 +1,28 @@ +#ifndef CPYCPPYY_CPPOPERATOR_H +#define CPYCPPYY_CPPOPERATOR_H + +// Bindings +#include "CPPMethod.h" + +// Standard +#include + + +namespace CPyCppyy { + +class CPPOperator : public CPPMethod { +public: + CPPOperator(Cppyy::TCppScope_t scope, Cppyy::TCppMethod_t method, const std::string& name); + +public: + virtual PyCallable* Clone() { return new CPPOperator(*this); } + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr); + +private: + binaryfunc fStub; +}; + +} // namespace CPyCppyy + +#endif // !CPYCPPYY_CPPOPERATOR_H diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx index 26da3277f7cca..f123b0bdb4f2c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.cxx @@ -2,11 +2,13 @@ #include "CPyCppyy.h" #include "CPyCppyy/Reflex.h" #include "structmember.h" // from Python -#if PY_VERSION_HEX < 0x02050000 -#include "compile.h" // from Python -#elif PY_VERSION_HEX < 0x030b0000 +#if PY_VERSION_HEX >= 0x02050000 +#if PY_VERSION_HEX < 0x030b0000 #include "code.h" // from Python #endif +#else +#include "compile.h" // from Python +#endif #ifndef CO_NOFREE // python2.2 does not have CO_NOFREE defined #define CO_NOFREE 0x0040 @@ -28,7 +30,7 @@ namespace CPyCppyy { namespace { // from CPython's instancemethod: Free list for method objects to safe malloc/free overhead -// The im_self element is used to chain the elements. +// The fSelf field is used to chain the elements. static CPPOverload* free_list; static int numfree = 0; #ifndef CPPOverload_MAXFREELIST @@ -36,7 +38,8 @@ static int numfree = 0; #endif -// TODO: only used here, but may be better off integrated with Pythonize.cxx callbacks +// TODO: only used in pythonizations to add Python-side overloads to existing +// C++ overloads, but may be better off integrated with Pythonize.cxx callbacks class TPythonCallback : public PyCallable { public: PyObject* fCallable; @@ -77,8 +80,8 @@ class TPythonCallback : public PyCallable { virtual PyObject* GetCoVarNames() { // TODO: pick these up from the callable Py_RETURN_NONE; } - virtual PyObject* GetArgDefault(int /* iarg */) { // TODO: pick these up from the callable - Py_RETURN_NONE; + virtual PyObject* GetArgDefault(int /* iarg */, bool /* silent */ =true) { + Py_RETURN_NONE; // TODO: pick these up from the callable } virtual PyObject* GetScopeProxy() { // should this be the module ?? @@ -91,9 +94,37 @@ class TPythonCallback : public PyCallable { virtual PyCallable* Clone() { return new TPythonCallback(*this); } - virtual PyObject* Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* /* ctxt = 0 */) { + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* /* ctxt = 0 */) { +#if PY_VERSION_HEX >= 0x03080000 + if (self) { + if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) { // mutation allowed? + std::swap(((PyObject**)args-1)[0], (PyObject*&)self); + nargsf &= ~PY_VECTORCALL_ARGUMENTS_OFFSET; + args = args-1; + } else { + Py_ssize_t nkwargs = kwds ? PyTuple_GET_SIZE(kwds) : 0; + Py_ssize_t totalargs = PyVectorcall_NARGS(nargsf)+nkwargs; + PyObject** newArgs = (PyObject**)PyMem_Malloc((totalargs+1) * sizeof(PyObject*)); + if (!newArgs) + return nullptr; + + newArgs[0] = (PyObject*)self; + if (0 < totalargs) + memcpy((void*)&newArgs[1], args, totalargs * sizeof(PyObject*)); + args = newArgs; + } + nargsf += 1; + } + + PyObject* result = CPyCppyy_PyObject_Call(fCallable, args, nargsf, kwds); + if (self) { + if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) + std::swap(((PyObject**)args-1)[0], (PyObject*&)self); + else PyMem_Free((void*)args); + } +#else PyObject* newArgs = nullptr; if (self) { Py_ssize_t nargs = PyTuple_Size(args); @@ -109,7 +140,10 @@ class TPythonCallback : public PyCallable { Py_INCREF(args); newArgs = args; } - return PyObject_Call(fCallable, newArgs, kwds); + PyObject* result = PyObject_Call(fCallable, newArgs, kwds); + Py_DECREF(newArgs); +#endif + return result; } }; @@ -120,83 +154,79 @@ static inline bool IsPseudoFunc(CPPOverload* pymeth) } // helper to sort on method priority -static int PriorityCmp(PyCallable* left, PyCallable* right) +static int PriorityCmp(const std::pair& left, const std::pair& right) { - return left->GetPriority() > right->GetPriority(); + return left.first > right.first; } // return helper -static inline void ResetCallState(CPPInstance*& selfnew, CPPInstance* selfold, bool clear) +static inline void ResetCallState(CPPInstance* descr_self, CPPInstance*& im_self) { - if (selfnew != selfold) { - Py_XDECREF(selfnew); - selfnew = selfold; +// reset self if needed, allowing simple re-use + if (descr_self != im_self) { + Py_XDECREF(im_self); + im_self = descr_self; } - - if (clear) - PyErr_Clear(); } -// helper to factor out return logic of mp_call +// helper to factor out return logic of mp_call / mp_vectorcall static inline PyObject* HandleReturn( - CPPOverload* pymeth, CPPInstance* oldSelf, PyObject* result) + CPPOverload* pymeth, CPPInstance* im_self, PyObject* result) { // special case for python exceptions, propagated through C++ layer - int ll_action = 0; if (result) { + CPPInstance* cppres = (CPPInstance*)(CPPInstance_Check(result) ? result : nullptr); // if this method creates new objects, always take ownership if (IsCreator(pymeth->fMethodInfo->fFlags)) { // either be a constructor with a fresh object proxy self ... if (IsConstructor(pymeth->fMethodInfo->fFlags)) { - if (pymeth->fSelf) - pymeth->fSelf->PythonOwns(); + if (im_self) + im_self->PythonOwns(); } - // ... or be a method with an object proxy return value - else if (CPPInstance_Check(result)) - ((CPPInstance*)result)->PythonOwns(); + // ... or be a regular method with an object proxy return value + else if (cppres) + cppres->PythonOwns(); } // if this new object falls inside self, make sure its lifetime is proper - if (pymeth->fMethodInfo->fFlags & CallContext::kSetLifeLine) - ll_action = 1; - else if (!(pymeth->fMethodInfo->fFlags & CallContext::kNeverLifeLine) && \ - CPPInstance_Check(pymeth->fSelf) && CPPInstance_Check(result)) { - // if self was a by-value return and result is not, pro-actively protect result; - // else if the return value falls within the memory of 'this', force a lifeline - CPPInstance* cppself = (CPPInstance*)pymeth->fSelf; - CPPInstance* cppres = (CPPInstance*)result; - if (!(cppres->fFlags & CPPInstance::kIsValue)) { // no need if the result is a full copy - if (cppself->fFlags & CPPInstance::kIsValue) - ll_action = 2; - else if (cppself->fFlags & CPPInstance::kHasLifeline) - ll_action = 3; - else { - ptrdiff_t offset = (ptrdiff_t)cppres->GetObject() - (ptrdiff_t)cppself->GetObject(); - if (0 <= offset && offset < (ptrdiff_t)Cppyy::SizeOf(cppself->ObjectIsA())) - ll_action = 4; + if (!(pymeth->fMethodInfo->fFlags & CallContext::kNeverLifeLine)) { + int ll_action = 0; + if ((PyObject*)im_self != result) { + if (pymeth->fMethodInfo->fFlags & CallContext::kSetLifeLine) + ll_action = 1; + else if (cppres && CPPInstance_Check(im_self)) { + // if self was a by-value return and result is not, pro-actively protect result; + // else if the return value falls within the memory of 'this', force a lifeline + if (!(cppres->fFlags & CPPInstance::kIsValue)) { // no need if the result is temporary + if (im_self->fFlags & CPPInstance::kIsValue) + ll_action = 2; + else if (im_self->fFlags & CPPInstance::kHasLifeLine) + ll_action = 3; + else { + ptrdiff_t offset = (ptrdiff_t)cppres->GetObject() - (ptrdiff_t)im_self->GetObject(); + if (0 <= offset && offset < (ptrdiff_t)Cppyy::SizeOf(im_self->ObjectIsA())) + ll_action = 4; + } + } } } - if (ll_action) cppres->fFlags |= CPPInstance::kHasLifeline; // for chaining - } - - if (!ll_action) - pymeth->fMethodInfo->fFlags |= CallContext::kNeverLifeLine; // assume invariant semantics - } - if (ll_action) { - if (PyObject_SetAttr(result, PyStrings::gLifeLine, (PyObject*)pymeth->fSelf) == -1) - PyErr_Clear(); // ignored - if (ll_action == 1 /* directly set */ && CPPInstance_Check(result)) - ((CPPInstance*)result)->fFlags |= CPPInstance::kHasLifeline; // for chaining - else - pymeth->fMethodInfo->fFlags |= CallContext::kSetLifeLine; // for next time + if (!ll_action) + pymeth->fMethodInfo->fFlags |= CallContext::kNeverLifeLine; // assume invariant semantics + else { + if (PyObject_SetAttr(result, PyStrings::gLifeLine, (PyObject*)im_self) == -1) + PyErr_Clear(); // ignored + if (cppres) cppres->fFlags |= CPPInstance::kHasLifeLine; // for chaining + pymeth->fMethodInfo->fFlags |= CallContext::kSetLifeLine; // for next time + } + } } // reset self as necessary to allow re-use of the CPPOverload - ResetCallState(pymeth->fSelf, oldSelf, false); + ResetCallState(pymeth->fSelf, im_self); return result; } @@ -218,6 +248,11 @@ static PyObject* mp_module(CPPOverload* /* pymeth */, void*) //---------------------------------------------------------------------------- static PyObject* mp_doc(CPPOverload* pymeth, void*) { + if (pymeth->fMethodInfo->fDoc) { + Py_INCREF(pymeth->fMethodInfo->fDoc); + return pymeth->fMethodInfo->fDoc; + } + // Build python document string ('__doc__') from all C++-side overloads. CPPOverload::Methods_t& methods = pymeth->fMethodInfo->fMethods; @@ -242,6 +277,14 @@ static PyObject* mp_doc(CPPOverload* pymeth, void*) return doc; } +static int mp_doc_set(CPPOverload* pymeth, PyObject *val, void *) +{ + Py_XDECREF(pymeth->fMethodInfo->fDoc); + Py_INCREF(val); + pymeth->fMethodInfo->fDoc = val; + return 0; +} + //---------------------------------------------------------------------------- static PyObject* mp_meth_func(CPPOverload* pymeth, void*) { @@ -299,8 +342,15 @@ static PyObject* mp_func_closure(CPPOverload* /* pymeth */, void*) Py_RETURN_NONE; } +// To declare a variable as unused only when compiling for Python 3. +#if PY_VERSION_HEX < 0x03000000 +#define CPyCppyy_Py3_UNUSED(name) name +#else +#define CPyCppyy_Py3_UNUSED(name) +#endif + //---------------------------------------------------------------------------- -static PyObject* mp_func_code(CPPOverload* pymeth, void*) +static PyObject* mp_func_code(CPPOverload* CPyCppyy_Py3_UNUSED(pymeth), void*) { // Code details are used in module inspect to fill out interactive help() #if PY_VERSION_HEX < 0x03000000 @@ -367,7 +417,6 @@ static PyObject* mp_func_code(CPPOverload* pymeth, void*) return code; #else // not important for functioning of most code, so not implemented for p3 for now (TODO) - (void)pymeth; Py_RETURN_NONE; #endif } @@ -391,6 +440,8 @@ static PyObject* mp_func_defaults(CPPOverload* pymeth, void*) PyObject* defvalue = methods[0]->GetArgDefault(iarg); if (defvalue) PyTuple_SET_ITEM(defaults, itup++, defvalue); + else + PyErr_Clear(); } _PyTuple_Resize(&defaults, itup); @@ -496,11 +547,27 @@ CPPYY_BOOLEAN_PROPERTY(threaded, CallContext::kReleaseGIL, "__release_gil__") CPPYY_BOOLEAN_PROPERTY(useffi, CallContext::kUseFFI, "__useffi__") CPPYY_BOOLEAN_PROPERTY(sig2exc, CallContext::kProtected, "__sig2exc__") +static PyObject* mp_getcppname(CPPOverload* pymeth, void*) +{ + if ((void*)pymeth == (void*)&CPPOverload_Type) + return CPyCppyy_PyText_FromString("CPPOverload_Type"); + + auto& methods = pymeth->fMethodInfo->fMethods; + if (methods.empty()) + return CPyCppyy_PyText_FromString("void (*)()"); // debatable + + if (methods.size() == 1) + return methods[0]->GetTypeName(); + + return CPyCppyy_PyText_FromString("void* (*)(...)"); // id. +} + + //---------------------------------------------------------------------------- static PyGetSetDef mp_getset[] = { {(char*)"__name__", (getter)mp_name, nullptr, nullptr, nullptr}, {(char*)"__module__", (getter)mp_module, nullptr, nullptr, nullptr}, - {(char*)"__doc__", (getter)mp_doc, nullptr, nullptr, nullptr}, + {(char*)"__doc__", (getter)mp_doc, (setter)mp_doc_set, nullptr, nullptr}, // to be more python-like, where these are duplicated as well; to actually // derive from the python method or function type is too memory-expensive, @@ -513,9 +580,11 @@ static PyGetSetDef mp_getset[] = { {(char*)"func_code", (getter)mp_func_code, nullptr, nullptr, nullptr}, {(char*)"func_defaults", (getter)mp_func_defaults, nullptr, nullptr, nullptr}, {(char*)"func_globals", (getter)mp_func_globals, nullptr, nullptr, nullptr}, - {(char*)"func_doc", (getter)mp_doc, nullptr, nullptr, nullptr}, + {(char*)"func_doc", (getter)mp_doc, (setter)mp_doc_set, nullptr, nullptr}, {(char*)"func_name", (getter)mp_name, nullptr, nullptr, nullptr}, + +// flags to control behavior {(char*)"__creates__", (getter)mp_getcreates, (setter)mp_setcreates, (char*)"For ownership rules of result: if true, objects are python-owned", nullptr}, {(char*)"__mempolicy__", (getter)mp_getmempolicy, (setter)mp_setmempolicy, @@ -528,18 +597,34 @@ static PyGetSetDef mp_getset[] = { (char*)"not implemented", nullptr}, {(char*)"__sig2exc__", (getter)mp_getsig2exc, (setter)mp_setsig2exc, (char*)"If true, turn signals into Python exceptions", nullptr}, + +// basic reflection information + {(char*)"__cpp_name__", (getter)mp_getcppname, nullptr, nullptr, nullptr}, + {(char*)nullptr, nullptr, nullptr, nullptr, nullptr} }; //= CPyCppyy method proxy function behavior ================================== +#if PY_VERSION_HEX >= 0x03080000 +static PyObject* mp_vectorcall( + CPPOverload* pymeth, PyObject* const *args, size_t nargsf, PyObject* kwds) +#else static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) +#endif { +#if PY_VERSION_HEX < 0x03080000 + size_t nargsf = PyTuple_GET_SIZE(args); +#endif + // Call the appropriate overload of this method. - CPPInstance* oldSelf = pymeth->fSelf; +// If called from a descriptor, then this could be a bound function with +// non-zero self; otherwise pymeth->fSelf is expected to always be nullptr. + + CPPInstance* im_self = pymeth->fSelf; // get local handles to proxy internals - auto& methods = pymeth->fMethodInfo->fMethods; + auto& methods = pymeth->fMethodInfo->fMethods; CPPOverload::Methods_t::size_type nMethods = methods.size(); @@ -550,26 +635,21 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) ctxt.fFlags |= (mflags & CallContext::kReleaseGIL); ctxt.fFlags |= (mflags & CallContext::kProtected); if (IsConstructor(pymeth->fMethodInfo->fFlags)) ctxt.fFlags |= CallContext::kIsConstructor; - ctxt.fPyContext = (PyObject*)pymeth->fSelf; // no Py_INCREF as no ownership - -// magic variable to prevent recursion passed by keyword? - if (kwds && PyDict_CheckExact(kwds) && PyDict_Size(kwds) != 0) { - if (PyDict_DelItem(kwds, PyStrings::gNoImplicit) == 0) { - ctxt.fFlags |= CallContext::kNoImplicit; - if (!PyDict_Size(kwds)) kwds = nullptr; - } else - PyErr_Clear(); - } + ctxt.fFlags |= (pymeth->fFlags & (CallContext::kCallDirect | CallContext::kFromDescr)); + ctxt.fPyContext = (PyObject*)im_self; // no Py_INCREF as no ownership + +// check implicit conversions status (may be disallowed to prevent recursion) + ctxt.fFlags |= (pymeth->fFlags & CallContext::kNoImplicit); // simple case if (nMethods == 1) { if (!NoImplicit(&ctxt)) ctxt.fFlags |= CallContext::kAllowImplicit; // no two rounds needed - PyObject* result = methods[0]->Call(pymeth->fSelf, args, kwds, &ctxt); - return HandleReturn(pymeth, oldSelf, result); + PyObject* result = methods[0]->Call(im_self, args, nargsf, kwds, &ctxt); + return HandleReturn(pymeth, im_self, result); } // otherwise, handle overloading - uint64_t sighash = HashSignature(args); + uint64_t sighash = HashSignature(args, nargsf); // look for known signatures ... auto& dispatchMap = pymeth->fMethodInfo->fDispatchMap; @@ -581,19 +661,30 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) } } if (memoized_pc) { - PyObject* result = memoized_pc->Call(pymeth->fSelf, args, kwds, &ctxt); - result = HandleReturn(pymeth, oldSelf, result); - + // it is necessary to enable implicit conversions as the memoized call may be from + // such a conversion case; if the call fails, the implicit flag is reset below + if (!NoImplicit(&ctxt)) ctxt.fFlags |= CallContext::kAllowImplicit; + PyObject* result = memoized_pc->Call(im_self, args, nargsf, kwds, &ctxt); if (result) - return result; + return HandleReturn(pymeth, im_self, result); // fall through: python is dynamic, and so, the hashing isn't infallible + ctxt.fFlags &= ~CallContext::kAllowImplicit; PyErr_Clear(); + ResetCallState(pymeth->fSelf, im_self); } - + // ... otherwise loop over all methods and find the one that does not fail if (!IsSorted(mflags)) { - std::stable_sort(methods.begin(), methods.end(), PriorityCmp); + // sorting is based on priority, which is not stored on the method as it is used + // only once, so copy the vector of methods into one where the priority can be + // stored during sorting + std::vector> pm; pm.reserve(methods.size()); + for (auto ptr : methods) + pm.emplace_back(ptr->GetPriority(), ptr); + std::stable_sort(pm.begin(), pm.end(), PriorityCmp); + for (CPPOverload::Methods_t::size_type i = 0; i < methods.size(); ++i) + methods[i] = pm[i].second; pymeth->fMethodInfo->fFlags |= CallContext::kIsSorted; } @@ -605,7 +696,7 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) if (stage && !implicit_possible[i]) continue; // did not set implicit conversion, so don't try again - PyObject* result = methods[i]->Call(pymeth->fSelf, args, kwds, &ctxt); + PyObject* result = methods[i]->Call(im_self, args, nargsf, kwds, &ctxt); if (result != 0) { // success: update the dispatch map for subsequent calls if (!memoized_pc) @@ -624,12 +715,13 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) // clear collected errors if (!errors.empty()) std::for_each(errors.begin(), errors.end(), Utility::PyError_t::Clear); - return HandleReturn(pymeth, oldSelf, result); + return HandleReturn(pymeth, im_self, result); } // else failure .. if (stage != 0) { - PyErr_Clear(); // first stage errors should be more informative + PyErr_Clear(); // first stage errors should be the more informative + ResetCallState(pymeth->fSelf, im_self); continue; } @@ -638,10 +730,14 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) // this should not happen; set an error to prevent core dump and report PyObject* sig = methods[i]->GetPrototype(); PyErr_Format(PyExc_SystemError, "%s =>\n %s", - CPyCppyy_PyText_AsString(sig), (char*)"nullptr result without error in mp_call"); + CPyCppyy_PyText_AsString(sig), (char*)"nullptr result without error in overload call"); Py_DECREF(sig); } - Utility::FetchError(errors); + + // retrieve, store, and clear errors + bool callee_error = ctxt.fFlags & (CallContext::kPyException | CallContext::kCppException); + ctxt.fFlags &= ~(CallContext::kPyException | CallContext::kCppException); + Utility::FetchError(errors, callee_error); if (HaveImplicit(&ctxt)) { bHaveImplicit = true; @@ -649,7 +745,7 @@ static PyObject* mp_call(CPPOverload* pymeth, PyObject* args, PyObject* kwds) ctxt.fFlags &= ~CallContext::kHaveImplicit; } else implicit_possible[i] = false; - ResetCallState(pymeth->fSelf, oldSelf, false); + ResetCallState(pymeth->fSelf, im_self); } // only move forward if implicit conversions are available @@ -678,36 +774,76 @@ static PyObject* mp_str(CPPOverload* cppinst) } //---------------------------------------------------------------------------- -static CPPOverload* mp_descrget(CPPOverload* pymeth, CPPInstance* pyobj, PyObject*) +static CPPOverload* mp_descr_get(CPPOverload* pymeth, CPPInstance* pyobj, PyObject*) { -// Descriptor; create and return a new bound method proxy (language requirement) if self - if (!pyobj) { +// Descriptor; create and return a new, possibly bound, method proxy. This method +// has evolved with versions of python as follows: +// +// Python version | Action +// <- py2.7 | bound methods need to be first-class objects, so create a new +// | method object if self is not nullptr or Py_None +// py3.0-py3.7 | bound methods are no longer a language requirement, but +// | still supported: for convenience, retain old behavior +// py3.8 <= | vector calls no longer call the descriptor, so when it is +// | called, the method is likely stored, so should be new object + +#if PY_VERSION_HEX < 0x03080000 + if (!pyobj || (PyObject*)pyobj == Py_None /* from unbound TemplateProxy */) { + Py_XDECREF(pymeth->fSelf); pymeth->fSelf = nullptr; + pymeth->fFlags |= CallContext::kCallDirect | CallContext::kFromDescr; Py_INCREF(pymeth); return pymeth; // unbound, e.g. free functions } +#endif -// else: bound +// create a new method object + bool gc_track = false; CPPOverload* newPyMeth = free_list; if (newPyMeth != NULL) { free_list = (CPPOverload*)(newPyMeth->fSelf); (void)PyObject_INIT(newPyMeth, &CPPOverload_Type); numfree--; - } - else { + } else { newPyMeth = PyObject_GC_New(CPPOverload, &CPPOverload_Type); if (!newPyMeth) return nullptr; + gc_track = true; } // method info is shared, as it contains the collected overload knowledge *pymeth->fMethodInfo->fRefCount += 1; newPyMeth->fMethodInfo = pymeth->fMethodInfo; +#if PY_VERSION_HEX >= 0x03080000 + newPyMeth->fVectorCall = pymeth->fVectorCall; + + if (pyobj && (PyObject*)pyobj != Py_None) { + Py_INCREF((PyObject*)pyobj); + newPyMeth->fSelf = pyobj; + newPyMeth->fFlags = CallContext::kNone; + } else { + newPyMeth->fSelf = nullptr; + newPyMeth->fFlags = CallContext::kCallDirect; + } + +// vector calls don't get here, unless a method is looked up on an instance, for +// e.g. class methods (C++ static); notify downstream to expect a 'self' + newPyMeth->fFlags |= CallContext::kFromDescr; + +#else // new method is to be bound to current object Py_INCREF((PyObject*)pyobj); newPyMeth->fSelf = pyobj; - PyObject_GC_Track(newPyMeth); +// reset flags of the new method, as there is a self (which may or may not have +// come in through direct call syntax, but that's now impossible to know, so this +// is the safer choice) + newPyMeth->fFlags = CallContext::kNone; +#endif + + if (gc_track) + PyObject_GC_Track(newPyMeth); + return newPyMeth; } @@ -718,6 +854,7 @@ static CPPOverload* mp_new(PyTypeObject*, PyObject*, PyObject*) // Create a new method proxy object. CPPOverload* pymeth = PyObject_GC_New(CPPOverload, &CPPOverload_Type); pymeth->fSelf = nullptr; + pymeth->fFlags = CallContext::kNone; pymeth->fMethodInfo = new CPPOverload::MethodInfo_t; PyObject_GC_Track(pymeth); @@ -740,8 +877,7 @@ static void mp_dealloc(CPPOverload* pymeth) pymeth->fSelf = (CPyCppyy::CPPInstance*)free_list; free_list = pymeth; numfree++; - } - else { + } else { PyObject_GC_Del(pymeth); } } @@ -795,7 +931,7 @@ static PyObject* mp_overload(CPPOverload* pymeth, PyObject* args) { // Select and call a specific C++ overload, based on its signature. const char* sigarg = nullptr; - PyObject* sigarg_tuple = nullptr; + PyObject* sigarg_tuple = nullptr; int want_const = -1; Py_ssize_t args_size = PyTuple_GET_SIZE(args); if (args_size && @@ -813,7 +949,6 @@ static PyObject* mp_overload(CPPOverload* pymeth, PyObject* args) } } -//= CPyCppyy method proxy access to internals ================================ static PyObject* mp_add_overload(CPPOverload* pymeth, PyObject* new_overload) { TPythonCallback* cb = new TPythonCallback(new_overload); @@ -849,58 +984,73 @@ static PyMethodDef mp_methods[] = { //= CPyCppyy method proxy type =============================================== PyTypeObject CPPOverload_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - (char*)"cppyy.CPPOverload", // tp_name - sizeof(CPPOverload), // tp_basicsize - 0, // tp_itemsize - (destructor)mp_dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - (hashfunc)mp_hash, // tp_hash - (ternaryfunc)mp_call, // tp_call - (reprfunc)mp_str, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags - (char*)"cppyy method proxy (internal)", // tp_doc - (traverseproc)mp_traverse, // tp_traverse - (inquiry)mp_clear, // tp_clear - (richcmpfunc)mp_richcompare, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - mp_methods, // tp_methods - 0, // tp_members - mp_getset, // tp_getset - 0, // tp_base - 0, // tp_dict - (descrgetfunc)mp_descrget, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - (newfunc)mp_new, // tp_new - 0, // tp_free - 0, // tp_is_gc - 0, // tp_bases - 0, // tp_mro - 0, // tp_cache - 0, // tp_subclasses - 0 // tp_weaklist + (char*)"cppyy.CPPOverload", // tp_name + sizeof(CPPOverload), // tp_basicsize + 0, // tp_itemsize + (destructor)mp_dealloc, // tp_dealloc +#if PY_VERSION_HEX >= 0x03080000 + offsetof(CPPOverload, fVectorCall), +#else + 0, // tp_vectorcall_offset / tp_print +#endif + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_as_async / tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + (hashfunc)mp_hash, // tp_hash +#if PY_VERSION_HEX >= 0x03080000 + (ternaryfunc)PyVectorcall_Call, // tp_call +#else + (ternaryfunc)mp_call, // tp_call +#endif + (reprfunc)mp_str, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC +#if PY_VERSION_HEX >= 0x03080000 + | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_METHOD_DESCRIPTOR +#endif + , // tp_flags + (char*)"cppyy method proxy (internal)", // tp_doc + (traverseproc)mp_traverse, // tp_traverse + (inquiry)mp_clear, // tp_clear + (richcmpfunc)mp_richcompare, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + mp_methods, // tp_methods + 0, // tp_members + mp_getset, // tp_getset + 0, // tp_base + 0, // tp_dict + (descrgetfunc)mp_descr_get, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + (newfunc)mp_new, // tp_new + 0, // tp_free + 0, // tp_is_gc + 0, // tp_bases + 0, // tp_mro + 0, // tp_cache + 0, // tp_subclasses + 0 // tp_weaklist #if PY_VERSION_HEX >= 0x02030000 - , 0 // tp_del + , 0 // tp_del #endif #if PY_VERSION_HEX >= 0x02060000 - , 0 // tp_version_tag + , 0 // tp_version_tag #endif #if PY_VERSION_HEX >= 0x03040000 - , 0 // tp_finalize + , 0 // tp_finalize +#endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall #endif }; @@ -923,6 +1073,10 @@ void CPyCppyy::CPPOverload::Set(const std::string& name, std::vectorfFlags |= CallContext::kIsCreator; + +#if PY_VERSION_HEX >= 0x03080000 + fVectorCall = (vectorcallfunc)mp_vectorcall; +#endif } //---------------------------------------------------------------------------- @@ -959,7 +1113,6 @@ PyObject* CPyCppyy::CPPOverload::FindOverload(const std::string& signature, int CPPOverload::Methods_t& methods = fMethodInfo->fMethods; for (auto& meth : methods) { - bool found = accept_any; if (!found) { PyObject* pysig2 = meth->GetSignature(false); @@ -1011,11 +1164,11 @@ PyObject* CPyCppyy::CPPOverload::FindOverload(const std::string& signature, int PyObject* CPyCppyy::CPPOverload::FindOverload(PyObject *args_tuple, int want_const) { Py_ssize_t n = PyTuple_Size(args_tuple); - + CPPOverload::Methods_t& methods = fMethodInfo->fMethods; // This value is set based on the maximum penalty in Cppyy::CompareMethodArgType - Py_ssize_t min_score = INT_MAX; + Py_ssize_t min_score = INT_MAX; bool found = false; size_t best_method = 0, method_index = 0; @@ -1054,7 +1207,7 @@ PyObject* CPyCppyy::CPPOverload::FindOverload(PyObject *args_tuple, int want_con PyErr_Format(PyExc_LookupError, "signature with arguments \"%s\" not found", sigargs.c_str()); return (PyObject*) nullptr; } - + CPPOverload* newmeth = mp_new(nullptr, nullptr, nullptr); CPPOverload::Methods_t vec; vec.push_back(methods[best_method]->Clone()); @@ -1065,7 +1218,7 @@ PyObject* CPyCppyy::CPPOverload::FindOverload(PyObject *args_tuple, int want_con newmeth->fSelf = fSelf; } newmeth->fMethodInfo->fFlags = fMethodInfo->fFlags; - + return (PyObject*) newmeth; } @@ -1078,6 +1231,7 @@ CPyCppyy::CPPOverload::MethodInfo_t::~MethodInfo_t() } fMethods.clear(); delete fRefCount; + Py_XDECREF(fDoc); } // TODO: something like PyMethod_Fini to clear up the free_list diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.h index ae46b1bcf789e..b392bf4ed7f20 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPOverload.h @@ -5,6 +5,7 @@ #include "PyCallable.h" // Standard +#include #include #include #include @@ -13,16 +14,16 @@ namespace CPyCppyy { // signature hashes are also used by TemplateProxy -inline uint64_t HashSignature(PyObject* args) +inline uint64_t HashSignature(CPyCppyy_PyArgs_t args, size_t nargsf) { // Build a hash from the types of the given python function arguments. uint64_t hash = 0; - int nargs = (int)PyTuple_GET_SIZE(args); - for (int i = 0; i < nargs; ++i) { + Py_ssize_t nargs = CPyCppyy_PyArgs_GET_SIZE(args, nargsf); + for (Py_ssize_t i = 0; i < nargs; ++i) { // TODO: hashing in the ref-count is for moves; resolve this together with the // improved overloads for implicit conversions - PyObject* pyobj = PyTuple_GET_ITEM(args, i); + PyObject* pyobj = CPyCppyy_PyArgs_GET_ITEM(args, i); hash += (uint64_t)Py_TYPE(pyobj); hash += (uint64_t)(pyobj->ob_refcnt == 1 ? 1 : 0); hash += (hash << 10); hash ^= (hash >> 6); @@ -39,13 +40,15 @@ class CPPOverload { typedef std::vector Methods_t; struct MethodInfo_t { - MethodInfo_t() : fFlags(CallContext::kNone) { fRefCount = new int(1); } + MethodInfo_t() : fDoc(nullptr), fFlags(CallContext::kNone) + { fRefCount = new int(1); } ~MethodInfo_t(); std::string fName; CPPOverload::DispatchMap_t fDispatchMap; CPPOverload::Methods_t fMethods; - uint64_t fFlags; + PyObject* fDoc; + uint32_t fFlags; int* fRefCount; @@ -70,6 +73,10 @@ class CPPOverload { PyObject_HEAD CPPInstance* fSelf; // must be first (same layout as TemplateProxy) MethodInfo_t* fMethodInfo; + uint32_t fFlags; +#if PY_VERSION_HEX >= 0x03080000 + vectorcallfunc fVectorCall; +#endif private: CPPOverload() = delete; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.cxx index 03c3befde0721..0750ce6c00513 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.cxx @@ -2,6 +2,7 @@ #include "CPyCppyy.h" #include "CPPScope.h" #include "CPPDataMember.h" +#include "CPPEnum.h" #include "CPPFunction.h" #include "CPPOverload.h" #include "CustomPyTypes.h" @@ -30,29 +31,21 @@ static inline PyObject* add_template(PyObject* pyclass, { // If templated, the user-facing function must be the template proxy, but the // specific lookup must be the current overload, if already found. - TemplateProxy* pytmpl = nullptr; const std::string& ncl = TypeManip::clean_type(name); - if (ncl != name) { - PyObject* pyncl = CPyCppyy_PyText_InternFromString(ncl.c_str()); - pytmpl = (TemplateProxy*)PyType_Type.tp_getattro((PyObject*)Py_TYPE(pyclass), pyncl); - if (!pytmpl) { - PyErr_Clear(); - pytmpl = TemplateProxy_New(ncl, ncl, pyclass); - // cache the template on its clean name - PyType_Type.tp_setattro((PyObject*)Py_TYPE(pyclass), pyncl, (PyObject*)pytmpl); - } + PyObject* pyncl = CPyCppyy_PyText_FromString(ncl.c_str()); + TemplateProxy* pytmpl = (TemplateProxy*)PyType_Type.tp_getattro(pyclass, pyncl); + if (!pytmpl) { + PyErr_Clear(); + pytmpl = TemplateProxy_New(ncl, ncl, pyclass); + // cache the template on its clean name + PyType_Type.tp_setattro(pyclass, pyncl, (PyObject*)pytmpl); Py_DECREF(pyncl); + } else if (!TemplateProxy_CheckExact((PyObject*)pytmpl)) { + Py_DECREF(pytmpl); + return nullptr; } - if (pytmpl) { - if (!TemplateProxy_CheckExact((PyObject*)pytmpl)) { - Py_DECREF(pytmpl); - return nullptr; - } - } else - pytmpl = TemplateProxy_New(ncl, ncl, pyclass); - if (overloads) { // adopt the new overloads if (ncl != name) @@ -61,6 +54,8 @@ static inline PyObject* add_template(PyObject* pyclass, for (auto clb : *overloads) pytmpl->AdoptMethod(clb); } +// the caller expects a method matching the full name, thus is a specialization +// was requested, do not return the template yet if (ncl == name) return (PyObject*)pytmpl; @@ -68,14 +63,6 @@ static inline PyObject* add_template(PyObject* pyclass, return nullptr; // so that caller caches the method on full name } -//---------------------------------------------------------------------------- -static int enum_setattro(PyObject* /* pyclass */, PyObject* /* pyname */, PyObject* /* pyval */) -{ -// Helper to make enums read-only. - PyErr_SetString(PyExc_TypeError, "enum values are read-only"); - return -1; -} - //= CPyCppyy type proxy construction/destruction ============================= static PyObject* meta_alloc(PyTypeObject* meta, Py_ssize_t nitems) @@ -185,6 +172,10 @@ static PyObject* meta_repr(CPPScope* scope) return PyType_Type.tp_repr((PyObject*)scope); } +// skip in case some Python-side derived meta class + if (!CPPScope_Check(scope) || !scope->fCppType) + return PyType_Type.tp_repr((PyObject*)scope); + // printing of C++ classes PyObject* modname = meta_getmodule(scope, nullptr); std::string clName = Cppyy::GetFinalName(scope->fCppType); @@ -216,9 +207,12 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) if (Cppyy::GetSmartPtrInfo(Cppyy::GetScopedFinalName(((CPPScope*)subtype)->fCppType), &raw, &deref)) subtype->tp_basicsize = sizeof(CPPSmartClass); } + CPPScope* result = (CPPScope*)PyType_Type.tp_new(subtype, args, kwds); - if (!result) + if (!CPPScope_Check(result)) { + // either failed or custom user-side metaclass that can't be handled here return nullptr; + } result->fFlags = CPPScope::kNone; result->fOperators = nullptr; @@ -231,8 +225,7 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) } // initialization of class (based on metatype) - const char* mp = strstr(subtype->tp_name, "_meta"); - if (!mp || !CPPScope_CheckExact(subtype)) { + if (!CPPScope_CheckExact(subtype) || !strstr(subtype->tp_name, "_meta") /* convention */) { // there has been a user meta class override in a derived class, so do // the consistent thing, thus allowing user control over naming result->fCppType = Cppyy::GetScope( @@ -254,6 +247,8 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) Py_ssize_t sz = PyDict_Size(dct); if (0 < sz && !Cppyy::IsNamespace(result->fCppType)) { result->fFlags |= CPPScope::kIsPython; + if (1 < PyTuple_GET_SIZE(PyTuple_GET_ITEM(args, 1))) + result->fFlags |= CPPScope::kIsMultiCross; std::ostringstream errmsg; if (!InsertDispatcher(result, PyTuple_GET_ITEM(args, 1), dct, errmsg)) { PyErr_Format(PyExc_TypeError, "no python-side overrides supported (%s)", errmsg.str().c_str()); @@ -272,6 +267,11 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) } } +// if the user messed with the metaclass, then we may not have a C++ type, +// simply return here before more damage gets done + if (!result->fCppType) + return (PyObject*)result; + // maps for using namespaces and tracking objects if (!Cppyy::IsNamespace(result->fCppType)) { static Cppyy::TCppType_t exc_type = (Cppyy::TCppType_t)Cppyy::GetScope("std::exception"); @@ -297,6 +297,7 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) Py_DECREF((PyObject*)result); return nullptr; } + return (PyObject*)result; } @@ -306,27 +307,52 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) { // normal type-based lookup PyObject* attr = PyType_Type.tp_getattro(pyclass, pyname); - if (attr || pyclass == (PyObject*)&CPPInstance_Type) + if (pyclass == (PyObject*)&CPPInstance_Type) return attr; + PyObject* possibly_shadowed = nullptr; + if (attr) { + if (CPPScope_Check(attr) && CPPScope_Check(pyclass) && !(((CPPScope*)pyclass)->fFlags & CPPScope::kIsNamespace)) { + // TODO: the goal here is to prevent typedefs that are shadowed in subclasses + // to be found as from the base class. The better approach would be to find + // all typedefs at class creation and insert placeholders to flag here. The + // current typedef loop in gInterpreter won't do, however. + PyObject* dct = PyObject_GetAttr(pyclass, PyStrings::gDict); + if (dct) { + PyObject* attr_from_dict = PyObject_GetItem(dct, pyname); + Py_DECREF(dct); + if (attr_from_dict) { + Py_DECREF(attr); + return attr_from_dict; + } + possibly_shadowed = attr; + attr = nullptr; + } + PyErr_Clear(); + } else + return attr; + } + if (!CPyCppyy_PyText_CheckExact(pyname) || !CPPScope_Check(pyclass)) - return nullptr; + return possibly_shadowed; // filter for python specials std::string name = CPyCppyy_PyText_AsString(pyname); - if (name.size() >= 2 && name.compare(0, 2, "__") == 0 && + if (name.size() >= 5 && name.compare(0, 2, "__") == 0 && name.compare(name.size()-2, name.size(), "__") == 0) - return nullptr; + return possibly_shadowed; // more elaborate search in case of failure (eg. for inner classes on demand) std::vector errors; Utility::FetchError(errors); attr = CreateScopeProxy(name, pyclass); + if (CPPScope_Check(attr) && (((CPPScope*)attr)->fFlags & CPPScope::kIsException)) { // Instead of the CPPScope, return a fresh exception class derived from CPPExcInstance. return CreateExcScopeProxy(attr, pyname, pyclass); } + bool templated_functions_checked = false; CPPScope* klass = ((CPPScope*)pyclass); if (!attr) { Utility::FetchError(errors); @@ -351,9 +377,9 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) // that it doesn't exist yet. if (Cppyy::ExistsMethodTemplate(scope, name)) attr = add_template(pyclass, name, &overloads); - - if (!attr) // add_template can fail if the method can not be added + else attr = (PyObject*)CPPOverload_New(name, overloads); + templated_functions_checked = true; } // tickle lazy lookup of data members @@ -368,22 +394,24 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) const std::string& lookup = Cppyy::GetScopedFinalName(klass->fCppType) + "::" + name; const std::string& resolved = Cppyy::ResolveName(lookup); if (resolved != lookup) { - const std::string& cpd = Utility::Compound(resolved); + const std::string& cpd = TypeManip::compound(resolved); if (cpd == "*") { const std::string& clean = TypeManip::clean_type(resolved, false, true); Cppyy::TCppType_t tcl = Cppyy::GetScope(clean); if (tcl) { typedefpointertoclassobject* tpc = PyObject_GC_New(typedefpointertoclassobject, &TypedefPointerToClass_Type); - tpc->fType = tcl; + tpc->fCppType = tcl; + tpc->fDict = PyDict_New(); attr = (PyObject*)tpc; } } } } - // function templates that have not been instantiated - if (!attr) { + // function templates that have not been instantiated (namespaces _may_ have already + // been taken care of, by their general function lookup above) + if (!attr && !templated_functions_checked) { if (Cppyy::ExistsMethodTemplate(scope, name)) attr = add_template(pyclass, name); else { @@ -396,62 +424,10 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) // enums types requested as type (rather than the constants) if (!attr) { // TODO: IsEnum should deal with the scope, using klass->GetListOfEnums()->FindObject() - if (Cppyy::IsEnum(scope == Cppyy::gGlobalScope ? name : Cppyy::GetScopedFinalName(scope)+"::"+name)) { + const std::string& ename = scope == Cppyy::gGlobalScope ? name : Cppyy::GetScopedFinalName(scope)+"::"+name; + if (Cppyy::IsEnum(ename)) { // enum types (incl. named and class enums) - Cppyy::TCppEnum_t etype = Cppyy::GetEnum(scope, name); - if (etype) { - // create new enum type with labeled values in place, with a meta-class - // to make sure the enum values are read-only - PyObject* pymetabases = PyTuple_New(1); - PyObject* btype = (PyObject*)Py_TYPE(&PyInt_Type); - Py_INCREF(btype); - PyTuple_SET_ITEM(pymetabases, 0, btype); - - PyObject* args = Py_BuildValue((char*)"sO{}", (name+"_meta").c_str(), pymetabases); - Py_DECREF(pymetabases); - PyObject* pymeta = PyType_Type.tp_new(Py_TYPE(&PyInt_Type), args, nullptr); - ((PyTypeObject*)pymeta)->tp_setattro = enum_setattro; - Py_DECREF(args); - - // prepare the base class - PyObject* pybases = PyTuple_New(1); - Py_INCREF(&PyInt_Type); - PyTuple_SET_ITEM(pybases, 0, (PyObject*)&PyInt_Type); - - // collect the enum values - Cppyy::TCppIndex_t ndata = Cppyy::GetNumEnumData(etype); - PyObject* dct = PyDict_New(); - for (Cppyy::TCppIndex_t idata = 0; idata < ndata; ++idata) { - PyObject* val = PyLong_FromLongLong(Cppyy::GetEnumDataValue(etype, idata)); - PyDict_SetItemString(dct, Cppyy::GetEnumDataName(etype, idata).c_str(), val); - Py_DECREF(val); - } - - // add the __cpp_name__ for templates - PyObject* cppname = nullptr; - if (scope == Cppyy::gGlobalScope) { - Py_INCREF(pyname); - cppname = pyname; - } else - cppname = CPyCppyy_PyText_FromString((Cppyy::GetScopedFinalName(scope)+"::"+name).c_str()); - PyDict_SetItem(dct, PyStrings::gCppName, cppname); - Py_DECREF(cppname); - - // create the actual enum class - args = Py_BuildValue((char*)"sOO", name.c_str(), pybases, dct); - Py_DECREF(pybases); - Py_DECREF(dct); - attr = ((PyTypeObject*)pymeta)->tp_new((PyTypeObject*)pymeta, args, nullptr); - - // final cleanup - Py_DECREF(args); - Py_DECREF(pymeta); - - } else { - // presumably not a class enum; simply pretend int - Py_INCREF(&PyInt_Type); - attr = (PyObject*)&PyInt_Type; - } + attr = (PyObject*)CPPEnum_New(name, scope); } else { // for completeness in error reporting PyErr_Format(PyExc_TypeError, "\'%s\' is not a known C++ enum", name.c_str()); @@ -465,6 +441,8 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) PyType_Type.tp_setattro((PyObject*)Py_TYPE(pyclass), pyname, attr); Py_DECREF(attr); attr = PyType_Type.tp_getattro(pyclass, pyname); + if (!attr && PyErr_Occurred()) + Utility::FetchError(errors); } else PyType_Type.tp_setattro(pyclass, pyname, attr); @@ -510,6 +488,19 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) } } +// if the attribute was not found but could possibly have been shadowed, insert it into +// the dict now, to short-circuit future lookups (TODO: this is part of the workaround +// described above of which the true solution is to loop over all typedefs at creation +// time for the class) + if (possibly_shadowed) { + if (attr) { + Py_DECREF(possibly_shadowed); + } else { + attr = possibly_shadowed; + PyType_Type.tp_setattro(pyclass, pyname, attr); + } + } + if (attr) { std::for_each(errors.begin(), errors.end(), Utility::PyError_t::Clear); PyErr_Clear(); @@ -562,10 +553,20 @@ static PyObject* meta_reflex(CPPScope* klass, PyObject* args) if (!PyArg_ParseTuple(args, const_cast("i|i:__cpp_reflex__"), &request, &format)) return nullptr; - if (request == Cppyy::Reflex::IS_NAMESPACE) { + switch (request) { + case Cppyy::Reflex::IS_NAMESPACE: if (klass->fFlags & CPPScope::kIsNamespace) Py_RETURN_TRUE; Py_RETURN_FALSE; + break; + case Cppyy::Reflex::IS_AGGREGATE: + // this is not the strict C++ definition of aggregates, but is closer to what + // is needed for Numba and C calling conventions (TODO: probably have to check + // for all public data types, too, and maybe for no padding?) + if (Cppyy::IsAggregate(klass->fCppType) || !Cppyy::HasVirtualDestructor(klass->fCppType)) + Py_RETURN_TRUE; + Py_RETURN_FALSE; + break; } PyErr_Format(PyExc_ValueError, "unsupported reflex request %d or format %d", request, format); @@ -599,16 +600,25 @@ static PyObject* meta_dir(CPPScope* klass) std::set cppnames; Cppyy::GetAllCppNames(klass->fCppType, cppnames); +// cleanup names + std::set dir_cppnames; + for (const std::string& name : cppnames) { + if (name.find("__", 0, 2) != std::string::npos || \ + name.find("<") != std::string::npos || \ + name.find("operator", 0, 8) != std::string::npos) continue; + dir_cppnames.insert(name); + } + // get rid of duplicates for (Py_ssize_t i = 0; i < PyList_GET_SIZE(dirlist); ++i) - cppnames.insert(CPyCppyy_PyText_AsString(PyList_GET_ITEM(dirlist, i))); + dir_cppnames.insert(CPyCppyy_PyText_AsString(PyList_GET_ITEM(dirlist, i))); Py_DECREF(dirlist); - dirlist = PyList_New(cppnames.size()); + dirlist = PyList_New(dir_cppnames.size()); // copy total onto python list Py_ssize_t i = 0; - for (const auto& name : cppnames) { + for (const auto& name : dir_cppnames) { PyList_SET_ITEM(dirlist, i++, CPyCppyy_PyText_FromString(name.c_str())); } return dirlist; @@ -638,10 +648,10 @@ PyTypeObject CPPScope_Type = { sizeof(CPyCppyy::CPPScope), // tp_basicsize 0, // tp_itemsize 0, // tp_dealloc - 0, // tp_print + 0, // tp_vectorcall_offset / tp_print 0, // tp_getattr 0, // tp_setattr - 0, // tp_compare + 0, // tp_as_async / tp_compare (reprfunc)meta_repr, // tp_repr 0, // tp_as_number 0, // tp_as_sequence @@ -655,6 +665,9 @@ PyTypeObject CPPScope_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE #if PY_VERSION_HEX >= 0x03040000 | Py_TPFLAGS_TYPE_SUBCLASS +#endif +#if PY_VERSION_HEX >= 0x03120000 + | Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_HAVE_GC #endif , // tp_flags (char*)"CPyCppyy metatype (internal)", // tp_doc @@ -691,6 +704,9 @@ PyTypeObject CPPScope_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.h index a9e2f8c0e021c..7a6959fa260e6 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPScope.h @@ -43,12 +43,17 @@ class CPPScope { kIsException = 0x0004, kIsSmart = 0x0008, kIsPython = 0x0010, - kIsInComplete = 0x0020 }; + kIsMultiCross = 0x0020, + kIsInComplete = 0x0040, + kNoImplicit = 0x0080, + kNoOSInsertion = 0x0100, + kGblOSInsertion = 0x0200, + kNoPrettyPrint = 0x0400 }; public: PyHeapTypeObject fType; Cppyy::TCppType_t fCppType; - int fFlags; + uint32_t fFlags; union { CppToPyMap_t* fCppObjects; // classes only std::vector* fUsing; // namespaces only @@ -75,7 +80,11 @@ extern PyTypeObject CPPScope_Type; template inline bool CPPScope_Check(T* object) { - return object && PyObject_TypeCheck(object, &CPPScope_Type); +// Short-circuit the type check by checking tp_new which all generated subclasses +// of CPPScope inherit. + return object && \ + (Py_TYPE(object)->tp_new == CPPScope_Type.tp_new || \ + PyObject_TypeCheck(object, &CPPScope_Type)); } template diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyy.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyy.h index a159a346f2765..d0f3b0fc5621a 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyy.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyy.h @@ -31,44 +31,22 @@ #endif // linux - +#define PY_SSIZE_T_CLEAN #include "Python.h" #include -// selected ROOT types from RtypesCore.h -#ifdef R__INT16 -typedef long Int_t; //Signed integer 4 bytes -typedef unsigned long UInt_t; //Unsigned integer 4 bytes -#else -typedef int Int_t; //Signed integer 4 bytes (int) -typedef unsigned int UInt_t; //Unsigned integer 4 bytes (unsigned int) -#endif -#ifdef R__B64 // Note: Long_t and ULong_t are currently not portable types -typedef long Long_t; //Signed long integer 8 bytes (long) -typedef unsigned long ULong_t; //Unsigned long integer 8 bytes (unsigned long) -#else -typedef long Long_t; //Signed long integer 4 bytes (long) -typedef unsigned long ULong_t; //Unsigned long integer 4 bytes (unsigned long) -#endif -typedef float Float16_t; //Float 4 bytes written with a truncated mantissa -typedef double Double32_t; //Double 8 bytes in memory, written as a 4 bytes float -typedef long double LongDouble_t;//Long Double -#ifdef _WIN32 -typedef __int64 Long64_t; //Portable signed long integer 8 bytes -typedef unsigned __int64 ULong64_t; //Portable unsigned long integer 8 bytes -#else -typedef long long Long64_t; //Portable signed long integer 8 bytes -typedef unsigned long long ULong64_t;//Portable unsigned long integer 8 bytes -#endif - -typedef Py_ssize_t dim_t; -typedef dim_t* dims_t; +namespace CPyCppyy { + typedef Py_ssize_t dim_t; +} // namespace CPyCppyy // for 3.3 support #if PY_VERSION_HEX < 0x03030000 typedef PyDictEntry* (*dict_lookup_func)(PyDictObject*, PyObject*, long); #else -#if PY_VERSION_HEX >= 0x03060000 +#if PY_VERSION_HEX >= 0x030b0000 + typedef Py_ssize_t (*dict_lookup_func)( + PyDictObject*, PyObject*, Py_hash_t, PyObject**); +#elif PY_VERSION_HEX >= 0x03060000 typedef Py_ssize_t (*dict_lookup_func)( PyDictObject*, PyObject*, Py_hash_t, PyObject***, Py_ssize_t*); #else @@ -84,6 +62,7 @@ typedef dim_t* dims_t; #define PyBytes_CheckExact PyString_CheckExact #define PyBytes_AS_STRING PyString_AS_STRING #define PyBytes_AsString PyString_AsString +#define PyBytes_AsStringAndSize PyString_AsStringAndSize #define PyBytes_GET_SIZE PyString_GET_SIZE #define PyBytes_Size PyString_Size #define PyBytes_FromFormat PyString_FromFormat @@ -297,6 +276,14 @@ inline Py_ssize_t PyNumber_AsSsize_t(PyObject* obj, PyObject*) { #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False #endif +#if PY_VERSION_HEX >= 0x03000000 && PY_VERSION_HEX < 0x03010000 +#define CPyCppyy_PyBuffer PyBuffer_Release +#else +inline void CPyCppyy_PyBuffer_Release(PyObject* /* unused */, Py_buffer* view) { + PyBuffer_Release(view); +} +#endif + // vector call support #if PY_VERSION_HEX >= 0x03090000 #define CPyCppyy_PyCFunction_Call PyObject_Call @@ -304,17 +291,82 @@ inline Py_ssize_t PyNumber_AsSsize_t(PyObject* obj, PyObject*) { #define CPyCppyy_PyCFunction_Call PyCFunction_Call #endif -// Py_TYPE is changed to an inline static function in 3.11 +// vector call support +#if PY_VERSION_HEX >= 0x03080000 +typedef PyObject* const* CPyCppyy_PyArgs_t; +static inline PyObject* CPyCppyy_PyArgs_GET_ITEM(CPyCppyy_PyArgs_t args, Py_ssize_t i) { + return args[i]; +} +static inline PyObject* CPyCppyy_PyArgs_SET_ITEM(CPyCppyy_PyArgs_t args, Py_ssize_t i, PyObject* item) { + return ((PyObject**)args)[i] = item; +} +static inline Py_ssize_t CPyCppyy_PyArgs_GET_SIZE(CPyCppyy_PyArgs_t, size_t nargsf) { + return PyVectorcall_NARGS(nargsf); +} +static inline CPyCppyy_PyArgs_t CPyCppyy_PyArgs_New(Py_ssize_t N) { + return (CPyCppyy_PyArgs_t)PyMem_Malloc(N*sizeof(PyObject*)); +} +static inline void CPyCppyy_PyArgs_DEL(CPyCppyy_PyArgs_t args) { + PyMem_Free((void*)args); +} +#if PY_VERSION_HEX >= 0x03090000 +#define CPyCppyy_PyObject_Call PyObject_Vectorcall +#else +#define CPyCppyy_PyObject_Call _PyObject_Vectorcall +#endif +inline PyObject* CPyCppyy_tp_call( + PyObject* cb, CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds) { + Py_ssize_t offset = Py_TYPE(cb)->tp_vectorcall_offset; + vectorcallfunc func = *(vectorcallfunc*)(((char*)cb) + offset); + return func(cb, args, nargsf, kwds); +} + +#ifndef Py_TPFLAGS_HAVE_VECTORCALL +#define Py_TPFLAGS_HAVE_VECTORCALL _Py_TPFLAGS_HAVE_VECTORCALL +#endif + +#else + +typedef PyObject* CPyCppyy_PyArgs_t; +static inline PyObject* CPyCppyy_PyArgs_GET_ITEM(CPyCppyy_PyArgs_t args, Py_ssize_t i) { + return PyTuple_GET_ITEM(args, i); +} +static inline PyObject* CPyCppyy_PyArgs_SET_ITEM(CPyCppyy_PyArgs_t args, Py_ssize_t i, PyObject* item) { + return PyTuple_SET_ITEM(args, i, item); +} +static inline Py_ssize_t CPyCppyy_PyArgs_GET_SIZE(CPyCppyy_PyArgs_t args, size_t) { + return PyTuple_GET_SIZE(args); +} +static inline CPyCppyy_PyArgs_t CPyCppyy_PyArgs_New(Py_ssize_t N) { + return PyTuple_New(N); +} +static inline void CPyCppyy_PyArgs_DEL(CPyCppyy_PyArgs_t args) { + Py_DECREF(args); +} +inline PyObject* CPyCppyy_PyObject_Call(PyObject* cb, PyObject* args, size_t, PyObject* kwds) { + return PyObject_Call(cb, args, kwds); +} +inline PyObject* CPyCppyy_tp_call(PyObject* cb, PyObject* args, size_t, PyObject* kwds) { + return Py_TYPE(cb)->tp_call(cb, args, kwds); +} +#endif + +// Py_TYPE as inline function #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE) static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } #define Py_SET_TYPE(ob, type) _Py_SET_TYPE((PyObject*)(ob), type) #endif +// py39 gained several faster (through vector call) equivalent method call API #if PY_VERSION_HEX < 0x03090000 static inline PyObject* PyObject_CallMethodNoArgs(PyObject* obj, PyObject* name) { return PyObject_CallMethodObjArgs(obj, name, nullptr); } + +static inline PyObject* PyObject_CallMethodOneArg(PyObject* obj, PyObject* name, PyObject* arg) { + return PyObject_CallMethodObjArgs(obj, name, arg, nullptr); +} #endif // C++ version of the cppyy API diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx index bb28714815da7..20b2c8fba6eef 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx @@ -17,6 +17,7 @@ #include "Utility.h" #define CPYCPPYY_INTERNAL 1 +#include "CPyCppyy/DispatchPtr.h" namespace CPyCppyy { void* Instance_AsVoidPtr(PyObject* pyobject); PyObject* Instance_FromVoidPtr( @@ -29,11 +30,17 @@ PyObject* Instance_FromVoidPtr( #include #include #include +#include #include #include #include +// Note: as of py3.11, dictionary objects no longer carry a function pointer for +// the lookup, so it can no longer be shimmed and "from cppyy.interactive import *" +// thus no longer works. +#if PY_VERSION_HEX < 0x030b0000 + //- from Python's dictobject.c ------------------------------------------------- #if PY_VERSION_HEX >= 0x03030000 typedef struct PyDictKeyEntry { @@ -73,6 +80,8 @@ PyObject* Instance_FromVoidPtr( #endif +#endif // PY_VERSION_HEX < 0x030b0000 + //- data ----------------------------------------------------------------------- static PyObject* nullptr_repr(PyObject*) { @@ -149,6 +158,46 @@ static PyTypeObject PyNullPtr_t_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +}; + + +static PyObject* default_repr(PyObject*) +{ + return CPyCppyy_PyText_FromString("type default"); +} + +static void default_dealloc(PyObject*) +{ + Py_FatalError("deallocating default"); +} + +static PyTypeObject PyDefault_t_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "default_t", // tp_name + sizeof(PyObject), // tp_basicsize + 0, // tp_itemsize + default_dealloc, // tp_dealloc (never called) + 0, 0, 0, 0, + default_repr, // tp_repr + 0, 0, 0, + (hashfunc)_Py_HashPointer, // tp_hash + 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +#if PY_VERSION_HEX >= 0x02030000 + , 0 // tp_del +#endif +#if PY_VERSION_HEX >= 0x02060000 + , 0 // tp_version_tag +#endif +#if PY_VERSION_HEX >= 0x03040000 + , 0 // tp_finalize +#endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; namespace { @@ -158,7 +207,12 @@ PyObject _CPyCppyy_NullPtrStruct = { 1, &PyNullPtr_t_Type }; -// TOOD: refactor with Converters.cxx +PyObject _CPyCppyy_DefaultStruct = { + _PyObject_EXTRA_INIT + 1, &PyDefault_t_Type +}; + +// TODO: refactor with Converters.cxx struct CPyCppyy_tagCDataObject { // non-public (but stable) PyObject_HEAD char* b_ptr; @@ -171,12 +225,15 @@ namespace CPyCppyy { PyObject* gThisModule = nullptr; PyObject* gPyTypeMap = nullptr; PyObject* gNullPtrObject = nullptr; + PyObject* gDefaultObject = nullptr; PyObject* gBusException = nullptr; PyObject* gSegvException = nullptr; PyObject* gIllException = nullptr; PyObject* gAbrtException = nullptr; std::map> gPythonizations; std::set gPinnedTypes; + std::ostringstream gCapturedError; + std::streambuf* gOldErrorBuffer = nullptr; } @@ -185,7 +242,9 @@ namespace { using namespace CPyCppyy; + //---------------------------------------------------------------------------- +#if PY_VERSION_HEX < 0x030b0000 namespace { class GblGetter { @@ -338,9 +397,12 @@ PyDictEntry* CPyCppyyLookDictString(PyDictObject* mp, PyObject* key, long hash) return ep; } +#endif // PY_VERSION_HEX < 0x030b0000 + //---------------------------------------------------------------------------- static PyObject* SetCppLazyLookup(PyObject*, PyObject* args) { +#if PY_VERSION_HEX < 0x030b0000 // Modify the given dictionary to install the lookup function that also // tries the global C++ namespace before failing. Called on a module's dictionary, // this allows for lazy lookups. This works fine for p3.2 and earlier, but should @@ -351,6 +413,12 @@ static PyObject* SetCppLazyLookup(PyObject*, PyObject* args) return nullptr; CPYCPPYY_GET_DICT_LOOKUP(dict) = CPyCppyyLookDictString; +#else +// As of py3.11, there is no longer a lookup function pointer in the dict object +// to replace. Since this feature is not widely advertised, it's simply dropped + PyErr_Warn(PyExc_RuntimeWarning, (char*)"lazy lookup is no longer supported"); + (void)args; // avoid warning about unused parameter +#endif Py_RETURN_NONE; } @@ -409,13 +477,13 @@ static void* GetCPPInstanceAddress(const char* fname, PyObject* args, PyObject* return nullptr; } - // this is an address of an address (i.e. &myobj, with myobj of type MyObj*) - // note that the return result may be null + // this is an address of an address (i.e. &myobj, with myobj of type MyObj*) + // note that the return result may be null if (!byref) return ((CPPInstance*)pyobj)->GetObject(); return &((CPPInstance*)pyobj)->GetObjectRaw(); } else if (CPyCppyy_PyText_Check(pyobj)) { - // special cases for acces to the CPyCppyy API + // special cases for access to the CPyCppyy API std::string req = CPyCppyy_PyText_AsString((PyObject*)pyobj); if (req == "Instance_AsVoidPtr") return (void*)&Instance_AsVoidPtr; @@ -454,8 +522,8 @@ static PyObject* addressof(PyObject* /* dummy */, PyObject* args, PyObject* kwds return nullptr; } - Cppyy::TCppFuncAddr_t addr = methods[0]->GetFunctionAddress(); - return PyLong_FromLongLong((intptr_t)addr); + Cppyy::TCppFuncAddr_t caddr = methods[0]->GetFunctionAddress(); + return PyLong_FromLongLong((intptr_t)caddr); } // C functions (incl. ourselves) @@ -494,7 +562,7 @@ static PyObject* AsCObject(PyObject* /* unused */, PyObject* args, PyObject* kwd } //---------------------------------------------------------------------------- -static PyObject* AsCapsule(PyObject* /* dummy */, PyObject* args, PyObject* kwds) +static PyObject* AsCapsule(PyObject* /* unused */, PyObject* args, PyObject* kwds) { // Return object proxy as an opaque PyCapsule. void* addr = GetCPPInstanceAddress("as_capsule", args, kwds); @@ -508,7 +576,7 @@ static PyObject* AsCapsule(PyObject* /* dummy */, PyObject* args, PyObject* kwds } //---------------------------------------------------------------------------- -static PyObject* AsCTypes(PyObject* /* dummy */, PyObject* args, PyObject* kwds) +static PyObject* AsCTypes(PyObject* /* unused */, PyObject* args, PyObject* kwds) { // Return object proxy as a ctypes c_void_p void* addr = GetCPPInstanceAddress("as_ctypes", args, kwds); @@ -533,6 +601,43 @@ static PyObject* AsCTypes(PyObject* /* dummy */, PyObject* args, PyObject* kwds) return ref; } +//---------------------------------------------------------------------------- +static PyObject* AsMemoryView(PyObject* /* unused */, PyObject* pyobject) +{ +// Return a raw memory view on arrays of PODs. + if (!CPPInstance_Check(pyobject)) { + PyErr_SetString(PyExc_TypeError, "C++ object proxy expected"); + return nullptr; + } + + CPPInstance* pyobj = (CPPInstance*)pyobject; + Cppyy::TCppType_t klass = ((CPPClass*)Py_TYPE(pyobject))->fCppType; + + Py_ssize_t array_len = pyobj->ArrayLength(); + + if (array_len < 0 || !Cppyy::IsAggregate(klass)) { + PyErr_SetString( + PyExc_TypeError, "object is not a proxy to an array of PODs of known size"); + return nullptr; + } + + Py_buffer view; + + view.obj = pyobject; + view.buf = pyobj->GetObject(); + view.itemsize = Cppyy::SizeOf(klass); + view.len = view.itemsize * array_len; + view.readonly = 0; + view.format = NULL; // i.e. "B" assumed + view.ndim = 1; + view.shape = NULL; + view.strides = NULL; + view.suboffsets = NULL; + view.internal = NULL; + + return PyMemoryView_FromBuffer(&view); +} + //---------------------------------------------------------------------------- static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds) { @@ -540,20 +645,121 @@ static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds) Py_ssize_t argc = PyTuple_GET_SIZE(args); if (argc != 2) { PyErr_Format(PyExc_TypeError, - "BindObject takes exactly 2 argumenst (" PY_SSIZE_T_FORMAT " given)", argc); + "bind_object takes 2 positional arguments but (" PY_SSIZE_T_FORMAT " were given)", argc); + return nullptr; + } + +// convert 2nd argument first (used for both pointer value and instance cases) + Cppyy::TCppType_t cast_type = 0; + PyObject* arg1 = PyTuple_GET_ITEM(args, 1); + if (!CPyCppyy_PyText_Check(arg1)) { // not string, then class + if (CPPScope_Check(arg1)) + cast_type = ((CPPClass*)arg1)->fCppType; + else + arg1 = PyObject_GetAttr(arg1, PyStrings::gName); + } else + Py_INCREF(arg1); + + if (!cast_type && arg1) { + cast_type = (Cppyy::TCppType_t)Cppyy::GetScope(CPyCppyy_PyText_AsString(arg1)); + Py_DECREF(arg1); + } + + if (!cast_type) { + PyErr_SetString(PyExc_TypeError, + "bind_object expects a valid class or class name as an argument"); return nullptr; } -// try to convert first argument: either PyCapsule/CObject or long integer - PyObject* pyaddr = PyTuple_GET_ITEM(args, 0); +// next, convert the first argument, some pointer value or a pre-existing instance + PyObject* arg0 = PyTuple_GET_ITEM(args, 0); + + if (CPPInstance_Check(arg0)) { + // if this instance's class has a relation to the requested one, calculate the + // offset, erase if from any caches, and update the pointer and type + CPPInstance* arg0_pyobj = (CPPInstance*)arg0; + Cppyy::TCppType_t cur_type = arg0_pyobj->ObjectIsA(false /* check_smart */); + + bool isPython = CPPScope_Check(arg1) && \ + (((CPPClass*)arg1)->fFlags & CPPScope::kIsPython); + + if (cur_type == cast_type && !isPython) { + Py_INCREF(arg0); // nothing to do + return arg0; + } + + int direction = 0; + Cppyy::TCppType_t base = 0, derived = 0; + if (Cppyy::IsSubtype(cast_type, cur_type)) { + derived = cast_type; + base = cur_type; + direction = -1; // down-cast + } else if (Cppyy::IsSubtype(cur_type, cast_type)) { + base = cast_type; + derived = cur_type; + direction = 1; // up-cast + } else { + PyErr_SetString(PyExc_TypeError, + "provided instance and provided target type are unrelated"); + return nullptr; + } + + Cppyy::TCppObject_t address = (Cppyy::TCppObject_t)arg0_pyobj->GetObject(); + ptrdiff_t offset = Cppyy::GetBaseOffset(derived, base, address, direction); + + // it's debatable whether a new proxy should be created rather than updating + // the old, but changing the old object would be changing the behavior of all + // code that has a reference to it, which may not be the intention if the cast + // is on a C++ data member; this probably is the "least surprise" option + // ownership is taken over as needed, again following the principle of "least + // surprise" as most likely only the cast object will be retained + bool owns = arg0_pyobj->fFlags & CPPInstance::kIsOwner; + + if (!isPython) { + // ordinary C++ class + PyObject* pyobj = BindCppObjectNoCast( + (void*)((intptr_t)address + offset), cast_type, owns ? CPPInstance::kIsOwner : 0); + if (owns && pyobj) arg0_pyobj->CppOwns(); + return pyobj; + + } else { + // rebinding to a Python-side class, create a fresh instance first to be able to + // perform a lookup of the original dispatch object and if found, return original + void* cast_address = (void*)((intptr_t)address + offset); + PyObject* pyobj = ((PyTypeObject*)arg1)->tp_new((PyTypeObject*)arg1, nullptr, nullptr); + ((CPPInstance*)pyobj)->GetObjectRaw() = cast_address; + + PyObject* dispproxy = CPyCppyy::GetScopeProxy(cast_type); + PyObject* res = PyObject_CallMethodOneArg(dispproxy, PyStrings::gDispGet, pyobj); + /* Note: the resultant object is borrowed */ + if (CPPInstance_Check(res) && ((CPPInstance*)res)->GetObject() == cast_address) { + ((CPPInstance*)pyobj)->CppOwns(); // make sure C++ object isn't deleted + Py_DECREF(pyobj); // on DECREF (is default, but still) + pyobj = res; + } else { + if (res) Py_DECREF(res); // most likely Py_None + else PyErr_Clear(); // should not happen + } + Py_DECREF(dispproxy); + + if (pyobj && owns) { + arg0_pyobj->CppOwns(); + ((CPPInstance*)pyobj)->PythonOwns(); + } + + return pyobj; + } + } + +// not a pre-existing object; get the address and bind void* addr = nullptr; - if (pyaddr != &_CPyCppyy_NullPtrStruct) { - addr = CPyCppyy_PyCapsule_GetPointer(pyaddr, nullptr); + if (arg0 != &_CPyCppyy_NullPtrStruct) { + addr = CPyCppyy_PyCapsule_GetPointer(arg0, nullptr); if (PyErr_Occurred()) { PyErr_Clear(); - addr = PyLong_AsVoidPtr(pyaddr); + addr = PyLong_AsVoidPtr(arg0); if (PyErr_Occurred()) { PyErr_Clear(); @@ -561,34 +767,13 @@ static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds) Py_ssize_t buflen = Utility::GetBuffer(PyTuple_GetItem(args, 0), '*', 1, addr, false); if (!addr || !buflen) { PyErr_SetString(PyExc_TypeError, - "BindObject requires a CObject or long integer as first argument"); + "bind_object requires a CObject/Capsule, long integer, buffer, or instance as first argument"); return nullptr; } } } } - Cppyy::TCppType_t klass = 0; - PyObject* pyname = PyTuple_GET_ITEM(args, 1); - if (!CPyCppyy_PyText_Check(pyname)) { // not string, then class - if (CPPScope_Check(pyname)) - klass = ((CPPClass*)pyname)->fCppType; - else - pyname = PyObject_GetAttr(pyname, PyStrings::gName); - } else - Py_INCREF(pyname); - - if (!klass && pyname) { - klass = (Cppyy::TCppType_t)Cppyy::GetScope(CPyCppyy_PyText_AsString(pyname)); - Py_DECREF(pyname); - } - - if (!klass) { - PyErr_SetString(PyExc_TypeError, - "BindObject expects a valid class or class name as an argument"); - return nullptr; - } - bool do_cast = false; if (kwds) { PyObject* cast = PyDict_GetItemString(kwds, "cast"); @@ -596,9 +781,9 @@ static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds) } if (do_cast) - return BindCppObject(addr, klass); + return BindCppObject(addr, cast_type); - return BindCppObjectNoCast(addr, klass); + return BindCppObjectNoCast(addr, cast_type); } //---------------------------------------------------------------------------- @@ -659,6 +844,34 @@ static PyObject* RemovePythonization(PyObject*, PyObject* args) Py_RETURN_FALSE; } +//---------------------------------------------------------------------------- +static PyObject* PinType(PyObject*, PyObject* pyclass) +{ +// Add a pinning so that objects of type `derived' are interpreted as +// objects of type `base'. + if (!CPPScope_Check(pyclass)) { + PyErr_SetString(PyExc_TypeError, "C++ class expected"); + return nullptr; + } + + gPinnedTypes.insert(((CPPClass*)pyclass)->fCppType); + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------- +static PyObject* AddTypeReducer(PyObject*, PyObject* args) +{ +// Add a type reducer to map type2 to type2 on function returns. + const char *reducable, *reduced; + if (!PyArg_ParseTuple(args, const_cast("ss"), &reducable, &reduced)) + return nullptr; + + Cppyy::AddTypeReducer(reducable, reduced); + + Py_RETURN_NONE; +} + //---------------------------------------------------------------------------- static PyObject* SetMemoryPolicy(PyObject*, PyObject* args) { @@ -721,36 +934,28 @@ static PyObject* AddSmartPtrType(PyObject*, PyObject* args) } //---------------------------------------------------------------------------- -static PyObject* PinType(PyObject*, PyObject* pyclass) +static PyObject* BeginCaptureStderr(PyObject*, PyObject*) { -// Add a pinning so that objects of type `derived' are interpreted as -// objects of type `base'. - if (!CPPScope_Check(pyclass)) { - PyErr_SetString(PyExc_TypeError, "C++ class expected"); - return nullptr; - } - - gPinnedTypes.insert(((CPPClass*)pyclass)->fCppType); + gOldErrorBuffer = std::cerr.rdbuf(); + std::cerr.rdbuf(gCapturedError.rdbuf()); Py_RETURN_NONE; } //---------------------------------------------------------------------------- -static PyObject* Cast(PyObject*, PyObject* args) +static PyObject* EndCaptureStderr(PyObject*, PyObject*) { -// Cast `obj' to type `type'. - CPPInstance* obj = nullptr; - CPPClass* type = nullptr; - if (!PyArg_ParseTuple(args, const_cast("O!O!"), - &CPPInstance_Type, &obj, - &CPPScope_Type, &type)) - return nullptr; -// TODO: this misses an offset calculation, and reference type must not -// be cast ... - return BindCppObjectNoCast(obj->GetObject(), type->fCppType, - obj->fFlags & CPPInstance::kIsReference); -} +// restore old rdbuf and return captured result + std::cerr.rdbuf(gOldErrorBuffer); + gOldErrorBuffer = nullptr; + + std::string capturedError = std::move(gCapturedError).str(); + gCapturedError.str(""); + gCapturedError.clear(); + + return Py_BuildValue("s", capturedError.c_str()); +} } // unnamed namespace @@ -772,6 +977,8 @@ static PyMethodDef gCPyCppyyMethods[] = { METH_VARARGS | METH_KEYWORDS, (char*)"Retrieve address of proxied object or field in a PyCapsule."}, {(char*) "as_ctypes", (PyCFunction)AsCTypes, METH_VARARGS | METH_KEYWORDS, (char*)"Retrieve address of proxied object or field in a ctypes c_void_p."}, + {(char*) "as_memoryview", (PyCFunction)AsMemoryView, + METH_O, (char*)"Represent an array of objects as raw memory."}, {(char*)"bind_object", (PyCFunction)BindObject, METH_VARARGS | METH_KEYWORDS, (char*) "Create an object of given type, from given address."}, {(char*) "move", (PyCFunction)Move, @@ -780,6 +987,10 @@ static PyMethodDef gCPyCppyyMethods[] = { METH_VARARGS, (char*)"Add a pythonizor."}, {(char*) "remove_pythonization", (PyCFunction)RemovePythonization, METH_VARARGS, (char*)"Remove a pythonizor."}, + {(char*) "_pin_type", (PyCFunction)PinType, + METH_O, (char*)"Install a type pinning."}, + {(char*) "_add_type_reducer", (PyCFunction)AddTypeReducer, + METH_VARARGS, (char*)"Add a type reducer."}, {(char*) "SetMemoryPolicy", (PyCFunction)SetMemoryPolicy, METH_VARARGS, (char*)"Determines object ownership model."}, {(char*) "SetGlobalSignalPolicy", (PyCFunction)SetGlobalSignalPolicy, @@ -788,10 +999,10 @@ static PyMethodDef gCPyCppyyMethods[] = { METH_VARARGS, (char*)"Modify held C++ object ownership."}, {(char*) "AddSmartPtrType", (PyCFunction)AddSmartPtrType, METH_VARARGS, (char*) "Add a smart pointer to the list of known smart pointer types."}, - {(char*) "_pin_type", (PyCFunction)PinType, - METH_O, (char*)"Install a type pinning."}, - {(char*) "Cast", (PyCFunction)Cast, - METH_VARARGS, (char*)"Cast the given object to the given type"}, + {(char*) "_begin_capture_stderr", (PyCFunction)BeginCaptureStderr, + METH_NOARGS, (char*) "Begin capturing stderr to a in memory buffer."}, + {(char*) "_end_capture_stderr", (PyCFunction)EndCaptureStderr, + METH_NOARGS, (char*) "End capturing stderr and returns the captured buffer."}, {nullptr, nullptr, 0, nullptr} }; @@ -815,6 +1026,7 @@ static int cpycppyymodule_clear(PyObject* m) return 0; } + static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "libcppyy", @@ -847,7 +1059,8 @@ extern "C" void initlibcppyy() PyEval_InitThreads(); #endif -// prepare for lazyness (the insert is needed to capture the most generic lookup +#if PY_VERSION_HEX < 0x030b0000 +// prepare for laziness (the insert is needed to capture the most generic lookup // function, just in case ...) PyObject* dict = PyDict_New(); PyObject* notstring = PyInt_FromLong(5); @@ -859,6 +1072,7 @@ extern "C" void initlibcppyy() gDictLookupOrg = (dict_lookup_func)((PyDictObject*)dict)->ma_lookup; #endif Py_DECREF(dict); +#endif // PY_VERSION_HEX < 0x030b0000 // setup this module #if PY_VERSION_HEX >= 0x03000000 @@ -904,40 +1118,45 @@ extern "C" void initlibcppyy() CPYCPPYY_INIT_ERROR; // inject custom data types +#if PY_VERSION_HEX < 0x03000000 if (!Utility::InitProxy(gThisModule, &RefFloat_Type, "Double")) CPYCPPYY_INIT_ERROR; if (!Utility::InitProxy(gThisModule, &RefInt_Type, "Long")) CPYCPPYY_INIT_ERROR; +#endif if (!Utility::InitProxy(gThisModule, &CustomInstanceMethod_Type, "InstanceMethod")) CPYCPPYY_INIT_ERROR; - if (!Utility::InitProxy(gThisModule, &TupleOfInstances_Type, "InstancesArray")) + if (!Utility::InitProxy(gThisModule, &TupleOfInstances_Type, "InstanceArray")) CPYCPPYY_INIT_ERROR; - if (!Utility::InitProxy(gThisModule, &InstanceArrayIter_Type, "instancearrayiter")) - CPYCPPYY_INIT_ERROR; + if (!Utility::InitProxy(gThisModule, &LowLevelView_Type, "LowLevelView")) + CPYCPPYY_INIT_ERROR; if (!Utility::InitProxy(gThisModule, &PyNullPtr_t_Type, "nullptr_t")) CPYCPPYY_INIT_ERROR; -// initialize low level ptr type, but do not inject in gThisModule - if (PyType_Ready(&LowLevelView_Type) < 0) +// custom iterators + if (PyType_Ready(&InstanceArrayIter_Type) < 0) CPYCPPYY_INIT_ERROR; -// custom iterators if (PyType_Ready(&IndexIter_Type) < 0) CPYCPPYY_INIT_ERROR; if (PyType_Ready(&VectorIter_Type) < 0) CPYCPPYY_INIT_ERROR; -// inject identifiable nullptr +// inject identifiable nullptr and default gNullPtrObject = (PyObject*)&_CPyCppyy_NullPtrStruct; Py_INCREF(gNullPtrObject); PyModule_AddObject(gThisModule, (char*)"nullptr", gNullPtrObject); + gDefaultObject = (PyObject*)&_CPyCppyy_DefaultStruct; + Py_INCREF(gDefaultObject); + PyModule_AddObject(gThisModule, (char*)"default", gDefaultObject); + // C++-specific exceptions PyObject* cppfatal = PyErr_NewException((char*)"cppyy.ll.FatalError", nullptr, nullptr); PyModule_AddObject(gThisModule, (char*)"FatalError", cppfatal); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CallContext.h b/bindings/pyroot/cppyy/CPyCppyy/src/CallContext.h index 674b8d41078de..3ea8f65cfba21 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CallContext.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CallContext.h @@ -12,6 +12,9 @@ namespace CPyCppyy { // small number that allows use of stack for argument passing const int SMALL_ARGS_N = 8; +// convention to pass flag for direct calls (similar to Python's vector calls) +#define DIRECT_CALL ((size_t)1 << (8 * sizeof(size_t) - 1)) + #ifndef CPYCPPYY_PARAMETER #define CPYCPPYY_PARAMETER // general place holder for function parameters @@ -43,28 +46,32 @@ struct Parameter { // extra call information struct CallContext { - CallContext() : fFlags(0), fCurScope(0), fPyContext(nullptr), + CallContext() : fCurScope(0), fPyContext(nullptr), fFlags(0), fArgsVec(nullptr), fNArgs(0), fTemps(nullptr) {} CallContext(const CallContext&) = delete; CallContext& operator=(const CallContext&) = delete; ~CallContext() { if (fTemps) Cleanup(); delete fArgsVec; } enum ECallFlags { - kNone = 0x0000, - kIsSorted = 0x0001, // if method overload priority determined - kIsCreator = 0x0002, // if method creates python-owned objects - kIsConstructor = 0x0004, // if method is a C++ constructor - kHaveImplicit = 0x0008, // indicate that implicit converters are available - kAllowImplicit = 0x0010, // indicate that implicit coversions are allowed - kNoImplicit = 0x0020, // disable implicit to prevent recursion - kUseHeuristics = 0x0040, // if method applies heuristics memory policy - kUseStrict = 0x0080, // if method applies strict memory policy - kReleaseGIL = 0x0100, // if method should release the GIL - kSetLifeLine = 0x0200, // if return value is part of 'this' - kNeverLifeLine = 0x0400, // if the return value is never part of 'this' - kProtected = 0x0800, // if method should return on signals - kUseFFI = 0x1000, // not implemented - kIsPseudoFunc = 0x2000, // internal, used for introspection + kNone = 0x000000, + kIsSorted = 0x000001, // if method overload priority determined + kIsCreator = 0x000002, // if method creates python-owned objects + kIsConstructor = 0x000004, // if method is a C++ constructor + kHaveImplicit = 0x000008, // indicate that implicit converters are available + kAllowImplicit = 0x000010, // indicate that implicit conversions are allowed + kNoImplicit = 0x000020, // disable implicit to prevent recursion + kCallDirect = 0x000040, // call wrapped method directly, no inheritance + kFromDescr = 0x000080, // initiated from a descriptor + kUseHeuristics = 0x000100, // if method applies heuristics memory policy + kUseStrict = 0x000200, // if method applies strict memory policy + kReleaseGIL = 0x000400, // if method should release the GIL + kSetLifeLine = 0x000800, // if return value is part of 'this' + kNeverLifeLine = 0x001000, // if the return value is never part of 'this' + kPyException = 0x002000, // Python exception during method execution + kCppException = 0x004000, // C++ exception during method execution + kProtected = 0x008000, // if method should return on signals + kUseFFI = 0x010000, // not implemented + kIsPseudoFunc = 0x020000, // internal, used for introspection }; // memory handling @@ -90,22 +97,24 @@ struct CallContext { if (fNArgs <= SMALL_ARGS_N) return fArgs; return fArgsVec->data(); } - + size_t GetSize() { return fNArgs; } + size_t GetEncodedSize() { return fNArgs | ((fFlags & kCallDirect) ? DIRECT_CALL : 0); } public: // info/status - uint64_t fFlags; Cppyy::TCppScope_t fCurScope; - PyObject* fPyContext; // used to set lifelines + PyObject* fPyContext; + uint32_t fFlags; private: + struct Temporary { PyObject* fPyObject; Temporary* fNext; }; + // payload - Parameter fArgs[SMALL_ARGS_N]; + Parameter fArgs[SMALL_ARGS_N]; std::vector* fArgsVec; - size_t fNArgs; - struct Temporary { PyObject* fPyObject; Temporary* fNext; }; - Temporary* fTemps; + size_t fNArgs; + Temporary* fTemps; }; inline bool IsSorted(uint64_t flags) { diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx index a437cccffb548..4ac294745cac9 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx @@ -16,19 +16,34 @@ #include "Utility.h" // Standard +#include #include #include // for ptrdiff_t #include +#include #include +#include // for wstring_convert +#include #include #include #if __cplusplus > 201402L #include -#endif #include - -#define UNKNOWN_SIZE -1 -#define UNKNOWN_ARRAY_SIZE -2 +#endif +// codecvt does not exist for gcc4.8.5 and is in principle deprecated; it is +// only used in py2 for char -> wchar_t conversion for std::wstring; if not +// available, the conversion is done through Python (requires an extra copy) +#if PY_VERSION_HEX < 0x03000000 +#if defined(__GNUC__) && !defined(__APPLE__) +# if __GNUC__ > 4 && __has_include("codecvt") +# include +# define HAS_CODECVT 1 +# endif +#else +#include +#define HAS_CODECVT 1 +#endif +#endif // py2 //- data _____________________________________________________________________ @@ -37,18 +52,26 @@ namespace CPyCppyy { // factories typedef std::map ConvFactories_t; static ConvFactories_t gConvFactories; + +// special objects extern PyObject* gNullPtrObject; + extern PyObject* gDefaultObject; +// regular expression for matching function pointer + static std::regex s_fnptr("\\(:*\\*&*\\)"); } #if PY_VERSION_HEX < 0x03000000 -const size_t MOVE_REFCOUNT_CUTOFF = 1; -#else +const Py_ssize_t MOVE_REFCOUNT_CUTOFF = 1; +#elif PY_VERSION_HEX < 0x03080000 // p3 has at least 2 ref-counts, as contrary to p2, it will create a descriptor // copy for the method holding self in the case of __init__; but there can also // be a reference held by the frame object, which is indistinguishable from a // local variable reference, so the cut-off has to remain 2. -const size_t MOVE_REFCOUNT_CUTOFF = 2; +const Py_ssize_t MOVE_REFCOUNT_CUTOFF = 2; +#else +// since py3.8, vector calls behave again as expected +const Py_ssize_t MOVE_REFCOUNT_CUTOFF = 1; #endif //- pretend-ctypes helpers --------------------------------------------------- @@ -70,8 +93,8 @@ struct CPyCppyy_tagPyCArgObject { // not public (but stable; note that olde PyObject* obj; }; -// indices of ctypes types into the array caches (not that c_complex does not -// exist as a type in ctypes) +// indices of ctypes types into the array caches (note that c_complex and c_fcomplex +// do not exist as types in ctypes) #define ct_c_bool 0 #define ct_c_char 1 #define ct_c_shar 1 @@ -97,14 +120,16 @@ struct CPyCppyy_tagPyCArgObject { // not public (but stable; note that olde #define ct_c_char_p 18 #define ct_c_wchar_p 19 #define ct_c_void_p 20 -#define ct_c_complex 21 -#define NTYPES 22 +#define ct_c_fcomplex 21 +#define ct_c_complex 22 +#define ct_c_pointer 23 +#define NTYPES 24 static std::array gCTypesNames = { "c_bool", "c_char", "c_wchar", "c_byte", "c_ubyte", "c_short", "c_ushort", "c_uint16", "c_int", "c_uint", "c_uint32", "c_long", "c_ulong", "c_longlong", "c_ulonglong", "c_float", "c_double", "c_longdouble", - "c_char_p", "c_wchar_p", "c_void_p", "c_complex" }; + "c_char_p", "c_wchar_p", "c_void_p", "c_fcomplex", "c_complex", "_Pointer" }; static std::array gCTypesTypes; static std::array gCTypesPtrTypes; @@ -207,13 +232,32 @@ static inline bool SetLifeLine(PyObject* holder, PyObject* target, intptr_t ref) // reuse when the C++ side is being overwritten std::ostringstream attr_name; attr_name << "__" << ref; - auto attr_name_str = attr_name.str(); - auto res = PyObject_SetAttrString(holder, attr_name_str.c_str(), target); + auto res = PyObject_SetAttrString(holder, (char*)attr_name.str().c_str(), target); return res != -1; } +static bool HasLifeLine(PyObject* holder, intptr_t ref) +{ +// determine if a lifeline was previously set for the ref on the holder + if (!holder) return false; + + std::ostringstream attr_name; + attr_name << "__" << ref; + PyObject* res = PyObject_GetAttrString(holder, (char*)attr_name.str().c_str()); + + if (res) { + Py_DECREF(res); + return true; + } + + PyErr_Clear(); + return false; +} + + //- helper to work with both CPPInstance and CPPExcInstance ------------------ -static inline CPyCppyy::CPPInstance* GetCppInstance(PyObject* pyobject) +static inline CPyCppyy::CPPInstance* GetCppInstance( + PyObject* pyobject, Cppyy::TCppType_t klass = (Cppyy::TCppType_t)0, bool accept_rvalue = false) { using namespace CPyCppyy; if (CPPInstance_Check(pyobject)) @@ -221,158 +265,141 @@ static inline CPyCppyy::CPPInstance* GetCppInstance(PyObject* pyobject) if (CPPExcInstance_Check(pyobject)) return (CPPInstance*)((CPPExcInstance*)pyobject)->fCppInstance; -#if PY_VERSION_HEX < 0x03090000 // this is not a C++ proxy; allow custom cast to C++ PyObject* castobj = PyObject_CallMethodNoArgs(pyobject, PyStrings::gCastCpp); if (castobj) { if (CPPInstance_Check(castobj)) return (CPPInstance*)castobj; + else if (klass && PyTuple_CheckExact(castobj)) { + // allow implicit conversion from a tuple of arguments + PyObject* pyclass = GetScopeProxy(klass); + if (pyclass) { + CPPInstance* pytmp = (CPPInstance*)PyObject_Call(pyclass, castobj, NULL); + Py_DECREF(pyclass); + if (CPPInstance_Check(pytmp)) { + if (accept_rvalue) + pytmp->fFlags |= CPPInstance::kIsRValue; + Py_DECREF(castobj); + return pytmp; + } + Py_XDECREF(pytmp); + } + } + Py_DECREF(castobj); return nullptr; } PyErr_Clear(); -#endif - return nullptr; } //- custom helpers to check ranges ------------------------------------------- -static inline bool CPyCppyy_PyLong_AsBool(PyObject* pyobject) +static inline bool ImplicitBool(PyObject* pyobject, CPyCppyy::CallContext* ctxt) { -// range-checking python integer to C++ bool conversion - long l = PyLong_AsLong(pyobject); -// fail to pass float -> bool; the problem is rounding (0.1 -> 0 -> False) - if (!(l == 0|| l == 1) || PyFloat_Check(pyobject)) { - PyErr_SetString(PyExc_ValueError, "boolean value should be bool, or integer 1 or 0"); - return (bool)-1; + using namespace CPyCppyy; + if (!AllowImplicit(ctxt) && PyBool_Check(pyobject)) { + if (!NoImplicit(ctxt)) ctxt->fFlags |= CallContext::kHaveImplicit; + return false; } - return (bool)l; -} - -static inline char CPyCppyy_PyText_AsChar(PyObject* pyobject) { -// python string to C++ char conversion - return (char)CPyCppyy_PyText_AsString(pyobject)[0]; + return true; } -static inline uint8_t CPyCppyy_PyLong_AsUInt8(PyObject* pyobject) +static inline bool StrictBool(PyObject* pyobject, CPyCppyy::CallContext* ctxt) { -// range-checking python integer to C++ uint8_t conversion (typically, an unsigned char) -// prevent p2.7 silent conversions and do a range check - if (!(PyLong_Check(pyobject) || PyInt_Check(pyobject))) { - PyErr_SetString(PyExc_TypeError, "short int conversion expects an integer object"); - return (uint8_t)-1; - } - long l = PyLong_AsLong(pyobject); - if (l < 0 || UCHAR_MAX < l) { - PyErr_Format(PyExc_ValueError, "integer %ld out of range for uint8_t", l); - return (uint8_t)-1; - + using namespace CPyCppyy; + if (!AllowImplicit(ctxt) && !PyBool_Check(pyobject)) { + if (!NoImplicit(ctxt)) ctxt->fFlags |= CallContext::kHaveImplicit; + return false; } - return (uint8_t)l; + return true; } -static inline int8_t CPyCppyy_PyLong_AsInt8(PyObject* pyobject) +static inline bool CPyCppyy_PyLong_AsBool(PyObject* pyobject) { -// range-checking python integer to C++ int8_t conversion (typically, an signed char) -// prevent p2.7 silent conversions and do a range check - if (!(PyLong_Check(pyobject) || PyInt_Check(pyobject))) { - PyErr_SetString(PyExc_TypeError, "short int conversion expects an integer object"); - return (int8_t)-1; - } +// range-checking python integer to C++ bool conversion long l = PyLong_AsLong(pyobject); - if (l < SCHAR_MIN || SCHAR_MAX < l) { - PyErr_Format(PyExc_ValueError, "integer %ld out of range for int8_t", l); - return (int8_t)-1; - +// fail to pass float -> bool; the problem is rounding (0.1 -> 0 -> False) + if (!(l == 0|| l == 1) || PyFloat_Check(pyobject)) { + PyErr_SetString(PyExc_ValueError, "boolean value should be bool, or integer 1 or 0"); + return (bool)-1; } - return (int8_t)l; + return (bool)l; } -static inline unsigned short CPyCppyy_PyLong_AsUShort(PyObject* pyobject) -{ -// range-checking python integer to C++ unsigend short int conversion -// prevent p2.7 silent conversions and do a range check - if (!(PyLong_Check(pyobject) || PyInt_Check(pyobject))) { - PyErr_SetString(PyExc_TypeError, "unsigned short conversion expects an integer object"); - return (unsigned short)-1; - } - long l = PyLong_AsLong(pyobject); - if (l < 0 || USHRT_MAX < l) { - PyErr_Format(PyExc_ValueError, "integer %ld out of range for unsigned short", l); - return (unsigned short)-1; - - } - return (unsigned short)l; +// range-checking python integer to C++ integer conversion (prevents p2.7 silent conversions) +#define CPPYY_PYLONG_AS_TYPE(name, type, limit_low, limit_high) \ +static inline type CPyCppyy_PyLong_As##name(PyObject* pyobject) \ +{ \ + if (!(PyLong_Check(pyobject) || PyInt_Check(pyobject))) { \ + if (pyobject == CPyCppyy::gDefaultObject) \ + return (type)0; \ + PyErr_SetString(PyExc_TypeError, #type" conversion expects an integer object");\ + return (type)-1; \ + } \ + long l = PyLong_AsLong(pyobject); \ + if (l < limit_low || limit_high < l) { \ + PyErr_Format(PyExc_ValueError, "integer %ld out of range for "#type, l);\ + return (type)-1; \ + } \ + return (type)l; \ } -static inline short CPyCppyy_PyLong_AsShort(PyObject* pyobject) -{ -// range-checking python integer to C++ short int conversion -// prevent p2.7 silent conversions and do a range check - if (!(PyLong_Check(pyobject) || PyInt_Check(pyobject))) { - PyErr_SetString(PyExc_TypeError, "short int conversion expects an integer object"); - return (short)-1; - } - long l = PyLong_AsLong(pyobject); - if (l < SHRT_MIN || SHRT_MAX < l) { - PyErr_Format(PyExc_ValueError, "integer %ld out of range for short int", l); - return (short)-1; - - } - return (short)l; -} +CPPYY_PYLONG_AS_TYPE(UInt8, uint8_t, 0, UCHAR_MAX) +CPPYY_PYLONG_AS_TYPE(Int8, int8_t, SCHAR_MIN, SCHAR_MAX) +CPPYY_PYLONG_AS_TYPE(UShort, unsigned short, 0, USHRT_MAX) +CPPYY_PYLONG_AS_TYPE(Short, short, SHRT_MIN, SHRT_MAX) +CPPYY_PYLONG_AS_TYPE(StrictInt, int, INT_MIN, INT_MAX) -static inline int CPyCppyy_PyLong_AsStrictInt(PyObject* pyobject) +static inline long CPyCppyy_PyLong_AsStrictLong(PyObject* pyobject) { -// strict python integer to C++ integer conversion +// strict python integer to C++ long integer conversion -// p2.7 and later silently converts floats to long, therefore require this -// check; earlier pythons may raise a SystemError which should be avoided as -// it is confusing +// prevent float -> long (see CPyCppyy_PyLong_AsStrictInt) if (!(PyLong_Check(pyobject) || PyInt_Check(pyobject))) { + if (pyobject == CPyCppyy::gDefaultObject) + return (long)0; PyErr_SetString(PyExc_TypeError, "int/long conversion expects an integer object"); - return -1; + return (long)-1; } - long l = PyLong_AsLong(pyobject); - if (l < INT_MIN || INT_MAX < l) { - PyErr_Format(PyExc_ValueError, "integer %ld out of range for int", l); - return (int)-1; - } - return (int)l; + return (long)PyLong_AsLong(pyobject); // already does long range check } -static inline long CPyCppyy_PyLong_AsStrictLong(PyObject* pyobject) +static inline PY_LONG_LONG CPyCppyy_PyLong_AsStrictLongLong(PyObject* pyobject) { -// strict python integer to C++ long integer conversion +// strict python integer to C++ long long integer conversion // prevent float -> long (see CPyCppyy_PyLong_AsStrictInt) if (!(PyLong_Check(pyobject) || PyInt_Check(pyobject))) { + if (pyobject == CPyCppyy::gDefaultObject) + return (PY_LONG_LONG)0; PyErr_SetString(PyExc_TypeError, "int/long conversion expects an integer object"); - return (long)-1; + return (PY_LONG_LONG)-1; } - return (long)PyLong_AsLong(pyobject); + + return PyLong_AsLongLong(pyobject); // already does long range check } //- helper for pointer/array/reference conversions --------------------------- -static inline bool CArraySetArg(PyObject* pyobject, CPyCppyy::Parameter& para, char tc, int size) +static inline bool CArraySetArg( + PyObject* pyobject, CPyCppyy::Parameter& para, char tc, int size, bool check=true) { // general case of loading a C array pointer (void* + type code) as function argument - if (pyobject == CPyCppyy::gNullPtrObject) + if (pyobject == CPyCppyy::gNullPtrObject || pyobject == CPyCppyy::gDefaultObject) para.fValue.fVoidp = nullptr; else { - Py_ssize_t buflen = CPyCppyy::Utility::GetBuffer(pyobject, tc, size, para.fValue.fVoidp); + Py_ssize_t buflen = CPyCppyy::Utility::GetBuffer(pyobject, tc, size, para.fValue.fVoidp, check); if (!buflen) { // stuck here as it's the least common if (CPyCppyy_PyLong_AsStrictInt(pyobject) == 0) para.fValue.fVoidp = nullptr; else { PyErr_Format(PyExc_TypeError, // ValueError? - "could not convert argument to buffer or nullptr"); + "could not convert argument to buffer or nullptr"); return false; } } @@ -383,23 +410,23 @@ static inline bool CArraySetArg(PyObject* pyobject, CPyCppyy::Parameter& para, c //- helper for implicit conversions ------------------------------------------ -static inline bool ConvertImplicit(Cppyy::TCppType_t klass, - PyObject* pyobject, CPyCppyy::Parameter& para, CPyCppyy::CallContext* ctxt) +static inline CPyCppyy::CPPInstance* ConvertImplicit(Cppyy::TCppType_t klass, + PyObject* pyobject, CPyCppyy::Parameter& para, CPyCppyy::CallContext* ctxt, bool manage=true) { using namespace CPyCppyy; // filter out copy and move constructors if (IsConstructor(ctxt->fFlags) && klass == ctxt->fCurScope && ctxt->GetSize() == 1) - return false; + return nullptr; // only proceed if implicit conversions are allowed (in "round 2") or if the // argument is exactly a tuple or list, as these are the equivalent of // initializer lists and thus "syntax" not a conversion if (!AllowImplicit(ctxt)) { PyTypeObject* pytype = (PyTypeObject*)Py_TYPE(pyobject); - if (!(pytype == &PyList_Type || pytype == &PyTuple_Type)) { + if (!(pytype == &PyList_Type || pytype == &PyTuple_Type)) {// || !CPPInstance_Check(pyobject))) { if (!NoImplicit(ctxt)) ctxt->fFlags |= CallContext::kHaveImplicit; - return false; + return nullptr; } } @@ -407,38 +434,36 @@ static inline bool ConvertImplicit(Cppyy::TCppType_t klass, PyObject* pyscope = CreateScopeProxy(klass); if (!CPPScope_Check(pyscope)) { Py_XDECREF(pyscope); - return false; + return nullptr; } -// add a pseudo-keyword argument to prevent recursion - PyObject* kwds = PyDict_New(); - PyDict_SetItem(kwds, PyStrings::gNoImplicit, Py_True); +// call constructor of argument type to attempt implicit conversion (disallow any +// implicit conversions by the scope's constructor itself) PyObject* args = PyTuple_New(1); Py_INCREF(pyobject); PyTuple_SET_ITEM(args, 0, pyobject); -// call constructor of argument type to attempt implicit conversion - CPPInstance* pytmp = (CPPInstance*)PyObject_Call(pyscope, args, kwds); + ((CPPScope*)pyscope)->fFlags |= CPPScope::kNoImplicit; + CPPInstance* pytmp = (CPPInstance*)PyObject_Call(pyscope, args, NULL); if (!pytmp && PyTuple_CheckExact(pyobject)) { // special case: allow implicit conversion from given set of arguments in tuple PyErr_Clear(); - PyDict_SetItem(kwds, PyStrings::gNoImplicit, Py_True); // was deleted - pytmp = (CPPInstance*)PyObject_Call(pyscope, pyobject, kwds); + pytmp = (CPPInstance*)PyObject_Call(pyscope, pyobject, NULL); } + ((CPPScope*)pyscope)->fFlags &= ~CPPScope::kNoImplicit; Py_DECREF(args); - Py_DECREF(kwds); Py_DECREF(pyscope); if (pytmp) { // implicit conversion succeeded! - ctxt->AddTemporary((PyObject*)pytmp); - para.fValue.fVoidp = pytmp->GetObject(); + if (manage) ctxt->AddTemporary((PyObject*)pytmp); + para.fValue.fVoidp = pytmp->GetObjectRaw(); para.fTypeCode = 'V'; - return true; + return pytmp; } PyErr_Clear(); - return false; + return nullptr; } @@ -466,10 +491,7 @@ bool CPyCppyy::Converter::ToMemory(PyObject*, void*, PyObject* /* ctxt */) //- helper macro's ----------------------------------------------------------- -#define CPPYY_IMPL_BASIC_CONVERTER(name, type, stype, ctype, F1, F2, tc) \ -bool CPyCppyy::name##Converter::SetArg( \ - PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) \ -{ \ +#define CPPYY_IMPL_BASIC_CONVERTER_BODY(name, type, stype, ctype, F1, F2, tc)\ /* convert to C++ 'type', set arg for call */ \ type val = (type)F2(pyobject); \ if (val == (type)-1 && PyErr_Occurred()) { \ @@ -483,39 +505,85 @@ bool CPyCppyy::name##Converter::SetArg( \ if (Py_TYPE(pyobject) == ctypes_type) { \ PyErr_Clear(); \ val = *((type*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr); \ + } else if (pyobject == CPyCppyy::gDefaultObject) { \ + PyErr_Clear(); \ + val = (type)0; \ } else \ return false; \ } \ para.fValue.f##name = val; \ para.fTypeCode = tc; \ - return true; \ -} \ - \ + return true; + +#define CPPYY_IMPL_BASIC_CONVERTER_METHODS(name, type, stype, ctype, F1, F2) \ PyObject* CPyCppyy::name##Converter::FromMemory(void* address) \ { \ return F1((stype)*((type*)address)); \ } \ \ -bool CPyCppyy::name##Converter::ToMemory(PyObject* value, void* address, \ - PyObject* /* ctxt */) \ +bool CPyCppyy::name##Converter::ToMemory( \ + PyObject* value, void* address, PyObject* /* ctxt */) \ { \ type s = (type)F2(value); \ - if (s == (type)-1 && PyErr_Occurred()) \ - return false; \ + if (s == (type)-1 && PyErr_Occurred()) { \ + if (value == CPyCppyy::gDefaultObject) { \ + PyErr_Clear(); \ + s = (type)0; \ + } else \ + return false; \ + } \ *((type*)address) = (type)s; \ return true; \ } +#define CPPYY_IMPL_BASIC_CONVERTER_NI(name, type, stype, ctype, F1, F2, tc) \ +bool CPyCppyy::name##Converter::SetArg( \ + PyObject* pyobject, Parameter& para, CallContext* ctxt) \ +{ \ + if (!StrictBool(pyobject, ctxt)) \ + return false; \ + CPPYY_IMPL_BASIC_CONVERTER_BODY(name, type, stype, ctype, F1, F2, tc) \ +} \ +CPPYY_IMPL_BASIC_CONVERTER_METHODS(name, type, stype, ctype, F1, F2) + +#define CPPYY_IMPL_BASIC_CONVERTER_IB(name, type, stype, ctype, F1, F2, tc) \ +bool CPyCppyy::name##Converter::SetArg( \ + PyObject* pyobject, Parameter& para, CallContext* ctxt) \ +{ \ + if (!ImplicitBool(pyobject, ctxt)) \ + return false; \ + CPPYY_IMPL_BASIC_CONVERTER_BODY(name, type, stype, ctype, F1, F2, tc) \ +} \ +CPPYY_IMPL_BASIC_CONVERTER_METHODS(name, type, stype, ctype, F1, F2) + +#define CPPYY_IMPL_BASIC_CONVERTER_NB(name, type, stype, ctype, F1, F2, tc) \ +bool CPyCppyy::name##Converter::SetArg( \ + PyObject* pyobject, Parameter& para, CallContext* /*ctxt*/) \ +{ \ + if (PyBool_Check(pyobject)) \ + return false; \ + CPPYY_IMPL_BASIC_CONVERTER_BODY(name, type, stype, ctype, F1, F2, tc) \ +} \ +CPPYY_IMPL_BASIC_CONVERTER_METHODS(name, type, stype, ctype, F1, F2) + //---------------------------------------------------------------------------- static inline int ExtractChar(PyObject* pyobject, const char* tname, int low, int high) { int lchar = -1; - if (CPyCppyy_PyText_Check(pyobject)) { + if (PyBytes_Check(pyobject)) { + if (PyBytes_GET_SIZE(pyobject) == 1) + lchar = (int)(PyBytes_AsString(pyobject)[0]); + else + PyErr_Format(PyExc_ValueError, "%s expected, got bytes of size " PY_SSIZE_T_FORMAT, + tname, PyBytes_GET_SIZE(pyobject)); + } else if (CPyCppyy_PyText_Check(pyobject)) { if (CPyCppyy_PyText_GET_SIZE(pyobject) == 1) - lchar = (int)CPyCppyy_PyText_AsChar(pyobject); + lchar = (int)(CPyCppyy_PyText_AsString(pyobject)[0]); else - PyErr_Format(PyExc_ValueError, "%s expected, got string of size " PY_SSIZE_T_FORMAT, + PyErr_Format(PyExc_ValueError, "%s expected, got str of size " PY_SSIZE_T_FORMAT, tname, CPyCppyy_PyText_GET_SIZE(pyobject)); + } else if (pyobject == CPyCppyy::gDefaultObject) { + lchar = (int)'\0'; } else if (!PyFloat_Check(pyobject)) { // don't allow truncating conversion lchar = (int)PyLong_AsLong(pyobject); if (lchar == -1 && PyErr_Occurred()) @@ -553,8 +621,13 @@ bool CPyCppyy::Const##name##RefConverter::SetArg( \ PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) \ { \ type val = (type)F1(pyobject); \ - if (val == (type)-1 && PyErr_Occurred()) \ - return false; \ + if (val == (type)-1 && PyErr_Occurred()) { \ + if (pyobject == CPyCppyy::gDefaultObject) { \ + PyErr_Clear(); \ + val = (type)0; \ + } else \ + return false; \ + } \ para.fValue.f##name = val; \ para.fRef = ¶.fValue.f##name; \ para.fTypeCode = 'r'; \ @@ -594,14 +667,19 @@ bool CPyCppyy::name##Converter::SetArg( \ \ PyObject* CPyCppyy::name##Converter::FromMemory(void* address) \ { \ + /* return char in "native" str type as that's more natural in use */ \ return CPyCppyy_PyText_FromFormat("%c", *((type*)address)); \ } \ \ -bool CPyCppyy::name##Converter::ToMemory(PyObject* value, void* address, \ - PyObject* /* ctxt */) \ +bool CPyCppyy::name##Converter::ToMemory( \ + PyObject* value, void* address, PyObject* /* ctxt */) \ { \ Py_ssize_t len; \ - const char* cstr = CPyCppyy_PyText_AsStringAndSize(value, &len); \ + const char* cstr = nullptr; \ + if (PyBytes_Check(value)) \ + PyBytes_AsStringAndSize(value, (char**)&cstr, &len); \ + else \ + cstr = CPyCppyy_PyText_AsStringAndSize(value, &len); \ if (cstr) { \ if (len != 1) { \ PyErr_Format(PyExc_TypeError, #type" expected, got string of size %zd", len);\ @@ -611,8 +689,13 @@ bool CPyCppyy::name##Converter::ToMemory(PyObject* value, void* address, \ } else { \ PyErr_Clear(); \ long l = PyLong_AsLong(value); \ - if (l == -1 && PyErr_Occurred()) \ - return false; \ + if (l == -1 && PyErr_Occurred()) { \ + if (value == CPyCppyy::gDefaultObject) { \ + PyErr_Clear(); \ + l = (long)0; \ + } else \ + return false; \ + } \ if (!(low <= l && l <= high)) { \ PyErr_Format(PyExc_ValueError, \ "integer to character: value %ld not in range [%d,%d]", l, low, high);\ @@ -625,7 +708,7 @@ bool CPyCppyy::name##Converter::ToMemory(PyObject* value, void* address, \ //- converters for built-ins ------------------------------------------------- -CPPYY_IMPL_BASIC_CONVERTER(Long, long, long, c_long, PyLong_FromLong, CPyCppyy_PyLong_AsStrictLong, 'l') +CPPYY_IMPL_BASIC_CONVERTER_IB(Long, long, long, c_long, PyLong_FromLong, CPyCppyy_PyLong_AsStrictLong, 'l') //---------------------------------------------------------------------------- bool CPyCppyy::LongRefConverter::SetArg( @@ -668,8 +751,8 @@ CPPYY_IMPL_BASIC_CONST_REFCONVERTER(Int, int, c_int, CPyCppy CPPYY_IMPL_BASIC_CONST_REFCONVERTER(UInt, unsigned int, c_uint, PyLongOrInt_AsULong) CPPYY_IMPL_BASIC_CONST_REFCONVERTER(Long, long, c_long, CPyCppyy_PyLong_AsStrictLong) CPPYY_IMPL_BASIC_CONST_REFCONVERTER(ULong, unsigned long, c_ulong, PyLongOrInt_AsULong) -CPPYY_IMPL_BASIC_CONST_REFCONVERTER(LLong, Long64_t, c_longlong, PyLong_AsLongLong) -CPPYY_IMPL_BASIC_CONST_REFCONVERTER(ULLong, ULong64_t, c_ulonglong, PyLongOrInt_AsULong64) +CPPYY_IMPL_BASIC_CONST_REFCONVERTER(LLong, PY_LONG_LONG, c_longlong, CPyCppyy_PyLong_AsStrictLongLong) +CPPYY_IMPL_BASIC_CONST_REFCONVERTER(ULLong, PY_ULONG_LONG, c_ulonglong, PyLongOrInt_AsULong64) //---------------------------------------------------------------------------- bool CPyCppyy::IntRefConverter::SetArg( @@ -747,13 +830,13 @@ CPPYY_IMPL_REFCONVERTER(LLong, c_longlong, long long, 'q'); CPPYY_IMPL_REFCONVERTER(ULLong, c_ulonglong, unsigned long long, 'Q'); CPPYY_IMPL_REFCONVERTER(Float, c_float, float, 'f'); CPPYY_IMPL_REFCONVERTER_FROM_MEMORY(Double, c_double); -CPPYY_IMPL_REFCONVERTER(LDouble, c_longdouble, LongDouble_t, 'D'); +CPPYY_IMPL_REFCONVERTER(LDouble, c_longdouble, PY_LONG_DOUBLE, 'g'); //---------------------------------------------------------------------------- // convert to C++ bool, allow int/long -> bool, set arg for call -CPPYY_IMPL_BASIC_CONVERTER( - Bool, bool, long, c_bool, PyInt_FromLong, CPyCppyy_PyLong_AsBool, 'l') +CPPYY_IMPL_BASIC_CONVERTER_NI( + Bool, bool, long, c_bool, PyBool_FromLong, CPyCppyy_PyLong_AsBool, 'l') //---------------------------------------------------------------------------- CPPYY_IMPL_BASIC_CHAR_CONVERTER(Char, char, CHAR_MIN, CHAR_MAX) @@ -884,22 +967,25 @@ bool CPyCppyy::Char32Converter::ToMemory(PyObject* value, void* address, PyObjec } //---------------------------------------------------------------------------- -CPPYY_IMPL_BASIC_CONVERTER( +CPPYY_IMPL_BASIC_CONVERTER_IB( Int8, int8_t, long, c_int8, PyInt_FromLong, CPyCppyy_PyLong_AsInt8, 'l') -CPPYY_IMPL_BASIC_CONVERTER( +CPPYY_IMPL_BASIC_CONVERTER_IB( UInt8, uint8_t, long, c_uint8, PyInt_FromLong, CPyCppyy_PyLong_AsUInt8, 'l') -CPPYY_IMPL_BASIC_CONVERTER( +CPPYY_IMPL_BASIC_CONVERTER_IB( Short, short, long, c_short, PyInt_FromLong, CPyCppyy_PyLong_AsShort, 'l') -CPPYY_IMPL_BASIC_CONVERTER( +CPPYY_IMPL_BASIC_CONVERTER_IB( UShort, unsigned short, long, c_ushort, PyInt_FromLong, CPyCppyy_PyLong_AsUShort, 'l') -CPPYY_IMPL_BASIC_CONVERTER( +CPPYY_IMPL_BASIC_CONVERTER_IB( Int, int, long, c_uint, PyInt_FromLong, CPyCppyy_PyLong_AsStrictInt, 'l') //---------------------------------------------------------------------------- bool CPyCppyy::ULongConverter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) + PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ unsigned long, set arg for call + if (!ImplicitBool(pyobject, ctxt)) + return false; + para.fValue.fULong = PyLongOrInt_AsULong(pyobject); if (para.fValue.fULong == (unsigned long)-1 && PyErr_Occurred()) return false; @@ -917,8 +1003,13 @@ bool CPyCppyy::ULongConverter::ToMemory(PyObject* value, void* address, PyObject { // convert to C++ unsigned long, write it at
unsigned long u = PyLongOrInt_AsULong(value); - if (u == (unsigned long)-1 && PyErr_Occurred()) - return false; + if (u == (unsigned long)-1 && PyErr_Occurred()) { + if (value == CPyCppyy::gDefaultObject) { + PyErr_Clear(); + u = (unsigned long)0; + } else + return false; + } *((unsigned long*)address) = u; return true; } @@ -927,33 +1018,33 @@ bool CPyCppyy::ULongConverter::ToMemory(PyObject* value, void* address, PyObject PyObject* CPyCppyy::UIntConverter::FromMemory(void* address) { // construct python object from C++ unsigned int read at
- return PyLong_FromUnsignedLong(*((UInt_t*)address)); + return PyLong_FromUnsignedLong(*((unsigned int*)address)); } bool CPyCppyy::UIntConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) { // convert to C++ unsigned int, write it at
- ULong_t u = PyLongOrInt_AsULong(value); + unsigned long u = PyLongOrInt_AsULong(value); if (u == (unsigned long)-1 && PyErr_Occurred()) return false; - if (u > (ULong_t)UINT_MAX) { + if (u > (unsigned long)UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "value too large for unsigned int"); return false; } - *((UInt_t*)address) = (UInt_t)u; + *((unsigned int*)address) = (unsigned int)u; return true; } //- floating point converters ------------------------------------------------ -CPPYY_IMPL_BASIC_CONVERTER( - Float, float, double, c_float, PyFloat_FromDouble, PyFloat_AsDouble, 'f') -CPPYY_IMPL_BASIC_CONVERTER( +CPPYY_IMPL_BASIC_CONVERTER_NB( + Float, float, double, c_float, PyFloat_FromDouble, PyFloat_AsDouble, 'f') +CPPYY_IMPL_BASIC_CONVERTER_NB( Double, double, double, c_double, PyFloat_FromDouble, PyFloat_AsDouble, 'd') -CPPYY_IMPL_BASIC_CONVERTER( - LDouble, LongDouble_t, LongDouble_t, c_longdouble, PyFloat_FromDouble, PyFloat_AsDouble, 'g') +CPPYY_IMPL_BASIC_CONVERTER_NB( + LDouble, PY_LONG_DOUBLE, PY_LONG_DOUBLE, c_longdouble, PyFloat_FromDouble, PyFloat_AsDouble, 'g') CPyCppyy::ComplexDConverter::ComplexDConverter(bool keepControl) : InstanceConverter(Cppyy::GetScope("std::complex"), keepControl) {} @@ -972,8 +1063,8 @@ bool CPyCppyy::ComplexDConverter::SetArg( } return this->InstanceConverter::SetArg(pyobject, para, ctxt); -} \ - \ +} + PyObject* CPyCppyy::ComplexDConverter::FromMemory(void* address) { std::complex* dc = (std::complex*)address; @@ -997,11 +1088,21 @@ bool CPyCppyy::DoubleRefConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) { // convert to C++ double&, set arg for call +#if PY_VERSION_HEX < 0x03000000 if (RefFloat_CheckExact(pyobject)) { para.fValue.fVoidp = (void*)&((PyFloatObject*)pyobject)->ob_fval; para.fTypeCode = 'V'; return true; } +#endif + +#if PY_VERSION_HEX >= 0x02050000 + if (Py_TYPE(pyobject) == GetCTypesType(ct_c_double)) { + para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr; + para.fTypeCode = 'V'; + return true; + } +#endif // alternate, pass pointer from buffer Py_ssize_t buflen = Utility::GetBuffer(pyobject, 'd', sizeof(double), para.fValue.fVoidp); @@ -1019,9 +1120,9 @@ bool CPyCppyy::DoubleRefConverter::SetArg( } //---------------------------------------------------------------------------- -CPPYY_IMPL_BASIC_CONST_REFCONVERTER(Float, float, c_float, PyFloat_AsDouble) -CPPYY_IMPL_BASIC_CONST_REFCONVERTER(Double, double, c_double, PyFloat_AsDouble) -CPPYY_IMPL_BASIC_CONST_REFCONVERTER(LDouble, LongDouble_t, c_longdouble, PyFloat_AsDouble) +CPPYY_IMPL_BASIC_CONST_REFCONVERTER(Float, float, c_float, PyFloat_AsDouble) +CPPYY_IMPL_BASIC_CONST_REFCONVERTER(Double, double, c_double, PyFloat_AsDouble) +CPPYY_IMPL_BASIC_CONST_REFCONVERTER(LDouble, PY_LONG_DOUBLE, c_longdouble, PyFloat_AsDouble) //---------------------------------------------------------------------------- bool CPyCppyy::VoidConverter::SetArg(PyObject*, Parameter&, CallContext*) @@ -1033,17 +1134,13 @@ bool CPyCppyy::VoidConverter::SetArg(PyObject*, Parameter&, CallContext*) //---------------------------------------------------------------------------- bool CPyCppyy::LLongConverter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) + PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ long long, set arg for call - if (PyFloat_Check(pyobject)) { - // special case: float implements nb_int, but allowing rounding conversions - // interferes with overloading - PyErr_SetString(PyExc_ValueError, "cannot convert float to long long"); + if (!ImplicitBool(pyobject, ctxt)) return false; - } - para.fValue.fLLong = PyLong_AsLongLong(pyobject); + para.fValue.fLLong = CPyCppyy_PyLong_AsStrictLongLong(pyobject); if (PyErr_Occurred()) return false; para.fTypeCode = 'q'; @@ -1053,24 +1150,32 @@ bool CPyCppyy::LLongConverter::SetArg( PyObject* CPyCppyy::LLongConverter::FromMemory(void* address) { // construct python object from C++ long long read at
- return PyLong_FromLongLong(*(Long64_t*)address); + return PyLong_FromLongLong(*(PY_LONG_LONG*)address); } bool CPyCppyy::LLongConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) { // convert to C++ long long, write it at
- Long64_t ll = PyLong_AsLongLong(value); - if (ll == -1 && PyErr_Occurred()) - return false; - *((Long64_t*)address) = ll; + PY_LONG_LONG ll = PyLong_AsLongLong(value); + if (ll == -1 && PyErr_Occurred()) { + if (value == CPyCppyy::gDefaultObject) { + PyErr_Clear(); + ll = (PY_LONG_LONG)0; + } else + return false; + } + *((PY_LONG_LONG*)address) = ll; return true; } //---------------------------------------------------------------------------- bool CPyCppyy::ULLongConverter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) + PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ unsigned long long, set arg for call + if (!ImplicitBool(pyobject, ctxt)) + return false; + para.fValue.fULLong = PyLongOrInt_AsULong64(pyobject); if (PyErr_Occurred()) return false; @@ -1081,22 +1186,27 @@ bool CPyCppyy::ULLongConverter::SetArg( PyObject* CPyCppyy::ULLongConverter::FromMemory(void* address) { // construct python object from C++ unsigned long long read at
- return PyLong_FromUnsignedLongLong(*(ULong64_t*)address); + return PyLong_FromUnsignedLongLong(*(PY_ULONG_LONG*)address); } bool CPyCppyy::ULLongConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) { // convert to C++ unsigned long long, write it at
- Long64_t ull = PyLongOrInt_AsULong64(value); - if (PyErr_Occurred()) - return false; - *((ULong64_t*)address) = ull; + PY_ULONG_LONG ull = PyLongOrInt_AsULong64(value); + if (PyErr_Occurred()) { + if (value == CPyCppyy::gDefaultObject) { + PyErr_Clear(); + ull = (PY_ULONG_LONG)0; + } else + return false; + } + *((PY_ULONG_LONG*)address) = ull; return true; } //---------------------------------------------------------------------------- bool CPyCppyy::CStringConverter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) + PyObject* pyobject, Parameter& para, CallContext* ctxt) { // construct a new string and copy it in new memory Py_ssize_t len; @@ -1106,6 +1216,7 @@ bool CPyCppyy::CStringConverter::SetArg( PyObject* pytype = 0, *pyvalue = 0, *pytrace = 0; PyErr_Fetch(&pytype, &pyvalue, &pytrace); if (Py_TYPE(pyobject) == GetCTypesType(ct_c_char_p)) { + SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this); para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr; para.fTypeCode = 'V'; Py_XDECREF(pytype); Py_XDECREF(pyvalue); Py_XDECREF(pytrace); @@ -1115,16 +1226,21 @@ bool CPyCppyy::CStringConverter::SetArg( return false; } - fBuffer = std::string(cstr, len); - // verify (too long string will cause truncation, no crash) - if (fMaxSize != -1 && fMaxSize < (long)fBuffer.size()) + if (fMaxSize != std::string::npos && fMaxSize < fBuffer.size()) PyErr_Warn(PyExc_RuntimeWarning, (char*)"string too long for char array (truncated)"); - else if (fMaxSize != -1) - fBuffer.resize(fMaxSize, '\0'); // padd remainder of buffer as needed + + if (!ctxt->fPyContext) { + // use internal buffer as workaround + fBuffer = std::string(cstr, len); + if (fMaxSize != std::string::npos) + fBuffer.resize(fMaxSize, '\0'); // pad remainder of buffer as needed + cstr = fBuffer.c_str(); + } else + SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this); // set the value and declare success - para.fValue.fVoidp = (void*)fBuffer.c_str(); + para.fValue.fVoidp = (void*)cstr; para.fTypeCode = 'p'; return true; } @@ -1132,12 +1248,14 @@ bool CPyCppyy::CStringConverter::SetArg( PyObject* CPyCppyy::CStringConverter::FromMemory(void* address) { // construct python object from C++ const char* read at
- if (address && *(char**)address) { - if (fMaxSize != -1) { // need to prevent reading beyond boundary - std::string buf(*(char**)address, fMaxSize); // cut on fMaxSize - return CPyCppyy_PyText_FromString(buf.c_str()); // cut on \0 - } + if (address && *(void**)address) { + if (fMaxSize != std::string::npos) // need to prevent reading beyond boundary + return CPyCppyy_PyText_FromStringAndSize(*(char**)address, (Py_ssize_t)fMaxSize); + if (*(void**)address == (void*)fBuffer.data()) // if we're buffering, we know the size + return CPyCppyy_PyText_FromStringAndSize((char*)fBuffer.data(), fBuffer.size()); + + // no idea about lentgth: cut on \0 return CPyCppyy_PyText_FromString(*(char**)address); } @@ -1146,7 +1264,7 @@ PyObject* CPyCppyy::CStringConverter::FromMemory(void* address) return PyStrings::gEmptyString; } -bool CPyCppyy::CStringConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) +bool CPyCppyy::CStringConverter::ToMemory(PyObject* value, void* address, PyObject* ctxt) { // convert to C++ const char*, write it at
Py_ssize_t len; @@ -1154,11 +1272,32 @@ bool CPyCppyy::CStringConverter::ToMemory(PyObject* value, void* address, PyObje if (!cstr) return false; // verify (too long string will cause truncation, no crash) - if (fMaxSize != -1 && fMaxSize < (long)len) + if (fMaxSize != std::string::npos && fMaxSize < (std::string::size_type)len) PyErr_Warn(PyExc_RuntimeWarning, (char*)"string too long for char array (truncated)"); - if (fMaxSize != -1) - strncpy(*(char**)address, cstr, fMaxSize); // padds remainder +// if address is available, and it wasn't set by this converter, assume a byte-wise copy; +// otherwise assume a pointer copy (this relies on the converter to be used for properties, +// or for argument passing, but not both at the same time; this is currently the case) + void* ptrval = *(void**)address; + if (ptrval == (void*)fBuffer.data()) { + fBuffer = std::string(cstr, len); + *(void**)address = (void*)fBuffer.data(); + return true; + } else if (ptrval && HasLifeLine(ctxt, (intptr_t)ptrval)) { + ptrval = nullptr; + // fall through; ptrval is nullptr means we're managing it + } + +// the string is (going to be) managed by us: assume pointer copy + if (!ptrval) { + SetLifeLine(ctxt, value, (intptr_t)address); + *(void**)address = (void*)cstr; + return true; + } + +// the pointer value is non-zero and not ours: assume byte copy + if (fMaxSize != std::string::npos) + strncpy(*(char**)address, cstr, fMaxSize); // pads remainder else // coverity[secure_coding] - can't help it, it's intentional. strcpy(*(char**)address, cstr); @@ -1191,8 +1330,8 @@ PyObject* CPyCppyy::WCStringConverter::FromMemory(void* address) { // construct python object from C++ wchar_t* read at
if (address && *(wchar_t**)address) { - if (fMaxSize != -1) // need to prevent reading beyond boundary - return PyUnicode_FromWideChar(*(wchar_t**)address, fMaxSize); + if (fMaxSize != std::wstring::npos) // need to prevent reading beyond boundary + return PyUnicode_FromWideChar(*(wchar_t**)address, (Py_ssize_t)fMaxSize); // with unknown size return PyUnicode_FromWideChar(*(wchar_t**)address, wcslen(*(wchar_t**)address)); } @@ -1210,12 +1349,12 @@ bool CPyCppyy::WCStringConverter::ToMemory(PyObject* value, void* address, PyObj return false; // verify (too long string will cause truncation, no crash) - if (fMaxSize != -1 && fMaxSize < len) + if (fMaxSize != std::wstring::npos && fMaxSize < (std::wstring::size_type)len) PyErr_Warn(PyExc_RuntimeWarning, (char*)"string too long for wchar_t array (truncated)"); Py_ssize_t res = -1; - if (fMaxSize != -1) - res = CPyCppyy_PyUnicode_AsWideChar(value, *(wchar_t**)address, fMaxSize); + if (fMaxSize != std::wstring::npos) + res = CPyCppyy_PyUnicode_AsWideChar(value, *(wchar_t**)address, (Py_ssize_t)fMaxSize); else // coverity[secure_coding] - can't help it, it's intentional. res = CPyCppyy_PyUnicode_AsWideChar(value, *(wchar_t**)address, len); @@ -1225,127 +1364,64 @@ bool CPyCppyy::WCStringConverter::ToMemory(PyObject* value, void* address, PyObj } //---------------------------------------------------------------------------- -bool CPyCppyy::CString16Converter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) -{ -// construct a new string and copy it in new memory - Py_ssize_t len = PyUnicode_GetSize(pyobject); - if (len == (Py_ssize_t)-1 && PyErr_Occurred()) - return false; - - PyObject* bstr = PyUnicode_AsUTF16String(pyobject); - if (!bstr) return false; - - fBuffer = (char16_t*)realloc(fBuffer, sizeof(char16_t)*(len+1)); - memcpy(fBuffer, PyBytes_AS_STRING(bstr) + sizeof(char16_t) /*BOM*/, len*sizeof(char16_t)); - Py_DECREF(bstr); - -// set the value and declare success - fBuffer[len] = u'\0'; - para.fValue.fVoidp = (void*)fBuffer; - para.fTypeCode = 'p'; - return true; -} - -PyObject* CPyCppyy::CString16Converter::FromMemory(void* address) -{ -// construct python object from C++ char16_t* read at
- if (address && *(char16_t**)address) { - if (fMaxSize != -1) // need to prevent reading beyond boundary - return PyUnicode_DecodeUTF16(*(const char**)address, fMaxSize, nullptr, nullptr); - // with unknown size - return PyUnicode_DecodeUTF16(*(const char**)address, - std::char_traits::length(*(char16_t**)address)*sizeof(char16_t), nullptr, nullptr); - } - -// empty string in case there's no valid address - char16_t w = u'\0'; - return PyUnicode_DecodeUTF16((const char*)&w, 0, nullptr, nullptr); -} - -bool CPyCppyy::CString16Converter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) -{ -// convert to C++ char16_t*, write it at
- Py_ssize_t len = PyUnicode_GetSize(value); - if (len == (Py_ssize_t)-1 && PyErr_Occurred()) - return false; - -// verify (too long string will cause truncation, no crash) - if (fMaxSize != -1 && fMaxSize < len) { - PyErr_Warn(PyExc_RuntimeWarning, (char*)"string too long for char16_t array (truncated)"); - len = fMaxSize-1; - } - - PyObject* bstr = PyUnicode_AsUTF16String(value); - if (!bstr) return false; - - memcpy(*((void**)address), PyBytes_AS_STRING(bstr) + sizeof(char16_t) /*BOM*/, len*sizeof(char16_t)); - Py_DECREF(bstr); - *((char16_t**)address)[len] = u'\0'; - return true; -} - -//---------------------------------------------------------------------------- -bool CPyCppyy::CString32Converter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) -{ -// construct a new string and copy it in new memory - Py_ssize_t len = PyUnicode_GetSize(pyobject); - if (len == (Py_ssize_t)-1 && PyErr_Occurred()) - return false; - - PyObject* bstr = PyUnicode_AsUTF32String(pyobject); - if (!bstr) return false; - - fBuffer = (char32_t*)realloc(fBuffer, sizeof(char32_t)*(len+1)); - memcpy(fBuffer, PyBytes_AS_STRING(bstr) + sizeof(char32_t) /*BOM*/, len*sizeof(char32_t)); - Py_DECREF(bstr); - -// set the value and declare success - fBuffer[len] = U'\0'; - para.fValue.fVoidp = (void*)fBuffer; - para.fTypeCode = 'p'; - return true; -} - -PyObject* CPyCppyy::CString32Converter::FromMemory(void* address) -{ -// construct python object from C++ char32_t* read at
- if (address && *(char32_t**)address) { - if (fMaxSize != -1) // need to prevent reading beyond boundary - return PyUnicode_DecodeUTF32(*(const char**)address, fMaxSize, nullptr, nullptr); - // with unknown size - return PyUnicode_DecodeUTF32(*(const char**)address, - std::char_traits::length(*(char32_t**)address)*sizeof(char32_t), nullptr, nullptr); - } - -// empty string in case there's no valid address - char32_t w = U'\0'; - return PyUnicode_DecodeUTF32((const char*)&w, 0, nullptr, nullptr); -} - -bool CPyCppyy::CString32Converter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) -{ -// convert to C++ char32_t*, write it at
- Py_ssize_t len = PyUnicode_GetSize(value); - if (len == (Py_ssize_t)-1 && PyErr_Occurred()) - return false; - -// verify (too long string will cause truncation, no crash) - if (fMaxSize != -1 && fMaxSize < len) { - PyErr_Warn(PyExc_RuntimeWarning, (char*)"string too long for char32_t array (truncated)"); - len = fMaxSize-1; - } - - PyObject* bstr = PyUnicode_AsUTF32String(value); - if (!bstr) return false; - - memcpy(*((void**)address), PyBytes_AS_STRING(bstr) + sizeof(char32_t) /*BOM*/, len*sizeof(char32_t)); - Py_DECREF(bstr); - *((char32_t**)address)[len] = U'\0'; - return true; +#define CPYCPPYY_WIDESTRING_CONVERTER(name, type, encode, decode, snull) \ +bool CPyCppyy::name##Converter::SetArg( \ + PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) \ +{ \ +/* change string encoding and copy into local buffer */ \ + PyObject* bstr = encode(pyobject); \ + if (!bstr) return false; \ + \ + Py_ssize_t len = PyBytes_GET_SIZE(bstr) - sizeof(type) /*BOM*/; \ + fBuffer = (type*)realloc(fBuffer, len + sizeof(type)); \ + memcpy(fBuffer, PyBytes_AS_STRING(bstr) + sizeof(type) /*BOM*/, len); \ + Py_DECREF(bstr); \ + \ + fBuffer[len/sizeof(type)] = snull; \ + para.fValue.fVoidp = (void*)fBuffer; \ + para.fTypeCode = 'p'; \ + return true; \ +} \ + \ +PyObject* CPyCppyy::name##Converter::FromMemory(void* address) \ +{ \ +/* construct python object from C++ * read at
*/ \ + if (address && *(type**)address) { \ + if (fMaxSize != std::wstring::npos) \ + return decode(*(const char**)address, (Py_ssize_t)fMaxSize*sizeof(type), nullptr, nullptr);\ + return decode(*(const char**)address, \ + std::char_traits::length(*(type**)address)*sizeof(type), nullptr, nullptr);\ + } \ + \ +/* empty string in case there's no valid address */ \ + type w = snull; \ + return decode((const char*)&w, 0, nullptr, nullptr); \ +} \ + \ +bool CPyCppyy::name##Converter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */)\ +{ \ +/* convert to C++ *, write it at
*/ \ + PyObject* bstr = encode(value); \ + if (!bstr) return false; \ + \ + Py_ssize_t len = PyBytes_GET_SIZE(bstr) - sizeof(type) /*BOM*/; \ + Py_ssize_t maxbytes = (Py_ssize_t)fMaxSize*sizeof(type); \ + \ +/* verify (too long string will cause truncation, no crash) */ \ + if (fMaxSize != std::wstring::npos && maxbytes < len) { \ + PyErr_Warn(PyExc_RuntimeWarning, (char*)"string too long for "#type" array (truncated)");\ + len = maxbytes; \ + } \ + \ + memcpy(*((void**)address), PyBytes_AS_STRING(bstr) + sizeof(type) /*BOM*/, len);\ + Py_DECREF(bstr); \ +/* debatable, but probably more convenient in most cases to null-terminate if enough space */\ + if (len/sizeof(type) < fMaxSize) (*(type**)address)[len/sizeof(type)] = snull;\ + return true; \ } +CPYCPPYY_WIDESTRING_CONVERTER(CString16, char16_t, PyUnicode_AsUTF16String, PyUnicode_DecodeUTF16, u'\0') +CPYCPPYY_WIDESTRING_CONVERTER(CString32, char32_t, PyUnicode_AsUTF32String, PyUnicode_DecodeUTF32, U'\0') //---------------------------------------------------------------------------- bool CPyCppyy::NonConstCStringConverter::SetArg( @@ -1364,8 +1440,8 @@ bool CPyCppyy::NonConstCStringConverter::SetArg( PyObject* CPyCppyy::NonConstCStringConverter::FromMemory(void* address) { // assume this is a buffer access if the size is known; otherwise assume string - if (fMaxSize != -1) - return CPyCppyy_PyText_FromStringAndSize(*(char**)address, fMaxSize); + if (fMaxSize != std::string::npos) + return CPyCppyy_PyText_FromStringAndSize(*(char**)address, (Py_ssize_t)fMaxSize); return this->CStringConverter::FromMemory(address); } @@ -1373,7 +1449,7 @@ PyObject* CPyCppyy::NonConstCStringConverter::FromMemory(void* address) bool CPyCppyy::VoidArrayConverter::GetAddressSpecialCase(PyObject* pyobject, void*& address) { // (1): C++11 style "null pointer" - if (pyobject == gNullPtrObject) { + if (pyobject == gNullPtrObject || pyobject == gDefaultObject) { address = nullptr; return true; } @@ -1457,11 +1533,11 @@ bool CPyCppyy::VoidArrayConverter::SetArg( PyObject* CPyCppyy::VoidArrayConverter::FromMemory(void* address) { // nothing sensible can be done, just return
as pylong - if (!address || *(ptrdiff_t*)address == 0) { + if (!address || *(uintptr_t*)address == 0) { Py_INCREF(gNullPtrObject); return gNullPtrObject; } - return CreatePointerView(*(ptrdiff_t**)address); + return CreatePointerView(*(uintptr_t**)address); } //---------------------------------------------------------------------------- @@ -1496,147 +1572,248 @@ bool CPyCppyy::VoidArrayConverter::ToMemory(PyObject* value, void* address, PyOb return true; } +namespace { -//---------------------------------------------------------------------------- -static inline void init_shape(Py_ssize_t defdim, dims_t dims, Py_ssize_t*& shape) +// Copy a buffer to memory address with an array converter. +template +bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt, + const void * buf, Py_ssize_t buflen, + CPyCppyy::dims_t& shape, bool isFixed) { - int nalloc = (dims && 0 < dims[0]) ? (int)dims[0]+1: defdim+1; - shape = new Py_ssize_t[nalloc]; - if (dims) { - for (int i = 0; i < nalloc; ++i) - shape[i] = (Py_ssize_t)dims[i]; - } else { - shape[0] = defdim; - for (int i = 1; i < nalloc; ++i) shape[i] = UNKNOWN_SIZE; + if (buflen == 0) + return false; + + Py_ssize_t oldsz = 1; + for (Py_ssize_t idim = 0; idim < shape.ndim(); ++idim) { + if (shape[idim] == CPyCppyy::UNKNOWN_SIZE) { + oldsz = -1; + break; + } + oldsz *= shape[idim]; + } + if (shape.ndim() != CPyCppyy::UNKNOWN_SIZE && 0 < oldsz && oldsz < buflen) { + PyErr_SetString(PyExc_ValueError, "buffer too large for value"); + return false; + } + + if (isFixed) + memcpy(*(type**)address, buf, (0 < buflen ? buflen : 1)*sizeof(type)); + else { + *(type**)address = (type*)buf; + shape.ndim(1); + shape[0] = buflen; + SetLifeLine(ctxt, owner, (intptr_t)address); } + return true; +} + } //---------------------------------------------------------------------------- -#define CPPYY_IMPL_ARRAY_CONVERTER(name, ctype, type, code) \ -CPyCppyy::name##ArrayConverter::name##ArrayConverter(dims_t dims, bool init) {\ - if (init) { \ - init_shape(1, dims, fShape); \ - fIsFixed = fShape[1] != UNKNOWN_SIZE; \ - } else fIsFixed = false; \ +#define CPPYY_IMPL_ARRAY_CONVERTER(name, ctype, type, code, suffix) \ +CPyCppyy::name##ArrayConverter::name##ArrayConverter(cdims_t dims) : \ + fShape(dims) { \ + fIsFixed = dims ? fShape[0] != UNKNOWN_SIZE : false; \ } \ \ bool CPyCppyy::name##ArrayConverter::SetArg( \ PyObject* pyobject, Parameter& para, CallContext* ctxt) \ { \ /* filter ctypes first b/c their buffer conversion will be wrong */ \ - bool res = false; \ - PyTypeObject* ctypes_type = GetCTypesType(ct_##ctype); \ - if (Py_TYPE(pyobject) == ctypes_type) { \ - para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ - para.fTypeCode = 'p'; \ - res = true; \ - } else if (Py_TYPE(pyobject) == GetCTypesPtrType(ct_##ctype)) { \ - para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ - para.fTypeCode = 'V'; \ - res = true; \ - } else if (IsPyCArgObject(pyobject)) { \ - CPyCppyy_tagPyCArgObject* carg = (CPyCppyy_tagPyCArgObject*)pyobject;\ - if (carg->obj && Py_TYPE(carg->obj) == ctypes_type) { \ - para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)carg->obj)->b_ptr;\ + bool convOk = false; \ + \ + /* 2-dim case: ptr-ptr types */ \ + if (fShape.ndim() == 2) { \ + if (Py_TYPE(pyobject) == GetCTypesPtrType(ct_##ctype)) { \ + para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ para.fTypeCode = 'p'; \ - res = true; \ + convOk = true; \ + } else if (Py_TYPE(pyobject) == GetCTypesType(ct_c_void_p)) { \ + /* special case: pass address of c_void_p buffer to return the address */\ + para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ + para.fTypeCode = 'p'; \ + convOk = true; \ + } else if (LowLevelView_Check(pyobject) && \ + ((LowLevelView*)pyobject)->fBufInfo.ndim == 2 && \ + strchr(((LowLevelView*)pyobject)->fBufInfo.format, code)) { \ + para.fValue.fVoidp = ((LowLevelView*)pyobject)->get_buf(); \ + para.fTypeCode = 'p'; \ + convOk = true; \ } \ } \ - if (!res) res = CArraySetArg(pyobject, para, code, sizeof(type)); \ - if (res) SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this); \ - return res; \ + \ + /* 1-dim (accept pointer), or unknown (accept pointer as cast) */ \ + if (!convOk) { \ + PyTypeObject* ctypes_type = GetCTypesType(ct_##ctype); \ + if (Py_TYPE(pyobject) == ctypes_type) { \ + para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ + para.fTypeCode = 'p'; \ + convOk = true; \ + } else if (Py_TYPE(pyobject) == GetCTypesPtrType(ct_##ctype)) { \ + para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ + para.fTypeCode = 'V'; \ + convOk = true; \ + } else if (IsPyCArgObject(pyobject)) { \ + CPyCppyy_tagPyCArgObject* carg = (CPyCppyy_tagPyCArgObject*)pyobject;\ + if (carg->obj && Py_TYPE(carg->obj) == ctypes_type) { \ + para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)carg->obj)->b_ptr;\ + para.fTypeCode = 'p'; \ + convOk = true; \ + } \ + } \ + } \ + \ + /* cast pointer type */ \ + if (!convOk) { \ + bool ismulti = fShape.ndim() > 1; \ + convOk = CArraySetArg(pyobject, para, code, ismulti ? sizeof(void*) : sizeof(type), true);\ + } \ + \ + /* memory management and offsetting */ \ + if (convOk) SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this); \ + \ + return convOk; \ } \ \ PyObject* CPyCppyy::name##ArrayConverter::FromMemory(void* address) \ { \ if (!fIsFixed) \ - return CreateLowLevelView((type**)address, fShape); \ - return CreateLowLevelView(*(type**)address, fShape); \ + return CreateLowLevelView##suffix((type**)address, fShape); \ + return CreateLowLevelView##suffix(*(type**)address, fShape); \ } \ \ bool CPyCppyy::name##ArrayConverter::ToMemory( \ PyObject* value, void* address, PyObject* ctxt) \ { \ - if (fShape[0] != 1) { \ - PyErr_SetString(PyExc_ValueError, "only 1-dim arrays supported"); \ - return false; \ - } \ - void* buf = nullptr; \ - Py_ssize_t buflen = Utility::GetBuffer(value, code, sizeof(type), buf); \ - if (buflen == 0) \ - return false; \ - if (fIsFixed) { \ - if (fShape[1] < buflen) { \ - PyErr_SetString(PyExc_ValueError, "buffer too large for value"); \ - return false; \ - } \ - memcpy(*(type**)address, buf, (0 < buflen ? buflen : 1)*sizeof(type));\ - } else { \ + if (fShape.ndim() <= 1 || fIsFixed) { \ + void* buf = nullptr; \ + Py_ssize_t buflen = Utility::GetBuffer(value, code, sizeof(type), buf);\ + return ToArrayFromBuffer(value, address, ctxt, buf, buflen, fShape, fIsFixed);\ + } else { /* multi-dim, non-flat array; assume structure matches */ \ + void* buf = nullptr; /* TODO: GetBuffer() assumes flat? */ \ + Py_ssize_t buflen = Utility::GetBuffer(value, code, sizeof(void*), buf);\ + if (buflen == 0) return false; \ *(type**)address = (type*)buf; \ - fShape[1] = buflen; \ + SetLifeLine(ctxt, value, (intptr_t)address); \ } \ - SetLifeLine(ctxt, value, (intptr_t)address); \ return true; \ -} \ - \ -bool CPyCppyy::name##ArrayPtrConverter::SetArg( \ - PyObject* pyobject, Parameter& para, CallContext* ctxt ) \ -{ \ - if (Py_TYPE(pyobject) == GetCTypesPtrType(ct_##ctype)) { \ - para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ - para.fTypeCode = 'p'; \ - return true; \ - } else if (Py_TYPE(pyobject) == GetCTypesType(ct_c_void_p)) { \ - /* special case: pass address of c_void_p buffer to return the address */\ - para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr;\ - para.fTypeCode = 'p'; \ - return true; \ - } \ - bool res = name##ArrayConverter::SetArg(pyobject, para, ctxt); \ - if (res && para.fTypeCode == 'p') { \ - para.fRef = para.fValue.fVoidp; \ - para.fValue.fVoidp = ¶.fRef; \ - return true; \ - } \ - return false; \ } //---------------------------------------------------------------------------- -CPPYY_IMPL_ARRAY_CONVERTER(Bool, c_bool, bool, '?') -CPPYY_IMPL_ARRAY_CONVERTER(SChar, c_char, signed char, 'b') -CPPYY_IMPL_ARRAY_CONVERTER(UChar, c_ubyte, unsigned char, 'B') +CPPYY_IMPL_ARRAY_CONVERTER(Bool, c_bool, bool, '?', ) +CPPYY_IMPL_ARRAY_CONVERTER(SChar, c_char, signed char, 'b', ) +CPPYY_IMPL_ARRAY_CONVERTER(UChar, c_ubyte, unsigned char, 'B', ) #if __cplusplus > 201402L -CPPYY_IMPL_ARRAY_CONVERTER(Byte, c_ubyte, std::byte, 'B') +CPPYY_IMPL_ARRAY_CONVERTER(Byte, c_ubyte, std::byte, 'B', ) +#endif +CPPYY_IMPL_ARRAY_CONVERTER(Int8, c_byte, int8_t, 'b', _i8) +CPPYY_IMPL_ARRAY_CONVERTER(UInt8, c_ubyte, uint8_t, 'B', _i8) +CPPYY_IMPL_ARRAY_CONVERTER(Short, c_short, short, 'h', ) +CPPYY_IMPL_ARRAY_CONVERTER(UShort, c_ushort, unsigned short, 'H', ) +CPPYY_IMPL_ARRAY_CONVERTER(Int, c_int, int, 'i', ) +CPPYY_IMPL_ARRAY_CONVERTER(UInt, c_uint, unsigned int, 'I', ) +CPPYY_IMPL_ARRAY_CONVERTER(Long, c_long, long, 'l', ) +CPPYY_IMPL_ARRAY_CONVERTER(ULong, c_ulong, unsigned long, 'L', ) +CPPYY_IMPL_ARRAY_CONVERTER(LLong, c_longlong, long long, 'q', ) +CPPYY_IMPL_ARRAY_CONVERTER(ULLong, c_ulonglong, unsigned long long, 'Q', ) +CPPYY_IMPL_ARRAY_CONVERTER(Float, c_float, float, 'f', ) +CPPYY_IMPL_ARRAY_CONVERTER(Double, c_double, double, 'd', ) +CPPYY_IMPL_ARRAY_CONVERTER(LDouble, c_longdouble, long double, 'g', ) +CPPYY_IMPL_ARRAY_CONVERTER(ComplexF, c_fcomplex, std::complex, 'z', ) +CPPYY_IMPL_ARRAY_CONVERTER(ComplexD, c_complex, std::complex, 'Z', ) + + +//---------------------------------------------------------------------------- +bool CPyCppyy::CStringArrayConverter::SetArg( + PyObject* pyobject, Parameter& para, CallContext* ctxt) +{ + if (Py_TYPE(pyobject) == GetCTypesPtrType(ct_c_char_p) || \ + (1 < fShape.ndim() && PyObject_IsInstance(pyobject, (PyObject*)GetCTypesType(ct_c_pointer)))) { + // 2nd predicate is ebatable: it's a catch-all for ctypes-styled multi-dimensional objects, + // which at this point does not check further dimensionality + para.fValue.fVoidp = (void*)((CPyCppyy_tagCDataObject*)pyobject)->b_ptr; + para.fTypeCode = 'V'; + return true; + + } else if (PySequence_Check(pyobject) && !CPyCppyy_PyText_Check(pyobject) +#if PY_VERSION_HEX >= 0x03000000 + && !PyBytes_Check(pyobject) #endif -CPPYY_IMPL_ARRAY_CONVERTER(Short, c_short, short, 'h') -CPPYY_IMPL_ARRAY_CONVERTER(UShort, c_ushort, unsigned short, 'H') -CPPYY_IMPL_ARRAY_CONVERTER(Int, c_int, int, 'i') -CPPYY_IMPL_ARRAY_CONVERTER(UInt, c_uint, unsigned int, 'I') -CPPYY_IMPL_ARRAY_CONVERTER(Long, c_long, long, 'l') -CPPYY_IMPL_ARRAY_CONVERTER(ULong, c_ulong, unsigned long, 'L') -CPPYY_IMPL_ARRAY_CONVERTER(LLong, c_longlong, long long, 'q') -CPPYY_IMPL_ARRAY_CONVERTER(ULLong, c_ulonglong, unsigned long long, 'Q') -CPPYY_IMPL_ARRAY_CONVERTER(Float, c_float, float, 'f') -CPPYY_IMPL_ARRAY_CONVERTER(Double, c_double, double, 'd') -CPPYY_IMPL_ARRAY_CONVERTER(LDouble, c_longdouble, long double, 'D') -CPPYY_IMPL_ARRAY_CONVERTER(ComplexD, c_complex, std::complex, 'Z') + ) { + //for (auto& p : fBuffer) free(p); + fBuffer.clear(); + + size_t len = (size_t)PySequence_Size(pyobject); + if (len == (size_t)-1) { + PyErr_SetString(PyExc_ValueError, "can not convert sequence object of unknown length"); + return false; + } + + fBuffer.reserve(len); + for (size_t i = 0; i < len; ++i) { + PyObject* item = PySequence_GetItem(pyobject, i); + if (item) { + Py_ssize_t sz; + const char* p = CPyCppyy_PyText_AsStringAndSize(item, &sz); + Py_DECREF(item); + + if (p) fBuffer.push_back(p); + else { + PyErr_Format(PyExc_TypeError, "could not convert item %d to string", (int)i); + return false; + } + + } else + return false; + } + + para.fValue.fVoidp = (void*)fBuffer.data(); + para.fTypeCode = 'p'; + return true; + } + + return SCharArrayConverter::SetArg(pyobject, para, ctxt); +} //---------------------------------------------------------------------------- PyObject* CPyCppyy::CStringArrayConverter::FromMemory(void* address) { - if (fShape[1] == UNKNOWN_SIZE) - return CreateLowLevelView((const char**)address, fShape); - return CreateLowLevelView(*(const char***)address, fShape); + if (fIsFixed) + return CreateLowLevelView(*(char**)address, fShape); + else if (fShape[0] == UNKNOWN_SIZE) + return CreateLowLevelViewString((const char**)address, fShape); + return CreateLowLevelViewString(*(const char***)address, fShape); } +//---------------------------------------------------------------------------- +bool CPyCppyy::CStringArrayConverter::ToMemory(PyObject* value, void* address, PyObject* ctxt) +{ +// As a special array converter, the CStringArrayConverter one can also copy strings in the array, +// and not only buffers. + Py_ssize_t len; + if (const char* cstr = CPyCppyy_PyText_AsStringAndSize(value, &len)) { + return ToArrayFromBuffer(value, address, ctxt, cstr, len, fShape, fIsFixed); + } + return SCharArrayConverter::ToMemory(value, address, ctxt); +} + +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::NonConstCStringArrayConverter::FromMemory(void* address) +{ + if (fIsFixed) + return CreateLowLevelView(*(char**)address, fShape); + else if (fShape[0] == UNKNOWN_SIZE) + return CreateLowLevelViewString((char**)address, fShape); + return CreateLowLevelViewString(*(char***)address, fShape); +} //- converters for special cases --------------------------------------------- bool CPyCppyy::NullptrConverter::SetArg(PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) { // Only allow C++11 style nullptr to pass - if (pyobject == gNullPtrObject) { + if (pyobject == gNullPtrObject || pyobject == gDefaultObject) { para.fValue.fVoidp = nullptr; para.fTypeCode = 'p'; return true; @@ -1646,6 +1823,33 @@ bool CPyCppyy::NullptrConverter::SetArg(PyObject* pyobject, Parameter& para, Cal //---------------------------------------------------------------------------- +template +static inline bool CPyCppyy_PyUnicodeAsBytes2Buffer(PyObject* pyobject, T& buffer) { + PyObject* pybytes = nullptr; + if (PyBytes_Check(pyobject)) { + Py_INCREF(pyobject); + pybytes = pyobject; + } else if (PyUnicode_Check(pyobject)) { +#if PY_VERSION_HEX < 0x03030000 + pybytes = PyUnicode_EncodeUTF8( + PyUnicode_AS_UNICODE(pyobject), CPyCppyy_PyUnicode_GET_SIZE(pyobject), nullptr); +#else + pybytes = PyUnicode_AsUTF8String(pyobject); +#endif + } + + if (pybytes) { + Py_ssize_t len; + const char* cstr = nullptr; + PyBytes_AsStringAndSize(pybytes, (char**)&cstr, &len); + if (cstr) buffer = T{cstr, (typename T::size_type)len}; + Py_DECREF(pybytes); + return (bool)cstr; + } + + return false; +} + #define CPPYY_IMPL_STRING_AS_PRIMITIVE_CONVERTER(name, type, F1, F2) \ CPyCppyy::name##Converter::name##Converter(bool keepControl) : \ InstanceConverter(Cppyy::GetScope(#type), keepControl) {} \ @@ -1653,11 +1857,8 @@ CPyCppyy::name##Converter::name##Converter(bool keepControl) : \ bool CPyCppyy::name##Converter::SetArg( \ PyObject* pyobject, Parameter& para, CallContext* ctxt) \ { \ - Py_ssize_t len; \ - const char* cstr = CPyCppyy_PyText_AsStringAndSize(pyobject, &len); \ - if (cstr) { \ - fBuffer = type(cstr, len); \ - para.fValue.fVoidp = &fBuffer; \ + if (CPyCppyy_PyUnicodeAsBytes2Buffer(pyobject, fStringBuffer)) { \ + para.fValue.fVoidp = &fStringBuffer; \ para.fTypeCode = 'V'; \ return true; \ } \ @@ -1668,36 +1869,41 @@ bool CPyCppyy::name##Converter::SetArg( \ para.fTypeCode = 'V'; \ return result; \ } \ + \ return false; \ } \ \ PyObject* CPyCppyy::name##Converter::FromMemory(void* address) \ { \ if (address) \ - return CPyCppyy_PyText_FromStringAndSize(((type*)address)->F1(), ((type*)address)->F2()); \ - Py_INCREF(PyStrings::gEmptyString); \ - return PyStrings::gEmptyString; \ + return InstanceConverter::FromMemory(address); \ + auto* empty = new type(); \ + return BindCppObjectNoCast(empty, fClass, CPPInstance::kIsOwner); \ } \ \ -bool CPyCppyy::name##Converter::ToMemory(PyObject* value, void* address, \ - PyObject* ctxt) \ +bool CPyCppyy::name##Converter::ToMemory( \ + PyObject* value, void* address, PyObject* ctxt) \ { \ - if (CPyCppyy_PyText_Check(value)) { \ - *((type*)address) = CPyCppyy_PyText_AsString(value); \ + if (CPyCppyy_PyUnicodeAsBytes2Buffer(value, *((type*)address))) \ return true; \ - } \ - \ return InstanceConverter::ToMemory(value, address, ctxt); \ } CPPYY_IMPL_STRING_AS_PRIMITIVE_CONVERTER(TString, TString, Data, Length) CPPYY_IMPL_STRING_AS_PRIMITIVE_CONVERTER(STLString, std::string, c_str, size) +#if __cplusplus > 201402L CPPYY_IMPL_STRING_AS_PRIMITIVE_CONVERTER(STLStringViewBase, std::string_view, data, size) bool CPyCppyy::STLStringViewConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { - if (this->STLStringViewBaseConverter::SetArg(pyobject, para, ctxt)) + if (this->STLStringViewBaseConverter::SetArg(pyobject, para, ctxt)) { + // One extra step compared to the regular std::string converter: + // Create a corresponding std::string_view and set the parameter value + // accordingly. + fStringView = *reinterpret_cast(para.fValue.fVoidp); + para.fValue.fVoidp = &fStringView; return true; + } if (!CPPInstance_Check(pyobject)) return false; @@ -1709,14 +1915,29 @@ bool CPyCppyy::STLStringViewConverter::SetArg( if (!ptr) return false; - fBuffer = *((std::string*)ptr); - para.fValue.fVoidp = &fBuffer; + // Copy the string to ensure the lifetime of the string_view and the + // underlying string is identical. + fStringBuffer = *((std::string*)ptr); + // Create the string_view on the copy + fStringView = fStringBuffer; + para.fValue.fVoidp = &fStringView; para.fTypeCode = 'V'; return true; } return false; } +bool CPyCppyy::STLStringViewConverter::ToMemory( + PyObject* value, void* address, PyObject* ctxt) +{ + if (CPyCppyy_PyUnicodeAsBytes2Buffer(value, fStringBuffer)) { + fStringView = fStringBuffer; + *reinterpret_cast(address) = fStringView; + return true; + } + return InstanceConverter::ToMemory(value, address, ctxt); +} +#endif CPyCppyy::STLWStringConverter::STLWStringConverter(bool keepControl) : InstanceConverter(Cppyy::GetScope("std::wstring"), keepControl) {} @@ -1726,15 +1947,32 @@ bool CPyCppyy::STLWStringConverter::SetArg( { if (PyUnicode_Check(pyobject)) { Py_ssize_t len = CPyCppyy_PyUnicode_GET_SIZE(pyobject); + fStringBuffer.resize(len); + CPyCppyy_PyUnicode_AsWideChar(pyobject, &fStringBuffer[0], len); + para.fValue.fVoidp = &fStringBuffer; + para.fTypeCode = 'V'; + return true; + } +#if PY_VERSION_HEX < 0x03000000 + else if (PyString_Check(pyobject)) { +#ifdef HAS_CODECVT + std::wstring_convert> cnv; + fBuffer = cnv.from_bytes(PyString_AS_STRING(pyobject)); +#else + PyObject* pyu = PyUnicode_FromString(PyString_AS_STRING(pyobject)); + if (!pyu) return false; + Py_ssize_t len = CPyCppyy_PyUnicode_GET_SIZE(pyu); fBuffer.resize(len); - CPyCppyy_PyUnicode_AsWideChar(pyobject, &fBuffer[0], len); + CPyCppyy_PyUnicode_AsWideChar(pyu, &fBuffer[0], len); +#endif para.fValue.fVoidp = &fBuffer; para.fTypeCode = 'V'; return true; } +#endif if (!(PyInt_Check(pyobject) || PyLong_Check(pyobject))) { - bool result = InstancePtrConverter::SetArg(pyobject, para, ctxt); + bool result = InstancePtrConverter::SetArg(pyobject, para, ctxt); para.fTypeCode = 'V'; return result; } @@ -1774,7 +2012,7 @@ bool CPyCppyy::STLStringMoveConverter::SetArg( if (pyobj->fFlags & CPPInstance::kIsRValue) { pyobj->fFlags &= ~CPPInstance::kIsRValue; moveit_reason = 2; - } else if (pyobject->ob_refcnt == MOVE_REFCOUNT_CUTOFF) { + } else if (pyobject->ob_refcnt <= MOVE_REFCOUNT_CUTOFF) { moveit_reason = 1; } else moveit_reason = 0; @@ -1793,11 +2031,12 @@ bool CPyCppyy::STLStringMoveConverter::SetArg( //---------------------------------------------------------------------------- -bool CPyCppyy::InstancePtrConverter::SetArg( +template +bool CPyCppyy::InstancePtrConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance*, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); + CPPInstance* pyobj = GetCppInstance(pyobject, ISCONST ? fClass : (Cppyy::TCppType_t)0); if (!pyobj) { if (GetAddressSpecialCase(pyobject, para.fValue.fVoidp)) { para.fTypeCode = 'p'; // allow special cases such as nullptr @@ -1821,12 +2060,12 @@ bool CPyCppyy::InstancePtrConverter::SetArg( // calculate offset between formal and actual arguments para.fValue.fVoidp = pyobj->GetObject(); - if (pyobj->ObjectIsA() != fClass) { + if (oisa != fClass) { para.fValue.fIntPtr += Cppyy::GetBaseOffset( - pyobj->ObjectIsA(), fClass, para.fValue.fVoidp, 1 /* up-cast */); + oisa, fClass, para.fValue.fVoidp, 1 /* up-cast */); } - // set pointer (may be null) and declare success + // set pointer (may be null) and declare success para.fTypeCode = 'p'; return true; } @@ -1835,17 +2074,21 @@ bool CPyCppyy::InstancePtrConverter::SetArg( } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::InstancePtrConverter::FromMemory(void* address) +template +PyObject* CPyCppyy::InstancePtrConverter::FromMemory(void* address) { // construct python object from C++ instance read at
- return BindCppObject(address, fClass, CPPInstance::kIsReference); + if (ISCONST) + return BindCppObject(*(void**)address, fClass); // by pointer value + return BindCppObject(address, fClass, CPPInstance::kIsReference); // modifiable } //---------------------------------------------------------------------------- -bool CPyCppyy::InstancePtrConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) +template +bool CPyCppyy::InstancePtrConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) { // convert to C++ instance, write it at
- CPPInstance* pyobj = GetCppInstance(value); + CPPInstance* pyobj = GetCppInstance(value, ISCONST ? fClass : (Cppyy::TCppType_t)0); if (!pyobj) { void* ptr = nullptr; if (GetAddressSpecialCase(value, ptr)) { @@ -1876,15 +2119,16 @@ bool CPyCppyy::InstanceConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); + CPPInstance* pyobj = GetCppInstance(pyobject, fClass); if (pyobj) { - if (pyobj->ObjectIsA() && Cppyy::IsSubtype(pyobj->ObjectIsA(), fClass)) { + auto oisa = pyobj->ObjectIsA(); + if (oisa && (oisa == fClass || Cppyy::IsSubtype(oisa, fClass))) { // calculate offset between formal and actual arguments para.fValue.fVoidp = pyobj->GetObject(); if (!para.fValue.fVoidp) return false; - if (pyobj->ObjectIsA() != fClass) { + if (oisa != fClass) { para.fValue.fIntPtr += Cppyy::GetBaseOffset( pyobj->ObjectIsA(), fClass, para.fValue.fVoidp, 1 /* up-cast */); } @@ -1894,13 +2138,17 @@ bool CPyCppyy::InstanceConverter::SetArg( } } - return ConvertImplicit(fClass, pyobject, para, ctxt); + return (bool)ConvertImplicit(fClass, pyobject, para, ctxt); } //---------------------------------------------------------------------------- PyObject* CPyCppyy::InstanceConverter::FromMemory(void* address) { - return BindCppObjectNoCast((Cppyy::TCppObject_t)address, fClass); +// This should not need a cast (ie. BindCppObjectNoCast), but performing the cast +// here means callbacks receive down-casted object when passed by-ptr, which is +// needed for object identity. The latter case is assumed to be more common than +// conversion of (global) objects. + return BindCppObject((Cppyy::TCppObject_t)address, fClass); } //---------------------------------------------------------------------------- @@ -1924,19 +2172,44 @@ bool CPyCppyy::InstanceRefConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance&, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); + CPPInstance* pyobj = GetCppInstance(pyobject, fIsConst ? fClass : (Cppyy::TCppType_t)0); if (pyobj) { // reject moves if (pyobj->fFlags & CPPInstance::kIsRValue) return false; - if (pyobj->ObjectIsA() && Cppyy::IsSubtype(pyobj->ObjectIsA(), fClass)) { + // smart pointers can end up here in case of a move, so preferentially match + // the smart type directly + bool argset = false; + Cppyy::TCppType_t cls = 0; + if (pyobj->IsSmart()) { + cls = pyobj->ObjectIsA(false); + if (cls && Cppyy::IsSubtype(cls, fClass)) { + para.fValue.fVoidp = pyobj->GetObjectRaw(); + argset = true; + } + } + + if (!argset) { + cls = pyobj->ObjectIsA(); + if (cls && Cppyy::IsSubtype(cls, fClass)) { + para.fValue.fVoidp = pyobj->GetObject(); + argset = true; + } + } + + if (argset) { + // do not allow null pointers through references + if (!para.fValue.fVoidp) { + PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer"); + return false; + } + // calculate offset between formal and actual arguments - para.fValue.fVoidp = pyobj->GetObject(); - if (pyobj->ObjectIsA() != fClass) { + if (cls != fClass) { para.fValue.fIntPtr += Cppyy::GetBaseOffset( - pyobj->ObjectIsA(), fClass, para.fValue.fVoidp, 1 /* up-cast */); + cls, fClass, para.fValue.fVoidp, 1 /* up-cast */); } para.fTypeCode = 'V'; @@ -1947,13 +2220,13 @@ bool CPyCppyy::InstanceRefConverter::SetArg( if (!fIsConst) // no implicit conversion possible return false; - return ConvertImplicit(fClass, pyobject, para, ctxt); + return (bool)ConvertImplicit(fClass, pyobject, para, ctxt); } //---------------------------------------------------------------------------- PyObject* CPyCppyy::InstanceRefConverter::FromMemory(void* address) { - return BindCppObjectNoCast((Cppyy::TCppObject_t)address, fClass); + return BindCppObjectNoCast((Cppyy::TCppObject_t)address, fClass, CPPInstance::kIsReference); } //---------------------------------------------------------------------------- @@ -1961,10 +2234,10 @@ bool CPyCppyy::InstanceMoveConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* ctxt) { // convert to C++ instance&&, set arg for call - CPPInstance* pyobj = GetCppInstance(pyobject); - if (!pyobj) { - // implicit conversion is fine as it the temporary by definition is moveable - return ConvertImplicit(fClass, pyobject, para, ctxt); + CPPInstance* pyobj = GetCppInstance(pyobject, fClass, true /* accept_rvalue */); + if (!pyobj || (pyobj->fFlags & CPPInstance::kIsLValue)) { + // implicit conversion is fine as the temporary by definition is moveable + return (bool)ConvertImplicit(fClass, pyobject, para, ctxt); } // moving is same as by-ref, but have to check that move is allowed @@ -1972,7 +2245,7 @@ bool CPyCppyy::InstanceMoveConverter::SetArg( if (pyobj->fFlags & CPPInstance::kIsRValue) { pyobj->fFlags &= ~CPPInstance::kIsRValue; moveit_reason = 2; - } else if (pyobject->ob_refcnt == MOVE_REFCOUNT_CUTOFF) { + } else if (pyobject->ob_refcnt <= MOVE_REFCOUNT_CUTOFF) { moveit_reason = 1; } @@ -1994,8 +2267,15 @@ bool CPyCppyy::InstancePtrPtrConverter::SetArg( { // convert to C++ instance**, set arg for call CPPInstance* pyobj = GetCppInstance(pyobject); - if (!pyobj) + if (!pyobj) { + if (!ISREFERENCE && (pyobject == gNullPtrObject || pyobject == gDefaultObject)) { + // allow nullptr as a special case + para.fValue.fVoidp = nullptr; + para.fTypeCode = 'p'; + return true; + } return false; // not a cppyy object (TODO: handle SWIG etc.) + } if (Cppyy::IsSubtype(pyobj->ObjectIsA(), fClass)) { // depending on memory policy, some objects need releasing when passed into functions @@ -2019,17 +2299,24 @@ template PyObject* CPyCppyy::InstancePtrPtrConverter::FromMemory(void* address) { // construct python object from C++ instance* read at
- return BindCppObject(address, fClass, CPPInstance::kIsReference); + return BindCppObject(*(void**)address, fClass, CPPInstance::kIsReference | CPPInstance::kIsPtrPtr); } //---------------------------------------------------------------------------- template -bool CPyCppyy::InstancePtrPtrConverter::ToMemory(PyObject* value, void* address, PyObject* /* ctxt */) +bool CPyCppyy::InstancePtrPtrConverter::ToMemory( + PyObject* value, void* address, PyObject* /* ctxt */) { // convert to C++ instance*, write it at
CPPInstance* pyobj = GetCppInstance(value); - if (!pyobj) + if (!pyobj) { + if (value == gNullPtrObject || value == gDefaultObject) { + // allow nullptr as a special case + *(void**)address = nullptr; + return true; + } return false; // not a cppyy object (TODO: handle SWIG etc.) + } if (Cppyy::IsSubtype(pyobj->ObjectIsA(), fClass)) { // depending on memory policy, some objects need releasing when passed into functions @@ -2050,6 +2337,8 @@ bool CPyCppyy::InstancePtrPtrConverter::ToMemory(PyObject* value, v namespace CPyCppyy { // Instantiate the templates + template class CPyCppyy::InstancePtrConverter; + template class CPyCppyy::InstancePtrConverter; template class CPyCppyy::InstancePtrPtrConverter; template class CPyCppyy::InstancePtrPtrConverter; } @@ -2085,11 +2374,12 @@ bool CPyCppyy::InstanceArrayConverter::SetArg( PyObject* CPyCppyy::InstanceArrayConverter::FromMemory(void* address) { // construct python tuple of instances from C++ array read at
- return BindCppObjectArray(*(char**)address, fClass, m_dims); + return BindCppObjectArray(*(char**)address, fClass, fShape); } //---------------------------------------------------------------------------- -bool CPyCppyy::InstanceArrayConverter::ToMemory(PyObject* /* value */, void* /* address */, PyObject* /* ctxt */) +bool CPyCppyy::InstanceArrayConverter::ToMemory( + PyObject* /* value */, void* /* address */, PyObject* /* ctxt */) { // convert to C++ array of instances, write it at
@@ -2131,6 +2421,12 @@ bool CPyCppyy::VoidPtrRefConverter::SetArg( return false; } +//---------------------------------------------------------------------------- +CPyCppyy::VoidPtrPtrConverter::VoidPtrPtrConverter(cdims_t dims) : + fShape(dims) { + fIsFixed = dims ? fShape[0] != UNKNOWN_SIZE : false; +} + //---------------------------------------------------------------------------- bool CPyCppyy::VoidPtrPtrConverter::SetArg( PyObject* pyobject, Parameter& para, CallContext* /* ctxt */) @@ -2167,12 +2463,14 @@ bool CPyCppyy::VoidPtrPtrConverter::SetArg( //---------------------------------------------------------------------------- PyObject* CPyCppyy::VoidPtrPtrConverter::FromMemory(void* address) { -// read a void** from address; since this is unknown, long is used (user can cast) +// read a void** from address; since this is unknown, uintptr_t is used (user can cast) if (!address || *(ptrdiff_t*)address == 0) { Py_INCREF(gNullPtrObject); return gNullPtrObject; } - return CreatePointerView(*(ptrdiff_t**)address, fSize); + if (!fIsFixed) + return CreatePointerView((uintptr_t**)address, fShape); + return CreatePointerView(*(uintptr_t**)address, fShape); } //---------------------------------------------------------------------------- @@ -2211,7 +2509,7 @@ bool CPyCppyy::PyObjectConverter::ToMemory(PyObject* value, void* address, PyObj //- function pointer converter ----------------------------------------------- static unsigned int sWrapperCounter = 0; // cache mapping signature/return type to python callable and corresponding wrapper -typedef std::pair RetSigKey_t; +typedef std::string RetSigKey_t; static std::map> sWrapperFree; static std::map> sWrapperLookup; static std::map> sWrapperWeakRefs; @@ -2221,16 +2519,25 @@ static PyObject* WrapperCacheEraser(PyObject*, PyObject* pyref) { auto ipos = sWrapperWeakRefs.find(pyref); if (ipos != sWrapperWeakRefs.end()) { - // disable this callback and store for possible re-use + auto key = ipos->second.second; + + // disable this callback and store on free list for possible re-use void* wpraddress = ipos->second.first; - *sWrapperReference[wpraddress] = nullptr; + PyObject** oldref = sWrapperReference[wpraddress]; + const auto& lookup = sWrapperLookup.find(key); + if (lookup != sWrapperLookup.end()) lookup->second.erase(*oldref); + *oldref = nullptr; // to detect deletions sWrapperFree[ipos->second.second].push_back(wpraddress); + + // clean up and remove weak reference from admin + Py_DECREF(ipos->first); + sWrapperWeakRefs.erase(ipos); } Py_RETURN_NONE; } static PyMethodDef gWrapperCacheEraserMethodDef = { - const_cast("interal_WrapperCacheEraser"), + const_cast("internal_WrapperCacheEraser"), (PyCFunction)WrapperCacheEraser, METH_O, nullptr }; @@ -2265,7 +2572,7 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, if (TemplateProxy_Check(pyobject)) { // get the actual underlying template matching the signature TemplateProxy* pytmpl = (TemplateProxy*)pyobject; - std::string fullname = CPyCppyy_PyText_AsString(pytmpl->fTI->fCppName); + std::string fullname = pytmpl->fTI->fCppName; if (pytmpl->fTemplateArgs) fullname += CPyCppyy_PyText_AsString(pytmpl->fTemplateArgs); Cppyy::TCppScope_t scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType; @@ -2282,7 +2589,7 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, void* wpraddress = nullptr; // re-use existing wrapper if possible - auto key = std::make_pair(rettype, signature); + auto key = rettype+signature; const auto& lookup = sWrapperLookup.find(key); if (lookup != sWrapperLookup.end()) { const auto& existing = lookup->second.find(pyobject); @@ -2296,7 +2603,8 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, if (freewrap != sWrapperFree.end() && !freewrap->second.empty()) { wpraddress = freewrap->second.back(); freewrap->second.pop_back(); - *sWrapperReference[wpraddress] = pyobject; + *(sWrapperReference[wpraddress]) = pyobject; + sWrapperLookup[key][pyobject] = wpraddress; PyObject* wref = PyWeakref_NewRef(pyobject, sWrapperCacheEraser); if (wref) sWrapperWeakRefs[wref] = std::make_pair(wpraddress, key); else PyErr_Clear(); // happens for builtins which don't need this @@ -2329,7 +2637,7 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, // start function body Utility::ConstructCallbackPreamble(rettype, argtypes, code); - // create a referencable pointer + // create a referenceable pointer PyObject** ref = new PyObject*{pyobject}; // function call itself and cleanup @@ -2372,10 +2680,10 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, } bool CPyCppyy::FunctionPointerConverter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /*ctxt*/) + PyObject* pyobject, Parameter& para, CallContext* ctxt) { // special case: allow nullptr singleton: - if (gNullPtrObject == pyobject) { + if (pyobject == gNullPtrObject || pyobject == gDefaultObject) { para.fValue.fVoidp = nullptr; para.fTypeCode = 'p'; return true; @@ -2384,6 +2692,7 @@ bool CPyCppyy::FunctionPointerConverter::SetArg( // normal case, get a function pointer void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature); if (fptr) { + SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this); para.fValue.fVoidp = fptr; para.fTypeCode = 'p'; return true; @@ -2403,10 +2712,11 @@ PyObject* CPyCppyy::FunctionPointerConverter::FromMemory(void* address) return nullptr; } -bool CPyCppyy::FunctionPointerConverter::ToMemory(PyObject* pyobject, void* address, PyObject* /* ctxt */) +bool CPyCppyy::FunctionPointerConverter::ToMemory( + PyObject* pyobject, void* address, PyObject* ctxt) { // special case: allow nullptr singleton: - if (gNullPtrObject == pyobject) { + if (pyobject == gNullPtrObject || pyobject == gDefaultObject) { *((void**)address) = nullptr; return true; } @@ -2414,6 +2724,7 @@ bool CPyCppyy::FunctionPointerConverter::ToMemory(PyObject* pyobject, void* addr // normal case, get a function pointer void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature); if (fptr) { + SetLifeLine(ctxt, pyobject, (intptr_t)address); *((void**)address) = fptr; return true; } @@ -2442,8 +2753,10 @@ bool CPyCppyy::StdFunctionConverter::SetArg( // then try normal conversion a second time PyObject* func = this->FunctionPointerConverter::FromMemory(¶.fValue.fVoidp); if (func) { - Py_XDECREF(fFuncWrap); fFuncWrap = func; - bool result = fConverter->SetArg(fFuncWrap, para, ctxt); + SetLifeLine(ctxt->fPyContext, func, (intptr_t)this); + bool result = fConverter->SetArg(func, para, ctxt); + if (result) ctxt->AddTemporary(func); + else Py_DECREF(func); if (!rf) ctxt->fFlags &= ~CallContext::kNoImplicit; return result; } @@ -2460,6 +2773,9 @@ PyObject* CPyCppyy::StdFunctionConverter::FromMemory(void* address) bool CPyCppyy::StdFunctionConverter::ToMemory(PyObject* value, void* address, PyObject* ctxt) { +// if the value is not an std::function<> but a generic Python callable, the +// conversion is done through the assignment, which may involve a temporary + if (address) SetLifeLine(ctxt, value, (intptr_t)address); return fConverter->ToMemory(value, address, ctxt); } @@ -2482,12 +2798,13 @@ bool CPyCppyy::SmartPtrConverter::SetArg( } CPPInstance* pyobj = (CPPInstance*)pyobject; + Cppyy::TCppType_t oisa = pyobj->ObjectIsA(); // for the case where we have a 'hidden' smart pointer: if (Cppyy::TCppType_t tsmart = pyobj->GetSmartIsA()) { if (Cppyy::IsSubtype(tsmart, fSmartPtrType)) { // depending on memory policy, some objects need releasing when passed into functions - if (fKeepControl && !UseStrictOwnership(ctxt)) + if (!fKeepControl && !UseStrictOwnership(ctxt)) ((CPPInstance*)pyobject)->CppOwns(); // calculate offset between formal and actual arguments @@ -2504,12 +2821,12 @@ bool CPyCppyy::SmartPtrConverter::SetArg( } // for the case where we have an 'exposed' smart pointer: - if (!pyobj->IsSmart() && Cppyy::IsSubtype(pyobj->ObjectIsA(), fSmartPtrType)) { + if (!pyobj->IsSmart() && Cppyy::IsSubtype(oisa, fSmartPtrType)) { // calculate offset between formal and actual arguments para.fValue.fVoidp = pyobj->GetObject(); - if (pyobj->ObjectIsA() != fSmartPtrType) { + if (oisa != fSmartPtrType) { para.fValue.fIntPtr += Cppyy::GetBaseOffset( - pyobj->ObjectIsA(), fSmartPtrType, para.fValue.fVoidp, 1 /* up-cast */); + oisa, fSmartPtrType, para.fValue.fVoidp, 1 /* up-cast */); } // set pointer (may be null) and declare success @@ -2517,8 +2834,27 @@ bool CPyCppyy::SmartPtrConverter::SetArg( return true; } +// for the case where we have an ordinary object to convert + if (!pyobj->IsSmart() && Cppyy::IsSubtype(oisa, fUnderlyingType)) { + // create the relevant smart pointer and make the pyobject "smart" + CPPInstance* pysmart = (CPPInstance*)ConvertImplicit(fSmartPtrType, pyobject, para, ctxt, false); + if (!CPPInstance_Check(pysmart)) { + Py_XDECREF(pysmart); + return false; + } + + // copy internals from the fresh smart object to the original, making it smart + pyobj->GetObjectRaw() = pysmart->GetSmartObject(); + pyobj->SetSmart(CreateScopeProxy(fSmartPtrType)); //(PyObject*)Py_TYPE(pysmart)); + pyobj->PythonOwns(); + pysmart->CppOwns(); + Py_DECREF(pysmart); + + return true; + } + // final option, try mapping pointer types held (TODO: do not allow for non-const ref) - if (pyobj->IsSmart() && Cppyy::IsSubtype(pyobj->ObjectIsA(), fUnderlyingType)) { + if (pyobj->IsSmart() && Cppyy::IsSubtype(oisa, fUnderlyingType)) { para.fValue.fVoidp = ((CPPInstance*)pyobject)->GetSmartObject(); para.fTypeCode = 'V'; return true; @@ -2561,43 +2897,88 @@ struct faux_initlist } // unnamed namespace +CPyCppyy::InitializerListConverter::InitializerListConverter(Cppyy::TCppType_t klass, std::string const &value_type) + + : InstanceConverter{klass}, + fValueTypeName{value_type}, + fValueType{Cppyy::GetScope(value_type)}, + fValueSize{Cppyy::SizeOf(value_type)} +{ +} + CPyCppyy::InitializerListConverter::~InitializerListConverter() { - if (fConverter && fConverter->HasState()) delete fConverter; + for (Converter *converter : fConverters) { + if (converter && converter->HasState()) delete converter; + } + if (fBuffer) Clear(); +} + +void CPyCppyy::InitializerListConverter::Clear() { + if (fValueType) { + faux_initlist* fake = (faux_initlist*)fBuffer; +#if defined (_LIBCPP_INITIALIZER_LIST) || defined(__GNUC__) + for (faux_initlist::size_type i = 0; i < fake->_M_len; ++i) { +#elif defined (_MSC_VER) + for (size_t i = 0; (fake->_M_array+i*fValueSize) != fake->_Last; ++i) { +#endif + void* memloc = (char*)fake->_M_array + i*fValueSize; + Cppyy::CallDestructor(fValueType, (Cppyy::TCppObject_t)memloc); + } + } + + free(fBuffer); + fBuffer = nullptr; } bool CPyCppyy::InitializerListConverter::SetArg( - PyObject* pyobject, Parameter& para, CallContext* /*ctxt*/) + PyObject* pyobject, Parameter& para, CallContext* ctxt) { #ifdef NO_KNOWN_INITIALIZER_LIST return false; #else + if (fBuffer) Clear(); + // convert the given argument to an initializer list temporary; this is purely meant // to be a syntactic thing, so only _python_ sequences are allowed; bound C++ proxies -// are therefore rejected (should go through eg. a copy constructor etc.) - if (CPPInstance_Check(pyobject) || !PySequence_Check(pyobject) || CPyCppyy_PyText_Check(pyobject) +// (likely explicitly created std::initializer_list, go through an instance converter + if (!PySequence_Check(pyobject) || CPyCppyy_PyText_Check(pyobject) #if PY_VERSION_HEX >= 0x03000000 || PyBytes_Check(pyobject) +#else + || PyUnicode_Check(pyobject) #endif ) return false; - void* buf; + if (CPPInstance_Check(pyobject)) + return this->InstanceConverter::SetArg(pyobject, para, ctxt); + + void* buf = nullptr; Py_ssize_t buflen = Utility::GetBuffer(pyobject, '*', (int)fValueSize, buf, true); faux_initlist* fake = nullptr; + size_t entries = 0; if (buf && buflen) { // dealing with an array here, pass on whole-sale fake = (faux_initlist*)malloc(sizeof(faux_initlist)); - fake->_M_array = (faux_initlist::iterator)buf; + fBuffer = (void*)fake; + fake->_M_array = (faux_initlist::iterator)buf; #if defined (_LIBCPP_INITIALIZER_LIST) || defined(__GNUC__) fake->_M_len = (faux_initlist::size_type)buflen; #elif defined (_MSC_VER) fake->_Last = fake->_M_array+buflen*fValueSize; #endif - } else { + } else if (fValueSize) { + // Remove any errors set by GetBuffer(); note that if the argument was an array + // that failed to extract because of a type mismatch, the following will perform + // a (rather inefficient) copy. No warning is issued b/c e.g. numpy doesn't do + // so either. + PyErr_Clear(); + // can only construct empty lists, so use a fake initializer list size_t len = (size_t)PySequence_Size(pyobject); fake = (faux_initlist*)malloc(sizeof(faux_initlist)+fValueSize*len); + fBuffer = (void*)fake; fake->_M_array = (faux_initlist::iterator)((char*)fake+sizeof(faux_initlist)); #if defined (_LIBCPP_INITIALIZER_LIST) || defined(__GNUC__) fake->_M_len = (faux_initlist::size_type)len; @@ -2609,29 +2990,55 @@ bool CPyCppyy::InitializerListConverter::SetArg( PyObject* item = PySequence_GetItem(pyobject, i); bool convert_ok = false; if (item) { - if (!fConverter) { + Converter *converter = CreateConverter(fValueTypeName); + if (!converter) { if (CPPInstance_Check(item)) { // by convention, use byte copy memcpy((char*)fake->_M_array + i*fValueSize, ((CPPInstance*)item)->GetObject(), fValueSize); convert_ok = true; } - } else - convert_ok = fConverter->ToMemory(item, (char*)fake->_M_array + i*fValueSize); + } else { + void* memloc = (char*)fake->_M_array + i*fValueSize; + if (fValueType) { + // we need to construct a default object for the constructor to assign into; this is + // clunky, but the use of a copy constructor isn't much better as the Python object + // need not be a C++ object + memloc = (void*)Cppyy::Construct(fValueType, memloc); + if (memloc) entries += 1; + else { + PyErr_SetString(PyExc_TypeError, + "default ctor needed for initializer list of objects"); + } + } + if (memloc) { + convert_ok = converter->ToMemory(item, memloc); + } + fConverters.emplace_back(converter); + } + Py_DECREF(item); } else PyErr_Format(PyExc_TypeError, "failed to get item %d from sequence", (int)i); if (!convert_ok) { - free((void*)fake); +#if defined (_LIBCPP_INITIALIZER_LIST) || defined(__GNUC__) + fake->_M_len = (faux_initlist::size_type)entries; +#elif defined (_MSC_VER) + fake->_Last = fake->_M_array+entries*fValueSize; +#endif + Clear(); return false; } } } + if (!fake) // no buffer and value size indeterminate + return false; + para.fValue.fVoidp = (void*)fake; - para.fTypeCode = 'X'; // means ptr that backend has to free after call + para.fTypeCode = 'V'; // means ptr that backend has to free after call return true; #endif } @@ -2646,8 +3053,8 @@ bool CPyCppyy::NotImplementedConverter::SetArg(PyObject*, Parameter&, CallContex //- helper to refactor some code from CreateConverter ------------------------ -static inline CPyCppyy::Converter* selectInstanceCnv( - Cppyy::TCppScope_t klass, const std::string& cpd, long size, dims_t dims, bool isConst, bool control) +static inline CPyCppyy::Converter* selectInstanceCnv(Cppyy::TCppScope_t klass, + const std::string& cpd, CPyCppyy::cdims_t dims, bool isConst, bool control) { using namespace CPyCppyy; Converter* result = nullptr; @@ -2656,13 +3063,15 @@ static inline CPyCppyy::Converter* selectInstanceCnv( result = new InstancePtrPtrConverter(klass, control); else if (cpd == "*&") result = new InstancePtrPtrConverter(klass, control); - else if (cpd == "*" && size <= 0) - result = new InstancePtrConverter(klass, control); + else if (cpd == "*" && dims.ndim() == UNKNOWN_SIZE) { + if (isConst) result = new InstancePtrConverter(klass, control); + else result = new InstancePtrConverter(klass, control); + } else if (cpd == "&") result = new InstanceRefConverter(klass, isConst); else if (cpd == "&&") result = new InstanceMoveConverter(klass); - else if (cpd == "[]" || size > 0) + else if (cpd == "[]" || dims) result = new InstanceArrayConverter(klass, dims, false); else if (cpd == "") // by value result = new InstanceConverter(klass, true); @@ -2672,7 +3081,7 @@ static inline CPyCppyy::Converter* selectInstanceCnv( //- factories ---------------------------------------------------------------- CPYCPPYY_EXPORT -CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, dims_t dims) +CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, cdims_t dims) { // The matching of the fulltype to a converter factory goes through up to five levels: // 1) full, exact match @@ -2683,12 +3092,11 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, dims // // If all fails, void is used, which will generate a run-time warning when used. - dim_t size = (dims && dims[0] != -1) ? dims[1] : -1; - // an exactly matching converter is best ConvFactories_t::iterator h = gConvFactories.find(fullType); - if (h != gConvFactories.end()) + if (h != gConvFactories.end()) { return (h->second)(dims); + } // resolve typedefs etc. const std::string& resolvedType = Cppyy::ResolveName(fullType); @@ -2702,50 +3110,57 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, dims //-- nothing? ok, collect information about the type and possible qualifiers/decorators bool isConst = strncmp(resolvedType.c_str(), "const", 5) == 0; - const std::string& cpd = Utility::Compound(resolvedType); + const std::string& cpd = TypeManip::compound(resolvedType); std::string realType = TypeManip::clean_type(resolvedType, false, true); // accept unqualified type (as python does not know about qualifiers) - h = gConvFactories.find(realType + cpd); + h = gConvFactories.find((isConst ? "const " : "") + realType + cpd); if (h != gConvFactories.end()) return (h->second)(dims); // drop const, as that is mostly meaningless to python (with the exception // of c-strings, but those are specialized in the converter map) if (isConst) { - realType = TypeManip::remove_const(realType); h = gConvFactories.find(realType + cpd); if (h != gConvFactories.end()) return (h->second)(dims); } //-- still nothing? try pointer instead of array (for builtins) - if (cpd == "[]") { - // simple array - h = gConvFactories.find(realType + "*"); - if (h != gConvFactories.end()) { - if (dims && dims[1] == UNKNOWN_SIZE) dims[1] = UNKNOWN_ARRAY_SIZE; - return (h->second)(dims); - } - } else if (cpd == "*[]") { - // array of pointers - h = gConvFactories.find(realType + "*"); + if (cpd.compare(0, 3, "*[]") == 0) { + // special case, array of pointers + h = gConvFactories.find(realType + " ptr"); if (h != gConvFactories.end()) { // upstream treats the pointer type as the array element type, but that pointer is - // treated as a low-level view as well, so adjust the dims - dim_t newdim = (dims && 0 < dims[0]) ? dims[0]+1 : 2; - dims_t newdims = new dim_t[newdim+1]; - newdims[0] = newdim; - newdims[1] = (0 < size ? size : UNKNOWN_ARRAY_SIZE); // the array - newdims[2] = UNKNOWN_SIZE; // the pointer - if (dims && 2 < newdim) { - for (int i = 2; i < (newdim-1); ++i) - newdims[i+1] = dims[i]; + // treated as a low-level view as well, unless it's a void*/char* so adjust the dims + if (realType != "void" && realType != "char") { + dim_t newdim = dims.ndim() == UNKNOWN_SIZE ? 2 : dims.ndim()+1; + dims_t newdims = dims_t(newdim); + // TODO: sometimes the array size is known and can thus be verified; however, + // currently the meta layer does not provide this information + newdims[0] = dims ? dims[0] : UNKNOWN_SIZE; // the array + newdims[1] = UNKNOWN_SIZE; // the pointer + if (2 < newdim) { + for (int i = 2; i < (newdim-1); ++i) + newdims[i] = dims[i-1]; + } + + return (h->second)(newdims); } - Converter* cnv = (h->second)(newdims); - delete [] newdims; - return cnv; + return (h->second)(dims); } + + } else if (!cpd.empty() && (std::string::size_type)std::count(cpd.begin(), cpd.end(), '*') == cpd.size()) { + // simple array; set or resize as necessary + h = gConvFactories.find(realType + " ptr"); + if (h != gConvFactories.end()) + return (h->second)((!dims && 1 < cpd.size()) ? dims_t(cpd.size()) : dims); + + } else if (2 <= cpd.size() && (std::string::size_type)std::count(cpd.begin(), cpd.end(), '[') == cpd.size() / 2) { + // fixed array, dims will have size if available + h = gConvFactories.find(realType + " ptr"); + if (h != gConvFactories.end()) + return (h->second)(dims); } //-- special case: initializer list @@ -2753,17 +3168,7 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, dims // get the type of the list and create a converter (TODO: get hold of value_type?) auto pos = realType.find('<'); std::string value_type = realType.substr(pos+1, realType.size()-pos-2); - Converter* cnv = nullptr; bool use_byte_cnv = false; - if (cpd == "" && Cppyy::GetScope(value_type)) { - // initializer list of object values does not work as the target is raw - // memory; simply use byte copies - - // by convention, leave cnv as nullptr - use_byte_cnv = true; - } else - cnv = CreateConverter(value_type); - if (cnv || use_byte_cnv) - return new InitializerListConverter(cnv, Cppyy::SizeOf(value_type)); + return new InitializerListConverter(Cppyy::GetScope(realType), value_type); } //-- still nothing? use a generalized converter @@ -2776,7 +3181,7 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, dims // get actual converter for normal passing Converter* cnv = selectInstanceCnv( - Cppyy::GetScope(realType), cpd, size, dims, isConst, control); + Cppyy::GetScope(realType), cpd, dims, isConst, control); if (cnv) { // get the type of the underlying (TODO: use target_type?) @@ -2802,34 +3207,29 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, dims result = new SmartPtrConverter(klass, raw, control); } else if (cpd == "&") { result = new SmartPtrConverter(klass, raw); - } else if (cpd == "*" && size <= 0) { + } else if (cpd == "*" && dims.ndim() == UNKNOWN_SIZE) { result = new SmartPtrConverter(klass, raw, control, true); } } if (!result) { // CLING WORKAROUND -- special case for STL iterators - if (realType.rfind("__gnu_cxx::__normal_iterator", 0) /* vector */ == 0 -#ifdef __APPLE__ - || realType.rfind("__wrap_iter", 0) == 0 -#endif - // TODO: Windows? - ) { + if (Utility::IsSTLIterator(realType)) { static STLIteratorConverter c; result = &c; } else // -- CLING WORKAROUND - result = selectInstanceCnv(klass, cpd, size, dims, isConst, control); + result = selectInstanceCnv(klass, cpd, dims, isConst, control); + } + } else { + std::smatch sm; + if (std::regex_search(resolvedType, sm, s_fnptr)) { + // this is a function pointer + auto pos1 = sm.position(0); + auto pos2 = resolvedType.rfind(')'); + result = new FunctionPointerConverter( + resolvedType.substr(0, pos1), resolvedType.substr(pos1+sm.length(), pos2-1)); } - } else if (resolvedType.find("(*)") != std::string::npos || - (resolvedType.find("::*)") != std::string::npos)) { - // this is a function function pointer - // TODO: find better way of finding the type - auto pos1 = resolvedType.find('('); - auto pos2 = resolvedType.find("*)"); - auto pos3 = resolvedType.rfind(')'); - result = new FunctionPointerConverter( - resolvedType.substr(0, pos1), resolvedType.substr(pos2+2, pos3-pos2-1)); } if (!result && cpd == "&&") { @@ -2847,7 +3247,7 @@ CPyCppyy::Converter* CPyCppyy::CreateConverter(const std::string& fullType, dims else if (!result) { // default to something reasonable, assuming "user knows best" if (cpd.size() == 2 && cpd != "&&") // "**", "*[]", "*&" - result = new VoidPtrPtrConverter(size); + result = new VoidPtrPtrConverter(dims.ndim()); else if (!cpd.empty()) result = new VoidArrayConverter(); // "user knows best" else @@ -2897,8 +3297,23 @@ namespace { using namespace CPyCppyy; +inline static +std::string::size_type dims2stringsz(cdims_t d) { + return (d && d.ndim() != UNKNOWN_SIZE) ? d[0] : std::string::npos; +} + #define STRINGVIEW "basic_string_view >" -#define WSTRING "basic_string,allocator >" +#define WSTRING1 "std::basic_string" +#define WSTRING2 "std::basic_string,std::allocator>" + +//-- aliasing special case: C complex (is binary compatible with C++ std::complex) +#ifndef _WIN32 +#define CCOMPLEX_D "_Complex double" +#define CCOMPLEX_F "_Complex float" +#else +#define CCOMPLEX_D "_C_double_complex" +#define CCOMPLEX_F "_C_float_complex" +#endif static struct InitConvFactories_t { public: @@ -2907,127 +3322,134 @@ static struct InitConvFactories_t { CPyCppyy::ConvFactories_t& gf = gConvFactories; // factories for built-ins - gf["bool"] = (cf_t)+[](dims_t) { static BoolConverter c{}; return &c; }; - gf["const bool&"] = (cf_t)+[](dims_t) { static ConstBoolRefConverter c{}; return &c; }; - gf["bool&"] = (cf_t)+[](dims_t) { static BoolRefConverter c{}; return &c; }; - gf["char"] = (cf_t)+[](dims_t) { static CharConverter c{}; return &c; }; - gf["const char&"] = (cf_t)+[](dims_t) { static ConstCharRefConverter c{}; return &c; }; - gf["char&"] = (cf_t)+[](dims_t) { static CharRefConverter c{}; return &c; }; - gf["signed char&"] = (cf_t)+[](dims_t) { static SCharRefConverter c{}; return &c; }; - gf["unsigned char"] = (cf_t)+[](dims_t) { static UCharConverter c{}; return &c; }; - gf["const unsigned char&"] = (cf_t)+[](dims_t) { static ConstUCharRefConverter c{}; return &c; }; - gf["unsigned char&"] = (cf_t)+[](dims_t) { static UCharRefConverter c{}; return &c; }; - gf["UCharAsInt"] = (cf_t)+[](dims_t) { static UCharAsIntConverter c{}; return &c; }; - gf["wchar_t"] = (cf_t)+[](dims_t) { static WCharConverter c{}; return &c; }; - gf["char16_t"] = (cf_t)+[](dims_t) { static Char16Converter c{}; return &c; }; - gf["char32_t"] = (cf_t)+[](dims_t) { static Char32Converter c{}; return &c; }; - gf["wchar_t&"] = (cf_t)+[](dims_t) { static WCharRefConverter c{}; return &c; }; - gf["char16_t&"] = (cf_t)+[](dims_t) { static Char16RefConverter c{}; return &c; }; - gf["char32_t&"] = (cf_t)+[](dims_t) { static Char32RefConverter c{}; return &c; }; - gf["int8_t"] = (cf_t)+[](dims_t) { static Int8Converter c{}; return &c; }; - gf["int8_t&"] = (cf_t)+[](dims_t) { static Int8RefConverter c{}; return &c; }; - gf["const int8_t&"] = (cf_t)+[](dims_t) { static ConstInt8RefConverter c{}; return &c; }; - gf["uint8_t"] = (cf_t)+[](dims_t) { static UInt8Converter c{}; return &c; }; - gf["const uint8_t&"] = (cf_t)+[](dims_t) { static ConstUInt8RefConverter c{}; return &c; }; - gf["uint8_t&"] = (cf_t)+[](dims_t) { static UInt8RefConverter c{}; return &c; }; - gf["short"] = (cf_t)+[](dims_t) { static ShortConverter c{}; return &c; }; - gf["const short&"] = (cf_t)+[](dims_t) { static ConstShortRefConverter c{}; return &c; }; - gf["short&"] = (cf_t)+[](dims_t) { static ShortRefConverter c{}; return &c; }; - gf["unsigned short"] = (cf_t)+[](dims_t) { static UShortConverter c{}; return &c; }; - gf["const unsigned short&"] = (cf_t)+[](dims_t) { static ConstUShortRefConverter c{}; return &c; }; - gf["unsigned short&"] = (cf_t)+[](dims_t) { static UShortRefConverter c{}; return &c; }; - gf["int"] = (cf_t)+[](dims_t) { static IntConverter c{}; return &c; }; - gf["int&"] = (cf_t)+[](dims_t) { static IntRefConverter c{}; return &c; }; - gf["const int&"] = (cf_t)+[](dims_t) { static ConstIntRefConverter c{}; return &c; }; - gf["unsigned int"] = (cf_t)+[](dims_t) { static UIntConverter c{}; return &c; }; - gf["const unsigned int&"] = (cf_t)+[](dims_t) { static ConstUIntRefConverter c{}; return &c; }; - gf["unsigned int&"] = (cf_t)+[](dims_t) { static UIntRefConverter c{}; return &c; }; - gf["long"] = (cf_t)+[](dims_t) { static LongConverter c{}; return &c; }; - gf["long&"] = (cf_t)+[](dims_t) { static LongRefConverter c{}; return &c; }; - gf["const long&"] = (cf_t)+[](dims_t) { static ConstLongRefConverter c{}; return &c; }; - gf["unsigned long"] = (cf_t)+[](dims_t) { static ULongConverter c{}; return &c; }; - gf["const unsigned long&"] = (cf_t)+[](dims_t) { static ConstULongRefConverter c{}; return &c; }; - gf["unsigned long&"] = (cf_t)+[](dims_t) { static ULongRefConverter c{}; return &c; }; - gf["long long"] = (cf_t)+[](dims_t) { static LLongConverter c{}; return &c; }; - gf["const long long&"] = (cf_t)+[](dims_t) { static ConstLLongRefConverter c{}; return &c; }; - gf["long long&"] = (cf_t)+[](dims_t) { static LLongRefConverter c{}; return &c; }; - gf["unsigned long long"] = (cf_t)+[](dims_t) { static ULLongConverter c{}; return &c; }; - gf["const unsigned long long&"] = (cf_t)+[](dims_t) { static ConstULLongRefConverter c{}; return &c; }; - gf["unsigned long long&"] = (cf_t)+[](dims_t) { static ULLongRefConverter c{}; return &c; }; - - gf["float"] = (cf_t)+[](dims_t) { static FloatConverter c{}; return &c; }; - gf["const float&"] = (cf_t)+[](dims_t) { static ConstFloatRefConverter c{}; return &c; }; - gf["float&"] = (cf_t)+[](dims_t) { static FloatRefConverter c{}; return &c; }; - gf["double"] = (cf_t)+[](dims_t) { static DoubleConverter c{}; return &c; }; - gf["double&"] = (cf_t)+[](dims_t) { static DoubleRefConverter c{}; return &c; }; - gf["const double&"] = (cf_t)+[](dims_t) { static ConstDoubleRefConverter c{}; return &c; }; - gf["long double"] = (cf_t)+[](dims_t) { static LDoubleConverter c{}; return &c; }; - gf["const long double&"] = (cf_t)+[](dims_t) { static ConstLDoubleRefConverter c{}; return &c; }; - gf["long double&"] = (cf_t)+[](dims_t) { static LDoubleRefConverter c{}; return &c; }; - gf["std::complex"] = (cf_t)+[](dims_t) { return new ComplexDConverter{}; }; - gf["complex"] = (cf_t)+[](dims_t) { return new ComplexDConverter{}; }; - gf["const std::complex&"] = (cf_t)+[](dims_t) { return new ComplexDConverter{}; }; - gf["const complex&"] = (cf_t)+[](dims_t) { return new ComplexDConverter{}; }; - gf["void"] = (cf_t)+[](dims_t) { static VoidConverter c{}; return &c; }; + gf["bool"] = (cf_t)+[](cdims_t) { static BoolConverter c{}; return &c; }; + gf["const bool&"] = (cf_t)+[](cdims_t) { static ConstBoolRefConverter c{}; return &c; }; + gf["bool&"] = (cf_t)+[](cdims_t) { static BoolRefConverter c{}; return &c; }; + gf["char"] = (cf_t)+[](cdims_t) { static CharConverter c{}; return &c; }; + gf["const char&"] = (cf_t)+[](cdims_t) { static ConstCharRefConverter c{}; return &c; }; + gf["char&"] = (cf_t)+[](cdims_t) { static CharRefConverter c{}; return &c; }; + gf["signed char&"] = (cf_t)+[](cdims_t) { static SCharRefConverter c{}; return &c; }; + gf["unsigned char"] = (cf_t)+[](cdims_t) { static UCharConverter c{}; return &c; }; + gf["const unsigned char&"] = (cf_t)+[](cdims_t) { static ConstUCharRefConverter c{}; return &c; }; + gf["unsigned char&"] = (cf_t)+[](cdims_t) { static UCharRefConverter c{}; return &c; }; + gf["UCharAsInt"] = (cf_t)+[](cdims_t) { static UCharAsIntConverter c{}; return &c; }; + gf["wchar_t"] = (cf_t)+[](cdims_t) { static WCharConverter c{}; return &c; }; + gf["char16_t"] = (cf_t)+[](cdims_t) { static Char16Converter c{}; return &c; }; + gf["char32_t"] = (cf_t)+[](cdims_t) { static Char32Converter c{}; return &c; }; + gf["wchar_t&"] = (cf_t)+[](cdims_t) { static WCharRefConverter c{}; return &c; }; + gf["char16_t&"] = (cf_t)+[](cdims_t) { static Char16RefConverter c{}; return &c; }; + gf["char32_t&"] = (cf_t)+[](cdims_t) { static Char32RefConverter c{}; return &c; }; + gf["int8_t"] = (cf_t)+[](cdims_t) { static Int8Converter c{}; return &c; }; + gf["const int8_t&"] = (cf_t)+[](cdims_t) { static ConstInt8RefConverter c{}; return &c; }; + gf["int8_t&"] = (cf_t)+[](cdims_t) { static Int8RefConverter c{}; return &c; }; + gf["uint8_t"] = (cf_t)+[](cdims_t) { static UInt8Converter c{}; return &c; }; + gf["const uint8_t&"] = (cf_t)+[](cdims_t) { static ConstUInt8RefConverter c{}; return &c; }; + gf["uint8_t&"] = (cf_t)+[](cdims_t) { static UInt8RefConverter c{}; return &c; }; + gf["short"] = (cf_t)+[](cdims_t) { static ShortConverter c{}; return &c; }; + gf["const short&"] = (cf_t)+[](cdims_t) { static ConstShortRefConverter c{}; return &c; }; + gf["short&"] = (cf_t)+[](cdims_t) { static ShortRefConverter c{}; return &c; }; + gf["unsigned short"] = (cf_t)+[](cdims_t) { static UShortConverter c{}; return &c; }; + gf["const unsigned short&"] = (cf_t)+[](cdims_t) { static ConstUShortRefConverter c{}; return &c; }; + gf["unsigned short&"] = (cf_t)+[](cdims_t) { static UShortRefConverter c{}; return &c; }; + gf["int"] = (cf_t)+[](cdims_t) { static IntConverter c{}; return &c; }; + gf["int&"] = (cf_t)+[](cdims_t) { static IntRefConverter c{}; return &c; }; + gf["const int&"] = (cf_t)+[](cdims_t) { static ConstIntRefConverter c{}; return &c; }; + gf["unsigned int"] = (cf_t)+[](cdims_t) { static UIntConverter c{}; return &c; }; + gf["const unsigned int&"] = (cf_t)+[](cdims_t) { static ConstUIntRefConverter c{}; return &c; }; + gf["unsigned int&"] = (cf_t)+[](cdims_t) { static UIntRefConverter c{}; return &c; }; + gf["long"] = (cf_t)+[](cdims_t) { static LongConverter c{}; return &c; }; + gf["long&"] = (cf_t)+[](cdims_t) { static LongRefConverter c{}; return &c; }; + gf["const long&"] = (cf_t)+[](cdims_t) { static ConstLongRefConverter c{}; return &c; }; + gf["unsigned long"] = (cf_t)+[](cdims_t) { static ULongConverter c{}; return &c; }; + gf["const unsigned long&"] = (cf_t)+[](cdims_t) { static ConstULongRefConverter c{}; return &c; }; + gf["unsigned long&"] = (cf_t)+[](cdims_t) { static ULongRefConverter c{}; return &c; }; + gf["long long"] = (cf_t)+[](cdims_t) { static LLongConverter c{}; return &c; }; + gf["const long long&"] = (cf_t)+[](cdims_t) { static ConstLLongRefConverter c{}; return &c; }; + gf["long long&"] = (cf_t)+[](cdims_t) { static LLongRefConverter c{}; return &c; }; + gf["unsigned long long"] = (cf_t)+[](cdims_t) { static ULLongConverter c{}; return &c; }; + gf["const unsigned long long&"] = (cf_t)+[](cdims_t) { static ConstULLongRefConverter c{}; return &c; }; + gf["unsigned long long&"] = (cf_t)+[](cdims_t) { static ULLongRefConverter c{}; return &c; }; + + gf["float"] = (cf_t)+[](cdims_t) { static FloatConverter c{}; return &c; }; + gf["const float&"] = (cf_t)+[](cdims_t) { static ConstFloatRefConverter c{}; return &c; }; + gf["float&"] = (cf_t)+[](cdims_t) { static FloatRefConverter c{}; return &c; }; + gf["double"] = (cf_t)+[](cdims_t) { static DoubleConverter c{}; return &c; }; + gf["double&"] = (cf_t)+[](cdims_t) { static DoubleRefConverter c{}; return &c; }; + gf["const double&"] = (cf_t)+[](cdims_t) { static ConstDoubleRefConverter c{}; return &c; }; + gf["long double"] = (cf_t)+[](cdims_t) { static LDoubleConverter c{}; return &c; }; + gf["const long double&"] = (cf_t)+[](cdims_t) { static ConstLDoubleRefConverter c{}; return &c; }; + gf["long double&"] = (cf_t)+[](cdims_t) { static LDoubleRefConverter c{}; return &c; }; + gf["std::complex"] = (cf_t)+[](cdims_t) { return new ComplexDConverter{}; }; + gf["const std::complex&"] = (cf_t)+[](cdims_t) { return new ComplexDConverter{}; }; + gf["void"] = (cf_t)+[](cdims_t) { static VoidConverter c{}; return &c; }; // pointer/array factories - gf["bool*"] = (cf_t)+[](dims_t d) { return new BoolArrayConverter{d}; }; - gf["bool**"] = (cf_t)+[](dims_t d) { return new BoolArrayPtrConverter{d}; }; - gf["const signed char[]"] = (cf_t)+[](dims_t d) { return new SCharArrayConverter{d}; }; - gf["signed char[]"] = (cf_t)+[](dims_t d) { return new SCharArrayConverter{d}; }; - gf["signed char**"] = (cf_t)+[](dims_t d) { return new SCharArrayPtrConverter{d}; }; - gf["const unsigned char*"] = (cf_t)+[](dims_t d) { return new UCharArrayConverter{d}; }; - gf["unsigned char*"] = (cf_t)+[](dims_t d) { return new UCharArrayConverter{d}; }; - gf["UCharAsInt*"] = (cf_t)+[](dims_t d) { return new UCharArrayConverter{d}; }; - gf["unsigned char**"] = (cf_t)+[](dims_t d) { return new UCharArrayPtrConverter{d}; }; + gf["bool ptr"] = (cf_t)+[](cdims_t d) { return new BoolArrayConverter{d}; }; + gf["const signed char[]"] = (cf_t)+[](cdims_t d) { return new SCharArrayConverter{d}; }; + gf["signed char[]"] = gf["const signed char[]"]; + gf["signed char**"] = (cf_t)+[](cdims_t) { return new SCharArrayConverter{{UNKNOWN_SIZE, UNKNOWN_SIZE}}; }; + gf["const unsigned char*"] = (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; }; + gf["unsigned char ptr"] = (cf_t)+[](cdims_t d) { return new UCharArrayConverter{d}; }; + gf["UCharAsInt*"] = gf["unsigned char ptr"]; + gf["UCharAsInt[]"] = gf["unsigned char ptr"]; #if __cplusplus > 201402L - gf["byte*"] = (cf_t)+[](dims_t d) { return new ByteArrayConverter{d}; }; - gf["byte**"] = (cf_t)+[](dims_t d) { return new ByteArrayPtrConverter{d}; }; + gf["std::byte ptr"] = (cf_t)+[](cdims_t d) { return new ByteArrayConverter{d}; }; #endif - gf["short*"] = (cf_t)+[](dims_t d) { return new ShortArrayConverter{d}; }; - gf["short**"] = (cf_t)+[](dims_t d) { return new ShortArrayPtrConverter{d}; }; - gf["unsigned short*"] = (cf_t)+[](dims_t d) { return new UShortArrayConverter{d}; }; - gf["unsigned short**"] = (cf_t)+[](dims_t d) { return new UShortArrayPtrConverter{d}; }; - gf["int*"] = (cf_t)+[](dims_t d) { return new IntArrayConverter{d}; }; - gf["int**"] = (cf_t)+[](dims_t d) { return new IntArrayPtrConverter{d}; }; - gf["unsigned int*"] = (cf_t)+[](dims_t d) { return new UIntArrayConverter{d}; }; - gf["unsigned int**"] = (cf_t)+[](dims_t d) { return new UIntArrayPtrConverter{d}; }; - gf["long*"] = (cf_t)+[](dims_t d) { return new LongArrayConverter{d}; }; - gf["long**"] = (cf_t)+[](dims_t d) { return new LongArrayPtrConverter{d}; }; - gf["unsigned long*"] = (cf_t)+[](dims_t d) { return new ULongArrayConverter{d}; }; - gf["unsigned long**"] = (cf_t)+[](dims_t d) { return new ULongArrayPtrConverter{d}; }; - gf["long long*"] = (cf_t)+[](dims_t d) { return new LLongArrayConverter{d}; }; - gf["long long**"] = (cf_t)+[](dims_t d) { return new LLongArrayPtrConverter{d}; }; - gf["unsigned long long*"] = (cf_t)+[](dims_t d) { return new ULLongArrayConverter{d}; }; - gf["unsigned long long**"] = (cf_t)+[](dims_t d) { return new ULLongArrayPtrConverter{d}; }; - gf["float*"] = (cf_t)+[](dims_t d) { return new FloatArrayConverter{d}; }; - gf["float**"] = (cf_t)+[](dims_t d) { return new FloatArrayPtrConverter{d}; }; - gf["double*"] = (cf_t)+[](dims_t d) { return new DoubleArrayConverter{d}; }; - gf["double**"] = (cf_t)+[](dims_t d) { return new DoubleArrayPtrConverter{d}; }; - gf["long double*"] = (cf_t)+[](dims_t d) { return new LDoubleArrayConverter{d}; }; - gf["long double**"] = (cf_t)+[](dims_t d) { return new LDoubleArrayPtrConverter{d}; }; - gf["std::complex*"] = (cf_t)+[](dims_t d) { return new ComplexDArrayConverter{d}; }; - gf["complex*"] = (cf_t)+[](dims_t d) { return new ComplexDArrayConverter{d}; }; - gf["std::complex**"] = (cf_t)+[](dims_t d) { return new ComplexDArrayPtrConverter{d}; }; - gf["void*"] = (cf_t)+[](dims_t d) { return new VoidArrayConverter{(bool)d}; }; + gf["int8_t ptr"] = (cf_t)+[](cdims_t d) { return new Int8ArrayConverter{d}; }; + gf["uint8_t ptr"] = (cf_t)+[](cdims_t d) { return new UInt8ArrayConverter{d}; }; + gf["short ptr"] = (cf_t)+[](cdims_t d) { return new ShortArrayConverter{d}; }; + gf["unsigned short ptr"] = (cf_t)+[](cdims_t d) { return new UShortArrayConverter{d}; }; + gf["int ptr"] = (cf_t)+[](cdims_t d) { return new IntArrayConverter{d}; }; + gf["unsigned int ptr"] = (cf_t)+[](cdims_t d) { return new UIntArrayConverter{d}; }; + gf["long ptr"] = (cf_t)+[](cdims_t d) { return new LongArrayConverter{d}; }; + gf["unsigned long ptr"] = (cf_t)+[](cdims_t d) { return new ULongArrayConverter{d}; }; + gf["long long ptr"] = (cf_t)+[](cdims_t d) { return new LLongArrayConverter{d}; }; + gf["unsigned long long ptr"] = (cf_t)+[](cdims_t d) { return new ULLongArrayConverter{d}; }; + gf["float ptr"] = (cf_t)+[](cdims_t d) { return new FloatArrayConverter{d}; }; + gf["double ptr"] = (cf_t)+[](cdims_t d) { return new DoubleArrayConverter{d}; }; + gf["long double ptr"] = (cf_t)+[](cdims_t d) { return new LDoubleArrayConverter{d}; }; + gf["std::complex ptr"] = (cf_t)+[](cdims_t d) { return new ComplexFArrayConverter{d}; }; + gf["std::complex ptr"] = (cf_t)+[](cdims_t d) { return new ComplexDArrayConverter{d}; }; + gf["void*"] = (cf_t)+[](cdims_t d) { return new VoidArrayConverter{(bool)d}; }; // aliases gf["signed char"] = gf["char"]; gf["const signed char&"] = gf["const char&"]; #if __cplusplus > 201402L - gf["byte"] = gf["uint8_t"]; - gf["const byte&"] = gf["const uint8_t&"]; - gf["byte&"] = gf["uint8&"]; + gf["std::byte"] = gf["uint8_t"]; + gf["const std::byte&"] = gf["const uint8_t&"]; + gf["std::byte&"] = gf["uint8_t&"]; #endif + gf["std::int8_t"] = gf["int8_t"]; + gf["const std::int8_t&"] = gf["const int8_t&"]; + gf["std::int8_t&"] = gf["int8_t&"]; + gf["std::uint8_t"] = gf["uint8_t"]; + gf["const std::uint8_t&"] = gf["const uint8_t&"]; + gf["std::uint8_t&"] = gf["uint8_t&"]; gf["internal_enum_type_t"] = gf["int"]; gf["internal_enum_type_t&"] = gf["int&"]; gf["const internal_enum_type_t&"] = gf["const int&"]; + gf["internal_enum_type_t ptr"] = gf["int ptr"]; +#ifdef _WIN32 + gf["__int64"] = gf["long long"]; + gf["const __int64&"] = gf["const long long&"]; + gf["__int64&"] = gf["long long&"]; + gf["__int64 ptr"] = gf["long long ptr"]; + gf["unsigned __int64"] = gf["unsigned long long"]; + gf["const unsigned __int64&"] = gf["const unsigned long long&"]; + gf["unsigned __int64&"] = gf["unsigned long long&"]; + gf["unsigned __int64 ptr"] = gf["unsigned long long ptr"]; +#endif + gf[CCOMPLEX_D] = gf["std::complex"]; + gf["const " CCOMPLEX_D "&"] = gf["const std::complex&"]; + gf[CCOMPLEX_F " ptr"] = gf["std::complex ptr"]; + gf[CCOMPLEX_D " ptr"] = gf["std::complex ptr"]; gf["Long64_t"] = gf["long long"]; - gf["Long64_t*"] = gf["long long*"]; + gf["Long64_t ptr"] = gf["long long ptr"]; gf["Long64_t&"] = gf["long long&"]; gf["const Long64_t&"] = gf["const long long&"]; gf["ULong64_t"] = gf["unsigned long long"]; - gf["ULong64_t*"] = gf["unsigned long long*"]; + gf["ULong64_t ptr"] = gf["unsigned long long ptr"]; gf["ULong64_t&"] = gf["unsigned long long&"]; gf["const ULong64_t&"] = gf["const unsigned long long&"]; gf["Float16_t"] = gf["float"]; @@ -3037,50 +3459,55 @@ static struct InitConvFactories_t { gf["const Double32_t&"] = gf["const double&"]; // factories for special cases - gf["TString"] = (cf_t)+[](dims_t) { return new TStringConverter{}; }; + gf["TString"] = (cf_t)+[](cdims_t) { return new TStringConverter{}; }; gf["TString&"] = gf["TString"]; gf["const TString&"] = gf["TString"]; - gf["nullptr_t"] = (cf_t)+[](dims_t) { static NullptrConverter c{}; return &c;}; - gf["const char*"] = (cf_t)+[](dims_t) { return new CStringConverter{}; }; + gf["nullptr_t"] = (cf_t)+[](cdims_t) { static NullptrConverter c{}; return &c;}; + gf["const char*"] = (cf_t)+[](cdims_t) { return new CStringConverter{}; }; gf["const signed char*"] = gf["const char*"]; - gf["const char[]"] = (cf_t)+[](dims_t) { return new CStringConverter{}; }; - gf["char*"] = (cf_t)+[](dims_t) { return new NonConstCStringConverter{}; }; + gf["const char*&&"] = gf["const char*"]; + gf["const char[]"] = (cf_t)+[](cdims_t) { return new CStringConverter{}; }; + gf["char*"] = (cf_t)+[](cdims_t d) { return new NonConstCStringConverter{dims2stringsz(d)}; }; + gf["char[]"] = (cf_t)+[](cdims_t d) { return new NonConstCStringArrayConverter{d, true}; }; gf["signed char*"] = gf["char*"]; - gf["wchar_t*"] = (cf_t)+[](dims_t) { return new WCStringConverter{}; }; - gf["char16_t*"] = (cf_t)+[](dims_t) { return new CString16Converter{}; }; - gf["char32_t*"] = (cf_t)+[](dims_t) { return new CString32Converter{}; }; + gf["wchar_t*"] = (cf_t)+[](cdims_t) { return new WCStringConverter{}; }; + gf["char16_t*"] = (cf_t)+[](cdims_t) { return new CString16Converter{}; }; + gf["char16_t[]"] = (cf_t)+[](cdims_t d) { return new CString16Converter{dims2stringsz(d)}; }; + gf["char32_t*"] = (cf_t)+[](cdims_t) { return new CString32Converter{}; }; + gf["char32_t[]"] = (cf_t)+[](cdims_t d) { return new CString32Converter{dims2stringsz(d)}; }; // TODO: the following are handled incorrectly upstream (char16_t** where char16_t* intended)?! gf["char16_t**"] = gf["char16_t*"]; gf["char32_t**"] = gf["char32_t*"]; - gf["const char**"] = (cf_t)+[](dims_t d) { return new CStringArrayConverter{d}; }; + gf["const char**"] = (cf_t)+[](cdims_t) { return new CStringArrayConverter{{UNKNOWN_SIZE, UNKNOWN_SIZE}, false}; }; gf["char**"] = gf["const char**"]; - gf["const char*[]"] = gf["const char**"]; - gf["char*[]"] = gf["const char*[]"]; - gf["std::string"] = (cf_t)+[](dims_t) { return new STLStringConverter{}; }; - gf["string"] = gf["std::string"]; + gf["const char*[]"] = (cf_t)+[](cdims_t d) { return new CStringArrayConverter{d, false}; }; + gf["char*[]"] = (cf_t)+[](cdims_t d) { return new NonConstCStringArrayConverter{d, false}; }; + gf["char ptr"] = gf["char*[]"]; + gf["std::string"] = (cf_t)+[](cdims_t) { return new STLStringConverter{}; }; gf["const std::string&"] = gf["std::string"]; + gf["string"] = gf["std::string"]; gf["const string&"] = gf["std::string"]; - gf["string&&"] = (cf_t)+[](dims_t) { return new STLStringMoveConverter{}; }; - gf["std::string&&"] = gf["string&&"]; - gf["std::string_view"] = (cf_t)+[](dims_t) { return new STLStringViewConverter{}; }; - gf["string_view"] = gf["std::string_view"]; + gf["std::string&&"] = (cf_t)+[](cdims_t) { return new STLStringMoveConverter{}; }; + gf["string&&"] = gf["std::string&&"]; +#if __cplusplus > 201402L + gf["std::string_view"] = (cf_t)+[](cdims_t) { return new STLStringViewConverter{}; }; gf[STRINGVIEW] = gf["std::string_view"]; - gf["experimental::" STRINGVIEW] = gf["std::string_view"]; gf["std::string_view&"] = gf["std::string_view"]; - gf["const string_view&"] = gf["std::string_view"]; + gf["const std::string_view&"] = gf["std::string_view"]; gf["const " STRINGVIEW "&"] = gf["std::string_view"]; - gf["std::wstring"] = (cf_t)+[](dims_t) { return new STLWStringConverter{}; }; - gf[WSTRING] = gf["std::wstring"]; - gf["std::" WSTRING] = gf["std::wstring"]; +#endif + gf["std::wstring"] = (cf_t)+[](cdims_t) { return new STLWStringConverter{}; }; + gf[WSTRING1] = gf["std::wstring"]; + gf[WSTRING2] = gf["std::wstring"]; gf["const std::wstring&"] = gf["std::wstring"]; - gf["const std::" WSTRING "&"] = gf["std::wstring"]; - gf["const " WSTRING "&"] = gf["std::wstring"]; - gf["void*&"] = (cf_t)+[](dims_t) { static VoidPtrRefConverter c{}; return &c; }; - gf["void**"] = (cf_t)+[](dims_t d) { return new VoidPtrPtrConverter{size_t((d && d[0] != -1) ? d[1] : -1)}; }; - gf["void*[]"] = (cf_t)+[](dims_t d) { return new VoidPtrPtrConverter{size_t((d && d[0] != -1) ? d[1] : -1)}; }; - gf["PyObject*"] = (cf_t)+[](dims_t) { static PyObjectConverter c{}; return &c; }; + gf["const " WSTRING1 "&"] = gf["std::wstring"]; + gf["const " WSTRING2 "&"] = gf["std::wstring"]; + gf["void*&"] = (cf_t)+[](cdims_t) { static VoidPtrRefConverter c{}; return &c; }; + gf["void**"] = (cf_t)+[](cdims_t d) { return new VoidPtrPtrConverter{d}; }; + gf["void ptr"] = gf["void**"]; + gf["PyObject*"] = (cf_t)+[](cdims_t) { static PyObjectConverter c{}; return &c; }; gf["_object*"] = gf["PyObject*"]; - gf["FILE*"] = (cf_t)+[](dims_t) { return new VoidArrayConverter{}; }; + gf["FILE*"] = (cf_t)+[](cdims_t) { return new VoidArrayConverter{}; }; } } initConvFactories_; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.h b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.h index 39d5cebe7f602..80b1d0c395822 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.h @@ -1,6 +1,9 @@ #ifndef CPYCPPYY_CONVERTERS_H #define CPYCPPYY_CONVERTERS_H +// Bindings +#include "Dimensions.h" + // Standard #include @@ -14,6 +17,13 @@ class CPYCPPYY_CLASS_EXPORT Converter { public: virtual ~Converter(); + Converter() = default; + + Converter(Converter const& other) = delete; + Converter(Converter && other) = delete; + Converter& operator=(Converter const& other) = delete; + Converter& operator=(Converter && other) = delete; + public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr) = 0; virtual PyObject* FromMemory(void* address); @@ -22,9 +32,9 @@ class CPYCPPYY_CLASS_EXPORT Converter { }; // create/destroy converter from fully qualified type (public API) -CPYCPPYY_EXPORT Converter* CreateConverter(const std::string& fullType, dims_t dims = nullptr); +CPYCPPYY_EXPORT Converter* CreateConverter(const std::string& fullType, cdims_t dims = 0); CPYCPPYY_EXPORT void DestroyConverter(Converter* p); -typedef Converter* (*cf_t)(dims_t d); +typedef Converter* (*cf_t)(cdims_t d); CPYCPPYY_EXPORT bool RegisterConverter(const std::string& name, cf_t fac); CPYCPPYY_EXPORT bool UnregisterConverter(const std::string& name); @@ -48,6 +58,7 @@ class VoidArrayConverter : public Converter { bool fKeepControl; }; +template class InstancePtrConverter : public VoidArrayConverter { public: InstancePtrConverter(Cppyy::TCppType_t klass, bool keepControl = false) : @@ -62,9 +73,9 @@ class InstancePtrConverter : public VoidArrayConverter { Cppyy::TCppType_t fClass; }; -class StrictInstancePtrConverter : public InstancePtrConverter { +class StrictInstancePtrConverter : public InstancePtrConverter { public: - using InstancePtrConverter::InstancePtrConverter; + using InstancePtrConverter::InstancePtrConverter; protected: virtual bool GetAddressSpecialCase(PyObject*, void*&) { return false; } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h b/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h index dd8a3b9c6b51e..fcaffa14f7474 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h @@ -15,6 +15,27 @@ #define CPPYY_IMPORT extern #endif +// some more types; assumes Cppyy.h follows Python.h +#ifndef PY_LONG_LONG +#ifdef _WIN32 +typedef __int64 PY_LONG_LONG; +#else +typedef long long PY_LONG_LONG; +#endif +#endif + +#ifndef PY_ULONG_LONG +#ifdef _WIN32 +typedef unsigned __int64 PY_ULONG_LONG; +#else +typedef unsigned long long PY_ULONG_LONG; +#endif +#endif + +#ifndef PY_LONG_DOUBLE +typedef long double PY_LONG_DOUBLE; +#endif + namespace Cppyy { @@ -29,7 +50,9 @@ namespace Cppyy { // direct interpreter access ------------------------------------------------- CPPYY_IMPORT - bool Compile(const std::string& code); + bool Compile(const std::string& code, bool silent = false); + CPPYY_IMPORT + std::string ToString(TCppType_t klass, TCppObject_t obj); // name to opaque C++ scope representation ----------------------------------- CPPYY_IMPORT @@ -59,7 +82,7 @@ namespace Cppyy { CPPYY_IMPORT void Deallocate(TCppType_t type, TCppObject_t instance); CPPYY_IMPORT - TCppObject_t Construct(TCppType_t type); + TCppObject_t Construct(TCppType_t type, void* arena = nullptr); CPPYY_IMPORT void Destruct(TCppType_t type, TCppObject_t instance); @@ -77,13 +100,13 @@ namespace Cppyy { CPPYY_IMPORT long CallL(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); CPPYY_IMPORT - Long64_t CallLL(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); + PY_LONG_LONG CallLL(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); CPPYY_IMPORT float CallF(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); CPPYY_IMPORT double CallD(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); CPPYY_IMPORT - LongDouble_t CallLD(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); + PY_LONG_DOUBLE CallLD(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); CPPYY_IMPORT void* CallR(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); CPPYY_IMPORT @@ -117,6 +140,10 @@ namespace Cppyy { bool IsAbstract(TCppType_t type); CPPYY_IMPORT bool IsEnum(const std::string& type_name); + CPPYY_IMPORT + bool IsAggregate(TCppType_t type); + CPPYY_IMPORT + bool IsDefaultConstructable(TCppType_t type); CPPYY_IMPORT void GetAllCppNames(TCppScope_t scope, std::set& cppnames); @@ -149,6 +176,9 @@ namespace Cppyy { CPPYY_IMPORT void AddSmartPtrType(const std::string&); + CPPYY_IMPORT + void AddTypeReducer(const std::string& reducable, const std::string& reduced); + // calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 CPPYY_IMPORT ptrdiff_t GetBaseOffset( @@ -156,7 +186,7 @@ namespace Cppyy { // method/function reflection information ------------------------------------ CPPYY_IMPORT - TCppIndex_t GetNumMethods(TCppScope_t scope); + TCppIndex_t GetNumMethods(TCppScope_t scope, bool accept_namespace = false); CPPYY_IMPORT std::vector GetMethodIndicesFromName(TCppScope_t scope, const std::string& name); @@ -191,7 +221,7 @@ namespace Cppyy { bool IsConstMethod(TCppMethod_t); CPPYY_IMPORT - TCppIndex_t GetNumTemplatedMethods(TCppScope_t scope); + TCppIndex_t GetNumTemplatedMethods(TCppScope_t scope, bool accept_namespace = false); CPPYY_IMPORT std::string GetTemplatedMethodName(TCppScope_t scope, TCppIndex_t imeth); CPPYY_IMPORT @@ -222,7 +252,7 @@ namespace Cppyy { // data member reflection information ---------------------------------------- CPPYY_IMPORT - TCppIndex_t GetNumDatamembers(TCppScope_t scope); + TCppIndex_t GetNumDatamembers(TCppScope_t scope, bool accept_namespace = false); CPPYY_IMPORT std::string GetDatamemberName(TCppScope_t scope, TCppIndex_t idata); CPPYY_IMPORT diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.cxx index 70b8330704cc7..69621b5c268a4 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.cxx @@ -20,9 +20,9 @@ #define CustomInstanceMethod_GET_CLASS(meth) PyMethod_GET_CLASS(meth) #endif - namespace CPyCppyy { +#if PY_VERSION_HEX < 0x03000000 //= float type allowed for reference passing ================================= PyTypeObject RefFloat_Type = { // python float is a C/C++ double PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -70,25 +70,91 @@ PyTypeObject RefInt_Type = { // python int is a C/C++ long , 0 // tp_finalize #endif }; +#endif //- custom type representing typedef to pointer of class --------------------- -static PyObject* tpc_call(typedefpointertoclassobject* self, PyObject* args, PyObject* /* kwds */) +static PyObject* tptc_call(typedefpointertoclassobject* self, PyObject* args, PyObject* /* kwds */) { long long addr = 0; if (!PyArg_ParseTuple(args, const_cast("|L"), &addr)) return nullptr; - return BindCppObjectNoCast((Cppyy::TCppObject_t)(intptr_t)addr, self->fType); + return BindCppObjectNoCast((Cppyy::TCppObject_t)(intptr_t)addr, self->fCppType); +} + +//----------------------------------------------------------------------------- +static PyObject* tptc_getcppname(typedefpointertoclassobject* self, void*) +{ + return CPyCppyy_PyText_FromString( + (Cppyy::GetScopedFinalName(self->fCppType)+"*").c_str()); } +//----------------------------------------------------------------------------- +static PyObject* tptc_name(typedefpointertoclassobject* self, void*) +{ + PyObject* pyclass = CPyCppyy::GetScopeProxy(self->fCppType); + if (pyclass) { + PyObject* pyname = PyObject_GetAttr(pyclass, PyStrings::gName); + Py_DECREF(pyclass); + return pyname; + } + + return CPyCppyy_PyText_FromString("*"); +} + +//----------------------------------------------------------------------------- +static PyGetSetDef tptc_getset[] = { + {(char*)"__name__", (getter)tptc_name, nullptr, nullptr, nullptr}, + {(char*)"__cpp_name__", (getter)tptc_getcppname, nullptr, nullptr, nullptr}, + {(char*)nullptr, nullptr, nullptr, nullptr, nullptr} +}; + + PyTypeObject TypedefPointerToClass_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) (char*)"cppyy.TypedefPointerToClass",// tp_name sizeof(typedefpointertoclassobject), // tp_basicsize - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - (ternaryfunc)tpc_call, // tp_call - 0, 0, 0, 0, - Py_TPFLAGS_DEFAULT, // tp_flags - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_vectorcall_offset + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_as_async + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + (ternaryfunc)tptc_call, // tp_call + 0, // tp_str + PyObject_GenericGetAttr, // tp_getattro + PyObject_GenericSetAttr, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + 0, // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + tptc_getset, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + offsetof(typedefpointertoclassobject, fDict), // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + 0, // tp_free + 0, // tp_is_gc + 0, // tp_bases + 0, // tp_mro + 0, // tp_cache + 0, // tp_subclasses + 0 // tp_weaklist #if PY_VERSION_HEX >= 0x02030000 , 0 // tp_del #endif @@ -98,6 +164,9 @@ PyTypeObject TypedefPointerToClass_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; //= instancemethod object with a more efficient call function ================ @@ -219,7 +288,6 @@ static PyObject* im_call(PyObject* meth, PyObject* args, PyObject* kw) //----------------------------------------------------------------------------- static PyObject* im_descr_get(PyObject* meth, PyObject* obj, PyObject* pyclass) { - // from instancemethod: don't rebind an already bound method, or an unbound method // of a class that's not a base class of pyclass if (CustomInstanceMethod_GET_SELF(meth) @@ -264,11 +332,15 @@ PyTypeObject CustomInstanceMethod_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; //= CPyCppyy custom iterator for performance ================================= static void indexiter_dealloc(indexiterobject* ii) { + PyObject_GC_UnTrack(ii); Py_XDECREF(ii->ii_container); PyObject_GC_Del(ii); } @@ -283,7 +355,7 @@ static PyObject* indexiter_iternext(indexiterobject* ii) { return nullptr; PyObject* pyindex = PyLong_FromSsize_t(ii->ii_pos); - PyObject* result = PyObject_CallMethodObjArgs((PyObject*)ii->ii_container, PyStrings::gGetItem, pyindex, nullptr); + PyObject* result = PyObject_CallMethodOneArg((PyObject*)ii->ii_container, PyStrings::gGetItem, pyindex); Py_DECREF(pyindex); ii->ii_pos += 1; @@ -314,6 +386,9 @@ PyTypeObject IndexIter_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; @@ -337,11 +412,11 @@ static PyObject* vectoriter_iternext(vectoriterobject* vi) { // (or at least not during the loop anyway). This gains 2x in performance. Cppyy::TCppObject_t cppobj = (Cppyy::TCppObject_t)((ptrdiff_t)vi->vi_data + vi->vi_stride * vi->ii_pos); result = CPyCppyy::BindCppObjectNoCast(cppobj, vi->vi_klass, CPyCppyy::CPPInstance::kNoMemReg); - if (vi->vi_flags && CPPInstance_Check(result)) + if ((vi->vi_flags & vectoriterobject::kNeedLifeLine) && result) PyObject_SetAttr(result, PyStrings::gLifeLine, vi->ii_container); } else { PyObject* pyindex = PyLong_FromSsize_t(vi->ii_pos); - result = PyObject_CallMethodObjArgs((PyObject*)vi->ii_container, PyStrings::gGetNoCheck, pyindex, nullptr); + result = PyObject_CallMethodOneArg((PyObject*)vi->ii_container, PyStrings::gGetNoCheck, pyindex); Py_DECREF(pyindex); } @@ -373,6 +448,9 @@ PyTypeObject VectorIter_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.h b/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.h index ec79669e2a267..a67e882d43877 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CustomPyTypes.h @@ -7,6 +7,7 @@ namespace CPyCppyy { performance. */ +#if PY_VERSION_HEX < 0x03000000 //- reference float object type and type verification ------------------------ extern PyTypeObject RefFloat_Type; @@ -36,11 +37,17 @@ inline bool RefInt_CheckExact(T* object) { return object && Py_TYPE(object) == &RefInt_Type; } +#endif //- custom type representing typedef to pointer of class --------------------- struct typedefpointertoclassobject { PyObject_HEAD - Cppyy::TCppType_t fType; + Cppyy::TCppType_t fCppType; + PyObject* fDict; + + ~typedefpointertoclassobject() { + Py_DECREF(fDict); + } }; extern PyTypeObject TypedefPointerToClass_Type; @@ -91,6 +98,11 @@ struct vectoriterobject : public indexiterobject { CPyCppyy::Converter* vi_converter; Cppyy::TCppType_t vi_klass; int vi_flags; + + enum EFlags { + kDefault = 0x0000, + kNeedLifeLine = 0x0001, + }; }; extern PyTypeObject VectorIter_Type; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h index 34e71c76f2c9d..24f3633614c5e 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h @@ -3,16 +3,16 @@ // Bindings #include "Converters.h" +#include "Dimensions.h" // Standard #include #include // ROOT -#include +#include "ROOT/RStringView.hxx" #include "TString.h" - namespace CPyCppyy { namespace { @@ -55,23 +55,16 @@ public: \ #define CPPYY_DECLARE_ARRAY_CONVERTER(name) \ class name##ArrayConverter : public Converter { \ public: \ - name##ArrayConverter(dims_t shape, bool init = true); \ + name##ArrayConverter(cdims_t dims); \ name##ArrayConverter(const name##ArrayConverter&) = delete; \ name##ArrayConverter& operator=(const name##ArrayConverter&) = delete; \ - virtual ~name##ArrayConverter() { delete [] fShape; } \ virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); \ virtual PyObject* FromMemory(void*); \ virtual bool ToMemory(PyObject*, void*, PyObject* = nullptr); \ virtual bool HasState() { return true; } \ protected: \ - Py_ssize_t* fShape; \ + dims_t fShape; \ bool fIsFixed; \ -}; \ - \ -class name##ArrayPtrConverter : public name##ArrayConverter { \ -public: \ - using name##ArrayConverter::name##ArrayConverter; \ - virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); \ }; @@ -129,7 +122,7 @@ class VoidConverter : public Converter { class CStringConverter : public Converter { public: - CStringConverter(long maxSize = -1) : fMaxSize(maxSize) {} + CStringConverter(std::string::size_type maxSize = std::string::npos) : fMaxSize(maxSize) {} public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); @@ -139,12 +132,12 @@ class CStringConverter : public Converter { protected: std::string fBuffer; - long fMaxSize; + std::string::size_type fMaxSize; }; class NonConstCStringConverter : public CStringConverter { public: - NonConstCStringConverter(long maxSize = -1) : CStringConverter(maxSize) {} + using CStringConverter::CStringConverter; public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); @@ -153,7 +146,8 @@ class NonConstCStringConverter : public CStringConverter { class WCStringConverter : public Converter { public: - WCStringConverter(long maxSize = -1) : fBuffer(nullptr), fMaxSize(maxSize) {} + WCStringConverter(std::wstring::size_type maxSize = std::wstring::npos) : + fBuffer(nullptr), fMaxSize(maxSize) {} WCStringConverter(const WCStringConverter&) = delete; WCStringConverter& operator=(const WCStringConverter&) = delete; virtual ~WCStringConverter() { free(fBuffer); } @@ -166,30 +160,32 @@ class WCStringConverter : public Converter { protected: wchar_t* fBuffer; - long fMaxSize; + std::wstring::size_type fMaxSize; }; class CString16Converter : public Converter { public: - CString16Converter(long maxSize = -1) : fBuffer(nullptr), fMaxSize(maxSize) {} + CString16Converter(std::wstring::size_type maxSize = std::wstring::npos) : + fBuffer(nullptr), fMaxSize(maxSize) {} CString16Converter(const CString16Converter&) = delete; CString16Converter& operator=(const CString16Converter&) = delete; virtual ~CString16Converter() { free(fBuffer); } - + public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); virtual PyObject* FromMemory(void* address); virtual bool ToMemory(PyObject* value, void* address, PyObject* = nullptr); virtual bool HasState() { return true; } - + protected: char16_t* fBuffer; - long fMaxSize; + std::wstring::size_type fMaxSize; }; class CString32Converter : public Converter { public: - CString32Converter(long maxSize = -1) : fBuffer(nullptr), fMaxSize(maxSize) {} + CString32Converter(std::wstring::size_type maxSize = std::wstring::npos) : + fBuffer(nullptr), fMaxSize(maxSize) {} CString32Converter(const CString32Converter&) = delete; CString32Converter& operator=(const CString32Converter&) = delete; virtual ~CString32Converter() { free(fBuffer); } @@ -202,7 +198,7 @@ class CString32Converter : public Converter { protected: char32_t* fBuffer; - long fMaxSize; + std::wstring::size_type fMaxSize; }; // pointer/array conversions @@ -212,6 +208,8 @@ CPPYY_DECLARE_ARRAY_CONVERTER(UChar); #if __cplusplus > 201402L CPPYY_DECLARE_ARRAY_CONVERTER(Byte); #endif +CPPYY_DECLARE_ARRAY_CONVERTER(Int8); +CPPYY_DECLARE_ARRAY_CONVERTER(UInt8); CPPYY_DECLARE_ARRAY_CONVERTER(Short); CPPYY_DECLARE_ARRAY_CONVERTER(UShort); CPPYY_DECLARE_ARRAY_CONVERTER(Int); @@ -223,14 +221,28 @@ CPPYY_DECLARE_ARRAY_CONVERTER(ULLong); CPPYY_DECLARE_ARRAY_CONVERTER(Float); CPPYY_DECLARE_ARRAY_CONVERTER(Double); CPPYY_DECLARE_ARRAY_CONVERTER(LDouble); +CPPYY_DECLARE_ARRAY_CONVERTER(ComplexF); CPPYY_DECLARE_ARRAY_CONVERTER(ComplexD); -class CStringArrayConverter : public SCharArrayPtrConverter { +class CStringArrayConverter : public SCharArrayConverter { public: - using SCharArrayPtrConverter::SCharArrayPtrConverter; + CStringArrayConverter(cdims_t dims, bool fixed) : SCharArrayConverter(dims) { + fIsFixed = fixed; // overrides SCharArrayConverter decision + } + using SCharArrayConverter::SCharArrayConverter; + virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); virtual PyObject* FromMemory(void* address); + virtual bool ToMemory(PyObject*, void*, PyObject* = nullptr); + +private: + std::vector fBuffer; }; +class NonConstCStringArrayConverter : public CStringArrayConverter { +public: + using CStringArrayConverter::CStringArrayConverter; + virtual PyObject* FromMemory(void* address); +}; // converters for special cases class NullptrConverter : public Converter { @@ -268,7 +280,7 @@ class InstanceMoveConverter : public InstanceRefConverter { }; template -class InstancePtrPtrConverter : public InstancePtrConverter { +class InstancePtrPtrConverter : public InstancePtrConverter { public: using InstancePtrConverter::InstancePtrConverter; @@ -278,21 +290,12 @@ class InstancePtrPtrConverter : public InstancePtrConverter { virtual bool ToMemory(PyObject* value, void* address, PyObject* = nullptr); }; -class InstanceArrayConverter : public InstancePtrConverter { -public: - InstanceArrayConverter(Cppyy::TCppType_t klass, dims_t dims, bool keepControl = false) : - InstancePtrConverter(klass, keepControl) { - dim_t size = (dims && 0 < dims[0]) ? dims[0]+1: 1; - m_dims = new dim_t[size]; - if (dims) { - for (int i = 0; i < size; ++i) m_dims[i] = dims[i]; - } else { - m_dims[0] = -1; - } - } +class InstanceArrayConverter : public InstancePtrConverter { +public: + InstanceArrayConverter(Cppyy::TCppType_t klass, cdims_t dims, bool keepControl = false) : + InstancePtrConverter(klass, keepControl), fShape(dims) { } InstanceArrayConverter(const InstanceArrayConverter&) = delete; InstanceArrayConverter& operator=(const InstanceArrayConverter&) = delete; - virtual ~InstanceArrayConverter() { delete [] m_dims; } public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); @@ -300,7 +303,7 @@ class InstanceArrayConverter : public InstancePtrConverter { virtual bool ToMemory(PyObject* value, void* address, PyObject* = nullptr); protected: - dims_t m_dims; + dims_t fShape; }; @@ -312,6 +315,7 @@ class ComplexDConverter: public InstanceConverter { virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); virtual PyObject* FromMemory(void* address); virtual bool ToMemory(PyObject* value, void* address, PyObject* = nullptr); + virtual bool HasState() { return true; } private: std::complex fBuffer; @@ -334,13 +338,14 @@ class VoidPtrRefConverter : public Converter { class VoidPtrPtrConverter : public Converter { public: - VoidPtrPtrConverter(size_t size) { fSize = size; } + VoidPtrPtrConverter(cdims_t dims); virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); virtual PyObject* FromMemory(void* address); virtual bool HasState() { return true; } protected: - size_t fSize; + dims_t fShape; + bool fIsFixed; }; CPPYY_DECLARE_BASIC_CONVERTER(PyObject); @@ -350,21 +355,29 @@ CPPYY_DECLARE_BASIC_CONVERTER(PyObject); class name##Converter : public InstanceConverter { \ public: \ name##Converter(bool keepControl = true); \ -public: \ virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); \ virtual PyObject* FromMemory(void* address); \ virtual bool ToMemory(PyObject*, void*, PyObject* = nullptr); \ + virtual bool HasState() { return true; } \ protected: \ - strtype fBuffer; \ + strtype fStringBuffer; \ } CPPYY_DECLARE_STRING_CONVERTER(TString, TString); CPPYY_DECLARE_STRING_CONVERTER(STLString, std::string); -CPPYY_DECLARE_STRING_CONVERTER(STLStringViewBase, std::string_view); +#if __cplusplus > 201402L +// The buffer type needs to be std::string also in the string_view case, +// otherwise the pointed-to string might not live long enough. See also: +// https://github.com/wlav/CPyCppyy/issues/13 +CPPYY_DECLARE_STRING_CONVERTER(STLStringViewBase, std::string); class STLStringViewConverter : public STLStringViewBaseConverter { public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); + virtual bool ToMemory(PyObject*, void*, PyObject* = nullptr); +private: + std::string_view fStringView; }; +#endif CPPYY_DECLARE_STRING_CONVERTER(STLWString, std::wstring); class STLStringMoveConverter : public STLStringConverter { @@ -397,10 +410,10 @@ class FunctionPointerConverter : public Converter { class StdFunctionConverter : public FunctionPointerConverter { public: StdFunctionConverter(Converter* cnv, const std::string& ret, const std::string& sig) : - FunctionPointerConverter(ret, sig), fConverter(cnv), fFuncWrap(nullptr) {} + FunctionPointerConverter(ret, sig), fConverter(cnv) {} StdFunctionConverter(const StdFunctionConverter&) = delete; StdFunctionConverter& operator=(const StdFunctionConverter&) = delete; - virtual ~StdFunctionConverter() { Py_XDECREF(fFuncWrap); delete fConverter; } + virtual ~StdFunctionConverter() { delete fConverter; } public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); @@ -409,7 +422,6 @@ class StdFunctionConverter : public FunctionPointerConverter { protected: Converter* fConverter; - PyObject* fFuncWrap; }; @@ -426,7 +438,7 @@ class SmartPtrConverter : public Converter { public: virtual bool SetArg(PyObject*, Parameter&, CallContext* = nullptr); virtual PyObject* FromMemory(void* address); - //virtual bool ToMemory(PyObject* value, void* address, PyObject* = nullptr); + //virtual bool ToMemory(PyObject*, void*, PyObject* = nullptr); virtual bool HasState() { return true; } protected: @@ -440,10 +452,9 @@ class SmartPtrConverter : public Converter { // initializer lists -class InitializerListConverter : public Converter { +class InitializerListConverter : public InstanceConverter { public: - InitializerListConverter(Converter* cnv, size_t sz) : - fConverter(cnv), fValueSize(sz) {} + InitializerListConverter(Cppyy::TCppType_t klass, std::string const& value_type); InitializerListConverter(const InitializerListConverter&) = delete; InitializerListConverter& operator=(const InitializerListConverter&) = delete; virtual ~InitializerListConverter(); @@ -453,8 +464,14 @@ class InitializerListConverter : public Converter { virtual bool HasState() { return true; } protected: - Converter* fConverter; - size_t fValueSize; + void Clear(); + +protected: + void* fBuffer = nullptr; + std::vector fConverters; + std::string fValueTypeName; + Cppyy::TCppType_t fValueType; + size_t fValueSize; }; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h index c4d9572fe2c13..f8167762f533b 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/DeclareExecutors.h @@ -4,7 +4,9 @@ // Bindings #include "Executors.h" #include "CallContext.h" +#include "Dimensions.h" +// Standard #if __cplusplus > 201402L #include #endif @@ -44,31 +46,43 @@ CPPYY_DECL_EXEC(Double); CPPYY_DECL_EXEC(LongDouble); CPPYY_DECL_EXEC(Void); CPPYY_DECL_EXEC(CString); +CPPYY_DECL_EXEC(CStringRef); CPPYY_DECL_EXEC(WCString); CPPYY_DECL_EXEC(CString16); CPPYY_DECL_EXEC(CString32); // pointer/array executors -CPPYY_DECL_EXEC(VoidArray); -CPPYY_DECL_EXEC(BoolArray); -CPPYY_DECL_EXEC(UCharArray); +#define CPPYY_ARRAY_DECL_EXEC(name) \ +class name##ArrayExecutor : public Executor { \ + dims_t fShape; \ +public: \ + name##ArrayExecutor(dims_t dims) : fShape(dims) {} \ + virtual PyObject* Execute( \ + Cppyy::TCppMethod_t, Cppyy::TCppObject_t, CallContext*); \ + virtual bool HasState() { return true; } \ +} +CPPYY_ARRAY_DECL_EXEC(Void); +CPPYY_ARRAY_DECL_EXEC(Bool); +CPPYY_ARRAY_DECL_EXEC(UChar); #if __cplusplus > 201402L -CPPYY_DECL_EXEC(ByteArray); +CPPYY_ARRAY_DECL_EXEC(Byte); #endif -CPPYY_DECL_EXEC(ShortArray); -CPPYY_DECL_EXEC(UShortArray); -CPPYY_DECL_EXEC(IntArray); -CPPYY_DECL_EXEC(UIntArray); -CPPYY_DECL_EXEC(LongArray); -CPPYY_DECL_EXEC(ULongArray); -CPPYY_DECL_EXEC(LLongArray); -CPPYY_DECL_EXEC(ULLongArray); -CPPYY_DECL_EXEC(FloatArray); -CPPYY_DECL_EXEC(DoubleArray); -CPPYY_DECL_EXEC(ComplexFArray); -CPPYY_DECL_EXEC(ComplexDArray); -CPPYY_DECL_EXEC(ComplexIArray); -CPPYY_DECL_EXEC(ComplexLArray); +CPPYY_ARRAY_DECL_EXEC(Int8); +CPPYY_ARRAY_DECL_EXEC(UInt8); +CPPYY_ARRAY_DECL_EXEC(Short); +CPPYY_ARRAY_DECL_EXEC(UShort); +CPPYY_ARRAY_DECL_EXEC(Int); +CPPYY_ARRAY_DECL_EXEC(UInt); +CPPYY_ARRAY_DECL_EXEC(Long); +CPPYY_ARRAY_DECL_EXEC(ULong); +CPPYY_ARRAY_DECL_EXEC(LLong); +CPPYY_ARRAY_DECL_EXEC(ULLong); +CPPYY_ARRAY_DECL_EXEC(Float); +CPPYY_ARRAY_DECL_EXEC(Double); +CPPYY_ARRAY_DECL_EXEC(ComplexF); +CPPYY_ARRAY_DECL_EXEC(ComplexD); +CPPYY_ARRAY_DECL_EXEC(ComplexI); +CPPYY_ARRAY_DECL_EXEC(ComplexL); // special cases CPPYY_DECL_EXEC(ComplexD); @@ -95,12 +109,14 @@ class InstanceExecutor : public Executor { protected: Cppyy::TCppType_t fClass; - unsigned int fFlags; + uint32_t fFlags; }; class IteratorExecutor : public InstanceExecutor { public: IteratorExecutor(Cppyy::TCppType_t klass); + virtual PyObject* Execute( + Cppyy::TCppMethod_t, Cppyy::TCppObject_t, CallContext*); }; CPPYY_DECL_EXEC(Constructor); @@ -159,13 +175,13 @@ class InstancePtrRefExecutor : public InstanceRefExecutor { class InstanceArrayExecutor : public InstancePtrExecutor { public: - InstanceArrayExecutor(Cppyy::TCppType_t klass, Py_ssize_t array_size) - : InstancePtrExecutor(klass), fArraySize(array_size) {} + InstanceArrayExecutor(Cppyy::TCppType_t klass, dim_t array_size) + : InstancePtrExecutor(klass), fSize(array_size) {} virtual PyObject* Execute( Cppyy::TCppMethod_t, Cppyy::TCppObject_t, CallContext*); protected: - Py_ssize_t fArraySize; + dim_t fSize; }; class FunctionPointerExecutor : public Executor { diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Dimensions.h b/bindings/pyroot/cppyy/CPyCppyy/src/Dimensions.h new file mode 100644 index 0000000000000..6a925033edb1d --- /dev/null +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Dimensions.h @@ -0,0 +1,84 @@ +#ifndef CPYCPPYY_DIMENSIONS_H +#define CPYCPPYY_DIMENSIONS_H + +// Standard +#include +#include + + +namespace CPyCppyy { + +static const dim_t UNKNOWN_SIZE = (dim_t)-1; + +class CPYCPPYY_CLASS_EXPORT Dimensions { + dim_t* fDims; + +public: + Dimensions(dim_t ndim = 0, dim_t* dims = nullptr) : fDims(nullptr) { + if (ndim && ndim != UNKNOWN_SIZE) { + fDims = new dim_t[ndim+1]; + fDims[0] = ndim; + if (dims) std::copy(dims, dims+ndim, fDims+1); + else std::fill_n(fDims+1, ndim, UNKNOWN_SIZE); + } + } + Dimensions(std::initializer_list l) { + fDims = new dim_t[l.size()+1]; + fDims[0] = l.size(); + std::copy(l.begin(), l.end(), fDims+1); + } + Dimensions(const Dimensions& d) : fDims(nullptr) { + if (d.fDims) { + fDims = new dim_t[d.fDims[0]+1]; + std::copy(d.fDims, d.fDims+d.fDims[0]+1, fDims); + } + } + Dimensions(Dimensions&& d) : fDims(d.fDims) { + d.fDims = nullptr; + } + Dimensions& operator=(const Dimensions& d) { + if (this != &d) { + if (!d.fDims) { + delete [] fDims; + fDims = nullptr; + } else { + if (!fDims || (fDims && fDims[0] != d.fDims[0])) { + delete [] fDims; + fDims = new dim_t[d.fDims[0]+1]; + } + std::copy(d.fDims, d.fDims+d.fDims[0]+1, fDims); + } + } + return *this; + } + ~Dimensions() { + delete [] fDims; + } + +public: + operator bool() const { return (bool)fDims; } + + dim_t ndim() const { return fDims ? fDims[0] : UNKNOWN_SIZE; } + void ndim(dim_t d) { + if (fDims) { + if (fDims[0] == d) return; + delete [] fDims; + } + + fDims = new dim_t[d+1]; + fDims[0] = d; + std::fill_n(fDims+1, d, UNKNOWN_SIZE); + } + + dim_t operator[](dim_t i) const { return fDims[i+1]; } + dim_t& operator[](dim_t i) { return fDims[i+1]; } + + Dimensions sub() const { return fDims ? Dimensions(fDims[0]-1, fDims+2) : Dimensions(); } +}; + +typedef Dimensions dims_t; +typedef const dims_t& cdims_t; + +} // namespace CPyCppyy + +#endif // !CPYCPPYY_DIMENSIONS_H diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/DispatchPtr.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/DispatchPtr.cxx index 1fbe14db25a78..43b73fb8f107c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/DispatchPtr.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/DispatchPtr.cxx @@ -1,24 +1,36 @@ -// Bindings +// Bindings #include "CPyCppyy.h" #define CPYCPPYY_INTERNAL 1 #include "CPyCppyy/DispatchPtr.h" #undef CPYCPPYY_INTERNAL #include "CPPInstance.h" +#include "CPPScope.h" //----------------------------------------------------------------------------- PyObject* CPyCppyy::DispatchPtr::Get() const { if (fPyHardRef) return fPyHardRef; - if (fPyWeakRef) return PyWeakref_GetObject(fPyWeakRef); + if (fPyWeakRef) { + PyObject* disp = PyWeakref_GetObject(fPyWeakRef); + if (disp != Py_None) // dispatcher object disappeared? + return disp; + } return nullptr; } //----------------------------------------------------------------------------- -CPyCppyy::DispatchPtr::DispatchPtr(PyObject* pyobj) : fPyHardRef(nullptr) +CPyCppyy::DispatchPtr::DispatchPtr(PyObject* pyobj, bool strong) : fPyHardRef(nullptr) { + if (strong) { + Py_INCREF(pyobj); + fPyHardRef = pyobj; + fPyWeakRef = nullptr; + } else { + fPyHardRef = nullptr; + fPyWeakRef = PyWeakref_NewRef(pyobj, nullptr); + } ((CPPInstance*)pyobj)->SetDispatchPtr(this); - fPyWeakRef = PyWeakref_NewRef(pyobj, nullptr); } //----------------------------------------------------------------------------- @@ -29,6 +41,23 @@ CPyCppyy::DispatchPtr::DispatchPtr(const DispatchPtr& other, void* cppinst) : fP if (fPyHardRef) ((CPPInstance*)fPyHardRef)->SetDispatchPtr(this); } +//----------------------------------------------------------------------------- +CPyCppyy::DispatchPtr::~DispatchPtr() { +// if we're holding a hard reference, or holding weak reference while being part +// of a dispatcher intermediate, then this delete is from the C++ side, and Python +// is "notified" by nulling out the reference and an exception will be raised on +// continued access + if (fPyWeakRef) { + PyObject* pyobj = PyWeakref_GetObject(fPyWeakRef); + if (pyobj && pyobj != Py_None && ((CPPScope*)Py_TYPE(pyobj))->fFlags & CPPScope::kIsPython) + ((CPPInstance*)pyobj)->GetObjectRaw() = nullptr; + Py_DECREF(fPyWeakRef); + } else if (fPyHardRef) { + ((CPPInstance*)fPyHardRef)->GetObjectRaw() = nullptr; + Py_DECREF(fPyHardRef); + } +} + //----------------------------------------------------------------------------- CPyCppyy::DispatchPtr& CPyCppyy::DispatchPtr::assign(const DispatchPtr& other, void* cppinst) { @@ -58,6 +87,7 @@ void CPyCppyy::DispatchPtr::CppOwns() // C++ maintains the hardref, keeping the PyObject alive w/o outstanding ref if (fPyWeakRef) { fPyHardRef = PyWeakref_GetObject(fPyWeakRef); + if (fPyHardRef == Py_None) fPyHardRef = nullptr; Py_XINCREF(fPyHardRef); Py_DECREF(fPyWeakRef); fPyWeakRef = nullptr; } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx index 43e5cc2e92dc4..db388575dbbd0 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx @@ -34,7 +34,26 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m if (Cppyy::IsConstMethod(method)) code << "const "; code << "{\n"; -// start function body +// on destruction, the Python object may go first, in which case provide a diagnostic +// warning (raising a PyException may not be possible as this could happen during +// program shutdown); note that this means that the actual result will be the default +// and the caller may need to act on that, but that's still an improvement over a +// possible crash + code << " PyObject* iself = (PyObject*)_internal_self;\n" + " if (!iself || iself == Py_None) {\n" + " PyErr_Warn(PyExc_RuntimeWarning, (char*)\"Call attempted on deleted python-side proxy\");\n" + " return"; + if (retType != "void") { + if (retType.back() != '*') + code << " " << CPyCppyy::TypeManip::remove_const(retType) << "{}"; + else + code << " nullptr"; + } + code << ";\n" + " }\n" + " Py_INCREF(iself);\n"; + +// start actual function body Utility::ConstructCallbackPreamble(retType, argtypes, code); // perform actual method call @@ -43,14 +62,13 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m #else code << " PyObject* mtPyName = PyUnicode_FromString(\"" << mtCppName << "\");\n" #endif - " PyObject* pyresult = PyObject_CallMethodObjArgs((PyObject*)_internal_self, mtPyName"; - + " PyObject* pyresult = PyObject_CallMethodObjArgs(iself, mtPyName"; for (Cppyy::TCppIndex_t i = 0; i < nArgs; ++i) code << ", pyargs[" << i << "]"; - code << ", NULL);\n Py_DECREF(mtPyName);\n"; + code << ", NULL);\n Py_DECREF(mtPyName);\n Py_DECREF(iself);\n"; // close - Utility::ConstructCallbackReturn(retType, nArgs, code); + Utility::ConstructCallbackReturn(retType, (int)nArgs, code); } //---------------------------------------------------------------------------- @@ -64,13 +82,93 @@ namespace { }; typedef std::vector BaseInfos_t; + typedef std::vector Ctors_t; + typedef std::vector AllCtors_t; + typedef std::vector> CtorInfos_t; +} // unnamed namespace + +static void build_constructors( + const std::string& derivedName, const BaseInfos_t& base_infos, const AllCtors_t& ctors, + std::ostringstream& code, const CtorInfos_t& methods = CtorInfos_t{}, size_t idx = 0) +{ + if (idx < ctors.size()) { + for (const auto& method : ctors[idx]) { + size_t argsmin = (size_t)Cppyy::GetMethodReqArgs(method); + size_t argsmax = (size_t)Cppyy::GetMethodNumArgs(method); + for (size_t i = argsmin; i <= argsmax; ++i) { + CtorInfos_t methods1{methods}; + methods1.emplace_back(method, i); + build_constructors(derivedName, base_infos, ctors, code, methods1, idx+1); + } + } + } else { + // this is as deep as we go; start writing + code << " " << derivedName << "("; + + // declare arguments + std::vector arg_tots; arg_tots.reserve(methods.size()); + for (Ctors_t::size_type i = 0; i < methods.size(); ++i) { + const auto& cinfo = methods[i]; + if (i != 0 && (arg_tots.back() || 1 < arg_tots.size())) code << ", "; + size_t nArgs = cinfo.second; + arg_tots.push_back(i == 0 ? nArgs : nArgs+arg_tots.back()); + + if (i != 0) code << "__cppyy_internal::Sep*"; + size_t offset = (i != 0 ? arg_tots[i-1] : 0); + for (size_t j = 0; j < nArgs; ++j) { + if (i != 0 || j != 0) code << ", "; + code << Cppyy::GetMethodArgType(cinfo.first, j) << " a" << (j+offset); + } + } + code << ") : "; + + // pass arguments to base constructors + for (BaseInfos_t::size_type i = 0; i < base_infos.size(); ++i) { + if (i != 0) code << ", "; + code << base_infos[i].bname << "("; + size_t first = (i != 0 ? arg_tots[i-1] : 0); + for (size_t j = first; j < arg_tots[i]; ++j) { + if (j != first) code << ", "; + bool isRValue = CPyCppyy::TypeManip::compound(\ + Cppyy::GetMethodArgType(methods[i].first, j-first)) == "&&"; + if (isRValue) code << "std::move("; + code << "a" << j; + if (isRValue) code << ")"; + } + code << ")"; + } + code << " {}\n"; + } +} + +namespace { + +using namespace Cppyy; + +static inline +std::vector FindBaseMethod(TCppScope_t tbase, const std::string mtCppName) +{ +// Recursively walk the inheritance tree to find the overloads of the named method + std::vector result; + result = GetMethodIndicesFromName(tbase, mtCppName); + if (result.empty()) { + for (TCppIndex_t ibase = 0; ibase < GetNumBases(tbase); ++ibase) { + TCppScope_t b = GetScope(GetBaseName(tbase, ibase)); + result = FindBaseMethod(b, mtCppName); + if (!result.empty()) + break; + } + } + return result; +} + } // unnamed namespace -//---------------------------------------------------------------------------- bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, std::ostringstream& err) { // Scan all methods in dct and where it overloads base methods in klass, create // dispatchers on the C++ side. Then interject the dispatcher class. + if (!PyTuple_Check(bases) || !PyTuple_GET_SIZE(bases) || !dct || !PyDict_Check(dct)) { err << "internal error: expected tuple of bases and proper dictionary"; return false; @@ -81,15 +179,15 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, return false; } +// collect all bases, error checking the hierarchy along the way const Py_ssize_t nBases = PyTuple_GET_SIZE(bases); BaseInfos_t base_infos; base_infos.reserve(nBases); for (Py_ssize_t ibase = 0; ibase < nBases; ++ibase) { - auto currentBase = PyTuple_GET_ITEM(bases, ibase); - if (!CPPScope_Check(currentBase)) - // Python base class + if (!CPPScope_Check(PyTuple_GET_ITEM(bases, ibase))) continue; - Cppyy::TCppType_t basetype = ((CPPScope*)currentBase)->fCppType; + Cppyy::TCppType_t basetype = ((CPPScope*)PyTuple_GET_ITEM(bases, ibase))->fCppType; + if (!basetype) { err << "base class is incomplete"; break; @@ -109,19 +207,8 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, basetype, TypeManip::template_base(Cppyy::GetFinalName(basetype)), Cppyy::GetScopedFinalName(basetype)); } - if ((Py_ssize_t)base_infos.size() == 0) - return false; - else if ((Py_ssize_t)base_infos.size() > 1) { - err << "multi cross-inheritance not supported"; - return false; - } - - const auto& binfo = base_infos[0]; - const Cppyy::TCppType_t baseType = binfo.btype; - const std::string& baseName = binfo.bname; - const std::string& baseNameScoped = binfo.bname_scoped; - - bool isDeepHierarchy = klass->fCppType && baseType != klass->fCppType; +// TODO: check deep hierarchy for multiple inheritance + bool isDeepHierarchy = klass->fCppType && base_infos.front().btype != klass->fCppType; // once classes can be extended, should consider re-use; for now, since derived // python classes can differ in what they override, simply use different shims @@ -135,15 +222,39 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, // start class declaration code << "namespace __cppyy_internal {\n" - << "class " << derivedName << " : public ::" << baseNameScoped << " {\n"; + << "class " << derivedName << " : "; + for (BaseInfos_t::size_type ibase = 0; ibase < base_infos.size(); ++ibase) { + if (ibase != 0) code << ", "; + code << "public ::" << base_infos[ibase].bname_scoped; + } + code << " {\n"; if (!isDeepHierarchy) code << "protected:\n CPyCppyy::DispatchPtr _internal_self;\n"; code << "public:\n"; -// add a virtual destructor for good measure - code << " virtual ~" << derivedName << "() {}\n"; - -// methods: first collect all callables, then get overrides from base class, for +// add a virtual destructor for good measure, which is allowed to be "overridden" by +// the conventional __destruct__ method (note that __del__ is always called, too, if +// provided, but only when the Python object goes away; furthermore, if the Python +// object goes before the C++ one, only __del__ is called) + if (PyMapping_HasKeyString(dct, (char*)"__destruct__")) { + code << " virtual ~" << derivedName << "() {\n" + " PyObject* iself = (PyObject*)_internal_self;\n" + " if (!iself || iself == Py_None)\n" + " return;\n" // safe, as destructor always returns void + " Py_INCREF(iself);\n" + " PyObject* mtPyName = PyUnicode_FromString(\"__destruct__\");\n" + " PyObject* pyresult = PyObject_CallMethodObjArgs(iself, mtPyName, NULL);\n" + " Py_DECREF(mtPyName);\n Py_DECREF(iself);\n"; + + // this being a destructor, print on exception rather than propagate using the + // magic C++ exception ... + code << " if (!pyresult) PyErr_Print();\n" + " else { Py_DECREF(pyresult); }\n" + " }\n"; + } else + code << " virtual ~" << derivedName << "() {}\n"; + +// methods: first collect all callables, then get overrides from base classes, for // those that are still missing, search the hierarchy PyObject* clbs = PyDict_New(); PyObject* items = PyDict_Items(dct); @@ -160,128 +271,187 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, // exposed on the Python side; so, collect their names as we go along std::set protected_names; -// simple case: methods from current class - bool has_default = false; - bool has_cctor = false; - bool has_constructors = false; - const Cppyy::TCppIndex_t nMethods = Cppyy::GetNumMethods(baseType); - for (Cppyy::TCppIndex_t imeth = 0; imeth < nMethods; ++imeth) { - Cppyy::TCppMethod_t method = Cppyy::GetMethod(baseType, imeth); - - if (Cppyy::IsConstructor(method) && (Cppyy::IsPublicMethod(method) || Cppyy::IsProtectedMethod(method))) { - has_constructors = true; - Cppyy::TCppIndex_t nreq = Cppyy::GetMethodReqArgs(method); - if (nreq == 0) - has_default = true; - else if (!has_cctor && nreq == 1) { - const std::string& argtype = Cppyy::GetMethodArgType(method, 0); - if (Utility::Compound(argtype) == "&" && TypeManip::clean_type(argtype, false) == baseNameScoped) - has_cctor = true; +// simple case: methods from current class (collect constructors along the way) + int has_default = 0, has_cctor = 0, has_ctors = 0, has_tmpl_ctors = 0; + AllCtors_t ctors{base_infos.size()}; + for (BaseInfos_t::size_type ibase = 0; ibase < base_infos.size(); ++ibase) { + const auto& binfo = base_infos[ibase]; + + const Cppyy::TCppIndex_t nMethods = Cppyy::GetNumMethods(binfo.btype); + bool cctor_found = false, default_found = false, any_ctor_found = false; + for (Cppyy::TCppIndex_t imeth = 0; imeth < nMethods; ++imeth) { + Cppyy::TCppMethod_t method = Cppyy::GetMethod(binfo.btype, imeth); + + if (Cppyy::IsConstructor(method)) { + any_ctor_found = true; + if (Cppyy::IsPublicMethod(method) || Cppyy::IsProtectedMethod(method)) { + Cppyy::TCppIndex_t nreq = Cppyy::GetMethodReqArgs(method); + if (nreq == 0) default_found = true; + else if (!cctor_found && nreq == 1) { + const std::string& argtype = Cppyy::GetMethodArgType(method, 0); + if (TypeManip::compound(argtype) == "&" && TypeManip::clean_type(argtype, false) == binfo.bname_scoped) + cctor_found = true; + } + ctors[ibase].push_back(method); + } + continue; } - continue; - } - - std::string mtCppName = Cppyy::GetMethodName(method); - PyObject* key = CPyCppyy_PyText_FromString(mtCppName.c_str()); - int contains = PyDict_Contains(dct, key); - if (contains == -1) PyErr_Clear(); - if (contains != 1) { - Py_DECREF(key); - // if the method is protected, we expose it through re-declaration and forwarding (using - // does not work here b/c there may be private overloads) - if (Cppyy::IsProtectedMethod(method)) { - protected_names.insert(mtCppName); - - code << " " << Cppyy::GetMethodResultType(method) << " " << mtCppName << "("; - Cppyy::TCppIndex_t nArgs = Cppyy::GetMethodNumArgs(method); - for (Cppyy::TCppIndex_t i = 0; i < nArgs; ++i) { - if (i != 0) code << ", "; - code << Cppyy::GetMethodArgType(method, i) << " arg" << i; - } - code << ") "; - if (Cppyy::IsConstMethod(method)) code << "const "; - code << "{\n return " << baseName << "::" << mtCppName << "("; - for (Cppyy::TCppIndex_t i = 0; i < nArgs; ++i) { - if (i != 0) code << ", "; - code << "arg" << i; + std::string mtCppName = Cppyy::GetMethodName(method); + PyObject* key = CPyCppyy_PyText_FromString(mtCppName.c_str()); + int contains = PyDict_Contains(dct, key); + if (contains == -1) PyErr_Clear(); + if (contains != 1) { + Py_DECREF(key); + + // if the method is protected, we expose it through re-declaration and forwarding (using + // does not work here b/c there may be private overloads) + if (Cppyy::IsProtectedMethod(method)) { + protected_names.insert(mtCppName); + + code << " " << Cppyy::GetMethodResultType(method) << " " << mtCppName << "("; + Cppyy::TCppIndex_t nArgs = Cppyy::GetMethodNumArgs(method); + for (Cppyy::TCppIndex_t i = 0; i < nArgs; ++i) { + if (i != 0) code << ", "; + code << Cppyy::GetMethodArgType(method, i) << " arg" << i; + } + code << ") "; + if (Cppyy::IsConstMethod(method)) code << "const "; + code << "{\n return " << binfo.bname << "::" << mtCppName << "("; + for (Cppyy::TCppIndex_t i = 0; i < nArgs; ++i) { + if (i != 0) code << ", "; + code << "arg" << i; + } + code << ");\n }\n"; } - code << ");\n }\n"; + + continue; } - continue; + InjectMethod(method, mtCppName, code); + + if (PyDict_DelItem(clbs, key) != 0) + PyErr_Clear(); // happens for overloads + Py_DECREF(key); } - InjectMethod(method, mtCppName, code); + // support for templated ctors in single inheritance (TODO: also multi possible?) + if (base_infos.size() == 1) { + const Cppyy::TCppIndex_t nTemplMethods = Cppyy::GetNumTemplatedMethods(binfo.btype); + for (Cppyy::TCppIndex_t imeth = 0; imeth < nTemplMethods; ++imeth) { + if (Cppyy::IsTemplatedConstructor(binfo.btype, imeth)) { + any_ctor_found = true; + has_tmpl_ctors += 1; + break; // one suffices to map as argument packs are used + } + } + } - if (PyDict_DelItem(clbs, key) != 0) - PyErr_Clear(); // happens for overloads - Py_DECREF(key); + // count the cctors and default ctors to determine whether each base has one + if (cctor_found || (!cctor_found && !any_ctor_found)) has_cctor += 1; + if (default_found || (!default_found && !any_ctor_found)) has_default += 1; + if (any_ctor_found && !has_tmpl_ctors) has_ctors += 1; } // try to locate left-overs in base classes - if (PyDict_Size(clbs)) { - size_t nbases = Cppyy::GetNumBases(baseType); - for (size_t ibase = 0; ibase < nbases; ++ibase) { - Cppyy::TCppScope_t tbase = (Cppyy::TCppScope_t)Cppyy::GetScope( \ - Cppyy::GetBaseName(baseType, ibase)); - - PyObject* keys = PyDict_Keys(clbs); - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(keys); ++i) { - // TODO: should probably invert this looping; but that makes handling overloads clunky - PyObject* key = PyList_GET_ITEM(keys, i); - std::string mtCppName = CPyCppyy_PyText_AsString(key); - const auto& v = Cppyy::GetMethodIndicesFromName(tbase, mtCppName); - for (auto idx : v) - InjectMethod(Cppyy::GetMethod(tbase, idx), mtCppName, code); - if (!v.empty()) { - if (PyDict_DelItem(clbs, key) != 0) PyErr_Clear(); + for (const auto& binfo : base_infos) { + if (PyDict_Size(clbs)) { + size_t nbases = Cppyy::GetNumBases(binfo.btype); + for (size_t ibase = 0; ibase < nbases; ++ibase) { + Cppyy::TCppScope_t tbase = (Cppyy::TCppScope_t)Cppyy::GetScope( \ + Cppyy::GetBaseName(binfo.btype, ibase)); + + PyObject* keys = PyDict_Keys(clbs); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(keys); ++i) { + // TODO: should probably invert this looping; but that makes handling overloads clunky + PyObject* key = PyList_GET_ITEM(keys, i); + std::string mtCppName = CPyCppyy_PyText_AsString(key); + const auto& v = FindBaseMethod(tbase, mtCppName); + for (auto idx : v) + InjectMethod(Cppyy::GetMethod(tbase, idx), mtCppName, code); + if (!v.empty()) { + if (PyDict_DelItem(clbs, key) != 0) PyErr_Clear(); + } } - } - Py_DECREF(keys); + Py_DECREF(keys); + } } } Py_DECREF(clbs); -// constructors: most are simply inherited, for use by the Python derived class - code << " using " << baseName << "::" << baseName << ";\n"; +// constructors: build up from the argument types of the base class, for use by the Python +// derived class (inheriting with/ "using" does not work b/c base class constructors may +// have been deleted), + build_constructors(derivedName, base_infos, ctors, code); // for working with C++ templates, additional constructors are needed to make // sure the python object is properly carried, but they can only be generated // if the base class supports them - if (has_default || !has_constructors) + if (1 < nBases && (!has_ctors || has_default == nBases)) code << " " << derivedName << "() {}\n"; - if (has_default || has_cctor || !has_constructors) { - code << " " << derivedName << "(const " << derivedName << "& other)"; - if (has_cctor) - code << " : " << baseName << "(other)"; - if (!isDeepHierarchy) { - if (has_cctor) code << ", "; - else code << " : "; - code << "_internal_self(other._internal_self, this)"; + if (has_cctor == nBases) { + code << " " << derivedName << "(const " << derivedName << "& other) : "; + for (BaseInfos_t::size_type ibase = 0; ibase < base_infos.size(); ++ibase) { + if (ibase != 0) code << ", "; + code << base_infos[ibase].bname << "(other)"; } + if (!isDeepHierarchy) + code << ", _internal_self(other._internal_self, this)"; code << " {}\n"; } + if (has_tmpl_ctors && base_infos.size() == 1) { + // support for templated ctors in single inheritance (TODO: also multi possible?) + code << " template\n " << derivedName << "(Args... args) : " + << base_infos[0].bname << "(args...) {}\n"; + } // destructor: default is fine // pull in data members that are protected - Cppyy::TCppIndex_t nData = Cppyy::GetNumDatamembers(baseType); - if (nData) code << "public:\n"; - for (Cppyy::TCppIndex_t idata = 0; idata < nData; ++idata) { - if (Cppyy::IsProtectedData(baseType, idata)) { - const std::string dm_name = Cppyy::GetDatamemberName(baseType, idata); - if (dm_name != "_internal_self") { - protected_names.insert(dm_name); - code << " using " << baseName << "::" << dm_name << ";\n"; + bool setPublic = false; + for (const auto& binfo : base_infos) { + Cppyy::TCppIndex_t nData = Cppyy::GetNumDatamembers(binfo.btype); + for (Cppyy::TCppIndex_t idata = 0; idata < nData; ++idata) { + if (Cppyy::IsProtectedData(binfo.btype, idata)) { + const std::string dm_name = Cppyy::GetDatamemberName(binfo.btype, idata); + if (dm_name != "_internal_self") { + const std::string& dname = Cppyy::GetDatamemberName(binfo.btype, idata); + protected_names.insert(dname); + if (!setPublic) { + code << "public:\n"; + setPublic = true; + } + code << " using " << binfo.bname << "::" << dname << ";\n"; + } } } } -// initialize the dispatch pointer for base - code << "public:\n static void _init_dispatchptr(" << derivedName << "* inst, PyObject* self) {\n" - " new ((void*)&inst->_internal_self) CPyCppyy::DispatchPtr{self};\n" - " }\n"; +// initialize the dispatch pointer for all direct bases that have one + BaseInfos_t::size_type disp_inited = 0; + code << "public:\n static void _init_dispatchptr(" << derivedName << "* inst, PyObject* self) {\n"; + if (1 < base_infos.size()) { + for (const auto& binfo : base_infos) { + if (Cppyy::GetDatamemberIndex(binfo.btype, "_internal_self") != (Cppyy::TCppIndex_t)-1) { + code << " " << binfo.bname << "::_init_dispatchptr(inst, self);\n"; + disp_inited += 1; + } + } + } +// The dispatch initializer is only used in constructors, and C++ object start out +// as owned by C++, with Python ownership explicitly set only later. To match, the +// dispatch pointer needs to start out with a hard reference, i.e. C++ ownership of +// the dispatch object. If the constructor has __creates__ set to True (default), +// then a call to PythonOwns() will switch the hard ref to a weak ref, preventing +// accidental circular references. + if (disp_inited != base_infos.size()) + code << " new ((void*)&inst->_internal_self) CPyCppyy::DispatchPtr{self, true};\n"; + code << " }"; + +// provide an accessor to re-initialize after round-tripping from C++ (internal) + code << "\n static PyObject* _get_dispatch(" << derivedName << "* inst) {\n" + " PyObject* res = (PyObject*)inst->_internal_self;\n" + " Py_XINCREF(res); return res;\n }"; // finish class declaration code << "};\n}"; @@ -304,7 +474,10 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct, // at this point, the dispatcher only lives in C++, as opposed to regular classes // that are part of the hierarchy in Python, so create it, which will cache it for // later use by e.g. the MemoryRegulator - PyObject* disp_proxy = CPyCppyy::CreateScopeProxy(disp); + unsigned int flags = (unsigned int)(klass->fFlags & CPPScope::kIsMultiCross); + PyObject* disp_proxy = CPyCppyy::CreateScopeProxy(disp, flags); + if (flags) ((CPPScope*)disp_proxy)->fFlags |= CPPScope::kIsMultiCross; + ((CPPScope*)disp_proxy)->fFlags |= CPPScope::kIsPython; // finally, to expose protected members, copy them over from the C++ dispatcher base // to the Python dictionary (the C++ dispatcher's Python proxy is not a base of the diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx index 6d1f220c3f929..70d8f72596d64 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Executors.cxx @@ -15,11 +15,11 @@ #include #include #include +#include //- data _____________________________________________________________________ namespace CPyCppyy { - typedef Executor* (*ef_t) (); typedef std::map ExecFactories_t; static ExecFactories_t gExecFactories; @@ -52,30 +52,30 @@ static inline rtype GILCall##tcode( \ Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CPyCppyy::CallContext* ctxt)\ { \ if (!ReleasesGIL(ctxt)) \ - return Cppyy::Call##tcode(method, self, ctxt->GetSize(), ctxt->GetArgs());\ + return Cppyy::Call##tcode(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs());\ GILControl gc{}; \ - return Cppyy::Call##tcode(method, self, ctxt->GetSize(), ctxt->GetArgs());\ + return Cppyy::Call##tcode(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs());\ } #else #define CPPYY_IMPL_GILCALL(rtype, tcode) \ static inline rtype GILCall##tcode( \ Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CPyCppyy::CallContext* ctxt)\ { \ - return Cppyy::Call##tcode(method, self, ctxt->GetSize(), ctxt->GetArgs());\ + return Cppyy::Call##tcode(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs());\ } #endif -CPPYY_IMPL_GILCALL(void, V) -CPPYY_IMPL_GILCALL(unsigned char, B) -CPPYY_IMPL_GILCALL(char, C) -CPPYY_IMPL_GILCALL(short, H) -CPPYY_IMPL_GILCALL(Int_t, I) -CPPYY_IMPL_GILCALL(Long_t, L) -CPPYY_IMPL_GILCALL(Long64_t, LL) -CPPYY_IMPL_GILCALL(float, F) -CPPYY_IMPL_GILCALL(double, D) -CPPYY_IMPL_GILCALL(LongDouble_t, LD) -CPPYY_IMPL_GILCALL(void*, R) +CPPYY_IMPL_GILCALL(void, V) +CPPYY_IMPL_GILCALL(unsigned char, B) +CPPYY_IMPL_GILCALL(char, C) +CPPYY_IMPL_GILCALL(short, H) +CPPYY_IMPL_GILCALL(int, I) +CPPYY_IMPL_GILCALL(long, L) +CPPYY_IMPL_GILCALL(PY_LONG_LONG, LL) +CPPYY_IMPL_GILCALL(float, F) +CPPYY_IMPL_GILCALL(double, D) +CPPYY_IMPL_GILCALL(PY_LONG_DOUBLE, LD) +CPPYY_IMPL_GILCALL(void*, R) static inline Cppyy::TCppObject_t GILCallO(Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CPyCppyy::CallContext* ctxt, Cppyy::TCppType_t klass) @@ -83,10 +83,10 @@ static inline Cppyy::TCppObject_t GILCallO(Cppyy::TCppMethod_t method, #ifdef WITH_THREAD if (!ReleasesGIL(ctxt)) #endif - return Cppyy::CallO(method, self, ctxt->GetSize(), ctxt->GetArgs(), klass); + return Cppyy::CallO(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs(), klass); #ifdef WITH_THREAD GILControl gc{}; - return Cppyy::CallO(method, self, ctxt->GetSize(), ctxt->GetArgs(), klass); + return Cppyy::CallO(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs(), klass); #endif } @@ -96,10 +96,10 @@ static inline Cppyy::TCppObject_t GILCallConstructor( #ifdef WITH_THREAD if (!ReleasesGIL(ctxt)) #endif - return Cppyy::CallConstructor(method, klass, ctxt->GetSize(), ctxt->GetArgs()); + return Cppyy::CallConstructor(method, klass, ctxt->GetEncodedSize(), ctxt->GetArgs()); #ifdef WITH_THREAD GILControl gc{}; - return Cppyy::CallConstructor(method, klass, ctxt->GetSize(), ctxt->GetArgs()); + return Cppyy::CallConstructor(method, klass, ctxt->GetEncodedSize(), ctxt->GetArgs()); #endif } @@ -262,7 +262,7 @@ PyObject* CPyCppyy::LongExecutor::Execute( Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) { // execute with argument , construct python long return value - return PyLong_FromLong((Long_t)GILCallL(method, self, ctxt)); + return PyLong_FromLong((long)GILCallL(method, self, ctxt)); } //---------------------------------------------------------------------------- @@ -270,7 +270,7 @@ PyObject* CPyCppyy::ULongExecutor::Execute( Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) { // execute with argument , construct python unsigned long return value - return PyLong_FromUnsignedLong((ULong_t)GILCallLL(method, self, ctxt)); + return PyLong_FromUnsignedLong((unsigned long)GILCallLL(method, self, ctxt)); } //---------------------------------------------------------------------------- @@ -278,7 +278,7 @@ PyObject* CPyCppyy::LongLongExecutor::Execute( Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) { // execute with argument , construct python long long return value - Long64_t result = GILCallLL(method, self, ctxt); + PY_LONG_LONG result = GILCallLL(method, self, ctxt); return PyLong_FromLongLong(result); } @@ -287,7 +287,7 @@ PyObject* CPyCppyy::ULongLongExecutor::Execute( Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) { // execute with argument , construct python unsigned long long return value - ULong64_t result = (ULong64_t)GILCallLL(method, self, ctxt); + PY_ULONG_LONG result = (PY_ULONG_LONG)GILCallLL(method, self, ctxt); return PyLong_FromUnsignedLongLong(result); } @@ -352,22 +352,22 @@ PyObject* CPyCppyy::name##RefExecutor::Execute( \ } \ } -CPPYY_IMPL_REFEXEC(Bool, bool, Long_t, CPyCppyy_PyBool_FromLong, PyLong_AsLong) -CPPYY_IMPL_REFEXEC(Char, char, Long_t, CPyCppyy_PyText_FromLong, PyLong_AsLong) -CPPYY_IMPL_REFEXEC(UChar, unsigned char, ULong_t, CPyCppyy_PyText_FromULong, PyLongOrInt_AsULong) -CPPYY_IMPL_REFEXEC(Int8, int8_t, Long_t, PyInt_FromLong, PyLong_AsLong) -CPPYY_IMPL_REFEXEC(UInt8, uint8_t, ULong_t, PyInt_FromLong, PyLongOrInt_AsULong) -CPPYY_IMPL_REFEXEC(Short, short, Long_t, PyInt_FromLong, PyLong_AsLong) -CPPYY_IMPL_REFEXEC(UShort, unsigned short, ULong_t, PyInt_FromLong, PyLongOrInt_AsULong) -CPPYY_IMPL_REFEXEC(Int, Int_t, Long_t, PyInt_FromLong, PyLong_AsLong) -CPPYY_IMPL_REFEXEC(UInt, UInt_t, ULong_t, PyLong_FromUnsignedLong, PyLongOrInt_AsULong) -CPPYY_IMPL_REFEXEC(Long, Long_t, Long_t, PyLong_FromLong, PyLong_AsLong) -CPPYY_IMPL_REFEXEC(ULong, ULong_t, ULong_t, PyLong_FromUnsignedLong, PyLongOrInt_AsULong) -CPPYY_IMPL_REFEXEC(LongLong, Long64_t, Long64_t, PyLong_FromLongLong, PyLong_AsLongLong) -CPPYY_IMPL_REFEXEC(ULongLong, ULong64_t, ULong64_t, PyLong_FromUnsignedLongLong, PyLongOrInt_AsULong64) -CPPYY_IMPL_REFEXEC(Float, float, double, PyFloat_FromDouble, PyFloat_AsDouble) -CPPYY_IMPL_REFEXEC(Double, double, double, PyFloat_FromDouble, PyFloat_AsDouble) -CPPYY_IMPL_REFEXEC(LongDouble, LongDouble_t, LongDouble_t, PyFloat_FromDouble, PyFloat_AsDouble) +CPPYY_IMPL_REFEXEC(Bool, bool, long, CPyCppyy_PyBool_FromLong, PyLong_AsLong) +CPPYY_IMPL_REFEXEC(Char, char, long, CPyCppyy_PyText_FromLong, PyLong_AsLong) +CPPYY_IMPL_REFEXEC(UChar, unsigned char, unsigned long, CPyCppyy_PyText_FromULong, PyLongOrInt_AsULong) +CPPYY_IMPL_REFEXEC(Int8, int8_t, long, PyInt_FromLong, PyLong_AsLong) +CPPYY_IMPL_REFEXEC(UInt8, uint8_t, unsigned long, PyInt_FromLong, PyLongOrInt_AsULong) +CPPYY_IMPL_REFEXEC(Short, short, long, PyInt_FromLong, PyLong_AsLong) +CPPYY_IMPL_REFEXEC(UShort, unsigned short, unsigned long, PyInt_FromLong, PyLongOrInt_AsULong) +CPPYY_IMPL_REFEXEC(Int, int, long, PyInt_FromLong, PyLong_AsLong) +CPPYY_IMPL_REFEXEC(UInt, unsigned int, unsigned long, PyLong_FromUnsignedLong, PyLongOrInt_AsULong) +CPPYY_IMPL_REFEXEC(Long, long, long, PyLong_FromLong, PyLong_AsLong) +CPPYY_IMPL_REFEXEC(ULong, unsigned long, unsigned long, PyLong_FromUnsignedLong, PyLongOrInt_AsULong) +CPPYY_IMPL_REFEXEC(LongLong, PY_LONG_LONG, PY_LONG_LONG, PyLong_FromLongLong, PyLong_AsLongLong) +CPPYY_IMPL_REFEXEC(ULongLong, PY_ULONG_LONG, PY_ULONG_LONG, PyLong_FromUnsignedLongLong, PyLongOrInt_AsULong64) +CPPYY_IMPL_REFEXEC(Float, float, double, PyFloat_FromDouble, PyFloat_AsDouble) +CPPYY_IMPL_REFEXEC(Double, double, double, PyFloat_FromDouble, PyFloat_AsDouble) +CPPYY_IMPL_REFEXEC(LongDouble, PY_LONG_DOUBLE, PY_LONG_DOUBLE, PyFloat_FromDouble, PyFloat_AsDouble) template static inline PyObject* PyComplex_FromComplex(const std::complex& c) { @@ -390,8 +390,9 @@ PyObject* CPyCppyy::STLStringRefExecutor::Execute( { // execute with argument , return python string return value std::string* result = (std::string*)GILCallR(method, self, ctxt); - if (!fAssignable) + if (!fAssignable) { return CPyCppyy_PyText_FromStringAndSize(result->c_str(), result->size()); + } *result = std::string( CPyCppyy_PyText_AsString(fAssignable), CPyCppyy_PyText_GET_SIZE(fAssignable)); @@ -408,6 +409,7 @@ PyObject* CPyCppyy::VoidExecutor::Execute( { // execute with argument , return None GILCallV(method, self, ctxt); + if (PyErr_Occurred()) return nullptr; Py_RETURN_NONE; } @@ -425,6 +427,20 @@ PyObject* CPyCppyy::CStringExecutor::Execute( return CPyCppyy_PyText_FromString(result); } +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::CStringRefExecutor::Execute( + Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) +{ +// execute with argument , construct python string return value + char** result = (char**)GILCallR(method, self, ctxt); + if (!result || !*result) { + Py_INCREF(PyStrings::gEmptyString); + return PyStrings::gEmptyString; + } + + return CPyCppyy_PyText_FromString(*result); +} + //---------------------------------------------------------------------------- PyObject* CPyCppyy::WCStringExecutor::Execute( Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) @@ -464,7 +480,7 @@ PyObject* CPyCppyy::CString32Executor::Execute( char32_t w = U'\0'; return PyUnicode_DecodeUTF32((const char*)&w, 0, nullptr, nullptr); } - + return PyUnicode_DecodeUTF32((const char*)result, std::char_traits::length(result)*sizeof(char32_t), nullptr, nullptr); } @@ -475,41 +491,43 @@ PyObject* CPyCppyy::VoidArrayExecutor::Execute( Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) { // execute with argument , construct python long return value - Long_t* result = (Long_t*)GILCallR(method, self, ctxt); + intptr_t* result = (intptr_t*)GILCallR(method, self, ctxt); if (!result) { Py_INCREF(gNullPtrObject); return gNullPtrObject; } - return CreatePointerView(result); + return CreatePointerView(result, fShape); } //---------------------------------------------------------------------------- -#define CPPYY_IMPL_ARRAY_EXEC(name, type) \ +#define CPPYY_IMPL_ARRAY_EXEC(name, type, suffix) \ PyObject* CPyCppyy::name##ArrayExecutor::Execute( \ Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) \ { \ - return CreateLowLevelView((type*)GILCallR(method, self, ctxt)); \ + return CreateLowLevelView##suffix((type*)GILCallR(method, self, ctxt), fShape); \ } -CPPYY_IMPL_ARRAY_EXEC(Bool, bool) -CPPYY_IMPL_ARRAY_EXEC(UChar, unsigned char) +CPPYY_IMPL_ARRAY_EXEC(Bool, bool, ) +CPPYY_IMPL_ARRAY_EXEC(UChar, unsigned char, ) #if __cplusplus > 201402L -CPPYY_IMPL_ARRAY_EXEC(Byte, std::byte) +CPPYY_IMPL_ARRAY_EXEC(Byte, std::byte, ) #endif -CPPYY_IMPL_ARRAY_EXEC(Short, short) -CPPYY_IMPL_ARRAY_EXEC(UShort, unsigned short) -CPPYY_IMPL_ARRAY_EXEC(Int, int) -CPPYY_IMPL_ARRAY_EXEC(UInt, unsigned int) -CPPYY_IMPL_ARRAY_EXEC(Long, long) -CPPYY_IMPL_ARRAY_EXEC(ULong, unsigned long) -CPPYY_IMPL_ARRAY_EXEC(LLong, long long) -CPPYY_IMPL_ARRAY_EXEC(ULLong, unsigned long long) -CPPYY_IMPL_ARRAY_EXEC(Float, float) -CPPYY_IMPL_ARRAY_EXEC(Double, double) -CPPYY_IMPL_ARRAY_EXEC(ComplexF, std::complex) -CPPYY_IMPL_ARRAY_EXEC(ComplexD, std::complex) -CPPYY_IMPL_ARRAY_EXEC(ComplexI, std::complex) -CPPYY_IMPL_ARRAY_EXEC(ComplexL, std::complex) +CPPYY_IMPL_ARRAY_EXEC(Int8, int8_t, _i8) +CPPYY_IMPL_ARRAY_EXEC(UInt8, uint8_t, _i8) +CPPYY_IMPL_ARRAY_EXEC(Short, short, ) +CPPYY_IMPL_ARRAY_EXEC(UShort, unsigned short, ) +CPPYY_IMPL_ARRAY_EXEC(Int, int, ) +CPPYY_IMPL_ARRAY_EXEC(UInt, unsigned int, ) +CPPYY_IMPL_ARRAY_EXEC(Long, long, ) +CPPYY_IMPL_ARRAY_EXEC(ULong, unsigned long, ) +CPPYY_IMPL_ARRAY_EXEC(LLong, long long, ) +CPPYY_IMPL_ARRAY_EXEC(ULLong, unsigned long long, ) +CPPYY_IMPL_ARRAY_EXEC(Float, float, ) +CPPYY_IMPL_ARRAY_EXEC(Double, double, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexF, std::complex, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexD, std::complex, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexI, std::complex, ) +CPPYY_IMPL_ARRAY_EXEC(ComplexL, std::complex, ) //- special cases ------------------------------------------------------------ @@ -605,7 +623,7 @@ PyObject* CPyCppyy::InstanceExecutor::Execute( return nullptr; // python ref counting will now control this object's life span; it will be -// deleted b/c it is marked as a by-value object (unless C++ ownership is set) +// deleted b/c it is marked as a by-value object owned by python (from fFlags) return pyobj; } @@ -617,6 +635,21 @@ CPyCppyy::IteratorExecutor::IteratorExecutor(Cppyy::TCppType_t klass) : fFlags |= CPPInstance::kNoWrapConv; // adds to flags from base class } +//---------------------------------------------------------------------------- +PyObject* CPyCppyy::IteratorExecutor::Execute( + Cppyy::TCppMethod_t method, Cppyy::TCppObject_t self, CallContext* ctxt) +{ + PyObject* iter = this->InstanceExecutor::Execute(method, self, ctxt); + if (iter && ctxt->fPyContext) { + // set life line to tie iterator life time to the container (which may + // be a temporary) + std::ostringstream attr_name; + attr_name << "__" << (intptr_t)iter; + if (PyObject_SetAttrString(ctxt->fPyContext, (char*)attr_name.str().c_str(), iter)) + PyErr_Clear(); + } + return iter; +} //---------------------------------------------------------------------------- PyObject* CPyCppyy::InstanceRefExecutor::Execute( @@ -722,8 +755,7 @@ PyObject* CPyCppyy::InstanceArrayExecutor::Execute( { // execute with argument , construct TupleOfInstances from // return value - dim_t dims[] = {1, (dim_t)fArraySize}; - return BindCppObjectArray((void*)GILCallR(method, self, ctxt), fClass, dims); + return BindCppObjectArray((void*)GILCallR(method, self, ctxt), fClass, {fSize}); } //---------------------------------------------------------------------------- @@ -760,7 +792,7 @@ PyObject* CPyCppyy::FunctionPointerExecutor::Execute( } //- factories ---------------------------------------------------------------- -CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType) +CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType, cdims_t dims) { // The matching of the fulltype to an executor factory goes through up to 4 levels: // 1) full, qualified match @@ -773,7 +805,7 @@ CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType) // an exactly matching executor is best ExecFactories_t::iterator h = gExecFactories.find(fullType); if (h != gExecFactories.end()) - return (h->second)(); + return (h->second)(dims); // resolve typedefs etc. const std::string& resolvedType = Cppyy::ResolveName(fullType); @@ -782,18 +814,18 @@ CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType) if (resolvedType != fullType) { h = gExecFactories.find(resolvedType); if (h != gExecFactories.end()) - return (h->second)(); + return (h->second)(dims); } //-- nothing? ok, collect information about the type and possible qualifiers/decorators bool isConst = strncmp(resolvedType.c_str(), "const", 5) == 0; - const std::string& cpd = Utility::Compound(resolvedType); + const std::string& cpd = TypeManip::compound(resolvedType); std::string realType = TypeManip::clean_type(resolvedType, false); // accept unqualified type (as python does not know about qualifiers) h = gExecFactories.find(realType + cpd); if (h != gExecFactories.end()) - return (h->second)(); + return (h->second)(dims); // drop const, as that is mostly meaningless to python (with the exception // of c-strings, but those are specialized in the converter map) @@ -801,20 +833,27 @@ CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType) realType = TypeManip::remove_const(realType); h = gExecFactories.find(realType + cpd); if (h != gExecFactories.end()) - return (h->second)(); + return (h->second)(dims); + } + +// simple array types + if (!cpd.empty() && (std::string::size_type)std::count(cpd.begin(), cpd.end(), '*') == cpd.size()) { + h = gExecFactories.find(realType + " ptr"); + if (h != gExecFactories.end()) + return (h->second)((!dims || dims.ndim() < (dim_t)cpd.size()) ? dims_t(cpd.size()) : dims); } //-- still nothing? try pointer instead of array (for builtins) if (cpd == "[]") { h = gExecFactories.find(realType + "*"); if (h != gExecFactories.end()) - return (h->second)(); // TODO: use array size + return (h->second)(dims); } // C++ classes and special cases Executor* result = 0; if (Cppyy::TCppType_t klass = Cppyy::GetScope(realType)) { - if (resolvedType.find("iterator") != std::string::npos || gIteratorTypes.find(fullType) != gIteratorTypes.end()) { + if (Utility::IsSTLIterator(realType) || gIteratorTypes.find(fullType) != gIteratorTypes.end()) { if (cpd == "") return new IteratorExecutor(klass); } @@ -828,7 +867,7 @@ CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType) else if (cpd == "*&") result = new InstancePtrRefExecutor(klass); else if (cpd == "[]") { - Py_ssize_t asize = Utility::ArraySize(resolvedType); + Py_ssize_t asize = TypeManip::array_size(resolvedType); if (0 < asize) result = new InstanceArrayExecutor(klass, asize); else @@ -846,12 +885,12 @@ CPyCppyy::Executor* CPyCppyy::CreateExecutor(const std::string& fullType) resolvedType.substr(0, pos1), resolvedType.substr(pos2+2, pos3-pos2-1)); } else { // unknown: void* may work ("user knows best"), void will fail on use of return value - h = (cpd == "") ? gExecFactories.find("void") : gExecFactories.find("void*"); + h = (cpd == "") ? gExecFactories.find("void") : gExecFactories.find("void ptr"); } if (!result && h != gExecFactories.end()) // executor factory available, use it to create executor - result = (h->second)(); + result = (h->second)(dims); return result; // may still be null } @@ -903,7 +942,17 @@ namespace { using namespace CPyCppyy; -#define WSTRING "basic_string,allocator >" +#define WSTRING1 "std::basic_string" +#define WSTRING2 "std::basic_string,std::allocator>" + +//-- aliasing special case: C complex (is binary compatible with C++ std::complex) +#ifndef _WIN32 +#define CCOMPLEX_D "_Complex double" +#define CCOMPLEX_F "_Complex float" +#else +#define CCOMPLEX_D "_C_double_complex" +#define CCOMPLEX_F "_C_float_complex" +#endif struct InitExecFactories_t { public: @@ -912,119 +961,143 @@ struct InitExecFactories_t { CPyCppyy::ExecFactories_t& gf = gExecFactories; // factories for built-ins - gf["bool"] = (ef_t)+[]() { static BoolExecutor e{}; return &e; }; - gf["bool&"] = (ef_t)+[]() { return new BoolRefExecutor{}; }; - gf["const bool&"] = (ef_t)+[]() { static BoolConstRefExecutor e{}; return &e; }; - gf["char"] = (ef_t)+[]() { static CharExecutor e{}; return &e; }; + gf["bool"] = (ef_t)+[](cdims_t) { static BoolExecutor e{}; return &e; }; + gf["bool&"] = (ef_t)+[](cdims_t) { return new BoolRefExecutor{}; }; + gf["const bool&"] = (ef_t)+[](cdims_t) { static BoolConstRefExecutor e{}; return &e; }; + gf["char"] = (ef_t)+[](cdims_t) { static CharExecutor e{}; return &e; }; gf["signed char"] = gf["char"]; - gf["unsigned char"] = (ef_t)+[]() { static UCharExecutor e{}; return &e; }; - gf["char&"] = (ef_t)+[]() { return new CharRefExecutor{}; }; + gf["unsigned char"] = (ef_t)+[](cdims_t) { static UCharExecutor e{}; return &e; }; + gf["char&"] = (ef_t)+[](cdims_t) { return new CharRefExecutor{}; }; gf["signed char&"] = gf["char&"]; - gf["unsigned char&"] = (ef_t)+[]() { return new UCharRefExecutor{}; }; - gf["const char&"] = (ef_t)+[]() { static CharConstRefExecutor e{}; return &e; }; + gf["unsigned char&"] = (ef_t)+[](cdims_t) { return new UCharRefExecutor{}; }; + gf["const char&"] = (ef_t)+[](cdims_t) { static CharConstRefExecutor e{}; return &e; }; gf["const signed char&"] = gf["const char&"]; - gf["const unsigned char&"] = (ef_t)+[]() { static UCharConstRefExecutor e{}; return &e; }; - gf["wchar_t"] = (ef_t)+[]() { static WCharExecutor e{}; return &e; }; - gf["char16_t"] = (ef_t)+[]() { static Char16Executor e{}; return &e; }; - gf["char32_t"] = (ef_t)+[]() { static Char32Executor e{}; return &e; }; - gf["int8_t"] = (ef_t)+[]() { static Int8Executor e{}; return &e; }; - gf["int8_t&"] = (ef_t)+[]() { return new Int8RefExecutor{}; }; - gf["const int8_t&"] = (ef_t)+[]() { static Int8RefExecutor e{}; return &e; }; - gf["uint8_t"] = (ef_t)+[]() { static UInt8Executor e{}; return &e; }; - gf["uint8_t&"] = (ef_t)+[]() { return new UInt8RefExecutor{}; }; - gf["const uint8_t&"] = (ef_t)+[]() { static UInt8RefExecutor e{}; return &e; }; - gf["short"] = (ef_t)+[]() { static ShortExecutor e{}; return &e; }; - gf["short&"] = (ef_t)+[]() { return new ShortRefExecutor{}; }; - gf["int"] = (ef_t)+[]() { static IntExecutor e{}; return &e; }; - gf["int&"] = (ef_t)+[]() { return new IntRefExecutor{}; }; + gf["const unsigned char&"] = (ef_t)+[](cdims_t) { static UCharConstRefExecutor e{}; return &e; }; + gf["wchar_t"] = (ef_t)+[](cdims_t) { static WCharExecutor e{}; return &e; }; + gf["char16_t"] = (ef_t)+[](cdims_t) { static Char16Executor e{}; return &e; }; + gf["char32_t"] = (ef_t)+[](cdims_t) { static Char32Executor e{}; return &e; }; + gf["int8_t"] = (ef_t)+[](cdims_t) { static Int8Executor e{}; return &e; }; + gf["int8_t&"] = (ef_t)+[](cdims_t) { return new Int8RefExecutor{}; }; + gf["const int8_t&"] = (ef_t)+[](cdims_t) { static Int8RefExecutor e{}; return &e; }; + gf["uint8_t"] = (ef_t)+[](cdims_t) { static UInt8Executor e{}; return &e; }; + gf["uint8_t&"] = (ef_t)+[](cdims_t) { return new UInt8RefExecutor{}; }; + gf["const uint8_t&"] = (ef_t)+[](cdims_t) { static UInt8RefExecutor e{}; return &e; }; + gf["short"] = (ef_t)+[](cdims_t) { static ShortExecutor e{}; return &e; }; + gf["short&"] = (ef_t)+[](cdims_t) { return new ShortRefExecutor{}; }; + gf["int"] = (ef_t)+[](cdims_t) { static IntExecutor e{}; return &e; }; + gf["int&"] = (ef_t)+[](cdims_t) { return new IntRefExecutor{}; }; gf["unsigned short"] = gf["int"]; - gf["unsigned short&"] = (ef_t)+[]() { return new UShortRefExecutor{}; }; - gf["unsigned long"] = (ef_t)+[]() { static ULongExecutor e{}; return &e; }; - gf["unsigned long&"] = (ef_t)+[]() { return new ULongRefExecutor{}; }; + gf["unsigned short&"] = (ef_t)+[](cdims_t) { return new UShortRefExecutor{}; }; + gf["unsigned long"] = (ef_t)+[](cdims_t) { static ULongExecutor e{}; return &e; }; + gf["unsigned long&"] = (ef_t)+[](cdims_t) { return new ULongRefExecutor{}; }; gf["unsigned int"] = gf["unsigned long"]; - gf["unsigned int&"] = (ef_t)+[]() { return new UIntRefExecutor{}; }; - gf["long"] = (ef_t)+[]() { static LongExecutor e{}; return &e; }; - gf["long&"] = (ef_t)+[]() { return new LongRefExecutor{}; }; - gf["unsigned long"] = (ef_t)+[]() { static ULongExecutor e{}; return &e; }; - gf["unsigned long&"] = (ef_t)+[]() { return new ULongRefExecutor{}; }; - gf["long long"] = (ef_t)+[]() { static LongLongExecutor e{}; return &e; }; - gf["long long&"] = (ef_t)+[]() { return new LongLongRefExecutor{}; }; - gf["unsigned long long"] = (ef_t)+[]() { static ULongLongExecutor e{}; return &e; }; - gf["unsigned long long&"] = (ef_t)+[]() { return new ULongLongRefExecutor{}; }; - - gf["float"] = (ef_t)+[]() { static FloatExecutor e{}; return &e; }; - gf["float&"] = (ef_t)+[]() { return new FloatRefExecutor{}; }; - gf["double"] = (ef_t)+[]() { static DoubleExecutor e{}; return &e; }; - gf["double&"] = (ef_t)+[]() { return new DoubleRefExecutor{}; }; - gf["long double"] = (ef_t)+[]() { static LongDoubleExecutor e{}; return &e; }; // TODO: lost precision - gf["long double&"] = (ef_t)+[]() { return new LongDoubleRefExecutor{}; }; - gf["void"] = (ef_t)+[]() { static VoidExecutor e{}; return &e; }; + gf["unsigned int&"] = (ef_t)+[](cdims_t) { return new UIntRefExecutor{}; }; + gf["long"] = (ef_t)+[](cdims_t) { static LongExecutor e{}; return &e; }; + gf["long&"] = (ef_t)+[](cdims_t) { return new LongRefExecutor{}; }; + gf["unsigned long"] = (ef_t)+[](cdims_t) { static ULongExecutor e{}; return &e; }; + gf["unsigned long&"] = (ef_t)+[](cdims_t) { return new ULongRefExecutor{}; }; + gf["long long"] = (ef_t)+[](cdims_t) { static LongLongExecutor e{}; return &e; }; + gf["long long&"] = (ef_t)+[](cdims_t) { return new LongLongRefExecutor{}; }; + gf["unsigned long long"] = (ef_t)+[](cdims_t) { static ULongLongExecutor e{}; return &e; }; + gf["unsigned long long&"] = (ef_t)+[](cdims_t) { return new ULongLongRefExecutor{}; }; + + gf["float"] = (ef_t)+[](cdims_t) { static FloatExecutor e{}; return &e; }; + gf["float&"] = (ef_t)+[](cdims_t) { return new FloatRefExecutor{}; }; + gf["double"] = (ef_t)+[](cdims_t) { static DoubleExecutor e{}; return &e; }; + gf["double&"] = (ef_t)+[](cdims_t) { return new DoubleRefExecutor{}; }; + gf["long double"] = (ef_t)+[](cdims_t) { static LongDoubleExecutor e{}; return &e; }; // TODO: lost precision + gf["long double&"] = (ef_t)+[](cdims_t) { return new LongDoubleRefExecutor{}; }; + gf["std::complex"] = (ef_t)+[](cdims_t) { static ComplexDExecutor e{}; return &e; }; + gf["std::complex&"] = (ef_t)+[](cdims_t) { return new ComplexDRefExecutor{}; }; + gf["void"] = (ef_t)+[](cdims_t) { static VoidExecutor e{}; return &e; }; // pointer/array factories - gf["void*"] = (ef_t)+[]() { static VoidArrayExecutor e{}; return &e; }; - gf["bool*"] = (ef_t)+[]() { static BoolArrayExecutor e{}; return &e; }; - gf["unsigned char*"] = (ef_t)+[]() { static UCharArrayExecutor e{}; return &e; }; - gf["const unsigned char*"] = gf["unsigned char*"]; + gf["void ptr"] = (ef_t)+[](cdims_t d) { return new VoidArrayExecutor{d}; }; + gf["bool ptr"] = (ef_t)+[](cdims_t d) { return new BoolArrayExecutor{d}; }; + gf["unsigned char ptr"] = (ef_t)+[](cdims_t d) { return new UCharArrayExecutor{d}; }; + gf["const unsigned char ptr"] = gf["unsigned char ptr"]; #if __cplusplus > 201402L - gf["byte*"] = (ef_t)+[]() { static ByteArrayExecutor e{}; return &e; }; - gf["const byte*"] = gf["byte*"]; + gf["std::byte ptr"] = (ef_t)+[](cdims_t d) { return new ByteArrayExecutor{d}; }; + gf["const std::byte ptr"] = gf["std::byte ptr"]; #endif - gf["short*"] = (ef_t)+[]() { static ShortArrayExecutor e{}; return &e; }; - gf["unsigned short*"] = (ef_t)+[]() { static UShortArrayExecutor e{}; return &e; }; - gf["int*"] = (ef_t)+[]() { static IntArrayExecutor e{}; return &e; }; - gf["unsigned int*"] = (ef_t)+[]() { static UIntArrayExecutor e{}; return &e; }; - gf["long*"] = (ef_t)+[]() { static LongArrayExecutor e{}; return &e; }; - gf["unsigned long*"] = (ef_t)+[]() { static ULongArrayExecutor e{}; return &e; }; - gf["long long*"] = (ef_t)+[]() { static LLongArrayExecutor e{}; return &e; }; - gf["unsigned long long*"] = (ef_t)+[]() { static ULLongArrayExecutor e{}; return &e; }; - gf["float*"] = (ef_t)+[]() { static FloatArrayExecutor e{}; return &e; }; - gf["double*"] = (ef_t)+[]() { static DoubleArrayExecutor e{}; return &e; }; - gf["complex*"] = (ef_t)+[]() { static ComplexFArrayExecutor e{}; return &e; }; - gf["complex*"] = (ef_t)+[]() { static ComplexDArrayExecutor e{}; return &e; }; - gf["complex*"] = (ef_t)+[]() { static ComplexIArrayExecutor e{}; return &e; }; - gf["complex*"] = (ef_t)+[]() { static ComplexLArrayExecutor e{}; return &e; }; + gf["int8_t ptr"] = (ef_t)+[](cdims_t d) { return new Int8ArrayExecutor{d}; }; + gf["uint8_t ptr"] = (ef_t)+[](cdims_t d) { return new UInt8ArrayExecutor{d}; }; + gf["short ptr"] = (ef_t)+[](cdims_t d) { return new ShortArrayExecutor{d}; }; + gf["unsigned short ptr"] = (ef_t)+[](cdims_t d) { return new UShortArrayExecutor{d}; }; + gf["int ptr"] = (ef_t)+[](cdims_t d) { return new IntArrayExecutor{d}; }; + gf["unsigned int ptr"] = (ef_t)+[](cdims_t d) { return new UIntArrayExecutor{d}; }; + gf["long ptr"] = (ef_t)+[](cdims_t d) { return new LongArrayExecutor{d}; }; + gf["unsigned long ptr"] = (ef_t)+[](cdims_t d) { return new ULongArrayExecutor{d}; }; + gf["long long ptr"] = (ef_t)+[](cdims_t d) { return new LLongArrayExecutor{d}; }; + gf["unsigned long long ptr"] = (ef_t)+[](cdims_t d) { return new ULLongArrayExecutor{d}; }; + gf["float ptr"] = (ef_t)+[](cdims_t d) { return new FloatArrayExecutor{d}; }; + gf["double ptr"] = (ef_t)+[](cdims_t d) { return new DoubleArrayExecutor{d}; }; + gf["std::complex ptr"] = (ef_t)+[](cdims_t d) { return new ComplexFArrayExecutor{d}; }; + gf["std::complex ptr"] = (ef_t)+[](cdims_t d) { return new ComplexDArrayExecutor{d}; }; + gf["std::complex ptr"] = (ef_t)+[](cdims_t d) { return new ComplexIArrayExecutor{d}; }; + gf["std::complex ptr"] = (ef_t)+[](cdims_t d) { return new ComplexLArrayExecutor{d}; }; // aliases gf["internal_enum_type_t"] = gf["int"]; gf["internal_enum_type_t&"] = gf["int&"]; - gf["internal_enum_type_t*"] = gf["int*"]; + gf["internal_enum_type_t ptr"] = gf["int ptr"]; #if __cplusplus > 201402L - gf["byte"] = gf["uint8_t"]; - gf["byte&"] = gf["uint8_t&"]; - gf["const byte&"] = gf["const uint8_t&"]; + gf["std::byte"] = gf["uint8_t"]; + gf["std::byte&"] = gf["uint8_t&"]; + gf["const std::byte&"] = gf["const uint8_t&"]; +#endif + gf["std::int8_t"] = gf["int8_t"]; + gf["std::int8_t&"] = gf["int8_t&"]; + gf["const std::int8_t&"] = gf["const int8_t&"]; + gf["std::int8_t ptr"] = gf["int8_t ptr"]; + gf["std::uint8_t"] = gf["uint8_t"]; + gf["std::uint8_t&"] = gf["uint8_t&"]; + gf["const std::uint8_t&"] = gf["const uint8_t&"]; + gf["std::uint8_t ptr"] = gf["uint8_t ptr"]; +#ifdef _WIN32 + gf["__int64"] = gf["long long"]; + gf["__int64&"] = gf["long long&"]; + gf["__int64 ptr"] = gf["long long ptr"]; + gf["unsigned __int64"] = gf["unsigned long long"]; + gf["unsigned __int64&"] = gf["unsigned long long&"]; + gf["unsigned __int64 ptr"] = gf["unsigned long long ptr"]; #endif + gf[CCOMPLEX_D] = gf["std::complex"]; + gf[CCOMPLEX_D "&"] = gf["std::complex&"]; + gf[CCOMPLEX_F " ptr"] = gf["std::complex ptr"]; + gf[CCOMPLEX_D " ptr"] = gf["std::complex ptr"]; gf["Long64_t"] = gf["long long"]; gf["Long64_t&"] = gf["long long&"]; - gf["Long64_t*"] = gf["long long*"]; + gf["Long64_t ptr"] = gf["long long ptr"]; gf["ULong64_t"] = gf["unsigned long long"]; gf["ULong64_t&"] = gf["unsigned long long&"]; - gf["ULong64_t*"] = gf["unsigned long long*"]; + gf["ULong64_t ptr"] = gf["unsigned long long ptr"]; gf["Float16_t"] = gf["float"]; gf["Float16_t&"] = gf["float&"]; gf["Double32_t"] = gf["double"]; gf["Double32_t&"] = gf["double&"]; // factories for special cases - gf["const char*"] = (ef_t)+[]() { static CStringExecutor e{}; return &e; }; + gf["const char*"] = (ef_t)+[](cdims_t) { static CStringExecutor e{}; return &e; }; gf["char*"] = gf["const char*"]; + gf["const char*&"] = (ef_t)+[](cdims_t) { static CStringRefExecutor e{}; return &e; }; + gf["char*&"] = gf["const char*&"]; gf["const signed char*"] = gf["const char*"]; gf["signed char*"] = gf["char*"]; - gf["wchar_t*"] = (ef_t)+[]() { static WCStringExecutor e{}; return &e;}; - gf["char16_t*"] = (ef_t)+[]() { static CString16Executor e{}; return &e;}; - gf["char32_t*"] = (ef_t)+[]() { static CString32Executor e{}; return &e;}; - gf["std::string"] = (ef_t)+[]() { static STLStringExecutor e{}; return &e; }; + gf["wchar_t*"] = (ef_t)+[](cdims_t) { static WCStringExecutor e{}; return &e;}; + gf["char16_t*"] = (ef_t)+[](cdims_t) { static CString16Executor e{}; return &e;}; + gf["char32_t*"] = (ef_t)+[](cdims_t) { static CString32Executor e{}; return &e;}; + gf["std::string"] = (ef_t)+[](cdims_t) { static STLStringExecutor e{}; return &e; }; gf["string"] = gf["std::string"]; - gf["std::string&"] = (ef_t)+[]() { return new STLStringRefExecutor{}; }; + gf["std::string&"] = (ef_t)+[](cdims_t) { return new STLStringRefExecutor{}; }; gf["string&"] = gf["std::string&"]; - gf["std::wstring"] = (ef_t)+[]() { static STLWStringExecutor e{}; return &e; }; - gf["std::" WSTRING] = gf["std::wstring"]; - gf[WSTRING] = gf["std::wstring"]; - gf["complex"] = (ef_t)+[]() { static ComplexDExecutor e{}; return &e; }; - gf["complex&"] = (ef_t)+[]() { return new ComplexDRefExecutor{}; }; - gf["__init__"] = (ef_t)+[]() { static ConstructorExecutor e{}; return &e; }; - gf["PyObject*"] = (ef_t)+[]() { static PyObjectExecutor e{}; return &e; }; + gf["std::wstring"] = (ef_t)+[](cdims_t) { static STLWStringExecutor e{}; return &e; }; + gf[WSTRING1] = gf["std::wstring"]; + gf[WSTRING2] = gf["std::wstring"]; + gf["__init__"] = (ef_t)+[](cdims_t) { static ConstructorExecutor e{}; return &e; }; + gf["PyObject*"] = (ef_t)+[](cdims_t) { static PyObjectExecutor e{}; return &e; }; gf["_object*"] = gf["PyObject*"]; - gf["FILE*"] = gf["void*"]; + gf["FILE*"] = gf["void ptr"]; } } initExecvFactories_; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Executors.h b/bindings/pyroot/cppyy/CPyCppyy/src/Executors.h index 22785d7776918..da4ac0f2f8558 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Executors.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Executors.h @@ -1,6 +1,9 @@ #ifndef CPYCPPYY_EXECUTORS_H #define CPYCPPYY_EXECUTORS_H +// Bindings +#include "Dimensions.h" + // Standard #include @@ -29,9 +32,9 @@ class RefExecutor : public Executor { }; // create/destroy executor from fully qualified type (public API) -CPYCPPYY_EXPORT Executor* CreateExecutor(const std::string& fullType); +CPYCPPYY_EXPORT Executor* CreateExecutor(const std::string& fullType, cdims_t = 0); CPYCPPYY_EXPORT void DestroyExecutor(Executor* p); -typedef Executor* (*ef_t) (); +typedef Executor* (*ef_t) (cdims_t); CPYCPPYY_EXPORT bool RegisterExecutor(const std::string& name, ef_t fac); CPYCPPYY_EXPORT bool UnregisterExecutor(const std::string& name); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx index e119774de918d..028873c3a2b7a 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.cxx @@ -2,10 +2,13 @@ #include "CPyCppyy.h" #include "LowLevelViews.h" #include "Converters.h" +#include "CustomPyTypes.h" +#include "PyStrings.h" // Standard #include #include +#include #include @@ -16,18 +19,36 @@ // converters, and typed results and assignments are supported. Code reused // under PSF License Version 2. + +//- helpers ------------------------------------------------------------------ +static inline void set_strides(Py_buffer& view, size_t itemsize, bool isfix) { + if (isfix) { + Py_ssize_t stride = itemsize; + for (Py_ssize_t idim = view.ndim-1; 0 <= idim; --idim) { + view.strides[idim] = stride; + stride *= view.shape[idim]; + } + } else { + view.strides[view.ndim-1] = itemsize; + for (Py_ssize_t idim = 0; idim < view.ndim-1; ++idim) + view.strides[idim] = view.itemsize; + } +} + + //= CPyCppyy low level view construction/destruction ========================= static CPyCppyy::LowLevelView* ll_new(PyTypeObject* subtype, PyObject*, PyObject*) { // Create a new low level ptr type CPyCppyy::LowLevelView* pyobj = (CPyCppyy::LowLevelView*)subtype->tp_alloc(subtype, 0); - if (!pyobj) { - PyErr_Print(); + if (!pyobj) return nullptr; - } + memset(&pyobj->fBufInfo, 0, sizeof(Py_buffer)); + (intptr_t&)pyobj->fBufInfo.internal |= CPyCppyy::LowLevelView::kIsCppArray; pyobj->fBuf = nullptr; pyobj->fConverter = nullptr; + pyobj->fElemCnv = nullptr; return pyobj; } @@ -38,64 +59,56 @@ static void ll_dealloc(CPyCppyy::LowLevelView* pyobj) // Destruction requires the deletion of the converter (if any) PyMem_Free(pyobj->fBufInfo.shape); PyMem_Free(pyobj->fBufInfo.strides); - if (pyobj->fConverter && pyobj->fConverter->HasState()) delete pyobj->fConverter; + if ((intptr_t)pyobj->fBufInfo.internal & CPyCppyy::LowLevelView::kIsOwner) { + if ((intptr_t)pyobj->fBufInfo.internal & CPyCppyy::LowLevelView::kIsCppArray) + delete [] pyobj->fBuf; + else + free(pyobj->fBuf); + } + + if (pyobj->fElemCnv != pyobj->fConverter &&\ + pyobj->fElemCnv && pyobj->fElemCnv->HasState()) + delete pyobj->fElemCnv; + + if (pyobj->fConverter && pyobj->fConverter->HasState()) + delete pyobj->fConverter; + Py_TYPE(pyobj)->tp_free((PyObject*)pyobj); } -//--------------------------------------------------------------------------- -static PyObject* ll_typecode(CPyCppyy::LowLevelView* self, void*) -{ - return CPyCppyy_PyText_FromString((char*)self->fBufInfo.format); +//---------------------------------------------------------------------------- +#define CPYCPPYY_LL_FLAG_GETSET(name, flag, doc) \ +static PyObject* ll_get##name(CPyCppyy::LowLevelView* pyobj) \ +{ \ + return PyBool_FromLong((long)((intptr_t)pyobj->fBufInfo.internal & flag));\ +} \ + \ +static int ll_set##name(CPyCppyy::LowLevelView* pyobj, PyObject* value, void*)\ +{ \ + long settrue = PyLong_AsLong(value); \ + if (settrue == -1 && PyErr_Occurred()) { \ + PyErr_SetString(PyExc_ValueError, #doc" should be either True or False");\ + return -1; \ + } \ + \ + if ((bool)settrue) \ + (intptr_t&)pyobj->fBufInfo.internal |= flag; \ + else \ + (intptr_t&)pyobj->fBufInfo.internal &= ~flag; \ + \ + return 0; \ } -//--------------------------------------------------------------------------- -static PyGetSetDef ll_getset[] = { - {(char*)"format", (getter)ll_typecode, nullptr, nullptr, nullptr}, - {(char*)"typecode", (getter)ll_typecode, nullptr, nullptr, nullptr}, - {(char*)nullptr, nullptr, nullptr, nullptr, nullptr } -}; - +CPYCPPYY_LL_FLAG_GETSET(ownership, CPyCppyy::LowLevelView::kIsOwner, __python_owns__) +CPYCPPYY_LL_FLAG_GETSET(cpparray, CPyCppyy::LowLevelView::kIsCppArray, __cpp_array__) //--------------------------------------------------------------------------- -static PyObject* ll_reshape(CPyCppyy::LowLevelView* self, PyObject* shape) +static PyObject* ll_typecode(CPyCppyy::LowLevelView* self, void*) { -// Allow the user to fix up the actual (type-strided) size of the buffer. - if (!PyTuple_Check(shape) || PyTuple_GET_SIZE(shape) != 1) { - if (shape) { - PyObject* pystr = PyObject_Str(shape); - if (pystr) { - PyErr_Format(PyExc_TypeError, "tuple object of length 1 expected, received %s", - CPyCppyy_PyText_AsStringChecked(pystr)); - Py_DECREF(pystr); - return nullptr; - } - } - PyErr_SetString(PyExc_TypeError, "tuple object of length 1 expected"); - return nullptr; - } - - Py_ssize_t nlen = PyInt_AsSsize_t(PyTuple_GET_ITEM(shape, 0)); - if (nlen == -1 && PyErr_Occurred()) - return nullptr; - - self->fBufInfo.len = nlen * self->fBufInfo.itemsize; - if (self->fBufInfo.ndim == 1 && self->fBufInfo.shape) - self->fBufInfo.shape[0] = nlen; - else { - PyErr_SetString(PyExc_TypeError, "unsupported buffer dimensions"); - return nullptr; - } - - Py_RETURN_NONE; + return CPyCppyy_PyText_FromString((char*)self->fBufInfo.format); } -//--------------------------------------------------------------------------- -static PyMethodDef ll_methods[] = { - {(char*)"reshape", (PyCFunction)ll_reshape, METH_O, nullptr}, - {(char*)nullptr, nullptr, 0, nullptr} -}; - //- Copy memoryview buffers ================================================= @@ -229,10 +242,23 @@ static char* lookup_dimension(Py_buffer& view, char* ptr, int dim, Py_ssize_t in assert(view.strides); nitems = view.shape[dim]; - if (index < 0) - index += nitems; + if (index < 0) { + if (nitems != CPyCppyy::UNKNOWN_SIZE) + index += nitems; + else { + PyErr_Format(PyExc_IndexError, + "negative index not supported on dimension %d with unknown size", dim + 1); + return nullptr; + } + } - if (index < 0 || index >= nitems) { + if (view.strides[dim] == CPyCppyy::UNKNOWN_SIZE) { + PyErr_Format(PyExc_IndexError, + "multi index not supported on dimension %d with unknown stride", dim + 1); + return nullptr; + } + + if (nitems != CPyCppyy::UNKNOWN_SIZE && (index < 0 || index >= nitems)) { PyErr_Format(PyExc_IndexError, "index out of bounds on dimension %d", dim + 1); return nullptr; @@ -257,6 +283,7 @@ static inline void* ptr_from_index(CPyCppyy::LowLevelView* llview, Py_ssize_t in static void* ptr_from_tuple(CPyCppyy::LowLevelView* llview, PyObject* tup) { Py_buffer& view = llview->fBufInfo; + Py_ssize_t nindices = PyTuple_GET_SIZE(tup); if (nindices > view.ndim) { PyErr_Format(PyExc_TypeError, @@ -271,9 +298,13 @@ static void* ptr_from_tuple(CPyCppyy::LowLevelView* llview, PyObject* tup) PyExc_IndexError); if (index == -1 && PyErr_Occurred()) return nullptr; + ptr = lookup_dimension(view, ptr, (int)dim, index); if (!ptr) return nullptr; + + if (!((intptr_t)view.internal & CPyCppyy::LowLevelView::kIsFixed) && dim != view.ndim-1) + ptr = *(char**)ptr; } return ptr; } @@ -372,8 +403,12 @@ static PyObject* ll_item(CPyCppyy::LowLevelView* self, Py_ssize_t index) } void* ptr = ptr_from_index(self, index); - if (ptr) - return self->fConverter->FromMemory(ptr); + if (ptr) { + bool isfix = (intptr_t)view.internal & CPyCppyy::LowLevelView::kIsFixed; + if (self->fBufInfo.ndim == 1 || !isfix) + return self->fConverter->FromMemory(ptr); + return self->fConverter->FromMemory((void*)&ptr); + } return nullptr; // error already set by lookup_dimension } @@ -393,9 +428,9 @@ static PyObject* ll_item_multi(CPyCppyy::LowLevelView* self, PyObject *tup) } void* ptr = ptr_from_tuple(self, tup); - if (!ptr) - return nullptr; - return self->fConverter->FromMemory(ptr); + +// if there's an error, it was already set by lookup_dimension + return ptr ? self->fElemCnv->FromMemory(ptr) : nullptr; } @@ -432,11 +467,38 @@ static PyObject* ll_subscript(CPyCppyy::LowLevelView* self, PyObject* key) return ll_item(self, index); } else if (PySlice_Check(key)) { - // TODO: handle slicing. This should be simpler than the memoryview - // case as there is no Python object holding the buffer. - PyErr_SetString(PyExc_NotImplementedError, - "multi-dimensional slicing is not implemented"); - return nullptr; + if (view.ndim == 1) { + Py_ssize_t start, stop, step, slicelen; + if (PySlice_Unpack(key, &start, &stop, &step) < 0) + return nullptr; + + slicelen = PySlice_AdjustIndices(view.shape[0], &start, &stop, step); + if (slicelen <= 0) + slicelen = view.shape[0]; + + char* buf = (char*)self->get_buf(); + char* slice_buf = new char[slicelen*view.itemsize]; + size_t isize = view.itemsize; + for (size_t i=0, cur=0; i < (size_t)slicelen; cur += step, ++i) { + for (size_t j=0; j < isize; ++j) + slice_buf[i*isize+j] = buf[(start+cur)*isize + j]; + } + + CPyCppyy::LowLevelView* ll = self->fCreator(slice_buf, {1, slicelen}); + if (!ll) + delete [] slice_buf; + else + (intptr_t&)ll->fBufInfo.internal |= CPyCppyy::LowLevelView::kIsOwner; + + return (PyObject*)ll; + + } else { + // TODO: handle slicing. This should be simpler than the memoryview + // case as there is no Python object holding the buffer. + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional slicing is not implemented"); + return nullptr; + } } else if (is_multiindex(key)) { return ll_item_multi(self, key); @@ -502,8 +564,10 @@ static int ll_ass_sub(CPyCppyy::LowLevelView* self, PyObject* key, PyObject* val int ret = -1; // rvalue must be an exporter - if (PyObject_GetBuffer(value, &src, PyBUF_FULL_RO) < 0) + if (PyObject_GetBuffer(value, &src, PyBUF_FULL_RO) < 0) { + if (src.obj) CPyCppyy_PyBuffer_Release(value, &src); return ret; + } dest = view; dest.shape = &arrays[0]; dest.shape[0] = view.shape[0]; @@ -516,11 +580,13 @@ static int ll_ass_sub(CPyCppyy::LowLevelView* self, PyObject* key, PyObject* val return -1; dest.len = dest.shape[0] * dest.itemsize; - return copy_single(&dest, &src); + ret = copy_single(&dest, &src); + CPyCppyy_PyBuffer_Release(value, &src); + return ret; } if (is_multiindex(key)) { - // TODO: implement + // TODO: implement if (PyTuple_GET_SIZE(key) < view.ndim) { PyErr_SetString(PyExc_NotImplementedError, "sub-views are not implemented"); @@ -529,7 +595,7 @@ static int ll_ass_sub(CPyCppyy::LowLevelView* self, PyObject* key, PyObject* val void* ptr = ptr_from_tuple(self, key); if (ptr == nullptr) return -1; - return self->fConverter->ToMemory(value, ptr) ? 0 : -1; + return self->fElemCnv->ToMemory(value, ptr) ? 0 : -1; } if (PySlice_Check(key) || is_multislice(key)) { @@ -611,6 +677,27 @@ static int ll_getbuf(CPyCppyy::LowLevelView* self, Py_buffer* view, int flags) } +//= iterator protocol ======================================================= +static PyObject* ll_iter(PyObject* self) { +// The index iterator indexes through getitem, just like python would do by +// default, except that it checks the size externally to raise StopIteration, +// rather than geitem failing. + + using namespace CPyCppyy; + + indexiterobject* ii = PyObject_GC_New(indexiterobject, &IndexIter_Type); + if (!ii) return nullptr; + + Py_INCREF(self); + ii->ii_container = self; + ii->ii_pos = 0; + ii->ii_len = ll_length((LowLevelView*)self); + + PyObject_GC_Track(ii); + return (PyObject*)ii; +} + + //- mapping methods --------------------------------------------------------- static PyMappingMethods ll_as_mapping = { (lenfunc) ll_length, // mp_length @@ -645,6 +732,168 @@ static PyBufferProcs ll_as_buffer = { }; + +//--------------------------------------------------------------------------- +static PyObject* ll_shape(CPyCppyy::LowLevelView* self) +{ + Py_buffer& view = self->fBufInfo; + + PyObject* shape = PyTuple_New(view.ndim); + for (Py_ssize_t idim = 0; idim < view.ndim; ++idim) + PyTuple_SET_ITEM(shape, idim, PyInt_FromSsize_t(view.shape[idim])); + + return shape; +} + +//--------------------------------------------------------------------------- +static PyObject* ll_reshape(CPyCppyy::LowLevelView* self, PyObject* shape) +{ +// Allow the user to fix up the actual (type-strided) size of the buffer. + if (!PyTuple_Check(shape)) { + if (shape) { + PyObject* pystr = PyObject_Str(shape); + if (pystr) { + PyErr_Format(PyExc_TypeError, "tuple object expected, received %s", + CPyCppyy_PyText_AsStringChecked(pystr)); + Py_DECREF(pystr); + return nullptr; + } + } + PyErr_SetString(PyExc_TypeError, "tuple object expected"); + return nullptr; + } + + Py_buffer& view = self->fBufInfo; + +// verify size match + Py_ssize_t oldsz = 0; + for (Py_ssize_t idim = 0; idim < view.ndim; ++idim) { + Py_ssize_t nlen = view.shape[idim]; + if (nlen == CPyCppyy::UNKNOWN_SIZE || nlen == INT_MAX/view.itemsize /* fake 'max' */) { + oldsz = -1; // meaning, unable to check size match + break; + } + oldsz += view.shape[idim]; + } + + if (0 < oldsz) { + Py_ssize_t newsz = 0; + for (Py_ssize_t idim = 0; idim < PyTuple_GET_SIZE(shape); ++idim) + newsz += PyInt_AsSsize_t(PyTuple_GET_ITEM(shape, idim)); + if (oldsz != newsz) { + PyObject* tas = PyObject_Str(shape); + PyErr_Format(PyExc_ValueError, + "cannot reshape array of size %ld into shape %s", (long)oldsz, CPyCppyy_PyText_AsString(tas)); + Py_DECREF(tas); + return nullptr; + } + } + +// reshape + size_t itemsize = view.strides[view.ndim-1]; + if (view.ndim != PyTuple_GET_SIZE(shape)) { + PyMem_Free(view.shape); + PyMem_Free(view.strides); + + view.ndim = (int)PyTuple_GET_SIZE(shape); + view.shape = (Py_ssize_t*)PyMem_Malloc(view.ndim * sizeof(Py_ssize_t)); + view.strides = (Py_ssize_t*)PyMem_Malloc(view.ndim * sizeof(Py_ssize_t)); + } + + for (Py_ssize_t idim = 0; idim < PyTuple_GET_SIZE(shape); ++idim) { + Py_ssize_t nlen = PyInt_AsSsize_t(PyTuple_GET_ITEM(shape, idim)); + if (nlen == -1 && PyErr_Occurred()) + return nullptr; + + if (idim == 0) view.len = nlen * view.itemsize; + + view.shape[idim] = nlen; + } + + set_strides(view, itemsize, false /* by definition not fixed */); + + Py_RETURN_NONE; +} + + +//--------------------------------------------------------------------------- +static PyObject* ll_array(CPyCppyy::LowLevelView* self, PyObject* args, PyObject* /* kwds */) +{ +// Construct a numpy array from the lowlevelview (w/o copy if possible); this +// uses the Python methods to avoid depending on numpy directly + +// Expect as most a dtype from the arguments; + static PyObject* ctmod = PyImport_ImportModule("numpy"); // ref-count kept + if (!ctmod) + return nullptr; + +// expect possible dtype from the arguments, otherwie take it from the type code + PyObject* dtype; + if (!args || PyTuple_GET_SIZE(args) != 1) { + PyObject* npdtype = PyObject_GetAttr(ctmod, CPyCppyy::PyStrings::gDType); + PyObject* typecode = ll_typecode(self, nullptr); + dtype = PyObject_CallFunctionObjArgs(npdtype, typecode, nullptr); + Py_DECREF(typecode); + Py_DECREF(npdtype); + } else { + dtype = PyTuple_GET_ITEM(args, 0); + Py_INCREF(dtype); + } + + if (!dtype) + return nullptr; + + PyObject* npfrombuf = PyObject_GetAttr(ctmod, CPyCppyy::PyStrings::gFromBuffer); + PyObject* view = PyObject_CallFunctionObjArgs(npfrombuf, (PyObject*)self, dtype, nullptr); + Py_DECREF(dtype); + Py_DECREF(npfrombuf); + + return view; +} + + +//--------------------------------------------------------------------------- +static PyObject* ll_as_string(CPyCppyy::LowLevelView* self) +{ +// Interpret memory as a null-terminated char string. + Py_buffer& view = self->fBufInfo; + + if (strcmp(view.format, "b") != 0 || view.ndim != 1) { + PyErr_Format(PyExc_TypeError, + "as_string only supported for 1-dim char strings (format: %s, dim: %d)", + view.format, (int)view.ndim); + return nullptr; + } + + char* buf = (char*)self->get_buf(); + size_t sz = strnlen(buf, (size_t)view.shape[0]); + return CPyCppyy_PyText_FromStringAndSize(buf, sz); +} + +//--------------------------------------------------------------------------- +static PyMethodDef ll_methods[] = { + {(char*)"reshape", (PyCFunction)ll_reshape, METH_O, + (char*)"change the shape (not layout) of the low level view"}, + {(char*)"as_string", (PyCFunction)ll_as_string, METH_NOARGS, + (char*)"interpret memory as a null-terminated char string and return Python str"}, + {(char*)"__array__", (PyCFunction)ll_array, METH_VARARGS | METH_KEYWORDS, + (char*)"return a numpy array from the low level view"}, + {(char*)nullptr, nullptr, 0, nullptr} +}; + +//--------------------------------------------------------------------------- +static PyGetSetDef ll_getset[] = { + {(char*)"__python_owns__", (getter)ll_getownership, (setter)ll_setownership, + (char*)"If true, python manages the life time of this buffer", nullptr}, + {(char*)"__cpp_array__", (getter)ll_getcpparray, (setter)ll_setcpparray, + (char*)"If true, this array was allocated with C++\'s new[]", nullptr}, + {(char*)"format", (getter)ll_typecode, nullptr, nullptr, nullptr}, + {(char*)"typecode", (getter)ll_typecode, nullptr, nullptr, nullptr}, + {(char*)"shape", (getter)ll_shape, (setter)ll_reshape, nullptr, nullptr}, + {(char*)nullptr, nullptr, nullptr, nullptr, nullptr } +}; + + namespace CPyCppyy { //= CPyCppyy low level view type ============================================ @@ -654,10 +903,10 @@ PyTypeObject LowLevelView_Type = { sizeof(CPyCppyy::LowLevelView),// tp_basicsize 0, // tp_itemsize (destructor)ll_dealloc, // tp_dealloc - 0, // tp_print + 0, // tp_vectorcall_offset / tp_print 0, // tp_getattr 0, // tp_setattr - 0, // tp_compare + 0, // itp_as_async / tp_compare 0, // tp_repr 0, // tp_as_number &ll_as_sequence, // tp_as_sequence @@ -675,7 +924,7 @@ PyTypeObject LowLevelView_Type = { 0, // tp_clear 0, // tp_richcompare 0, // tp_weaklistoffset - 0, // tp_iter + (getiterfunc)ll_iter, // tp_iter 0, // tp_iternext ll_methods, // tp_methods 0, // tp_members @@ -704,6 +953,9 @@ PyTypeObject LowLevelView_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; } // namespace CPyCppyy @@ -713,6 +965,8 @@ namespace { template struct typecode_traits {}; template<> struct typecode_traits { static constexpr const char* format = "?"; static constexpr const char* name = "bool"; }; +template<> struct typecode_traits { + static constexpr const char* format = "b"; static constexpr const char* name = "char"; }; template<> struct typecode_traits { static constexpr const char* format = "b"; static constexpr const char* name = "signed char"; }; template<> struct typecode_traits { @@ -721,6 +975,8 @@ template<> struct typecode_traits { template<> struct typecode_traits { static constexpr const char* format = "B"; static constexpr const char* name = "UCharAsInt"; }; #endif +template<> struct typecode_traits { + static constexpr const char* format = "b"; static constexpr const char* name = "char*"; }; template<> struct typecode_traits { static constexpr const char* format = "b"; static constexpr const char* name = "const char*"; }; template<> struct typecode_traits { @@ -732,13 +988,7 @@ template<> struct typecode_traits { template<> struct typecode_traits { static constexpr const char* format = "I"; static constexpr const char* name = "unsigned int"; }; template<> struct typecode_traits { - static constexpr const char* format = "l"; static constexpr const char* -#if PY_VERSION_HEX < 0x03000000 - name = "int"; -#else - name = "long"; -#endif -}; + static constexpr const char* format = "l"; static constexpr const char* name = "long"; }; template<> struct typecode_traits { static constexpr const char* format = "L"; static constexpr const char* name = "unsigned long"; }; template<> struct typecode_traits { @@ -763,12 +1013,27 @@ template<> struct typecode_traits> { } // unnamed namespace +//--------------------------------------------------------------------------- +bool CPyCppyy::LowLevelView::resize(size_t sz) +{ + Py_buffer& bi = this->fBufInfo; + if (bi.ndim == 1 && bi.shape) { + bi.len = sz * bi.itemsize; + bi.shape[0] = sz; + return true; + } + + return false; +} + //--------------------------------------------------------------------------- template -static inline PyObject* CreateLowLevelViewT(T* address, Py_ssize_t* shape) +static inline CPyCppyy::LowLevelView* CreateLowLevelViewT( + T* address, CPyCppyy::cdims_t shape, const char* format = nullptr, const char* name = nullptr, Py_ssize_t itemsize = -1) { using namespace CPyCppyy; - Py_ssize_t nx = (shape && 0 <= shape[1]) ? shape[1] : INT_MAX/sizeof(T); + Py_ssize_t nx = (shape.ndim() != UNKNOWN_SIZE) ? shape[0] : INT_MAX/sizeof(T); + if (nx == UNKNOWN_SIZE) nx = INT_MAX/sizeof(T); PyObject* args = PyTuple_New(0); LowLevelView* llp = (LowLevelView*)LowLevelView_Type.tp_new(&LowLevelView_Type, args, nullptr); @@ -778,57 +1043,70 @@ static inline PyObject* CreateLowLevelViewT(T* address, Py_ssize_t* shape) view.buf = address; view.obj = nullptr; view.readonly = 0; - view.format = (char*)typecode_traits::format; - view.ndim = shape ? (int)shape[0] : 1; + view.format = (char*)(format ? format : typecode_traits::format); + view.ndim = int(shape.ndim() != UNKNOWN_SIZE ? shape.ndim() : 1); view.shape = (Py_ssize_t*)PyMem_Malloc(view.ndim * sizeof(Py_ssize_t)); view.shape[0] = nx; // view.len / view.itemsize view.strides = (Py_ssize_t*)PyMem_Malloc(view.ndim * sizeof(Py_ssize_t)); view.suboffsets = nullptr; - view.internal = nullptr; + (intptr_t&)view.internal = CPyCppyy::LowLevelView::kIsCppArray; // assumed + bool isfix = shape.ndim() != UNKNOWN_SIZE; + if (isfix) { + for (int i = 0; i < shape.ndim(); ++i) + isfix = isfix && (shape[i] != UNKNOWN_SIZE); + if (isfix) (intptr_t&)view.internal |= CPyCppyy::LowLevelView::kIsFixed; + } + llp->fElemCnv = CreateConverter(name ? name : typecode_traits::name); if (view.ndim == 1) { // simple 1-dim array of the declared type view.len = nx * sizeof(T); - view.itemsize = sizeof(T); - llp->fConverter = CreateConverter(typecode_traits::name); + view.itemsize = (itemsize > 0 ? (size_t)itemsize : sizeof(T)); + llp->fConverter = llp->fElemCnv; } else { // multi-dim array; sub-views are projected by using more LLViews view.len = nx * sizeof(void*); view.itemsize = sizeof(void*); + for (Py_ssize_t idim = 1; idim < view.ndim; ++idim) + view.shape[idim] = shape[idim]; // peel off one dimension and create a new LLView converter - Py_ssize_t res = shape[1]; - shape[1] = shape[0] - 1; - std::string tname{typecode_traits::name}; - tname.append("*"); // make sure to ask for another array + std::string tname{name ? name : typecode_traits::name}; + tname.append("[]"); // make sure to ask for another array // TODO: although this will work, it means that "naive" loops are expensive - llp->fConverter = CreateConverter(tname, &shape[1]); - shape[1] = res; + llp->fConverter = CreateConverter(tname, shape.sub()); } - view.strides[0] = view.itemsize; + set_strides(view, sizeof(T), isfix); - return (PyObject*)llp; + return llp; } //--------------------------------------------------------------------------- template -static inline PyObject* CreateLowLevelViewT(T** address, Py_ssize_t* shape) +static inline CPyCppyy::LowLevelView* CreateLowLevelViewT( + T** address, CPyCppyy::cdims_t shape, const char* format = nullptr, const char* name = nullptr) { using namespace CPyCppyy; - T* buf = address ? *address : nullptr; - LowLevelView* llp = (LowLevelView*)CreateLowLevelViewT(buf, shape); + LowLevelView* llp = (LowLevelView*)CreateLowLevelViewT((T*)address, shape, format, name); llp->set_buf((void**)address); - return (PyObject*)llp; + return llp; } //--------------------------------------------------------------------------- +#define CPPYY_RET_W_CREATOR(type, fname) \ + PyObject* (*c)(type, cdims_t) = &fname; \ + ll->fCreator = (LowLevelView::Creator_t)c; \ + return (PyObject*)ll + #define CPPYY_IMPL_VIEW_CREATOR(type) \ -PyObject* CPyCppyy::CreateLowLevelView(type* address, Py_ssize_t* shape) { \ - return CreateLowLevelViewT(address, shape); \ +PyObject* CPyCppyy::CreateLowLevelView(type* address, cdims_t shape) { \ + LowLevelView* ll = CreateLowLevelViewT(address, shape); \ + CPPYY_RET_W_CREATOR(type*, CreateLowLevelView); \ } \ -PyObject* CPyCppyy::CreateLowLevelView(type** address, Py_ssize_t* shape) { \ - return CreateLowLevelViewT(address, shape); \ +PyObject* CPyCppyy::CreateLowLevelView(type** address, cdims_t shape) { \ + LowLevelView* ll = CreateLowLevelViewT(address, shape); \ + CPPYY_RET_W_CREATOR(type**, CreateLowLevelView); \ } CPPYY_IMPL_VIEW_CREATOR(bool); @@ -853,6 +1131,42 @@ CPPYY_IMPL_VIEW_CREATOR(std::complex); CPPYY_IMPL_VIEW_CREATOR(std::complex); CPPYY_IMPL_VIEW_CREATOR(std::complex); -PyObject* CPyCppyy::CreateLowLevelView(const char** address, Py_ssize_t* shape) { - return CreateLowLevelViewT(address, shape); +PyObject* CPyCppyy::CreateLowLevelView(char* address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape); + CPPYY_RET_W_CREATOR(char*, CreateLowLevelView); +} + +PyObject* CPyCppyy::CreateLowLevelView(char** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape); + CPPYY_RET_W_CREATOR(char**, CreateLowLevelView); +} + +PyObject* CPyCppyy::CreateLowLevelViewString(char** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, nullptr, nullptr, sizeof(char)); + CPPYY_RET_W_CREATOR(char**, CreateLowLevelViewString); +} + +PyObject* CPyCppyy::CreateLowLevelViewString(const char** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, nullptr, nullptr, sizeof(char)); + CPPYY_RET_W_CREATOR(const char**, CreateLowLevelViewString); +} + +PyObject* CPyCppyy::CreateLowLevelView_i8(int8_t* address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "b", "int8_t"); + CPPYY_RET_W_CREATOR(int8_t*, CreateLowLevelView_i8); +} + +PyObject* CPyCppyy::CreateLowLevelView_i8(int8_t** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "b", "int8_t"); + CPPYY_RET_W_CREATOR(int8_t**, CreateLowLevelView_i8); +} + +PyObject* CPyCppyy::CreateLowLevelView_i8(uint8_t* address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "B", "uint8_t"); + CPPYY_RET_W_CREATOR(uint8_t*, CreateLowLevelView_i8); +} + +PyObject* CPyCppyy::CreateLowLevelView_i8(uint8_t** address, cdims_t shape) { + LowLevelView* ll = CreateLowLevelViewT(address, shape, "B", "uint8_t"); + CPPYY_RET_W_CREATOR(uint8_t**, CreateLowLevelView_i8); } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.h b/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.h index 8002c7f3cbbbd..002e60b72df27 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/LowLevelViews.h @@ -1,9 +1,12 @@ #ifndef CPYCPPYY_LOWLEVELVIEWS_H #define CPYCPPYY_LOWLEVELVIEWS_H +// Bindings +#include "Dimensions.h" + +// Standard #include #include - #if __cplusplus > 201402L #include #endif @@ -14,27 +17,45 @@ namespace CPyCppyy { class Converter; class LowLevelView { +public: + enum EFlags { + kDefault = 0x0000, + kIsCppArray = 0x0001, + kIsFixed = 0x0002, + kIsOwner = 0x0004 }; + public: PyObject_HEAD Py_buffer fBufInfo; void** fBuf; Converter* fConverter; + Converter* fElemCnv; + + typedef LowLevelView* (*Creator_t)(void*, cdims_t); + Creator_t fCreator; // for slicing, which requires copying public: void* get_buf() { return fBuf ? *fBuf : fBufInfo.buf; } void set_buf(void** buf) { fBuf = buf; fBufInfo.buf = get_buf(); } + + bool resize(size_t sz); }; #define CPPYY_DECL_VIEW_CREATOR(type) \ - PyObject* CreateLowLevelView(type*, Py_ssize_t* shape = nullptr); \ - PyObject* CreateLowLevelView(type**, Py_ssize_t* shape = nullptr) + PyObject* CreateLowLevelView(type*, cdims_t shape); \ + PyObject* CreateLowLevelView(type**, cdims_t shape) CPPYY_DECL_VIEW_CREATOR(bool); +CPPYY_DECL_VIEW_CREATOR(char); CPPYY_DECL_VIEW_CREATOR(signed char); CPPYY_DECL_VIEW_CREATOR(unsigned char); #if __cplusplus > 201402L CPPYY_DECL_VIEW_CREATOR(std::byte); #endif +PyObject* CreateLowLevelView_i8(int8_t*, cdims_t shape); +PyObject* CreateLowLevelView_i8(int8_t**, cdims_t shape); +PyObject* CreateLowLevelView_i8(uint8_t*, cdims_t shape); +PyObject* CreateLowLevelView_i8(uint8_t**, cdims_t shape); CPPYY_DECL_VIEW_CREATOR(short); CPPYY_DECL_VIEW_CREATOR(unsigned short); CPPYY_DECL_VIEW_CREATOR(int); @@ -51,10 +72,10 @@ CPPYY_DECL_VIEW_CREATOR(std::complex); CPPYY_DECL_VIEW_CREATOR(std::complex); CPPYY_DECL_VIEW_CREATOR(std::complex); -PyObject* CreateLowLevelView(const char**, Py_ssize_t* shape = nullptr); +PyObject* CreateLowLevelViewString(char**, cdims_t shape); +PyObject* CreateLowLevelViewString(const char**, cdims_t shape); -inline PyObject* CreatePointerView(void* ptr, size_t size = (size_t)-1) { - Py_ssize_t shape[] = {1, (Py_ssize_t)size}; +inline PyObject* CreatePointerView(void* ptr, cdims_t shape = 0) { return CreateLowLevelView((uintptr_t*)ptr, shape); } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx index ed26b702baa46..37cc394a5c23d 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/MemoryRegulator.cxx @@ -99,7 +99,7 @@ CPyCppyy::MemoryRegulator::MemoryRegulator() bool CPyCppyy::MemoryRegulator::RecursiveRemove( Cppyy::TCppObject_t cppobj, Cppyy::TCppType_t klass) { -// if registerd by the framework, called whenever a cppobj gets destroyed +// if registered by the framework, called whenever a cppobj gets destroyed if (!cppobj) return false; @@ -135,10 +135,7 @@ bool CPyCppyy::MemoryRegulator::RecursiveRemove( CPyCppyy_NoneType.tp_traverse = Py_TYPE(pyobj)->tp_traverse; CPyCppyy_NoneType.tp_clear = Py_TYPE(pyobj)->tp_clear; CPyCppyy_NoneType.tp_free = Py_TYPE(pyobj)->tp_free; -#ifdef Py_TPFLAGS_MANAGED_DICT - CPyCppyy_NoneType.tp_flags |= (Py_TYPE(pyobj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); -#endif - CPyCppyy_NoneType.tp_flags |= (Py_TYPE(pyobj)->tp_flags & Py_TPFLAGS_HAVE_GC); + CPyCppyy_NoneType.tp_flags |= Py_TYPE(pyobj)->tp_flags; } else if (CPyCppyy_NoneType.tp_traverse != Py_TYPE(pyobj)->tp_traverse) { // TODO: SystemError? std::cerr << "in CPyCppyy::MemoryRegulater, unexpected object of type: " @@ -188,16 +185,18 @@ bool CPyCppyy::MemoryRegulator::RegisterPyObject( CppToPyMap_t* cppobjs = ((CPPClass*)Py_TYPE(pyobj))->fImp.fCppObjects; if (!cppobjs) - return false; + return false; - CppToPyMap_t::iterator ppo = cppobjs->find(cppobj); - if (ppo == cppobjs->end()) { - cppobjs->insert(std::make_pair(cppobj, (PyObject*)pyobj)); - pyobj->fFlags |= CPPInstance::kIsRegulated; - return true; +// if an address was already associated with a different object, then stop following +// the old and force insert the new proxy for following + const auto& res = cppobjs->insert(std::make_pair(cppobj, (PyObject*)pyobj)); + if (!res.second) { + ((CPPInstance*)res.first->second)->fFlags &= ~CPPInstance::kIsRegulated; + (*cppobjs)[cppobj] = (PyObject*)pyobj; } - return false; + pyobj->fFlags |= CPPInstance::kIsRegulated; + return true; } //----------------------------------------------------------------------------- diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx index 210fe4ea8b21f..801337717e081 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx @@ -9,6 +9,7 @@ #include "CPPGetSetItem.h" #include "CPPInstance.h" #include "CPPMethod.h" +#include "CPPOperator.h" #include "CPPOverload.h" #include "CPPScope.h" #include "MemoryRegulator.h" @@ -69,10 +70,8 @@ static PyObject* CreateNewCppProxyClass(Cppyy::TCppScope_t klass, PyObject* pyba PyObject* pymeta = (PyObject*)CPPScopeMeta_New(klass, args); Py_DECREF(args); - if (!pymeta) { - PyErr_Print(); + if (!pymeta) return nullptr; - } // alright, and now we really badly want to get rid of the dummy ... PyObject* dictproxy = PyObject_GetAttr(pymeta, PyStrings::gDict); @@ -153,7 +152,7 @@ static inline void sync_templates( Py_DECREF(pyname); } -static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) +static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass, const unsigned int flags) { // Collect methods and data for the given scope, and add them to the given python // proxy object. @@ -179,6 +178,10 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) for (Cppyy::TCppIndex_t imeth = 0; imeth < nMethods; ++imeth) { Cppyy::TCppMethod_t method = Cppyy::GetMethod(scope, imeth); + // do not expose non-public methods as the Cling wrappers as those won't compile + if (!Cppyy::IsPublicMethod(method)) + continue; + // process the method based on its name std::string mtCppName = Cppyy::GetMethodName(method); @@ -186,6 +189,7 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) bool setupSetItem = false; bool isConstructor = Cppyy::IsConstructor(method); bool isTemplate = isConstructor ? false : Cppyy::IsMethodTemplate(scope, imeth); + bool isStubbedOperator = false; // filter empty names (happens for namespaces, is bug?) if (mtCppName == "") @@ -196,7 +200,8 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) continue; // translate operators - std::string mtName = Utility::MapOperatorName(mtCppName, Cppyy::GetMethodNumArgs(method)); + std::string mtName = Utility::MapOperatorName( + mtCppName, Cppyy::GetMethodNumArgs(method), &isStubbedOperator); if (mtName.empty()) continue; @@ -204,24 +209,20 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) bool isCall = mtName == "__call__"; if (isCall || mtName == "__getitem__") { const std::string& qual_return = Cppyy::ResolveName(Cppyy::GetMethodResultType(method)); - const std::string& cpd = Utility::Compound(qual_return); - if (!cpd.empty() && cpd[cpd.size()- 1] == '&' && \ + const std::string& cpd = TypeManip::compound(qual_return); + if (!cpd.empty() && cpd[cpd.size()-1] == '&' && \ qual_return.find("const", 0, 5) == std::string::npos) { if (isCall && !potGetItem) potGetItem = method; setupSetItem = true; // will add methods as overloads } else if (isCall && 1 < Cppyy::GetMethodNumArgs(method)) { // not a non-const by-ref return, thus better __getitem__ candidate; the // requirement for multiple arguments is that there is otherwise no benefit - // over the use of normal __getitem__ (this allows multi-indexing argumenst, + // over the use of normal __getitem__ (this allows multi-indexing arguments, // which is clean in Python, but not allowed in C++) potGetItem = method; } } - // do not expose private methods as the Cling wrappers for them won't compile - if (!Cppyy::IsPublicMethod(method)) - continue; - // template members; handled by adding a dispatcher to the class bool storeOnTemplate = isTemplate ? true : (!isConstructor && Cppyy::ExistsMethodTemplate(scope, mtCppName)); @@ -240,10 +241,15 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) else if (isConstructor) { // ctor mtName = "__init__"; hasConstructor = true; - if (!isAbstract) - pycall = new CPPConstructor(scope, method); - else + if (!isAbstract) { + if (flags & CPPScope::kIsMultiCross) { + pycall = new CPPMultiConstructor(scope, method); + } else + pycall = new CPPConstructor(scope, method); + } else pycall = new CPPAbstractClassConstructor(scope, method); + } else if (isStubbedOperator) { + pycall = new CPPOperator(scope, method, mtName); } else // member function pycall = new CPPMethod(scope, method); @@ -318,7 +324,7 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) ((CPPScope*)pyclass)->fFlags |= CPPScope::kIsInComplete; defctor = new CPPIncompleteClassConstructor(scope, (Cppyy::TCppMethod_t)0); } else - defctor = new CPPConstructor(scope, (Cppyy::TCppMethod_t)0); + defctor = new CPPAllPrivateClassConstructor(scope, (Cppyy::TCppMethod_t)0); cache["__init__"].push_back(defctor); } @@ -380,9 +386,10 @@ static int BuildScopeProxyDict(Cppyy::TCppScope_t scope, PyObject* pyclass) PyErr_Clear(); - // it could still be that this is an unnamed enum, which is not in the list + // it could still be that this is an anonymous enum, which is not in the list // provided by the class - if (strstr(Cppyy::GetDatamemberType(scope, idata).c_str(), "(unnamed)") != 0) { + if (strstr(Cppyy::GetDatamemberType(scope, idata).c_str(), "(anonymous)") != 0 || + strstr(Cppyy::GetDatamemberType(scope, idata).c_str(), "(unnamed)") != 0) { AddPropertyToClass(pyclass, scope, idata); continue; } @@ -503,14 +510,14 @@ PyObject* CPyCppyy::GetScopeProxy(Cppyy::TCppScope_t scope) } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::CreateScopeProxy(Cppyy::TCppScope_t scope) +PyObject* CPyCppyy::CreateScopeProxy(Cppyy::TCppScope_t scope, const unsigned flags) { // Convenience function with a lookup first through the known existing proxies. PyObject* pyclass = GetScopeProxy(scope); if (pyclass) return pyclass; - return CreateScopeProxy(Cppyy::GetScopedFinalName(scope)); + return CreateScopeProxy(Cppyy::GetScopedFinalName(scope), nullptr, flags); } //---------------------------------------------------------------------------- @@ -521,11 +528,11 @@ PyObject* CPyCppyy::CreateScopeProxy(PyObject*, PyObject* args) if (PyErr_Occurred()) return nullptr; - return CreateScopeProxy(cname); + return CreateScopeProxy(cname); } //---------------------------------------------------------------------------- -PyObject* CPyCppyy::CreateScopeProxy(const std::string& name, PyObject* parent) +PyObject* CPyCppyy::CreateScopeProxy(const std::string& name, PyObject* parent, const unsigned flags) { // Build a python shadow class for the named C++ class or namespace. @@ -572,7 +579,7 @@ PyObject* CPyCppyy::CreateScopeProxy(const std::string& name, PyObject* parent) } if (!(bool)klass) { - // could be an enum, which are treated seperately in CPPScope (TODO: maybe they + // could be an enum, which are treated separately in CPPScope (TODO: maybe they // should be handled here instead anyway??) if (Cppyy::IsEnum(lookup)) return nullptr; @@ -660,11 +667,6 @@ PyObject* CPyCppyy::CreateScopeProxy(const std::string& name, PyObject* parent) } } - if (!parent && name.size() > 0) { - // Didn't find any scope in the name, so must be global - parent = CreateScopeProxy(Cppyy::gGlobalScope); - } - if (parent && !CPPScope_Check(parent)) { // Special case: parent found is not one of ours (it's e.g. a pure Python module), so // continuing would fail badly. One final lookup, then out of here ... @@ -694,7 +696,7 @@ PyObject* CPyCppyy::CreateScopeProxy(const std::string& name, PyObject* parent) // fill the dictionary, if successful if (pyscope) { - if (BuildScopeProxyDict(klass, pyscope)) { + if (BuildScopeProxyDict(klass, pyscope, flags)) { // something failed in building the dictionary Py_DECREF(pyscope); pyscope = nullptr; @@ -868,8 +870,8 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address, // bind, register and return if successful if (pyobj != 0) { // fill proxy value? - unsigned objflags = - (isRef ? CPPInstance::kIsReference : 0) | (isValue ? CPPInstance::kIsValue : 0) | (flags & CPPInstance::kIsOwner); + unsigned objflags = flags & \ + (CPPInstance::kIsReference | CPPInstance::kIsPtrPtr | CPPInstance::kIsValue | CPPInstance::kIsOwner | CPPInstance::kIsActual); pyobj->Set(address, (CPPInstance::EFlags)objflags); if (smart_type) @@ -889,7 +891,6 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address, } Py_DECREF(pyclass); - return (PyObject*)pyobj; } @@ -909,30 +910,35 @@ PyObject* CPyCppyy::BindCppObject(Cppyy::TCppObject_t address, bool isRef = flags & CPPInstance::kIsReference; -// get actual class for recycling checking and/or downcasting - Cppyy::TCppType_t clActual = isRef ? 0 : Cppyy::GetActualClass(klass, address); - // downcast to real class for object returns, unless pinned - if (clActual && klass != clActual) { - auto pci = gPinnedTypes.find(klass); - if (pci == gPinnedTypes.end()) { - intptr_t offset = Cppyy::GetBaseOffset( - clActual, klass, address, -1 /* down-cast */, true /* report errors */); - if (offset != -1) { // may fail if clActual not fully defined - address = (void*)((intptr_t)address + offset); - klass = clActual; +// TODO: should the memory regulator for klass be searched first, so that if +// successful, no down-casting is attempted? +// TODO: optimize for final classes + unsigned new_flags = flags; + if (!isRef && (gPinnedTypes.empty() || gPinnedTypes.find(klass) == gPinnedTypes.end())) { + Cppyy::TCppType_t clActual = Cppyy::GetActualClass(klass, address); + + if (clActual) { + if (clActual != klass) { + intptr_t offset = Cppyy::GetBaseOffset( + clActual, klass, address, -1 /* down-cast */, true /* report errors */); + if (offset != -1) { // may fail if clActual not fully defined + address = (void*)((intptr_t)address + offset); + klass = clActual; + } } + new_flags |= CPPInstance::kIsActual; } } // actual binding (returned object may be zero w/ a python exception set) - return BindCppObjectNoCast(address, klass, flags); + return BindCppObjectNoCast(address, klass, new_flags); } //---------------------------------------------------------------------------- PyObject* CPyCppyy::BindCppObjectArray( - Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, dims_t dims) + Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, cdims_t dims) { // TODO: this function exists for symmetry; need to figure out if it's useful - return TupleOfInstances_New(address, klass, dims[0], dims+1); + return TupleOfInstances_New(address, klass, dims); } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.h b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.h index a462064fa3583..cfa4360294d08 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.h @@ -1,6 +1,9 @@ #ifndef CPYCPPYY_PROXYWRAPPERS_H #define CPYCPPYY_PROXYWRAPPERS_H +// Bindings +#include "Dimensions.h" + // Standard #include @@ -9,10 +12,10 @@ namespace CPyCppyy { // construct a Python shadow class for the named C++ class PyObject* GetScopeProxy(Cppyy::TCppScope_t); -PyObject* CreateScopeProxy(Cppyy::TCppScope_t); +PyObject* CreateScopeProxy(Cppyy::TCppScope_t, const unsigned flags = 0); PyObject* CreateScopeProxy(PyObject*, PyObject* args); PyObject* CreateScopeProxy( - const std::string& scope_name, PyObject* parent = nullptr); + const std::string& scope_name, PyObject* parent = nullptr, const unsigned flags = 0); // C++ exceptions form a special case b/c they have to derive from BaseException PyObject* CreateExcScopeProxy(PyObject* pyscope, PyObject* pyname, PyObject* parent); @@ -23,7 +26,7 @@ PyObject* BindCppObjectNoCast(Cppyy::TCppObject_t object, PyObject* BindCppObject(Cppyy::TCppObject_t object, Cppyy::TCppType_t klass, const unsigned flags = 0); PyObject* BindCppObjectArray( - Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, Py_ssize_t* dims); + Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, cdims_t dims); } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/PyCallable.h b/bindings/pyroot/cppyy/CPyCppyy/src/PyCallable.h index b324bc69f3c52..62ce3aa1af693 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/PyCallable.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/PyCallable.h @@ -19,6 +19,7 @@ class PyCallable { public: virtual PyObject* GetSignature(bool show_formalargs = true) = 0; virtual PyObject* GetPrototype(bool show_formalargs = true) = 0; + virtual PyObject* GetTypeName() { return GetPrototype(false); } virtual PyObject* GetDocString() { return GetPrototype(); } virtual PyObject* Reflex(Cppyy::Reflex::RequestId_t request, Cppyy::Reflex::FormatId_t format = Cppyy::Reflex::OPTIMAL) { @@ -31,9 +32,9 @@ class PyCallable { virtual int GetMaxArgs() = 0; virtual PyObject* GetCoVarNames() = 0; - virtual PyObject* GetArgDefault(int /* iarg */) = 0; + virtual PyObject* GetArgDefault(int /* iarg */, bool silent=true) = 0; virtual bool IsConst() { return false; } - + virtual PyObject* GetScopeProxy() = 0; virtual Cppyy::TCppFuncAddr_t GetFunctionAddress() = 0; @@ -42,8 +43,8 @@ class PyCallable { virtual int GetArgMatchScore(PyObject* /* args_tuple */) { return INT_MAX; } public: - virtual PyObject* Call( - CPPInstance*& self, PyObject* args, PyObject* kwds, CallContext* ctxt = nullptr) = 0; + virtual PyObject* Call(CPPInstance*& self, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr) = 0; }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/PyException.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/PyException.cxx index 4d2b6c8da1bbd..2929683fe0667 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/PyException.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/PyException.cxx @@ -1,3 +1,6 @@ +// Standard +#include + // Bindings #include "CPyCppyy.h" #define CPYCPPYY_INTERNAL 1 @@ -21,7 +24,91 @@ //- constructors/destructor -------------------------------------------------- CPyCppyy::PyException::PyException() { -// default constructor +#ifdef WITH_THREAD + PyGILState_STATE state = PyGILState_Ensure(); +#endif + + PyObject* pytype = nullptr, *pyvalue = nullptr, *pytrace = nullptr; + PyErr_Fetch(&pytype, &pyvalue, &pytrace); + if (pytype && pyvalue) { + const char* tname = PyExceptionClass_Name(pytype); + if (tname) { + char* dot = strrchr((char*)tname, '.'); + if (dot) tname = dot+1; + fMsg += tname; + fMsg += ": "; + } + + PyObject* msg = PyObject_Str(pyvalue); + if (msg) { + fMsg += CPyCppyy_PyText_AsString(msg); + Py_DECREF(msg); + } + } + + PyObject* traceback = pytrace; // to keep the original unchanged + Py_XINCREF(traceback); + + std::string locName; + std::string locFile; + int locLine = 0; + + while (traceback && traceback != Py_None) { + PyObject* frame = PyObject_GetAttrString(traceback, "tb_frame"); + PyObject* code = PyObject_GetAttrString(frame, "f_code"); + Py_DECREF(frame); + + PyObject* filename = PyObject_GetAttrString(code, "co_filename"); + Py_DECREF(code); + + PyObject* filenameStr = PyObject_Str(filename); + locFile = CPyCppyy_PyText_AsString(filenameStr); + Py_DECREF(filenameStr); + Py_DECREF(filename); + + PyObject* name = PyObject_GetAttrString(code, "co_name"); + PyObject* nameStr = PyObject_Str(name); + locName = CPyCppyy_PyText_AsString(nameStr); + Py_DECREF(nameStr); + Py_DECREF(name); + + PyObject* lineno = PyObject_GetAttrString(traceback, "tb_lineno"); + locLine = PyLong_AsLong(lineno); + Py_DECREF(lineno); + + if (locFile == "") { // these are not that useful, skipping + PyObject* nextTraceback = PyObject_GetAttrString(traceback, "tb_next"); + Py_DECREF(traceback); + traceback = nextTraceback; + continue; + } + + break; + } + + Py_XDECREF(traceback); + + PyErr_Restore(pytype, pyvalue, pytrace); + + if (fMsg.empty()) + fMsg = "python exception"; + + if (!locFile.empty()) { + + // only keeping the filename, not the full path + locFile = locFile.substr(locFile.find_last_of("/\\") + 1); + + fMsg += " (at " + locFile + ":" + std::to_string(locLine); + + if (locName != "") + fMsg += " in " + locName; + + fMsg += ")"; + } + +#ifdef WITH_THREAD + PyGILState_Release(state); +#endif } CPyCppyy::PyException::~PyException() noexcept @@ -34,5 +121,11 @@ CPyCppyy::PyException::~PyException() noexcept const char* CPyCppyy::PyException::what() const noexcept { // Return reason for throwing this exception: a python exception was raised. - return "python exception"; + return fMsg.c_str(); +} + +void CPyCppyy::PyException::clear() const noexcept +{ +// clear Python error, to allow full error handling C++ side + PyErr_Clear(); } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.cxx index 45dd47e1939f3..e918d44fc0d54 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.cxx @@ -7,8 +7,11 @@ PyObject* CPyCppyy::PyStrings::gAssign = nullptr; PyObject* CPyCppyy::PyStrings::gBases = nullptr; PyObject* CPyCppyy::PyStrings::gBase = nullptr; +PyObject* CPyCppyy::PyStrings::gCppBool = nullptr; PyObject* CPyCppyy::PyStrings::gCppName = nullptr; +PyObject* CPyCppyy::PyStrings::gAnnotations = nullptr; PyObject* CPyCppyy::PyStrings::gCastCpp = nullptr; +PyObject* CPyCppyy::PyStrings::gCType = nullptr; PyObject* CPyCppyy::PyStrings::gDeref = nullptr; PyObject* CPyCppyy::PyStrings::gPreInc = nullptr; PyObject* CPyCppyy::PyStrings::gPostInc = nullptr; @@ -27,10 +30,15 @@ PyObject* CPyCppyy::PyStrings::gModule = nullptr; PyObject* CPyCppyy::PyStrings::gMRO = nullptr; PyObject* CPyCppyy::PyStrings::gName = nullptr; PyObject* CPyCppyy::PyStrings::gNe = nullptr; +PyObject* CPyCppyy::PyStrings::gRepr = nullptr; +PyObject* CPyCppyy::PyStrings::gCppRepr = nullptr; +PyObject* CPyCppyy::PyStrings::gStr = nullptr; +PyObject* CPyCppyy::PyStrings::gCppStr = nullptr; PyObject* CPyCppyy::PyStrings::gTypeCode = nullptr; PyObject* CPyCppyy::PyStrings::gCTypesType = nullptr; PyObject* CPyCppyy::PyStrings::gUnderlying = nullptr; +PyObject* CPyCppyy::PyStrings::gRealInit = nullptr; PyObject* CPyCppyy::PyStrings::gAdd = nullptr; PyObject* CPyCppyy::PyStrings::gSub = nullptr; @@ -48,18 +56,25 @@ PyObject* CPyCppyy::PyStrings::gSecond = nullptr; PyObject* CPyCppyy::PyStrings::gSize = nullptr; PyObject* CPyCppyy::PyStrings::gTemplate = nullptr; PyObject* CPyCppyy::PyStrings::gVectorAt = nullptr; +PyObject* CPyCppyy::PyStrings::gInsert = nullptr; +PyObject* CPyCppyy::PyStrings::gValueType = nullptr; +PyObject* CPyCppyy::PyStrings::gValueSize = nullptr; PyObject* CPyCppyy::PyStrings::gCppReal = nullptr; PyObject* CPyCppyy::PyStrings::gCppImag = nullptr; PyObject* CPyCppyy::PyStrings::gThisModule = nullptr; -PyObject* CPyCppyy::PyStrings::gNoImplicit = nullptr; PyObject* CPyCppyy::PyStrings::gDispInit = nullptr; +PyObject* CPyCppyy::PyStrings::gDispGet = nullptr; PyObject* CPyCppyy::PyStrings::gExPythonize = nullptr; PyObject* CPyCppyy::PyStrings::gPythonize = nullptr; +PyObject* CPyCppyy::PyStrings::gArray = nullptr; +PyObject* CPyCppyy::PyStrings::gDType = nullptr; +PyObject* CPyCppyy::PyStrings::gFromBuffer = nullptr; + //----------------------------------------------------------------------------- #define CPPYY_INITIALIZE_STRING(var, str) \ @@ -72,8 +87,15 @@ bool CPyCppyy::CreatePyStrings() { CPPYY_INITIALIZE_STRING(gAssign, __assign__); CPPYY_INITIALIZE_STRING(gBases, __bases__); CPPYY_INITIALIZE_STRING(gBase, __base__); +#if PY_VERSION_HEX < 0x03000000 + CPPYY_INITIALIZE_STRING(gCppBool, __cpp_nonzero__); +#else + CPPYY_INITIALIZE_STRING(gCppBool, __cpp_bool__); +#endif CPPYY_INITIALIZE_STRING(gCppName, __cpp_name__); + CPPYY_INITIALIZE_STRING(gAnnotations, __annotations__); CPPYY_INITIALIZE_STRING(gCastCpp, __cast_cpp__); + CPPYY_INITIALIZE_STRING(gCType, __ctype__); CPPYY_INITIALIZE_STRING(gDeref, __deref__); CPPYY_INITIALIZE_STRING(gPreInc, __preinc__); CPPYY_INITIALIZE_STRING(gPostInc, __postinc__); @@ -93,10 +115,15 @@ bool CPyCppyy::CreatePyStrings() { CPPYY_INITIALIZE_STRING(gMRO, __mro__); CPPYY_INITIALIZE_STRING(gName, __name__); CPPYY_INITIALIZE_STRING(gNe, __ne__); + CPPYY_INITIALIZE_STRING(gRepr, __repr__); + CPPYY_INITIALIZE_STRING(gCppRepr, __cpp_repr); + CPPYY_INITIALIZE_STRING(gStr, __str__); + CPPYY_INITIALIZE_STRING(gCppStr, __cpp_str); CPPYY_INITIALIZE_STRING(gTypeCode, typecode); CPPYY_INITIALIZE_STRING(gCTypesType, _type_); CPPYY_INITIALIZE_STRING(gUnderlying, __underlying); + CPPYY_INITIALIZE_STRING(gRealInit, __real_init); CPPYY_INITIALIZE_STRING(gAdd, __add__); CPPYY_INITIALIZE_STRING(gSub, __sub__); @@ -114,18 +141,25 @@ bool CPyCppyy::CreatePyStrings() { CPPYY_INITIALIZE_STRING(gSize, size); CPPYY_INITIALIZE_STRING(gTemplate, Template); CPPYY_INITIALIZE_STRING(gVectorAt, _vector__at); + CPPYY_INITIALIZE_STRING(gInsert, insert); + CPPYY_INITIALIZE_STRING(gValueType, value_type); + CPPYY_INITIALIZE_STRING(gValueSize, value_size); CPPYY_INITIALIZE_STRING(gCppReal, __cpp_real); CPPYY_INITIALIZE_STRING(gCppImag, __cpp_imag); CPPYY_INITIALIZE_STRING(gThisModule, cppyy); - CPPYY_INITIALIZE_STRING(gNoImplicit, __cppyy_no_implicit); CPPYY_INITIALIZE_STRING(gDispInit, _init_dispatchptr); + CPPYY_INITIALIZE_STRING(gDispGet, _get_dispatch); CPPYY_INITIALIZE_STRING(gExPythonize, __cppyy_explicit_pythonize__); CPPYY_INITIALIZE_STRING(gPythonize, __cppyy_pythonize__); + CPPYY_INITIALIZE_STRING(gArray, __array__); + CPPYY_INITIALIZE_STRING(gDType, dtype); + CPPYY_INITIALIZE_STRING(gFromBuffer, frombuffer); + return true; } @@ -135,7 +169,10 @@ PyObject* CPyCppyy::DestroyPyStrings() { // Remove all cached python strings. Py_DECREF(PyStrings::gBases); PyStrings::gBases = nullptr; Py_DECREF(PyStrings::gBase); PyStrings::gBase = nullptr; + Py_DECREF(PyStrings::gCppBool); PyStrings::gCppBool = nullptr; Py_DECREF(PyStrings::gCppName); PyStrings::gCppName = nullptr; + Py_DECREF(PyStrings::gAnnotations); PyStrings::gAnnotations = nullptr; + Py_DECREF(PyStrings::gCType); PyStrings::gCType = nullptr; Py_DECREF(PyStrings::gDeref); PyStrings::gDeref = nullptr; Py_DECREF(PyStrings::gPreInc); PyStrings::gPreInc = nullptr; Py_DECREF(PyStrings::gPostInc); PyStrings::gPostInc = nullptr; @@ -158,6 +195,7 @@ PyObject* CPyCppyy::DestroyPyStrings() { Py_DECREF(PyStrings::gCTypesType); PyStrings::gCTypesType = nullptr; Py_DECREF(PyStrings::gUnderlying); PyStrings::gUnderlying = nullptr; + Py_DECREF(PyStrings::gRealInit); PyStrings::gRealInit = nullptr; Py_DECREF(PyStrings::gAdd); PyStrings::gAdd = nullptr; Py_DECREF(PyStrings::gSub); PyStrings::gSub = nullptr; @@ -175,17 +213,24 @@ PyObject* CPyCppyy::DestroyPyStrings() { Py_DECREF(PyStrings::gSize); PyStrings::gSize = nullptr; Py_DECREF(PyStrings::gTemplate); PyStrings::gTemplate = nullptr; Py_DECREF(PyStrings::gVectorAt); PyStrings::gVectorAt = nullptr; + Py_DECREF(PyStrings::gInsert); PyStrings::gInsert = nullptr; + Py_DECREF(PyStrings::gValueType); PyStrings::gValueType = nullptr; + Py_DECREF(PyStrings::gValueSize); PyStrings::gValueSize = nullptr; Py_DECREF(PyStrings::gCppReal); PyStrings::gCppReal = nullptr; Py_DECREF(PyStrings::gCppImag); PyStrings::gCppImag = nullptr; Py_DECREF(PyStrings::gThisModule); PyStrings::gThisModule = nullptr; - Py_DECREF(PyStrings::gNoImplicit); PyStrings::gNoImplicit = nullptr; Py_DECREF(PyStrings::gDispInit); PyStrings::gDispInit = nullptr; + Py_DECREF(PyStrings::gDispGet); PyStrings::gDispGet = nullptr; Py_DECREF(PyStrings::gExPythonize); PyStrings::gExPythonize = nullptr; Py_DECREF(PyStrings::gPythonize); PyStrings::gPythonize = nullptr; + Py_DECREF(PyStrings::gArray); PyStrings::gArray = nullptr; + Py_DECREF(PyStrings::gDType); PyStrings::gDType = nullptr; + Py_DECREF(PyStrings::gFromBuffer); PyStrings::gFromBuffer = nullptr; + Py_RETURN_NONE; } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.h b/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.h index 4a952646e4094..7012b89ce1620 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/PyStrings.h @@ -10,8 +10,11 @@ namespace PyStrings { extern PyObject* gAssign; extern PyObject* gBases; extern PyObject* gBase; + extern PyObject* gCppBool; extern PyObject* gCppName; + extern PyObject* gAnnotations; extern PyObject* gCastCpp; + extern PyObject* gCType; extern PyObject* gDeref; extern PyObject* gPreInc; extern PyObject* gPostInc; @@ -30,10 +33,15 @@ namespace PyStrings { extern PyObject* gMRO; extern PyObject* gName; extern PyObject* gNe; + extern PyObject* gRepr; + extern PyObject* gCppRepr; + extern PyObject* gStr; + extern PyObject* gCppStr; extern PyObject* gTypeCode; extern PyObject* gCTypesType; extern PyObject* gUnderlying; + extern PyObject* gRealInit; extern PyObject* gAdd; extern PyObject* gSub; @@ -51,18 +59,25 @@ namespace PyStrings { extern PyObject* gSize; extern PyObject* gTemplate; extern PyObject* gVectorAt; + extern PyObject* gInsert; + extern PyObject* gValueType; + extern PyObject* gValueSize; extern PyObject* gCppReal; extern PyObject* gCppImag; extern PyObject* gThisModule; - extern PyObject* gNoImplicit; extern PyObject* gDispInit; + extern PyObject* gDispGet; extern PyObject* gExPythonize; extern PyObject* gPythonize; + extern PyObject* gArray; + extern PyObject* gDType; + extern PyObject* gFromBuffer; + } // namespace PyStrings bool CreatePyStrings(); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx index f587f41907db3..ae0e31cac85ff 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Pythonize.cxx @@ -3,6 +3,7 @@ #include "Pythonize.h" #include "Converters.h" #include "CPPInstance.h" +#include "CPPFunction.h" #include "CPPOverload.h" #include "CustomPyTypes.h" #include "LowLevelViews.h" @@ -51,6 +52,17 @@ bool HasAttrDirect(PyObject* pyclass, PyObject* pyname, bool mustBeCPyCppyy = fa return false; } +PyObject* GetAttrDirect(PyObject* pyclass, PyObject* pyname) { +// get an attribute without causing getattr lookups + PyObject* dct = PyObject_GetAttr(pyclass, PyStrings::gDict); + if (dct) { + PyObject* attr = PyObject_GetItem(dct, pyname); + Py_DECREF(dct); + return attr; + } + return nullptr; +} + //----------------------------------------------------------------------------- inline bool IsTemplatedSTLClass(const std::string& name, const std::string& klass) { // Scan the name of the class and determine whether it is a template instantiation. @@ -129,7 +141,7 @@ inline PyObject* CallSelfIndex(CPPInstance* self, PyObject* idx, PyObject* pymet return nullptr; } - PyObject* result = PyObject_CallMethodObjArgs((PyObject*)self, pymeth, pyindex, nullptr); + PyObject* result = PyObject_CallMethodOneArg((PyObject*)self, pymeth, pyindex); Py_DECREF(pyindex); Py_DECREF((PyObject*)self); return result; @@ -151,7 +163,7 @@ PyObject* DeRefGetAttr(PyObject* self, PyObject* name) if (!CPyCppyy_PyText_Check(name)) PyErr_SetString(PyExc_TypeError, "getattr(): attribute name must be string"); - PyObject* pyptr = PyObject_CallMethodObjArgs(self, PyStrings::gDeref, nullptr); + PyObject* pyptr = PyObject_CallMethodNoArgs(self, PyStrings::gDeref); if (!pyptr) return nullptr; @@ -181,7 +193,7 @@ PyObject* FollowGetAttr(PyObject* self, PyObject* name) if (!CPyCppyy_PyText_Check(name)) PyErr_SetString(PyExc_TypeError, "getattr(): attribute name must be string"); - PyObject* pyptr = PyObject_CallMethodObjArgs(self, PyStrings::gFollow, nullptr); + PyObject* pyptr = PyObject_CallMethodNoArgs(self, PyStrings::gFollow); if (!pyptr) return nullptr; @@ -190,6 +202,19 @@ PyObject* FollowGetAttr(PyObject* self, PyObject* name) return result; } +//- pointer checking bool converter ------------------------------------------- +PyObject* NullCheckBool(PyObject* self) +{ + if (!CPPInstance_Check(self)) { + PyErr_SetString(PyExc_TypeError, "C++ object proxy expected"); + return nullptr; + } + + if (!((CPPInstance*)self)->GetObject()) + Py_RETURN_FALSE; + + return PyObject_CallMethodNoArgs(self, PyStrings::gCppBool); +} //- vector behavior as primitives ---------------------------------------------- #if PY_VERSION_HEX < 0x03040000 @@ -257,158 +282,204 @@ struct IterItemGetter : public ItemGetter { virtual PyObject* get() { return (*(Py_TYPE(fPyObject)->tp_iternext))(fPyObject); } }; -PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) +static ItemGetter* GetGetter(PyObject* args) { -// Specialized vector constructor to allow construction from containers; allowing -// such construction from initializer_list instead would possible, but can be -// error-prone. This use case is common enough for std::vector to implement it -// directly, except for arrays (which can be passed wholesale) and strings (which -// won't convert properly as they'll be seen as buffers) - +// Create an ItemGetter to loop over the iterable argument, if any. ItemGetter* getter = nullptr; + if (PyTuple_GET_SIZE(args) == 1) { PyObject* fi = PyTuple_GET_ITEM(args, 0); - if (CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi)) { - PyErr_SetString(PyExc_TypeError, "can not convert string to vector"); - return nullptr; - } + if (CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi)) + return nullptr; // do not accept string to fill std::vector + // TODO: this only tests for new-style buffers, which is too strict, but a // generic check for Py_TYPE(fi)->tp_as_buffer is too loose (note that the // main use case is numpy, which offers the new interface) - if (!PyObject_CheckBuffer(fi)) { - if (PyTuple_CheckExact(fi)) - getter = new TupleItemGetter(fi); - else if (PyList_CheckExact(fi)) - getter = new ListItemGetter(fi); - else if (PySequence_Check(fi)) - getter = new SequenceItemGetter(fi); - else { - PyObject* iter = PyObject_GetIter(fi); - if (iter) { - getter = new IterItemGetter{iter}; - Py_DECREF(iter); - } - else PyErr_Clear(); + if (PyObject_CheckBuffer(fi)) + return nullptr; + + if (PyTuple_CheckExact(fi)) + getter = new TupleItemGetter(fi); + else if (PyList_CheckExact(fi)) + getter = new ListItemGetter(fi); + else if (PySequence_Check(fi)) + getter = new SequenceItemGetter(fi); + else { + PyObject* iter = PyObject_GetIter(fi); + if (iter) { + getter = new IterItemGetter{iter}; + Py_DECREF(iter); } + else PyErr_Clear(); } } - if (getter) { - // construct an empty vector, then back-fill it - PyObject* mname = CPyCppyy_PyText_FromString("__real_init"); - PyObject* result = PyObject_CallMethodObjArgs(self, mname, nullptr); - Py_DECREF(mname); - if (!result) { - delete getter; - return result; - } - - Py_ssize_t sz = getter->size(); - if (sz < 0) { - delete getter; - return nullptr; - } + return getter; +} - // reserve memory as appliable - if (0 < sz) { - PyObject* res = PyObject_CallMethod(self, (char*)"reserve", (char*)"n", sz); - Py_DECREF(res); - } else { // empty container - delete getter; - return result; - } +static bool FillVector(PyObject* vecin, PyObject* args, ItemGetter* getter) +{ + Py_ssize_t sz = getter->size(); + if (sz < 0) + return false; - bool fill_ok = true; - - // two main options: a list of lists (or tuples), or a list of objects; the former - // are emplace_back'ed, the latter push_back'ed - PyObject* fi = PySequence_GetItem(PyTuple_GET_ITEM(args, 0), 0); - if (!fi) PyErr_Clear(); - if (fi && (PyTuple_CheckExact(fi) || PyList_CheckExact(fi))) { - // use emplace_back to construct the vector entries one by one - PyObject* eb_call = PyObject_GetAttrString(self, (char*)"emplace_back"); - PyObject* vtype = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "value_type"); - bool value_is_vector = false; - if (vtype && CPyCppyy_PyText_Check(vtype)) { - // if the value_type is a vector, then allow for initialization from sequences - if (std::string(CPyCppyy_PyText_AsString(vtype)).rfind("std::vector", 0) != std::string::npos) - value_is_vector = true; - } else - PyErr_Clear(); - Py_XDECREF(vtype); - - if (eb_call) { - PyObject* eb_args; - for (int i = 0; /* until break */; ++i) { - PyObject* item = getter->get(); - if (item) { - if (value_is_vector && PySequence_Check(item)) { - eb_args = PyTuple_New(1); - PyTuple_SET_ITEM(eb_args, 0, item); - } else if (PyTuple_CheckExact(item)) { +// reserve memory as applicable + if (0 < sz) { + PyObject* res = PyObject_CallMethod(vecin, (char*)"reserve", (char*)"n", sz); + Py_DECREF(res); + } else // i.e. sz == 0, so empty container: done + return true; + + bool fill_ok = true; + +// two main options: a list of lists (or tuples), or a list of objects; the former +// are emplace_back'ed, the latter push_back'ed + PyObject* fi = PySequence_GetItem(PyTuple_GET_ITEM(args, 0), 0); + if (!fi) PyErr_Clear(); + if (fi && (PyTuple_CheckExact(fi) || PyList_CheckExact(fi))) { + // use emplace_back to construct the vector entries one by one + PyObject* eb_call = PyObject_GetAttrString(vecin, (char*)"emplace_back"); + PyObject* vtype = GetAttrDirect((PyObject*)Py_TYPE(vecin), PyStrings::gValueType); + bool value_is_vector = false; + if (vtype && CPyCppyy_PyText_Check(vtype)) { + // if the value_type is a vector, then allow for initialization from sequences + if (std::string(CPyCppyy_PyText_AsString(vtype)).rfind("std::vector", 0) != std::string::npos) + value_is_vector = true; + } else + PyErr_Clear(); + Py_XDECREF(vtype); + + if (eb_call) { + PyObject* eb_args; + for (int i = 0; /* until break */; ++i) { + PyObject* item = getter->get(); + if (item) { + if (value_is_vector && PySequence_Check(item)) { + eb_args = PyTuple_New(1); + PyTuple_SET_ITEM(eb_args, 0, item); + } else if (PyTuple_CheckExact(item)) { eb_args = item; - } else if (PyList_CheckExact(item)) { - Py_ssize_t isz = PyList_GET_SIZE(item); - eb_args = PyTuple_New(isz); - for (Py_ssize_t j = 0; j < isz; ++j) { - PyObject* iarg = PyList_GET_ITEM(item, j); - Py_INCREF(iarg); - PyTuple_SET_ITEM(eb_args, j, iarg); - } - Py_DECREF(item); - } else { - Py_DECREF(item); - PyErr_Format(PyExc_TypeError, "argument %d is not a tuple or list", i); - fill_ok = false; - break; - } - PyObject* ebres = PyObject_CallObject(eb_call, eb_args); - Py_DECREF(eb_args); - if (!ebres) { - fill_ok = false; - break; + } else if (PyList_CheckExact(item)) { + Py_ssize_t isz = PyList_GET_SIZE(item); + eb_args = PyTuple_New(isz); + for (Py_ssize_t j = 0; j < isz; ++j) { + PyObject* iarg = PyList_GET_ITEM(item, j); + Py_INCREF(iarg); + PyTuple_SET_ITEM(eb_args, j, iarg); } - Py_DECREF(ebres); + Py_DECREF(item); } else { - if (PyErr_Occurred()) { - if (!(PyErr_ExceptionMatches(PyExc_IndexError) || - PyErr_ExceptionMatches(PyExc_StopIteration))) - fill_ok = false; - else { PyErr_Clear(); } - } + Py_DECREF(item); + PyErr_Format(PyExc_TypeError, "argument %d is not a tuple or list", i); + fill_ok = false; + break; + } + PyObject* ebres = PyObject_CallObject(eb_call, eb_args); + Py_DECREF(eb_args); + if (!ebres) { + fill_ok = false; break; } + Py_DECREF(ebres); + } else { + if (PyErr_Occurred()) { + if (!(PyErr_ExceptionMatches(PyExc_IndexError) || + PyErr_ExceptionMatches(PyExc_StopIteration))) + fill_ok = false; + else { PyErr_Clear(); } + } + break; } - Py_DECREF(eb_call); } - } else { - // use push_back to add the vector entries one by one - PyObject* pb_call = PyObject_GetAttrString(self, (char*)"push_back"); - if (pb_call) { - for (;;) { - PyObject* item = getter->get(); - if (item) { - PyObject* pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); - Py_DECREF(item); - if (!pbres) { - fill_ok = false; - break; - } - Py_DECREF(pbres); - } else { - if (PyErr_Occurred()) { - if (!(PyErr_ExceptionMatches(PyExc_IndexError) || - PyErr_ExceptionMatches(PyExc_StopIteration))) - fill_ok = false; - else { PyErr_Clear(); } - } + Py_DECREF(eb_call); + } + } else { + // use push_back to add the vector entries one by one + PyObject* pb_call = PyObject_GetAttrString(vecin, (char*)"push_back"); + if (pb_call) { + for (;;) { + PyObject* item = getter->get(); + if (item) { + PyObject* pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); + Py_DECREF(item); + if (!pbres) { + fill_ok = false; break; } + Py_DECREF(pbres); + } else { + if (PyErr_Occurred()) { + if (!(PyErr_ExceptionMatches(PyExc_IndexError) || + PyErr_ExceptionMatches(PyExc_StopIteration))) + fill_ok = false; + else { PyErr_Clear(); } + } + break; } - Py_DECREF(pb_call); } + Py_DECREF(pb_call); } - Py_XDECREF(fi); + } + Py_XDECREF(fi); + + return fill_ok; +} + +PyObject* VectorIAdd(PyObject* self, PyObject* args, PyObject* /* kwds */) +{ +// Implement fast __iadd__ on std::vector (generic __iadd__ is in Python) + ItemGetter* getter = GetGetter(args); + + if (getter) { + bool fill_ok = FillVector(self, args, getter); + delete getter; + + if (!fill_ok) + return nullptr; + + Py_INCREF(self); + return self; + } + +// if no getter, it could still be b/c we have a buffer (e.g. numpy); looping over +// a buffer here is slow, so use insert() instead + if (PyTuple_GET_SIZE(args) == 1) { + PyObject* fi = PyTuple_GET_ITEM(args, 0); + if (PyObject_CheckBuffer(fi) && !(CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi))) { + PyObject* vend = PyObject_CallMethodNoArgs(self, PyStrings::gEnd); + if (vend) { + PyObject* result = PyObject_CallMethodObjArgs(self, PyStrings::gInsert, vend, fi, nullptr); + Py_DECREF(vend); + return result; + } + } + } + + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, "argument is not iterable"); + return nullptr; // error already set +} + + +PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) +{ +// Specialized vector constructor to allow construction from containers; allowing +// such construction from initializer_list instead would possible, but can be +// error-prone. This use case is common enough for std::vector to implement it +// directly, except for arrays (which can be passed wholesale) and strings (which +// won't convert properly as they'll be seen as buffers) + + ItemGetter* getter = GetGetter(args); + + if (getter) { + // construct an empty vector, then back-fill it + PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!result) { + delete getter; + return nullptr; + } + + bool fill_ok = FillVector(self, args, getter); delete getter; if (!fill_ok) { @@ -420,7 +491,7 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) } // The given argument wasn't iterable: simply forward to regular constructor - PyObject* realInit = PyObject_GetAttrString(self, "__real_init"); + PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); if (realInit) { PyObject* result = PyObject_Call(realInit, args, nullptr); Py_DECREF(realInit); @@ -434,9 +505,10 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) PyObject* VectorData(PyObject* self, PyObject*) { PyObject* pydata = CallPyObjMethod(self, "__real_data"); - if (!LowLevelView_Check(pydata)) return pydata; + if (!LowLevelView_Check(pydata) && !CPPInstance_Check(pydata)) + return pydata; - PyObject* pylen = PyObject_CallMethodObjArgs(self, PyStrings::gSize, nullptr); + PyObject* pylen = PyObject_CallMethodNoArgs(self, PyStrings::gSize); if (!pylen) { PyErr_Clear(); return pydata; @@ -445,16 +517,26 @@ PyObject* VectorData(PyObject* self, PyObject*) long clen = PyInt_AsLong(pylen); Py_DECREF(pylen); -// TODO: should be a LowLevelView helper - Py_buffer& bi = ((LowLevelView*)pydata)->fBufInfo; - bi.len = clen * bi.itemsize; - if (bi.ndim == 1 && bi.shape) - bi.shape[0] = clen; + if (CPPInstance_Check(pydata)) { + ((CPPInstance*)pydata)->CastToArray(clen); + return pydata; + } + ((LowLevelView*)pydata)->resize((size_t)clen); return pydata; } +//--------------------------------------------------------------------------- +PyObject* VectorArray(PyObject* self, PyObject* /* args */) +{ + PyObject* pydata = VectorData(self, nullptr); + PyObject* view = PyObject_CallMethodNoArgs(pydata, PyStrings::gArray); + Py_DECREF(pydata); + return view; +} + + //----------------------------------------------------------------------------- static PyObject* vector_iter(PyObject* v) { vectoriterobject* vi = PyObject_GC_New(vectoriterobject, &VectorIter_Type); @@ -462,33 +544,58 @@ static PyObject* vector_iter(PyObject* v) { Py_INCREF(v); vi->ii_container = v; - vi->vi_flags = v->ob_refcnt <= 2 ? 1 : 0; // 2, b/c of preceding INCREF - - PyObject* pyvalue_type = PyObject_GetAttrString((PyObject*)Py_TYPE(v), "value_type"); - PyObject* pyvalue_size = PyObject_GetAttrString((PyObject*)Py_TYPE(v), "value_size"); - - vi->vi_klass = 0; - if (pyvalue_type && pyvalue_size) { - PyObject* pydata = CallPyObjMethod(v, "data"); - if (!pydata || Utility::GetBuffer(pydata, '*', 1, vi->vi_data, false) == 0) { - if (CPPInstance_Check(pydata)) { - vi->vi_data = ((CPPInstance*)pydata)->GetObjectRaw(); - vi->vi_klass = ((CPPInstance*)pydata)->ObjectIsA(false); + +// tell the iterator code to set a life line if this container is a temporary + vi->vi_flags = vectoriterobject::kDefault; + if (v->ob_refcnt <= 2 || (((CPPInstance*)v)->fFlags & CPPInstance::kIsValue)) + vi->vi_flags = vectoriterobject::kNeedLifeLine; + + PyObject* pyvalue_type = PyObject_GetAttr((PyObject*)Py_TYPE(v), PyStrings::gValueType); + if (pyvalue_type) { + PyObject* pyvalue_size = GetAttrDirect((PyObject*)Py_TYPE(v), PyStrings::gValueSize); + if (pyvalue_size) { + vi->vi_stride = PyLong_AsLong(pyvalue_size); + Py_DECREF(pyvalue_size); + } else { + PyErr_Clear(); + vi->vi_stride = 0; + } + + if (CPyCppyy_PyText_Check(pyvalue_type)) { + std::string value_type = CPyCppyy_PyText_AsString(pyvalue_type); + vi->vi_klass = Cppyy::GetScope(value_type); + if (vi->vi_klass) { + vi->vi_converter = nullptr; + if (!vi->vi_flags) { + value_type = Cppyy::ResolveName(value_type); + if (value_type.back() != '*') // meaning, object stored by-value + vi->vi_flags = vectoriterobject::kNeedLifeLine; + } } else - vi->vi_data = nullptr; + vi->vi_converter = CPyCppyy::CreateConverter(value_type); + if (!vi->vi_stride) vi->vi_stride = Cppyy::SizeOf(value_type); + + } else if (CPPScope_Check(pyvalue_type)) { + vi->vi_klass = ((CPPClass*)pyvalue_type)->fCppType; + vi->vi_converter = nullptr; + if (!vi->vi_stride) vi->vi_stride = Cppyy::SizeOf(vi->vi_klass); + if (!vi->vi_flags) vi->vi_flags = vectoriterobject::kNeedLifeLine; } + + PyObject* pydata = CallPyObjMethod(v, "__real_data"); + if (!pydata || Utility::GetBuffer(pydata, '*', 1, vi->vi_data, false) == 0) + vi->vi_data = CPPInstance_Check(pydata) ? ((CPPInstance*)pydata)->GetObjectRaw() : nullptr; Py_XDECREF(pydata); - vi->vi_converter = vi->vi_klass ? nullptr : CPyCppyy::CreateConverter(CPyCppyy_PyText_AsString(pyvalue_type)); - vi->vi_stride = PyLong_AsLong(pyvalue_size); } else { PyErr_Clear(); vi->vi_data = nullptr; - vi->vi_converter = nullptr; vi->vi_stride = 0; + vi->vi_converter = nullptr; + vi->vi_klass = 0; + vi->vi_flags = 0; } - Py_XDECREF(pyvalue_size); Py_XDECREF(pyvalue_type); vi->ii_pos = 0; @@ -520,7 +627,7 @@ PyObject* VectorGetItem(CPPInstance* self, PySliceObject* index) const Py_ssize_t sign = step < 0 ? -1 : 1; for (Py_ssize_t i = start; i*sign < stop*sign; i += step) { PyObject* pyidx = PyInt_FromSsize_t(i); - PyObject* item = PyObject_CallMethodObjArgs((PyObject*)self, PyStrings::gGetNoCheck, pyidx, nullptr); + PyObject* item = PyObject_CallMethodOneArg((PyObject*)self, PyStrings::gGetNoCheck, pyidx); CallPyObjMethod(nseq, "push_back", item); Py_DECREF(item); Py_DECREF(pyidx); @@ -564,7 +671,7 @@ PyObject* VectorBoolGetItem(CPPInstance* self, PyObject* idx) const Py_ssize_t sign = step < 0 ? -1 : 1; for (Py_ssize_t i = start; i*sign < stop*sign; i += step) { PyObject* pyidx = PyInt_FromSsize_t(i); - PyObject* item = PyObject_CallMethodObjArgs((PyObject*)self, PyStrings::gGetItem, pyidx, nullptr); + PyObject* item = PyObject_CallMethodOneArg((PyObject*)self, PyStrings::gGetItem, pyidx); CallPyObjMethod(nseq, "push_back", item); Py_DECREF(item); Py_DECREF(pyidx); @@ -625,15 +732,148 @@ PyObject* VectorBoolSetItem(CPPInstance* self, PyObject* args) Py_RETURN_NONE; } + +//- array behavior as primitives ---------------------------------------------- +PyObject* ArrayInit(PyObject* self, PyObject* args, PyObject* /* kwds */) +{ +// std::array is normally only constructed using aggregate initialization, which +// is a concept that does not exist in python, so use this custom constructor to +// to fill the array using setitem + + if (args && PyTuple_GET_SIZE(args) == 1 && PySequence_Check(PyTuple_GET_ITEM(args, 0))) { + // construct the empty array, then fill it + PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!result) + return nullptr; + + PyObject* items = PyTuple_GET_ITEM(args, 0); + Py_ssize_t fillsz = PySequence_Size(items); + if (PySequence_Size(self) != fillsz) { + PyErr_Format(PyExc_ValueError, "received sequence of size %zd where %zd expected", + fillsz, PySequence_Size(self)); + Py_DECREF(result); + return nullptr; + } + + PyObject* si_call = PyObject_GetAttr(self, PyStrings::gSetItem); + for (Py_ssize_t i = 0; i < fillsz; ++i) { + PyObject* item = PySequence_GetItem(items, i); + PyObject* index = PyInt_FromSsize_t(i); + PyObject* sires = PyObject_CallFunctionObjArgs(si_call, index, item, nullptr); + Py_DECREF(index); + Py_DECREF(item); + if (!sires) { + Py_DECREF(si_call); + Py_DECREF(result); + return nullptr; + } else + Py_DECREF(sires); + } + Py_DECREF(si_call); + + return result; + } else + PyErr_Clear(); + +// The given argument wasn't iterable: simply forward to regular constructor + PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); + if (realInit) { + PyObject* result = PyObject_Call(realInit, args, nullptr); + Py_DECREF(realInit); + return result; + } + + return nullptr; +} + + //- map behavior as primitives ------------------------------------------------ -PyObject* MapContains(PyObject* self, PyObject* obj) +static PyObject* MapFromPairs(PyObject* self, PyObject* pairs) +{ +// construct an empty map, then fill it with the key, value pairs + PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!result) + return nullptr; + + PyObject* si_call = PyObject_GetAttr(self, PyStrings::gSetItem); + for (Py_ssize_t i = 0; i < PySequence_Size(pairs); ++i) { + PyObject* pair = PySequence_GetItem(pairs, i); + PyObject* sires = nullptr; + if (pair && PySequence_Check(pair) && PySequence_Size(pair) == 2) { + PyObject* key = PySequence_GetItem(pair, 0); + PyObject* value = PySequence_GetItem(pair, 1); + sires = PyObject_CallFunctionObjArgs(si_call, key, value, nullptr); + Py_DECREF(value); + Py_DECREF(key); + } + Py_DECREF(pair); + if (!sires) { + Py_DECREF(si_call); + Py_DECREF(result); + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, "Failed to fill map (argument not a dict or sequence of pairs)"); + return nullptr; + } else + Py_DECREF(sires); + } + Py_DECREF(si_call); + + return result; +} + +PyObject* MapInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { -// Implement python's __contains__ for std::map<>s +// Specialized map constructor to allow construction from mapping containers and +// from tuples of pairs ("initializer_list style"). + +// PyMapping_Check is not very discriminatory, as it basically only checks for the +// existence of __getitem__, hence the most common cases of tuple and list are +// dropped straight-of-the-bat (the PyMapping_Items call will fail on them). + if (PyTuple_GET_SIZE(args) == 1 && PyMapping_Check(PyTuple_GET_ITEM(args, 0)) && \ + !(PyTuple_Check(PyTuple_GET_ITEM(args, 0)) || PyList_Check(PyTuple_GET_ITEM(args, 0)))) { + PyObject* assoc = PyTuple_GET_ITEM(args, 0); +#if PY_VERSION_HEX < 0x03000000 + // to prevent warning about literal string, expand macro + PyObject* items = PyObject_CallMethod(assoc, (char*)"items", nullptr); +#else + // in p3, PyMapping_Items isn't a macro, but a function that short-circuits dict + PyObject* items = PyMapping_Items(assoc); +#endif + if (items && PySequence_Check(items)) { + PyObject* result = MapFromPairs(self, items); + Py_DECREF(items); + return result; + } + + Py_XDECREF(items); + PyErr_Clear(); + + // okay to fall through as long as 'self' has not been created (is done in MapFromPairs) + } + +// tuple of pairs case (some mapping types are sequences) + if (PyTuple_GET_SIZE(args) == 1 && PySequence_Check(PyTuple_GET_ITEM(args, 0))) + return MapFromPairs(self, PyTuple_GET_ITEM(args, 0)); + +// The given argument wasn't a mapping or tuple of pairs: forward to regular constructor + PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); + if (realInit) { + PyObject* result = PyObject_Call(realInit, args, nullptr); + Py_DECREF(realInit); + return result; + } + + return nullptr; +} + +PyObject* STLContainsWithFind(PyObject* self, PyObject* obj) +{ +// Implement python's __contains__ for std::map/std::set PyObject* result = nullptr; PyObject* iter = CallPyObjMethod(self, "find", obj); if (CPPInstance_Check(iter)) { - PyObject* end = PyObject_CallMethodObjArgs(self, PyStrings::gEnd, nullptr); + PyObject* end = PyObject_CallMethodNoArgs(self, PyStrings::gEnd); if (CPPInstance_Check(end)) { if (!PyObject_RichCompareBool(iter, end, Py_EQ)) { Py_INCREF(Py_True); @@ -654,17 +894,92 @@ PyObject* MapContains(PyObject* self, PyObject* obj) } +//- set behavior as primitives ------------------------------------------------ +PyObject* SetInit(PyObject* self, PyObject* args, PyObject* /* kwds */) +{ +// Specialized set constructor to allow construction from Python sets. + if (PyTuple_GET_SIZE(args) == 1 && PySet_Check(PyTuple_GET_ITEM(args, 0))) { + PyObject* pyset = PyTuple_GET_ITEM(args, 0); + + // construct an empty set, then fill it + PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!result) + return nullptr; + + PyObject* iter = PyObject_GetIter(pyset); + if (iter) { + PyObject* ins_call = PyObject_GetAttrString(self, (char*)"insert"); + + IterItemGetter getter{iter}; + Py_DECREF(iter); + + PyObject* item = getter.get(); + while (item) { + PyObject* insres = PyObject_CallFunctionObjArgs(ins_call, item, nullptr); + Py_DECREF(item); + if (!insres) { + Py_DECREF(ins_call); + Py_DECREF(result); + return nullptr; + } else + Py_DECREF(insres); + item = getter.get(); + } + Py_DECREF(ins_call); + } + + return result; + } + +// The given argument wasn't iterable: simply forward to regular constructor + PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); + if (realInit) { + PyObject* result = PyObject_Call(realInit, args, nullptr); + Py_DECREF(realInit); + return result; + } + + return nullptr; +} + + //- STL container iterator support -------------------------------------------- static const ptrdiff_t PS_END_ADDR = 7; // non-aligned address, so no clash static const ptrdiff_t PS_FLAG_ADDR = 11; // id. static const ptrdiff_t PS_COLL_ADDR = 13; // id. -PyObject* StlSequenceIter(PyObject* self) +PyObject* LLSequenceIter(PyObject* self) +{ +// Implement python's __iter__ for low level views used through STL-type begin()/end() + PyObject* iter = PyObject_CallMethodNoArgs(self, PyStrings::gBegin); + + if (LowLevelView_Check(iter)) { + // builtin pointer iteration: can only succeed if a size is available + Py_ssize_t sz = PySequence_Size(self); + if (sz == -1) { + Py_DECREF(iter); + return nullptr; + } + PyObject* lliter = Py_TYPE(iter)->tp_iter(iter); + ((indexiterobject*)lliter)->ii_len = sz; + Py_DECREF(iter); + return lliter; + } + + if (iter) { + Py_DECREF(iter); + PyErr_SetString(PyExc_TypeError, "unrecognized iterator type for low level views"); + } + + return nullptr; +} + +PyObject* STLSequenceIter(PyObject* self) { // Implement python's __iter__ for std::iterator<>s - PyObject* iter = PyObject_CallMethodObjArgs(self, PyStrings::gBegin, nullptr); + PyObject* iter = PyObject_CallMethodNoArgs(self, PyStrings::gBegin); if (iter) { - PyObject* end = PyObject_CallMethodObjArgs(self, PyStrings::gEnd, nullptr); + PyObject* end = PyObject_CallMethodNoArgs(self, PyStrings::gEnd); if (end) { if (CPPInstance_Check(iter)) { // use the data member cache to store extra state on the iterator object, @@ -711,13 +1026,13 @@ PyObject* CheckedGetItem(PyObject* self, PyObject* obj) { // Implement a generic python __getitem__ for STL-like classes that are missing the // reflection info for their iterators. This is then used for iteration by means of -// consecutive indeces, it such index is of integer type. +// consecutive indices, it such index is of integer type. Py_ssize_t size = PySequence_Size(self); Py_ssize_t idx = PyInt_AsSsize_t(obj); if ((size == (Py_ssize_t)-1 || idx == (Py_ssize_t)-1) && PyErr_Occurred()) { // argument conversion problem: let method itself resolve anew and report PyErr_Clear(); - return PyObject_CallMethodObjArgs(self, PyStrings::gGetNoCheck, obj, nullptr); + return PyObject_CallMethodOneArg(self, PyStrings::gGetNoCheck, obj); } bool inbounds = false; @@ -726,13 +1041,14 @@ PyObject* CheckedGetItem(PyObject* self, PyObject* obj) inbounds = true; if (inbounds) - return PyObject_CallMethodObjArgs(self, PyStrings::gGetNoCheck, obj, nullptr); + return PyObject_CallMethodOneArg(self, PyStrings::gGetNoCheck, obj); else PyErr_SetString( PyExc_IndexError, "index out of range" ); return nullptr; }*/ + //- pair as sequence to allow tuple unpacking -------------------------------- PyObject* PairUnpack(PyObject* self, PyObject* pyindex) { @@ -762,16 +1078,18 @@ PyObject* ReturnTwo(CPPInstance*, PyObject*) { } -//- shared_ptr behavior -------------------------------------------------------- -PyObject* SharedPtrInit(PyObject* self, PyObject* args, PyObject* /* kwds */) +//- shared/unique_ptr behavior ----------------------------------------------- +PyObject* SmartPtrInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { -// since the shared pointer will take ownership, we need to relinquish it - PyObject* realInit = PyObject_GetAttrString(self, "__real_init"); +// since the shared/unique pointer will take ownership, we need to relinquish it + PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); if (realInit) { PyObject* result = PyObject_Call(realInit, args, nullptr); Py_DECREF(realInit); - if (result && PyTuple_GET_SIZE(args) == 1 && CPPInstance_Check(PyTuple_GET_ITEM(args, 0))) - PyObject_SetAttrString(PyTuple_GET_ITEM(args, 0), "__python_owns__", Py_False); + if (result && PyTuple_GET_SIZE(args) == 1 && CPPInstance_Check(PyTuple_GET_ITEM(args, 0))) { + CPPInstance* cppinst = (CPPInstance*)PyTuple_GET_ITEM(args, 0); + if (!(cppinst->fFlags & CPPInstance::kIsSmartPtr)) cppinst->CppOwns(); + } return result; } return nullptr; @@ -785,32 +1103,59 @@ static int PyObject_Compare(PyObject* one, PyObject* other) { return !PyObject_RichCompareBool(one, other, Py_EQ); } #endif -static inline PyObject* CPyCppyy_PyString_FromCppString(std::string* s) { - return CPyCppyy_PyText_FromStringAndSize(s->c_str(), s->size()); +static inline +PyObject* CPyCppyy_PyString_FromCppString(std::string* s, bool native=true) { + if (native) + return PyBytes_FromStringAndSize(s->data(), s->size()); + return CPyCppyy_PyText_FromStringAndSize(s->data(), s->size()); } -static inline PyObject* CPyCppyy_PyString_FromCppString(std::wstring* s) { - return PyUnicode_FromWideChar(s->c_str(), s->size()); +static inline +PyObject* CPyCppyy_PyString_FromCppString(std::wstring* s, bool native=true) { + PyObject* pyobj = PyUnicode_FromWideChar(s->data(), s->size()); + if (pyobj && native) { + PyObject* pybytes = PyUnicode_AsEncodedString(pyobj, "UTF-8", "strict"); + Py_DECREF(pyobj); + pyobj = pybytes; + } + return pyobj; } #define CPPYY_IMPL_STRING_PYTHONIZATION(type, name) \ -static PyObject* name##StringGetData(PyObject* self) \ +static inline \ +PyObject* name##StringGetData(PyObject* self, bool native=true) \ { \ if (CPyCppyy::CPPInstance_Check(self)) { \ type* obj = ((type*)((CPPInstance*)self)->GetObject()); \ - if (obj) { \ - return CPyCppyy_PyString_FromCppString(obj); \ - } else { \ - return CPPInstance_Type.tp_str(self); \ - } \ + if (obj) return CPyCppyy_PyString_FromCppString(obj, native); \ } \ PyErr_Format(PyExc_TypeError, "object mismatch (%s expected)", #type); \ return nullptr; \ } \ \ +PyObject* name##StringStr(PyObject* self) \ +{ \ + PyObject* pyobj = name##StringGetData(self, false); \ + if (!pyobj) { \ + /* do a native conversion to make printing possible (debatable) */ \ + PyErr_Clear(); \ + PyObject* pybytes = name##StringGetData(self, true); \ + if (pybytes) { /* should not fail */ \ + pyobj = PyObject_Str(pybytes); \ + Py_DECREF(pybytes); \ + } \ + } \ + return pyobj; \ +} \ + \ +PyObject* name##StringBytes(PyObject* self) \ +{ \ + return name##StringGetData(self, true); \ +} \ + \ PyObject* name##StringRepr(PyObject* self) \ { \ - PyObject* data = name##StringGetData(self); \ + PyObject* data = name##StringGetData(self, true); \ if (data) { \ PyObject* repr = PyObject_Repr(data); \ Py_DECREF(data); \ @@ -821,7 +1166,7 @@ PyObject* name##StringRepr(PyObject* self) \ \ PyObject* name##StringIsEqual(PyObject* self, PyObject* obj) \ { \ - PyObject* data = name##StringGetData(self); \ + PyObject* data = name##StringGetData(self, PyBytes_Check(obj)); \ if (data) { \ PyObject* result = PyObject_RichCompare(data, obj, Py_EQ); \ Py_DECREF(data); \ @@ -832,7 +1177,7 @@ PyObject* name##StringIsEqual(PyObject* self, PyObject* obj) \ \ PyObject* name##StringIsNotEqual(PyObject* self, PyObject* obj) \ { \ - PyObject* data = name##StringGetData(self); \ + PyObject* data = name##StringGetData(self, PyBytes_Check(obj)); \ if (data) { \ PyObject* result = PyObject_RichCompare(data, obj, Py_NE); \ Py_DECREF(data); \ @@ -841,12 +1186,12 @@ PyObject* name##StringIsNotEqual(PyObject* self, PyObject* obj) \ return nullptr; \ } -// Only define StlStringCompare: +// Only define STLStringCompare: #define CPPYY_IMPL_STRING_PYTHONIZATION_CMP(type, name) \ CPPYY_IMPL_STRING_PYTHONIZATION(type, name) \ PyObject* name##StringCompare(PyObject* self, PyObject* obj) \ { \ - PyObject* data = name##StringGetData(self); \ + PyObject* data = name##StringGetData(self, PyBytes_Check(obj)); \ int result = 0; \ if (data) { \ result = PyObject_Compare(data, obj); \ @@ -857,22 +1202,210 @@ PyObject* name##StringCompare(PyObject* self, PyObject* obj) \ return PyInt_FromLong(result); \ } -CPPYY_IMPL_STRING_PYTHONIZATION_CMP(std::string, Stl) -CPPYY_IMPL_STRING_PYTHONIZATION_CMP(std::wstring, StlW) +CPPYY_IMPL_STRING_PYTHONIZATION_CMP(std::string, STL) +CPPYY_IMPL_STRING_PYTHONIZATION_CMP(std::wstring, STLW) + +static inline std::string* GetSTLString(CPPInstance* self) { + if (!CPPInstance_Check(self)) { + PyErr_SetString(PyExc_TypeError, "std::string object expected"); + return nullptr; + } + + std::string* obj = (std::string*)self->GetObject(); + if (!obj) + PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer"); + + return obj; +} + +PyObject* STLStringDecode(CPPInstance* self, PyObject* args, PyObject* kwds) +{ + std::string* obj = GetSTLString(self); + if (!obj) + return nullptr; + + char* keywords[] = {(char*)"encoding", (char*)"errors", (char*)nullptr}; + const char* encoding; const char* errors; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + const_cast("s|s"), keywords, &encoding, &errors)) + return nullptr; + + return PyUnicode_Decode(obj->data(), obj->size(), encoding, errors); +} + +PyObject* STLStringContains(CPPInstance* self, PyObject* pyobj) +{ + std::string* obj = GetSTLString(self); + if (!obj) + return nullptr; + + const char* needle = CPyCppyy_PyText_AsString(pyobj); + if (!needle) + return nullptr; + + if (obj->find(needle) != std::string::npos) { + Py_RETURN_TRUE; + } + + Py_RETURN_FALSE; +} + +PyObject* STLStringReplace(CPPInstance* self, PyObject* args, PyObject* /*kwds*/) +{ + std::string* obj = GetSTLString(self); + if (!obj) + return nullptr; + +// both str and std::string have a method "replace", but the Python version only +// accepts strings and takes no keyword arguments, whereas the C++ version has no +// overload that takes a string + + if (2 <= PyTuple_GET_SIZE(args) && CPyCppyy_PyText_Check(PyTuple_GET_ITEM(args, 0))) { + PyObject* pystr = CPyCppyy_PyText_FromStringAndSize(obj->data(), obj->size()); + PyObject* meth = PyObject_GetAttrString(pystr, (char*)"replace"); + Py_DECREF(pystr); + PyObject* result = PyObject_CallObject(meth, args); + Py_DECREF(meth); + return result; + } + + PyObject* cppreplace = PyObject_GetAttrString((PyObject*)self, (char*)"__cpp_replace"); + if (cppreplace) { + PyObject* result = PyObject_Call(cppreplace, args, nullptr); + Py_DECREF(cppreplace); + return result; + } + + PyErr_SetString(PyExc_AttributeError, "\'std::string\' object has no attribute \'replace\'"); + return nullptr; +} + +#define CPYCPPYY_STRING_FINDMETHOD(name, cppname, pyname) \ +PyObject* STLString##name(CPPInstance* self, PyObject* args, PyObject* /*kwds*/) \ +{ \ + std::string* obj = GetSTLString(self); \ + if (!obj) \ + return nullptr; \ + \ + PyObject* cppmeth = PyObject_GetAttrString((PyObject*)self, (char*)#cppname);\ + if (cppmeth) { \ + PyObject* result = PyObject_Call(cppmeth, args, nullptr); \ + Py_DECREF(cppmeth); \ + if (result) { \ + if (PyLongOrInt_AsULong64(result) == (PY_ULONG_LONG)std::string::npos) {\ + Py_DECREF(result); \ + return PyInt_FromLong(-1); \ + } \ + return result; \ + } \ + PyErr_Clear(); \ + } \ + \ + PyObject* pystr = CPyCppyy_PyText_FromStringAndSize(obj->data(), obj->size());\ + PyObject* pymeth = PyObject_GetAttrString(pystr, (char*)#pyname); \ + Py_DECREF(pystr); \ + PyObject* result = PyObject_CallObject(pymeth, args); \ + Py_DECREF(pymeth); \ + return result; \ +} + +// both str and std::string have method "find" and "rfin"; try the C++ version first +// and fall back on the Python one in case of failure +CPYCPPYY_STRING_FINDMETHOD( Find, __cpp_find, find) +CPYCPPYY_STRING_FINDMETHOD(RFind, __cpp_rfind, rfind) + +PyObject* STLStringGetAttr(CPPInstance* self, PyObject* attr_name) +{ + std::string* obj = GetSTLString(self); + if (!obj) + return nullptr; + + PyObject* pystr = CPyCppyy_PyText_FromStringAndSize(obj->data(), obj->size()); + PyObject* attr = PyObject_GetAttr(pystr, attr_name); + Py_DECREF(pystr); + return attr; +} + -Py_hash_t StlStringHash(PyObject* self) +#if 0 +PyObject* UTF8Repr(PyObject* self) +{ +// force C++ string types conversion to Python str per Python __repr__ requirements + PyObject* res = PyObject_CallMethodNoArgs(self, PyStrings::gCppRepr); + if (!res || CPyCppyy_PyText_Check(res)) + return res; + PyObject* str_res = PyObject_Str(res); + Py_DECREF(res); + return str_res; +} + +PyObject* UTF8Str(PyObject* self) +{ +// force C++ string types conversion to Python str per Python __str__ requirements + PyObject* res = PyObject_CallMethodNoArgs(self, PyStrings::gCppStr); + if (!res || CPyCppyy_PyText_Check(res)) + return res; + PyObject* str_res = PyObject_Str(res); + Py_DECREF(res); + return str_res; +} +#endif + +Py_hash_t STLStringHash(PyObject* self) { // std::string objects hash to the same values as Python strings to allow // matches in dictionaries etc. - PyObject* data = StlStringGetData(self); + PyObject* data = STLStringGetData(self, false); Py_hash_t h = CPyCppyy_PyText_Type.tp_hash(data); Py_DECREF(data); return h; } +//- string_view behavior as primitive ---------------------------------------- +PyObject* StringViewInit(PyObject* self, PyObject* args, PyObject* /* kwds */) +{ +// if constructed from a Python unicode object, the constructor will convert it +// to a temporary byte string, which is likely to go out of scope too soon; so +// buffer it as needed + PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); + if (realInit) { + PyObject *strbuf = nullptr, *newArgs = nullptr; + if (PyTuple_GET_SIZE(args) == 1) { + PyObject* arg0 = PyTuple_GET_ITEM(args, 0); + if (PyUnicode_Check(arg0)) { + // convert to the expected bytes array to control the temporary + strbuf = PyUnicode_AsEncodedString(arg0, "UTF-8", "strict"); + newArgs = PyTuple_New(1); + Py_INCREF(strbuf); + PyTuple_SET_ITEM(newArgs, 0, strbuf); + } else if (PyBytes_Check(arg0)) { + // tie the life time of the provided string to the string_view + Py_INCREF(arg0); + strbuf = arg0; + } + } + + PyObject* result = PyObject_Call(realInit, newArgs ? newArgs : args, nullptr); + + Py_XDECREF(newArgs); + Py_DECREF(realInit); + + // if construction was successful and a string buffer was used, add a + // life line to it from the string_view bound object + if (result && self && strbuf) + PyObject_SetAttr(self, PyStrings::gLifeLine, strbuf); + Py_XDECREF(strbuf); + + return result; + } + return nullptr; +} + + + //- STL iterator behavior ---------------------------------------------------- -PyObject* StlIterNext(PyObject* self) +PyObject* STLIterNext(PyObject* self) { // Python iterator protocol __next__ for STL forward iterators. bool mustIncrement = true; @@ -902,18 +1435,18 @@ PyObject* StlIterNext(PyObject* self) if (mustIncrement) { // prefer preinc, but allow post-inc; in both cases, it is "self" that has // the updated state to dereference - PyObject* iter = PyObject_CallMethodObjArgs(self, PyStrings::gPreInc, nullptr); + PyObject* iter = PyObject_CallMethodNoArgs(self, PyStrings::gPreInc); if (!iter) { PyErr_Clear(); static PyObject* dummy = PyInt_FromLong(1l); - iter = PyObject_CallMethodObjArgs(self, PyStrings::gPostInc, dummy, nullptr); + iter = PyObject_CallMethodOneArg(self, PyStrings::gPostInc, dummy); } iter_valid = iter && PyObject_RichCompareBool(last, self, Py_NE); Py_XDECREF(iter); } if (iter_valid) { - next = PyObject_CallMethodObjArgs(self, PyStrings::gDeref, nullptr); + next = PyObject_CallMethodNoArgs(self, PyStrings::gDeref); if (!next) PyErr_Clear(); } } @@ -928,10 +1461,10 @@ PyObject* StlIterNext(PyObject* self) //- STL complex behavior -------------------------------------------------- #define COMPLEX_METH_GETSET(name, cppname) \ static PyObject* name##ComplexGet(PyObject* self, void*) { \ - return PyObject_CallMethodObjArgs(self, cppname, nullptr); \ + return PyObject_CallMethodNoArgs(self, cppname); \ } \ static int name##ComplexSet(PyObject* self, PyObject* value, void*) { \ - PyObject* result = PyObject_CallMethodObjArgs(self, cppname, value, nullptr);\ + PyObject* result = PyObject_CallMethodOneArg(self, cppname, value); \ if (result) { \ Py_DECREF(result); \ return 0; \ @@ -944,14 +1477,14 @@ COMPLEX_METH_GETSET(real, PyStrings::gCppReal) COMPLEX_METH_GETSET(imag, PyStrings::gCppImag) static PyObject* ComplexComplex(PyObject* self) { - PyObject* real = PyObject_CallMethodObjArgs(self, PyStrings::gCppReal, nullptr); + PyObject* real = PyObject_CallMethodNoArgs(self, PyStrings::gCppReal); if (!real) return nullptr; double r = PyFloat_AsDouble(real); Py_DECREF(real); if (r == -1. && PyErr_Occurred()) return nullptr; - PyObject* imag = PyObject_CallMethodObjArgs(self, PyStrings::gCppImag, nullptr); + PyObject* imag = PyObject_CallMethodNoArgs(self, PyStrings::gCppImag); if (!imag) return nullptr; double i = PyFloat_AsDouble(imag); Py_DECREF(imag); @@ -962,14 +1495,14 @@ static PyObject* ComplexComplex(PyObject* self) { } static PyObject* ComplexRepr(PyObject* self) { - PyObject* real = PyObject_CallMethodObjArgs(self, PyStrings::gCppReal, nullptr); + PyObject* real = PyObject_CallMethodNoArgs(self, PyStrings::gCppReal); if (!real) return nullptr; double r = PyFloat_AsDouble(real); Py_DECREF(real); if (r == -1. && PyErr_Occurred()) return nullptr; - PyObject* imag = PyObject_CallMethodObjArgs(self, PyStrings::gCppImag, nullptr); + PyObject* imag = PyObject_CallMethodNoArgs(self, PyStrings::gCppImag); if (!imag) return nullptr; double i = PyFloat_AsDouble(imag); Py_DECREF(imag); @@ -1030,6 +1563,27 @@ namespace CPyCppyy { std::set gIteratorTypes; } +static inline +bool run_pythonizors(PyObject* pyclass, PyObject* pyname, const std::vector& v) +{ + PyObject* args = PyTuple_New(2); + Py_INCREF(pyclass); PyTuple_SET_ITEM(args, 0, pyclass); + Py_INCREF(pyname); PyTuple_SET_ITEM(args, 1, pyname); + + bool pstatus = true; + for (auto pythonizor : v) { + PyObject* result = PyObject_CallObject(pythonizor, args); + if (!result) { + pstatus = false; // TODO: detail the error handling + break; + } + Py_DECREF(result); + } + Py_DECREF(args); + + return pstatus; +} + bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) { // Add pre-defined pythonizations (for STL and ROOT) to classes based on their @@ -1050,6 +1604,16 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) else if (HasAttrDirect(pyclass, PyStrings::gFollow) && !Cppyy::IsSmartPtr(klass->fCppType)) Utility::AddToClass(pyclass, "__getattr__", (PyCFunction)FollowGetAttr, METH_O); +// for pre-check of nullptr for boolean types + if (HasAttrDirect(pyclass, PyStrings::gCppBool)) { +#if PY_VERSION_HEX >= 0x03000000 + const char* pybool_name = "__bool__"; +#else + const char* pybool_name = "__nonzero__"; +#endif + Utility::AddToClass(pyclass, pybool_name, (PyCFunction)NullCheckBool, METH_NOARGS); + } + // for STL containers, and user classes modeled after them if (HasAttrDirect(pyclass, PyStrings::gSize)) Utility::AddToClass(pyclass, "__len__", "size"); @@ -1064,22 +1628,34 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) // types to add the "next" method on use Cppyy::TCppMethod_t meth = Cppyy::GetMethod(klass->fCppType, v[0]); const std::string& resname = Cppyy::GetMethodResultType(meth); - if (Cppyy::GetScope(resname)) { + bool isIterator = gIteratorTypes.find(resname) != gIteratorTypes.end(); + if (!isIterator && Cppyy::GetScope(resname)) { if (resname.find("iterator") == std::string::npos) gIteratorTypes.insert(resname); + isIterator = true; + } + if (isIterator) { // install iterator protocol a la STL - ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)StlSequenceIter; - Utility::AddToClass(pyclass, "__iter__", (PyCFunction)StlSequenceIter, METH_NOARGS); + ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)STLSequenceIter; + Utility::AddToClass(pyclass, "__iter__", (PyCFunction)STLSequenceIter, METH_NOARGS); + } else { + // still okay if this is some pointer type of builtin persuasion (general class + // won't work: the return type needs to understand the iterator protocol) + std::string resolved = Cppyy::ResolveName(resname); + if (resolved.back() == '*' && Cppyy::IsBuiltin(resolved.substr(0, resolved.size()-1))) { + ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)LLSequenceIter; + Utility::AddToClass(pyclass, "__iter__", (PyCFunction)LLSequenceIter, METH_NOARGS); + } } } } if (!((PyTypeObject*)pyclass)->tp_iter && // no iterator resolved - HasAttrDirect(pyclass, PyStrings::gGetItem) && HasAttrDirect(pyclass, PyStrings::gLen)) { + HasAttrDirect(pyclass, PyStrings::gGetItem) && PyObject_HasAttr(pyclass, PyStrings::gLen)) { // Python will iterate over __getitem__ using integers, but C++ operator[] will never raise // a StopIteration. A checked getitem (raising IndexError if beyond size()) works in some // cases but would mess up if operator[] is meant to implement an associative container. So, - // this has to be implemented as an interator protocol. + // this has to be implemented as an iterator protocol. ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)index_iter; Utility::AddToClass(pyclass, "__iter__", (PyCFunction)index_iter, METH_NOARGS); } @@ -1088,7 +1664,8 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) // operator==/!= are used in op_richcompare of CPPInstance, which subsequently allows // comparisons to None; if no operator is available, a hook is installed for lazy // lookups in the global and/or class namespace - if (HasAttrDirect(pyclass, PyStrings::gEq, true)) { + if (HasAttrDirect(pyclass, PyStrings::gEq, true) && \ + Cppyy::GetMethodIndicesFromName(klass->fCppType, "__eq__").empty()) { PyObject* cppol = PyObject_GetAttr(pyclass, PyStrings::gEq); if (!klass->fOperators) klass->fOperators = new Utility::PyOperators(); klass->fOperators->fEq = cppol; @@ -1104,7 +1681,8 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) PyObject_SetAttr(pyclass, PyStrings::gEq, top_eq); } - if (HasAttrDirect(pyclass, PyStrings::gNe, true)) { + if (HasAttrDirect(pyclass, PyStrings::gNe, true) && \ + Cppyy::GetMethodIndicesFromName(klass->fCppType, "__ne__").empty()) { PyObject* cppol = PyObject_GetAttr(pyclass, PyStrings::gNe); if (!klass->fOperators) klass->fOperators = new Utility::PyOperators(); klass->fOperators->fNe = cppol; @@ -1119,6 +1697,91 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) PyObject_SetAttr(pyclass, PyStrings::gNe, top_ne); } +#if 0 + if (HasAttrDirect(pyclass, PyStrings::gRepr, true)) { + // guarantee that the result of __repr__ is a Python string + Utility::AddToClass(pyclass, "__cpp_repr", "__repr__"); + Utility::AddToClass(pyclass, "__repr__", (PyCFunction)UTF8Repr, METH_NOARGS); + } + + if (HasAttrDirect(pyclass, PyStrings::gStr, true)) { + // guarantee that the result of __str__ is a Python string + Utility::AddToClass(pyclass, "__cpp_str", "__str__"); + Utility::AddToClass(pyclass, "__str__", (PyCFunction)UTF8Str, METH_NOARGS); + } +#endif + + // This pythonization is disabled for ROOT because it is a bit buggy +#if 0 + if (Cppyy::IsAggregate(((CPPClass*)pyclass)->fCppType) && name.compare(0, 5, "std::", 5) != 0) { + // create a pseudo-constructor to allow initializer-style object creation + Cppyy::TCppType_t kls = ((CPPClass*)pyclass)->fCppType; + Cppyy::TCppIndex_t ndata = Cppyy::GetNumDatamembers(kls); + if (ndata) { + std::string rname = name; + TypeManip::cppscope_to_legalname(rname); + + std::ostringstream initdef; + initdef << "namespace __cppyy_internal {\n" + << "void init_" << rname << "(" << name << "*& self"; + bool codegen_ok = true; + std::vector arg_types, arg_names, arg_defaults; + arg_types.reserve(ndata); arg_names.reserve(ndata); arg_defaults.reserve(ndata); + for (Cppyy::TCppIndex_t i = 0; i < ndata; ++i) { + if (Cppyy::IsStaticData(kls, i) || !Cppyy::IsPublicData(kls, i)) + continue; + + const std::string& txt = Cppyy::GetDatamemberType(kls, i); + const std::string& res = Cppyy::IsEnum(txt) ? txt : Cppyy::ResolveName(txt); + const std::string& cpd = TypeManip::compound(res); + std::string res_clean = TypeManip::clean_type(res, false, true); + + if (res_clean == "internal_enum_type_t") + res_clean = txt; // restore (properly scoped name) + + if (res.rfind(']') == std::string::npos && res.rfind(')') == std::string::npos) { + if (!cpd.empty()) arg_types.push_back(res_clean+cpd); + else arg_types.push_back("const "+res_clean+"&"); + arg_names.push_back(Cppyy::GetDatamemberName(kls, i)); + if ((!cpd.empty() && cpd.back() == '*') || Cppyy::IsBuiltin(res_clean)) + arg_defaults.push_back("0"); + else { + Cppyy::TCppScope_t klsid = Cppyy::GetScope(res_clean); + if (Cppyy::IsDefaultConstructable(klsid)) arg_defaults.push_back(res_clean+"{}"); + } + } else { + codegen_ok = false; // TODO: how to support arrays, anonymous enums, etc? + break; + } + } + + if (codegen_ok && !arg_types.empty()) { + bool defaults_ok = arg_defaults.size() == arg_types.size(); + for (std::vector::size_type i = 0; i < arg_types.size(); ++i) { + initdef << ", " << arg_types[i] << " " << arg_names[i]; + if (defaults_ok) initdef << " = " << arg_defaults[i]; + } + initdef << ") {\n self = new " << name << "{"; + for (std::vector::size_type i = 0; i < arg_names.size(); ++i) { + if (i != 0) initdef << ", "; + initdef << arg_names[i]; + } + initdef << "};\n} }"; + + if (Cppyy::Compile(initdef.str(), true /* silent */)) { + Cppyy::TCppScope_t cis = Cppyy::GetScope("__cppyy_internal"); + const auto& mix = Cppyy::GetMethodIndicesFromName(cis, "init_"+rname); + if (mix.size()) { + if (!Utility::AddToClass(pyclass, "__init__", + new CPPFunction(cis, Cppyy::GetMethod(cis, mix[0])))) + PyErr_Clear(); + } + } + } + } + } +#endif + //- class name based pythonization ------------------------------------------- @@ -1138,6 +1801,9 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) Utility::AddToClass(pyclass, "__real_data", "data"); Utility::AddToClass(pyclass, "data", (PyCFunction)VectorData); + // numpy array conversion + Utility::AddToClass(pyclass, "__array__", (PyCFunction)VectorArray); + // checked getitem if (HasAttrDirect(pyclass, PyStrings::gLen)) { Utility::AddToClass(pyclass, "_getitem__unchecked", "__getitem__"); @@ -1145,25 +1811,48 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) } // vector-optimized iterator protocol - ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)vector_iter; + ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)vector_iter; + + // optimized __iadd__ + Utility::AddToClass(pyclass, "__iadd__", (PyCFunction)VectorIAdd, METH_VARARGS | METH_KEYWORDS); // helpers for iteration const std::string& vtype = Cppyy::ResolveName(name+"::value_type"); - size_t typesz = Cppyy::SizeOf(vtype); + if (vtype.rfind("value_type") == std::string::npos) { // actually resolved? + PyObject* pyvalue_type = CPyCppyy_PyText_FromString(vtype.c_str()); + PyObject_SetAttr(pyclass, PyStrings::gValueType, pyvalue_type); + Py_DECREF(pyvalue_type); + } + + size_t typesz = Cppyy::SizeOf(name+"::value_type"); if (typesz) { PyObject* pyvalue_size = PyLong_FromSsize_t(typesz); - PyObject_SetAttrString(pyclass, "value_size", pyvalue_size); + PyObject_SetAttr(pyclass, PyStrings::gValueSize, pyvalue_size); Py_DECREF(pyvalue_size); - - PyObject* pyvalue_type = CPyCppyy_PyText_FromString(vtype.c_str()); - PyObject_SetAttrString(pyclass, "value_type", pyvalue_type); - Py_DECREF(pyvalue_type); } } } - else if (IsTemplatedSTLClass(name, "map")) { - Utility::AddToClass(pyclass, "__contains__", (PyCFunction)MapContains, METH_O); + else if (IsTemplatedSTLClass(name, "array")) { + // constructor that takes python associative collections + Utility::AddToClass(pyclass, "__real_init", "__init__"); + Utility::AddToClass(pyclass, "__init__", (PyCFunction)ArrayInit, METH_VARARGS | METH_KEYWORDS); + } + + else if (IsTemplatedSTLClass(name, "map") || IsTemplatedSTLClass(name, "unordered_map")) { + // constructor that takes python associative collections + Utility::AddToClass(pyclass, "__real_init", "__init__"); + Utility::AddToClass(pyclass, "__init__", (PyCFunction)MapInit, METH_VARARGS | METH_KEYWORDS); + + Utility::AddToClass(pyclass, "__contains__", (PyCFunction)STLContainsWithFind, METH_O); + } + + else if (IsTemplatedSTLClass(name, "set")) { + // constructor that takes python associative collections + Utility::AddToClass(pyclass, "__real_init", "__init__"); + Utility::AddToClass(pyclass, "__init__", (PyCFunction)SetInit, METH_VARARGS | METH_KEYWORDS); + + Utility::AddToClass(pyclass, "__contains__", (PyCFunction)STLContainsWithFind, METH_O); } else if (IsTemplatedSTLClass(name, "pair")) { @@ -1171,34 +1860,52 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) Utility::AddToClass(pyclass, "__len__", (PyCFunction)ReturnTwo, METH_NOARGS); } - if (IsTemplatedSTLClass(name, "shared_ptr")) { + if (IsTemplatedSTLClass(name, "shared_ptr") || IsTemplatedSTLClass(name, "unique_ptr")) { Utility::AddToClass(pyclass, "__real_init", "__init__"); - Utility::AddToClass(pyclass, "__init__", (PyCFunction)SharedPtrInit, METH_VARARGS | METH_KEYWORDS); + Utility::AddToClass(pyclass, "__init__", (PyCFunction)SmartPtrInit, METH_VARARGS | METH_KEYWORDS); } - else if (name.find("iterator") != std::string::npos || gIteratorTypes.find(name) != gIteratorTypes.end()) { - ((PyTypeObject*)pyclass)->tp_iternext = (iternextfunc)StlIterNext; - Utility::AddToClass(pyclass, CPPYY__next__, (PyCFunction)StlIterNext, METH_NOARGS); + else if (!((PyTypeObject*)pyclass)->tp_iter && \ + (name.find("iterator") != std::string::npos || gIteratorTypes.find(name) != gIteratorTypes.end())) { + ((PyTypeObject*)pyclass)->tp_iternext = (iternextfunc)STLIterNext; + Utility::AddToClass(pyclass, CPPYY__next__, (PyCFunction)STLIterNext, METH_NOARGS); ((PyTypeObject*)pyclass)->tp_iter = (getiterfunc)PyObject_SelfIter; Utility::AddToClass(pyclass, "__iter__", (PyCFunction)PyObject_SelfIter, METH_NOARGS); } else if (name == "string" || name == "std::string") { // TODO: ask backend as well - Utility::AddToClass(pyclass, "__repr__", (PyCFunction)StlStringRepr, METH_NOARGS); - Utility::AddToClass(pyclass, "__str__", (PyCFunction)StlStringGetData, METH_NOARGS); - Utility::AddToClass(pyclass, "__cmp__", (PyCFunction)StlStringCompare, METH_O); - Utility::AddToClass(pyclass, "__eq__", (PyCFunction)StlStringIsEqual, METH_O); - Utility::AddToClass(pyclass, "__ne__", (PyCFunction)StlStringIsNotEqual, METH_O); - ((PyTypeObject*)pyclass)->tp_hash = (hashfunc)StlStringHash; + Utility::AddToClass(pyclass, "__repr__", (PyCFunction)STLStringRepr, METH_NOARGS); + Utility::AddToClass(pyclass, "__str__", (PyCFunction)STLStringStr, METH_NOARGS); + Utility::AddToClass(pyclass, "__bytes__", (PyCFunction)STLStringBytes, METH_NOARGS); + Utility::AddToClass(pyclass, "__cmp__", (PyCFunction)STLStringCompare, METH_O); + Utility::AddToClass(pyclass, "__eq__", (PyCFunction)STLStringIsEqual, METH_O); + Utility::AddToClass(pyclass, "__ne__", (PyCFunction)STLStringIsNotEqual, METH_O); + Utility::AddToClass(pyclass, "__contains__", (PyCFunction)STLStringContains, METH_O); + Utility::AddToClass(pyclass, "decode", (PyCFunction)STLStringDecode, METH_VARARGS | METH_KEYWORDS); + Utility::AddToClass(pyclass, "__cpp_find", "find"); + Utility::AddToClass(pyclass, "find", (PyCFunction)STLStringFind, METH_VARARGS | METH_KEYWORDS); + Utility::AddToClass(pyclass, "__cpp_rfind", "rfind"); + Utility::AddToClass(pyclass, "rfind", (PyCFunction)STLStringRFind, METH_VARARGS | METH_KEYWORDS); + Utility::AddToClass(pyclass, "__cpp_replace", "replace"); + Utility::AddToClass(pyclass, "replace", (PyCFunction)STLStringReplace, METH_VARARGS | METH_KEYWORDS); + Utility::AddToClass(pyclass, "__getattr__", (PyCFunction)STLStringGetAttr, METH_O); + + // to allow use of std::string in dictionaries and findable with str + ((PyTypeObject*)pyclass)->tp_hash = (hashfunc)STLStringHash; + } + + else if (name == "basic_string_view" || name == "std::basic_string_view") { + Utility::AddToClass(pyclass, "__real_init", "__init__"); + Utility::AddToClass(pyclass, "__init__", (PyCFunction)StringViewInit, METH_VARARGS | METH_KEYWORDS); } - else if (name == "basic_string,allocator >" || \ - name == "std::basic_string,allocator >") { - Utility::AddToClass(pyclass, "__repr__", (PyCFunction)StlWStringRepr, METH_NOARGS); - Utility::AddToClass(pyclass, "__str__", (PyCFunction)StlWStringGetData, METH_NOARGS); - Utility::AddToClass(pyclass, "__cmp__", (PyCFunction)StlWStringCompare, METH_O); - Utility::AddToClass(pyclass, "__eq__", (PyCFunction)StlWStringIsEqual, METH_O); - Utility::AddToClass(pyclass, "__ne__", (PyCFunction)StlWStringIsNotEqual, METH_O); + else if (name == "basic_string,allocator >" || name == "std::basic_string,std::allocator >") { + Utility::AddToClass(pyclass, "__repr__", (PyCFunction)STLWStringRepr, METH_NOARGS); + Utility::AddToClass(pyclass, "__str__", (PyCFunction)STLWStringStr, METH_NOARGS); + Utility::AddToClass(pyclass, "__bytes__", (PyCFunction)STLWStringBytes, METH_NOARGS); + Utility::AddToClass(pyclass, "__cmp__", (PyCFunction)STLWStringCompare, METH_O); + Utility::AddToClass(pyclass, "__eq__", (PyCFunction)STLWStringIsEqual, METH_O); + Utility::AddToClass(pyclass, "__ne__", (PyCFunction)STLWStringIsNotEqual, METH_O); } else if (name == "complex" || name == "std::complex") { @@ -1214,7 +1921,7 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) Utility::AddToClass(pyclass, "__cpp_real", "real"); PyObject_SetAttrString(pyclass, "real", PyDescr_NewGetSet((PyTypeObject*)pyclass, &realComplex)); Utility::AddToClass(pyclass, "__cpp_imag", "imag"); - PyObject_SetAttrString(pyclass, "imag", PyDescr_NewGetSet((PyTypeObject*)pyclass, &imagComplex)); + PyObject_SetAttrString(pyclass, "imag", PyDescr_NewGetSet((PyTypeObject*)pyclass, &imagComplex)); Utility::AddToClass(pyclass, "__complex__", (PyCFunction)ComplexComplex, METH_NOARGS); Utility::AddToClass(pyclass, "__repr__", (PyCFunction)ComplexRepr, METH_NOARGS); } @@ -1241,40 +1948,30 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) return false; } else { Py_XDECREF(res); - // pyname handed to tuple below + // pyname handed to args tuple below } -// call registered pythonizors, if any - PyObject* args = PyTuple_New(2); - Py_INCREF(pyclass); - PyTuple_SET_ITEM(args, 0, pyclass); - - std::string outer_scope = TypeManip::extract_namespace(name); - +// call registered pythonizors, if any: first run the namespace-specific pythonizors, then +// the global ones (the idea is to allow writing a pythonizor that see all classes) bool pstatus = true; - auto p = outer_scope.empty() ? gPythonizations.end() : gPythonizations.find(outer_scope); - if (p == gPythonizations.end()) { - p = gPythonizations.find(""); - PyTuple_SET_ITEM(args, 1, pyname); - } else { - PyTuple_SET_ITEM(args, 1, CPyCppyy_PyText_FromString( - name.substr(outer_scope.size()+2, std::string::npos).c_str())); - Py_DECREF(pyname); + std::string outer_scope = TypeManip::extract_namespace(name); + if (!outer_scope.empty()) { + auto p = gPythonizations.find(outer_scope); + if (p != gPythonizations.end()) { + PyObject* subname = CPyCppyy_PyText_FromString( + name.substr(outer_scope.size()+2, std::string::npos).c_str()); + pstatus = run_pythonizors(pyclass, subname, p->second); + Py_DECREF(subname); + } } - if (p != gPythonizations.end()) { - for (auto pythonizor : p->second) { - PyObject* result = PyObject_CallObject(pythonizor, args); - if (!result) { - // TODO: detail error handling for the pythonizors - pstatus = false; - break; - } - Py_DECREF(result); - } + if (pstatus) { + auto p = gPythonizations.find(""); + if (p != gPythonizations.end()) + pstatus = run_pythonizors(pyclass, pyname, p->second); } - Py_DECREF(args); + Py_DECREF(pyname); // phew! all done ... return pstatus; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/SignalTryCatch.h b/bindings/pyroot/cppyy/CPyCppyy/src/SignalTryCatch.h index 95b3fae18e03b..5d5e6301a32c7 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/SignalTryCatch.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/SignalTryCatch.h @@ -18,6 +18,13 @@ #define NEED_SIGJMP 1 #endif +// By default, the ExceptionContext_t class is expected in the namespace +// CppyyLegacy, If it is expected in no namespace, one can explicitly define +// NO_CPPYY_LEGACY_NAMESPACE at build time (e.g. if one wants to use ROOT). + +#ifndef NO_CPPYY_LEGACY_NAMESPACE +namespace CppyyLegacy { +#endif struct ExceptionContext_t { #ifdef NEED_SIGJMP sigjmp_buf fBuf; @@ -25,38 +32,45 @@ struct ExceptionContext_t { jmp_buf fBuf; #endif }; +#ifndef NO_CPPYY_LEGACY_NAMESPACE +} + +using CppyyExceptionContext_t = CppyyLegacy::ExceptionContext_t; +#else +using CppyyExceptionContext_t = ExceptionContext_t; +#endif #ifdef NEED_SIGJMP -# define SETJMP(buf) sigsetjmp(buf,1) +# define CLING_EXCEPTION_SETJMP(buf) sigsetjmp(buf,1) #else -# define SETJMP(buf) setjmp(buf) +# define CLING_EXCEPTION_SETJMP(buf) setjmp(buf) #endif -#define RETRY \ +#define CLING_EXCEPTION_RETRY \ { \ - static ExceptionContext_t R__curr, *R__old = gException; \ + static CppyyExceptionContext_t R__curr, *R__old = gException; \ int R__code; \ gException = &R__curr; \ - R__code = SETJMP(gException->fBuf); if (R__code) { }; { + R__code = CLING_EXCEPTION_SETJMP(gException->fBuf); if (R__code) { }; { -#define TRY \ +#define CLING_EXCEPTION_TRY \ { \ - static ExceptionContext_t R__curr, *R__old = gException; \ + static CppyyExceptionContext_t R__curr, *R__old = gException; \ int R__code; \ gException = &R__curr; \ - if ((R__code = SETJMP(gException->fBuf)) == 0) { + if ((R__code = CLING_EXCEPTION_SETJMP(gException->fBuf)) == 0) { -#define CATCH(n) \ +#define CLING_EXCEPTION_CATCH(n) \ gException = R__old; \ } else { \ int n = R__code; \ gException = R__old; -#define ENDTRY \ +#define CLING_EXCEPTION_ENDTRY \ } \ gException = R__old; \ } -CPYCPPYY_IMPORT ExceptionContext_t *gException; +CPYCPPYY_IMPORT CppyyExceptionContext_t *gException; #endif diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.cxx index 0dae07015e86d..a80a0ef172c29 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.cxx @@ -38,6 +38,8 @@ static PyObject* TC2CppName(PyObject* pytc, const char* cpd, bool allow_voidp) case 'f': name = "float"; break; case 'd': name = "double"; break; case 'g': name = "long double"; break; + case 'z': // special case for C strings, ignore cpd + return CPyCppyy_PyText_FromString(std::string{"const char*"}.c_str()); default: name = (allow_voidp ? "void*" : nullptr); break; } } @@ -48,8 +50,8 @@ static PyObject* TC2CppName(PyObject* pytc, const char* cpd, bool allow_voidp) } //---------------------------------------------------------------------------- -TemplateInfo::TemplateInfo() : fCppName(nullptr), fPyName(nullptr), fPyClass(nullptr), - fNonTemplated(nullptr), fTemplated(nullptr), fLowPriority(nullptr) +TemplateInfo::TemplateInfo() : fPyClass(nullptr), fNonTemplated(nullptr), + fTemplated(nullptr), fLowPriority(nullptr), fDoc(nullptr) { /* empty */ } @@ -57,10 +59,9 @@ TemplateInfo::TemplateInfo() : fCppName(nullptr), fPyName(nullptr), fPyClass(nul //---------------------------------------------------------------------------- TemplateInfo::~TemplateInfo() { - Py_XDECREF(fCppName); - Py_XDECREF(fPyName); Py_XDECREF(fPyClass); + Py_XDECREF(fDoc); Py_DECREF(fNonTemplated); Py_DECREF(fTemplated); Py_DECREF(fLowPriority); @@ -72,23 +73,6 @@ TemplateInfo::~TemplateInfo() } } -//---------------------------------------------------------------------------- -void TemplateProxy::Set(const std::string& cppname, const std::string& pyname, PyObject* pyclass) -{ -// Initialize the proxy for the given 'pyclass.' - fSelf = nullptr; - fTemplateArgs = nullptr; - - fTI->fCppName = CPyCppyy_PyText_FromString(const_cast(cppname.c_str())); - fTI->fPyName = CPyCppyy_PyText_FromString(const_cast(pyname.c_str())); - Py_XINCREF(pyclass); - fTI->fPyClass = pyclass; - - std::vector dummy; - fTI->fNonTemplated = CPPOverload_New(pyname, dummy); - fTI->fTemplated = CPPOverload_New(pyname, dummy); - fTI->fLowPriority = CPPOverload_New(pyname, dummy); -} //---------------------------------------------------------------------------- void TemplateProxy::MergeOverload(CPPOverload* mp) { @@ -119,16 +103,24 @@ void TemplateProxy::AdoptTemplate(PyCallable* pc) //---------------------------------------------------------------------------- PyObject* TemplateProxy::Instantiate(const std::string& fname, - PyObject* args, Utility::ArgPreference pref, int* pcnt) + CPyCppyy_PyArgs_t args, size_t nargsf, Utility::ArgPreference pref, int* pcnt) { // Instantiate (and cache) templated methods, return method if any std::string proto = ""; - Py_ssize_t nArgs = PyTuple_GET_SIZE(args); - if (nArgs != 0) { - PyObject* tpArgs = PyTuple_New(nArgs); - for (int i = 0; i < nArgs; ++i) { - PyObject* itemi = PyTuple_GET_ITEM(args, i); +#if PY_VERSION_HEX >= 0x03080000 + bool isNS = (((CPPScope*)fTI->fPyClass)->fFlags & CPPScope::kIsNamespace); + if (!isNS && !fSelf && CPyCppyy_PyArgs_GET_SIZE(args, nargsf)) { + args += 1; + nargsf -= 1; + } +#endif + + Py_ssize_t argc = CPyCppyy_PyArgs_GET_SIZE(args, nargsf); + if (argc != 0) { + PyObject* tpArgs = PyTuple_New(argc); + for (Py_ssize_t i = 0; i < argc; ++i) { + PyObject* itemi = CPyCppyy_PyArgs_GET_ITEM(args, i); bool bArgSet = false; @@ -178,7 +170,21 @@ PyObject* TemplateProxy::Instantiate(const std::string& fname, } } - const std::string& name_v1 = Utility::ConstructTemplateArgs(nullptr, tpArgs, args, pref, 0, pcnt); +#if PY_VERSION_HEX >= 0x03080000 + PyObject* pyargs = PyTuple_New(argc); + for (Py_ssize_t i = 0; i < argc; ++i) { + PyObject* item = CPyCppyy_PyArgs_GET_ITEM(args, i); + Py_INCREF(item); + PyTuple_SET_ITEM(pyargs, i, item); + } +#else + Py_INCREF(args); + PyObject* pyargs = args; +#endif + const std::string& name_v1 = \ + Utility::ConstructTemplateArgs(nullptr, tpArgs, pyargs, pref, 0, pcnt); + + Py_DECREF(pyargs); Py_DECREF(tpArgs); if (name_v1.size()) proto = name_v1.substr(1, name_v1.size()-2); @@ -205,10 +211,10 @@ PyObject* TemplateProxy::Instantiate(const std::string& fname, // the argument types. If it did, replace with vector and lookup anew. if (resname.find("initializer_list") != std::string::npos) { auto pos = proto.find("initializer_list"); - while (pos != std::string::npos) { - proto.replace(pos, 16, "vector"); - pos = proto.find("initializer_list", pos + 6); - } + while (pos != std::string::npos) { + proto.replace(pos, 16, "vector"); + pos = proto.find("initializer_list", pos + 6); + } Cppyy::TCppMethod_t m2 = Cppyy::GetMethodTemplate(scope, fname, proto); if (m2 && m2 != cppmeth) { @@ -384,6 +390,11 @@ static int tpp_traverse(TemplateProxy* pytmpl, visitproc visit, void* arg) //---------------------------------------------------------------------------- static PyObject* tpp_doc(TemplateProxy* pytmpl, void*) { + if (pytmpl->fTI->fDoc) { + Py_INCREF(pytmpl->fTI->fDoc); + return pytmpl->fTI->fDoc; + } + // Forward to method proxies to doc all overloads PyObject* doc = nullptr; if (pytmpl->fTI->fNonTemplated->HasMethods()) @@ -413,16 +424,23 @@ static PyObject* tpp_doc(TemplateProxy* pytmpl, void*) return CPyCppyy_PyText_FromString(TemplateProxy_Type.tp_doc); } -//---------------------------------------------------------------------------- -static PyObject* tpp_repr(TemplateProxy* pytmpl) +static int tpp_doc_set(TemplateProxy* pytmpl, PyObject *val, void *) { -// Simply return the doc string as that's the most useful info (this will appear -// on clsses on calling help()). - return tpp_doc(pytmpl, nullptr); + Py_XDECREF(pytmpl->fTI->fDoc); + Py_INCREF(val); + pytmpl->fTI->fDoc = val; + return 0; } +//---------------------------------------------------------------------------- //= CPyCppyy template proxy callable behavior ================================ + +#define TPPCALL_RETURN \ +{ if (!errors.empty()) \ + std::for_each(errors.begin(), errors.end(), Utility::PyError_t::Clear);\ + return result; } + static inline std::string targs2str(TemplateProxy* pytmpl) { if (!pytmpl || !pytmpl->fTemplateArgs) return ""; @@ -431,8 +449,8 @@ static inline std::string targs2str(TemplateProxy* pytmpl) static inline void UpdateDispatchMap(TemplateProxy* pytmpl, bool use_targs, uint64_t sighash, CPPOverload* pymeth) { -// memoize a method in the dispatch map after successful call; replace old if need be (may be -// with the same CPPOverload, just with more methods) +// Memoize a method in the dispatch map after successful call; replace old if need be (may be +// with the same CPPOverload, just with more methods). bool bInserted = false; auto& v = pytmpl->fTI->fDispatchMap[use_targs ? targs2str(pytmpl) : ""]; @@ -447,29 +465,51 @@ static inline void UpdateDispatchMap(TemplateProxy* pytmpl, bool use_targs, uint if (!bInserted) v.push_back(std::make_pair(sighash, pymeth)); } +static inline PyObject* SelectAndForward(TemplateProxy* pytmpl, CPPOverload* pymeth, + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, + bool implicitOkay, bool use_targs, uint64_t sighash, std::vector& errors) +{ +// Forward a call to known overloads, if any. + if (pymeth->HasMethods()) { + PyObject* pycall = CPPOverload_Type.tp_descr_get( + (PyObject*)pymeth, pytmpl->fSelf, (PyObject*)&CPPOverload_Type); + + if (!implicitOkay) + ((CPPOverload*)pycall)->fFlags |= CallContext::kNoImplicit; + + // now call the method with the arguments (loops internally) + PyObject* result = CPyCppyy_tp_call(pycall, args, nargsf, kwds); + Py_DECREF(pycall); + if (result) { + UpdateDispatchMap(pytmpl, use_targs, sighash, pymeth); + TPPCALL_RETURN; + } + Utility::FetchError(errors); + } + + return nullptr; +} + static inline PyObject* CallMethodImp(TemplateProxy* pytmpl, PyObject*& pymeth, - PyObject* args, PyObject* kwds, bool impOK, uint64_t sighash) + CPyCppyy_PyArgs_t args, size_t nargsf, PyObject* kwds, bool impOK, uint64_t sighash) { // Actual call of a given overload: takes care of handlign of "self" and // dereferences the overloaded method after use. + PyObject* result; - if (!impOK) PyDict_SetItem(kwds, PyStrings::gNoImplicit, Py_True); + if (!impOK && CPPOverload_Check(pymeth)) + ((CPPOverload*)pymeth)->fFlags |= CallContext::kNoImplicit; bool isNS = (((CPPScope*)pytmpl->fTI->fPyClass)->fFlags & CPPScope::kIsNamespace); - if (isNS && pytmpl->fSelf) { + if (isNS && pytmpl->fSelf && pytmpl->fSelf != Py_None) { // this is a global method added a posteriori to the class - Py_ssize_t sz = PyTuple_GET_SIZE(args); - PyObject* newArgs = PyTuple_New(sz+1); - for (int i = 0; i < sz; ++i) { - PyObject* item = PyTuple_GET_ITEM(args, i); - Py_INCREF(item); - PyTuple_SET_ITEM(newArgs, i+1, item); - } - Py_INCREF((PyObject*)pytmpl->fSelf); - PyTuple_SET_ITEM(newArgs, 0, (PyObject*)pytmpl->fSelf); - result = CPPOverload_Type.tp_call(pymeth, newArgs, kwds); - Py_DECREF(newArgs); - } else - result = CPPOverload_Type.tp_call(pymeth, args, kwds); + PyCallArgs cargs{(CPPInstance*&)pytmpl->fSelf, args, nargsf, kwds}; + AdjustSelf(cargs); + result = CPyCppyy_tp_call(pymeth, cargs.fArgs, cargs.fNArgsf, cargs.fKwds); + } else { + if (!pytmpl->fSelf && CPPOverload_Check(pymeth)) + ((CPPOverload*)pymeth)->fFlags &= ~CallContext::kFromDescr; + result = CPyCppyy_tp_call(pymeth, args, nargsf, kwds); + } if (result) { Py_XDECREF(((CPPOverload*)pymeth)->fSelf); ((CPPOverload*)pymeth)->fSelf = nullptr; // unbind @@ -480,13 +520,12 @@ static inline PyObject* CallMethodImp(TemplateProxy* pytmpl, PyObject*& pymeth, return result; } -#define TPPCALL_RETURN \ -{ if (!errors.empty()) \ - std::for_each(errors.begin(), errors.end(), Utility::PyError_t::Clear);\ - Py_DECREF(kwds); \ - return result; } - +#if PY_VERSION_HEX >= 0x03080000 +static PyObject* tpp_vectorcall( + TemplateProxy* pytmpl, PyObject* const *args, size_t nargsf, PyObject* kwds) +#else static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) +#endif { // Dispatcher to the actual member method, several uses possible; in order: // @@ -516,18 +555,20 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) // TODO: should previously instantiated templates be considered first? -// container for collecting errors - std::vector errors; +#if PY_VERSION_HEX < 0x03080000 + size_t nargsf = PyTuple_GET_SIZE(args); +#endif - PyObject* pymeth = nullptr, *result = nullptr; + PyObject *pymeth = nullptr, *result = nullptr; // short-cut through memoization map - uint64_t sighash = HashSignature(args); + Py_ssize_t argc = CPyCppyy_PyArgs_GET_SIZE(args, nargsf); + uint64_t sighash = HashSignature(args, argc); + CPPOverload* ol = nullptr; if (!pytmpl->fTemplateArgs) { // look for known signatures ... - CPPOverload* ol = nullptr; - auto& v = pytmpl->fTI->fDispatchMap[targs2str(pytmpl)]; + auto& v = pytmpl->fTI->fDispatchMap[""]; for (const auto& p : v) { if (p.first == sighash) { ol = p.second; @@ -536,41 +577,40 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) } if (ol != nullptr) { - if (!pytmpl->fSelf) { - result = CPPOverload_Type.tp_call((PyObject*)ol, args, kwds); + if (!pytmpl->fSelf || pytmpl->fSelf == Py_None) { + result = CPyCppyy_tp_call((PyObject*)ol, args, nargsf, kwds); } else { pymeth = CPPOverload_Type.tp_descr_get( (PyObject*)ol, pytmpl->fSelf, (PyObject*)&CPPOverload_Type); - result = CPPOverload_Type.tp_call(pymeth, args, kwds); + result = CPyCppyy_tp_call(pymeth, args, nargsf, kwds); Py_DECREF(pymeth); pymeth = nullptr; } if (result) return result; - Utility::FetchError(errors); } } -// do not mix template instantiations with implicit conversions - if (!kwds) kwds = PyDict_New(); - else { - Py_INCREF(kwds); - } +// container for collecting errors + std::vector errors; + if (ol) Utility::FetchError(errors); // case 1: explicit template previously selected through subscript if (pytmpl->fTemplateArgs) { // instantiate explicitly - PyObject* pyfullname = CPyCppyy_PyText_FromString( - CPyCppyy_PyText_AsString(pytmpl->fTI->fCppName)); + PyObject* pyfullname = CPyCppyy_PyText_FromString(pytmpl->fTI->fCppName.c_str()); CPyCppyy_PyText_Append(&pyfullname, pytmpl->fTemplateArgs); // first, lookup by full name, if previously stored bool isNS = (((CPPScope*)pytmpl->fTI->fPyClass)->fFlags & CPPScope::kIsNamespace); - pymeth = PyObject_GetAttr((pytmpl->fSelf && !isNS) ? pytmpl->fSelf : pytmpl->fTI->fPyClass, pyfullname); + if (pytmpl->fSelf && pytmpl->fSelf != Py_None && !isNS) + pymeth = PyObject_GetAttr(pytmpl->fSelf, pyfullname); + else // by-passes custom scope getattr that searches into Cling + pymeth = PyType_Type.tp_getattro(pytmpl->fTI->fPyClass, pyfullname); // attempt call if found (this may fail if there are specializations) if (CPPOverload_Check(pymeth)) { // since the template args are fully explicit, allow implicit conversion of arguments - result = CallMethodImp(pytmpl, pymeth, args, kwds, true, sighash); + result = CallMethodImp(pytmpl, pymeth, args, nargsf, kwds, true, sighash); if (result) { Py_DECREF(pyfullname); TPPCALL_RETURN; @@ -578,7 +618,7 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) Utility::FetchError(errors); } else if (pymeth && PyCallable_Check(pymeth)) { // something different (user provided?) - result = PyObject_CallObject(pymeth, args); + result = CPyCppyy_PyObject_Call(pymeth, args, nargsf, kwds); Py_DECREF(pymeth); if (result) { Py_DECREF(pyfullname); @@ -590,10 +630,10 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) // not cached or failed call; try instantiation pymeth = pytmpl->Instantiate( - CPyCppyy_PyText_AsString(pyfullname), args, Utility::kNone); + CPyCppyy_PyText_AsString(pyfullname), args, nargsf, Utility::kNone); if (pymeth) { - // attempt actuall call; same as above, allow implicit conversion of arguments - result = CallMethodImp(pytmpl, pymeth, args, kwds, true, sighash); + // attempt actual call; same as above, allow implicit conversion of arguments + result = CallMethodImp(pytmpl, pymeth, args, nargsf, kwds, true, sighash); if (result) { Py_DECREF(pyfullname); TPPCALL_RETURN; @@ -602,57 +642,35 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) // no drop through if failed (if implicit was desired, don't provide template args) Utility::FetchError(errors); - PyObject* topmsg = CPyCppyy_PyText_FromFormat("Could not instantiate %s:", CPyCppyy_PyText_AsString(pyfullname)); + PyObject* topmsg = CPyCppyy_PyText_FromFormat( + "Could not find \"%s\" (set cppyy.set_debug() for C++ errors):", CPyCppyy_PyText_AsString(pyfullname)); Py_DECREF(pyfullname); Utility::SetDetailedException(errors, topmsg /* steals */, PyExc_TypeError /* default error */); - Py_DECREF(kwds); return nullptr; } // case 2: select known non-template overload - if (pytmpl->fTI->fNonTemplated->HasMethods()) { - // simply forward the call: all non-templated methods are defined on class definition - // and thus already available - pymeth = CPPOverload_Type.tp_descr_get( - (PyObject*)pytmpl->fTI->fNonTemplated, pytmpl->fSelf, (PyObject*)&CPPOverload_Type); - // now call the method with the arguments (loops internally and implicit is okay as - // these are not templated methods that should match exactly) - result = CPPOverload_Type.tp_call(pymeth, args, kwds); - Py_DECREF(pymeth); pymeth = nullptr; - if (result) { - UpdateDispatchMap(pytmpl, false, sighash, pytmpl->fTI->fNonTemplated); - TPPCALL_RETURN; - } - Utility::FetchError(errors); - } + result = SelectAndForward(pytmpl, pytmpl->fTI->fNonTemplated, args, nargsf, kwds, + true /* implicitOkay */, false /* use_targs */, sighash, errors); + if (result) + TPPCALL_RETURN; // case 3: select known template overload - if (pytmpl->fTI->fTemplated->HasMethods()) { - // simply forward the call - pymeth = CPPOverload_Type.tp_descr_get( - (PyObject*)pytmpl->fTI->fTemplated, pytmpl->fSelf, (PyObject*)&CPPOverload_Type); - // now call the method with the arguments (loops internally) - PyDict_SetItem(kwds, PyStrings::gNoImplicit, Py_True); - result = CPPOverload_Type.tp_call(pymeth, args, kwds); - Py_DECREF(pymeth); pymeth = nullptr; - if (result) { - UpdateDispatchMap(pytmpl, true, sighash, pytmpl->fTI->fTemplated); - TPPCALL_RETURN; - } - Utility::FetchError(errors); - } + result = SelectAndForward(pytmpl, pytmpl->fTI->fTemplated, args, nargsf, kwds, + false /* implicitOkay */, true /* use_targs */, sighash, errors); + if (result) + TPPCALL_RETURN; // case 4: auto-instantiation from types of arguments for (auto pref : {Utility::kReference, Utility::kPointer, Utility::kValue}) { // TODO: no need to loop if there are no non-instance arguments; also, should any // failed lookup be removed? int pcnt = 0; - pymeth = pytmpl->Instantiate( - CPyCppyy_PyText_AsString(pytmpl->fTI->fCppName), args, pref, &pcnt); + pymeth = pytmpl->Instantiate(pytmpl->fTI->fCppName, args, nargsf, pref, &pcnt); if (pymeth) { - // attempt actuall call; argument based, so do not allow implicit conversions - result = CallMethodImp(pytmpl, pymeth, args, kwds, false, sighash); + // attempt actual call; argument based, so do not allow implicit conversions + result = CallMethodImp(pytmpl, pymeth, args, nargsf, kwds, false, sighash); if (result) TPPCALL_RETURN; } Utility::FetchError(errors); @@ -660,20 +678,10 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) } // case 5: low priority methods, such as ones that take void* arguments - if (pytmpl->fTI->fLowPriority->HasMethods()) { - // simply forward the call - pymeth = CPPOverload_Type.tp_descr_get( - (PyObject*)pytmpl->fTI->fLowPriority, pytmpl->fSelf, (PyObject*)&CPPOverload_Type); - // now call the method with the arguments (loops internally) - PyDict_SetItem(kwds, PyStrings::gNoImplicit, Py_True); - result = CPPOverload_Type.tp_call(pymeth, args, kwds); - Py_DECREF(pymeth); pymeth = nullptr; - if (result) { - UpdateDispatchMap(pytmpl, false, sighash, pytmpl->fTI->fLowPriority); - TPPCALL_RETURN; - } - Utility::FetchError(errors); - } + result = SelectAndForward(pytmpl, pytmpl->fTI->fLowPriority, args, nargsf, kwds, + false /* implicitOkay */, false /* use_targs */, sighash, errors); + if (result) + TPPCALL_RETURN; // error reporting is fraud, given the numerous steps taken, but more details seems better if (!errors.empty()) { @@ -681,22 +689,26 @@ static PyObject* tpp_call(TemplateProxy* pytmpl, PyObject* args, PyObject* kwds) Utility::SetDetailedException(errors, topmsg /* steals */, PyExc_TypeError /* default error */); } else { PyErr_Format(PyExc_TypeError, "cannot resolve method template call for \'%s\'", - CPyCppyy_PyText_AsString(pytmpl->fTI->fPyName)); + pytmpl->fTI->fCppName.c_str()); } - Py_DECREF(kwds); return nullptr; } //---------------------------------------------------------------------------- -static TemplateProxy* tpp_descrget(TemplateProxy* pytmpl, PyObject* pyobj, PyObject*) +static TemplateProxy* tpp_descr_get(TemplateProxy* pytmpl, PyObject* pyobj, PyObject*) { // create and use a new template proxy (language requirement) TemplateProxy* newPyTmpl = (TemplateProxy*)TemplateProxy_Type.tp_alloc(&TemplateProxy_Type, 0); // new method is to be bound to current object (may be nullptr) - Py_XINCREF(pyobj); - newPyTmpl->fSelf = pyobj; + if (pyobj) { + Py_INCREF(pyobj); + newPyTmpl->fSelf = pyobj; + } else { + Py_INCREF(Py_None); + newPyTmpl->fSelf = Py_None; + } Py_XINCREF(pytmpl->fTemplateArgs); newPyTmpl->fTemplateArgs = pytmpl->fTemplateArgs; @@ -704,16 +716,21 @@ static TemplateProxy* tpp_descrget(TemplateProxy* pytmpl, PyObject* pyobj, PyObj // copy name, class, etc. pointers new (&newPyTmpl->fTI) std::shared_ptr{pytmpl->fTI}; +#if PY_VERSION_HEX >= 0x03080000 + newPyTmpl->fVectorCall = pytmpl->fVectorCall; +#endif + return newPyTmpl; } + //---------------------------------------------------------------------------- static PyObject* tpp_subscript(TemplateProxy* pytmpl, PyObject* args) { // Explicit template member lookup/instantiation; works by re-bounding. This method can // not cache overloads as instantiations need not be unique for the argument types due // to template specializations. - TemplateProxy* typeBoundMethod = tpp_descrget(pytmpl, pytmpl->fSelf, nullptr); + TemplateProxy* typeBoundMethod = tpp_descr_get(pytmpl, pytmpl->fSelf, nullptr); Py_XDECREF(typeBoundMethod->fTemplateArgs); typeBoundMethod->fTemplateArgs = CPyCppyy_PyText_FromString( Utility::ConstructTemplateArgs(nullptr, args).c_str()); @@ -722,15 +739,15 @@ static PyObject* tpp_subscript(TemplateProxy* pytmpl, PyObject* args) //----------------------------------------------------------------------------- static PyObject* tpp_getuseffi(CPPOverload*, void*) -{ +{ return PyInt_FromLong(0); // dummy (__useffi__ unused) -} - +} + //----------------------------------------------------------------------------- static int tpp_setuseffi(CPPOverload*, PyObject*, void*) -{ +{ return 0; // dummy (__useffi__ unused) -} +} //---------------------------------------------------------------------------- @@ -739,13 +756,35 @@ static PyMappingMethods tpp_as_mapping = { }; static PyGetSetDef tpp_getset[] = { - {(char*)"__doc__", (getter)tpp_doc, nullptr, nullptr, nullptr}, + {(char*)"__doc__", (getter)tpp_doc, (setter)tpp_doc_set, nullptr, nullptr}, {(char*)"__useffi__", (getter)tpp_getuseffi, (setter)tpp_setuseffi, (char*)"unused", nullptr}, {(char*)nullptr, nullptr, nullptr, nullptr, nullptr} }; +//---------------------------------------------------------------------------- +void TemplateProxy::Set(const std::string& cppname, const std::string& pyname, PyObject* pyclass) +{ +// Initialize the proxy for the given 'pyclass.' + fSelf = nullptr; + fTemplateArgs = nullptr; + + fTI->fCppName = cppname; + Py_XINCREF(pyclass); + fTI->fPyClass = pyclass; + + std::vector dummy; + fTI->fNonTemplated = CPPOverload_New(pyname, dummy); + fTI->fTemplated = CPPOverload_New(pyname, dummy); + fTI->fLowPriority = CPPOverload_New(pyname, dummy); + +#if PY_VERSION_HEX >= 0x03080000 + fVectorCall = (vectorcallfunc)tpp_vectorcall; +#endif +} + + //= CPyCppyy method proxy access to internals ================================ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) { @@ -754,6 +793,8 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) PyObject* sigarg_tuple = nullptr; int want_const = -1; + Cppyy::TCppScope_t scope = (Cppyy::TCppScope_t) 0; + Cppyy::TCppMethod_t cppmeth = (Cppyy::TCppMethod_t) 0; std::string proto; if (PyArg_ParseTuple(args, const_cast("s|i:__overload__"), &sigarg, &want_const)) { @@ -770,6 +811,10 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) if (ol) return ol; proto = Utility::ConstructTemplateArgs(nullptr, args); + + scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType; + cppmeth = Cppyy::GetMethodTemplate( + scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2)); } else if (PyArg_ParseTuple(args, const_cast("O|i:__overload__"), &sigarg_tuple, &want_const)) { PyErr_Clear(); want_const = PyTuple_GET_SIZE(args) == 1 ? -1 : want_const; @@ -798,6 +843,10 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) proto.push_back(','); } proto.push_back('>'); + + scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType; + cppmeth = Cppyy::GetMethodTemplate( + scope, pytmpl->fTI->fCppName, proto.substr(1, proto.size()-2)); } else { PyErr_Format(PyExc_TypeError, "Unexpected arguments to __overload__"); return nullptr; @@ -807,11 +856,6 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) PyObject* pytype = 0, *pyvalue = 0, *pytrace = 0; PyErr_Fetch(&pytype, &pyvalue, &pytrace); - Cppyy::TCppScope_t scope = ((CPPClass*)pytmpl->fTI->fPyClass)->fCppType; - Cppyy::TCppMethod_t cppmeth = Cppyy::GetMethodTemplate( - scope, CPyCppyy_PyText_AsString(pytmpl->fTI->fCppName), - proto.substr(1, proto.size()-2)); - if (!cppmeth) { PyErr_Restore(pytype, pyvalue, pytrace); return nullptr; @@ -832,7 +876,7 @@ static PyObject* tpp_overload(TemplateProxy* pytmpl, PyObject* args) } else meth = new CPPMethod(scope, cppmeth); - return (PyObject*)CPPOverload_New(CPyCppyy_PyText_AsString(pytmpl->fTI->fCppName) + proto, meth); + return (PyObject*)CPPOverload_New(pytmpl->fTI->fCppName+proto, meth); } static PyMethodDef tpp_methods[] = { @@ -841,61 +885,80 @@ static PyMethodDef tpp_methods[] = { {(char*)nullptr, nullptr, 0, nullptr } }; + //= CPyCppyy template proxy type ============================================= PyTypeObject TemplateProxy_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - (char*)"cppyy.TemplateProxy", // tp_name - sizeof(TemplateProxy), // tp_basicsize - 0, // tp_itemsize - (destructor)tpp_dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - (reprfunc)tpp_repr, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - &tpp_as_mapping, // tp_as_mapping - (hashfunc)tpp_hash, // tp_hash - (ternaryfunc)tpp_call, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags - (char*)"cppyy template proxy (internal)", // tp_doc - (traverseproc)tpp_traverse,// tp_traverse - (inquiry)tpp_clear, // tp_clear - (richcmpfunc)tpp_richcompare,// tp_richcompare - offsetof(TemplateProxy, fWeakrefList), // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - tpp_methods, // tp_methods - 0, // tp_members - tpp_getset, // tp_getset - 0, // tp_base - 0, // tp_dict - (descrgetfunc)tpp_descrget,// tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - (newfunc)tpp_new, // tp_new - 0, // tp_free - 0, // tp_is_gc - 0, // tp_bases - 0, // tp_mro - 0, // tp_cache - 0, // tp_subclasses - 0 // tp_weaklist + PyVarObject_HEAD_INIT(&PyType_Type, 0) + (char*)"cppyy.TemplateProxy", // tp_name + sizeof(TemplateProxy), // tp_basicsize + 0, // tp_itemsize + (destructor)tpp_dealloc, // tp_dealloc +#if PY_VERSION_HEX >= 0x03080000 + offsetof(TemplateProxy, fVectorCall), +#else + 0, // tp_vectorcall_offset / tp_print +#endif + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_as_async / tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &tpp_as_mapping, // tp_as_mapping + (hashfunc)tpp_hash, // tp_hash +#if PY_VERSION_HEX >= 0x03080000 + (ternaryfunc)PyVectorcall_Call, // tp_call +#else + (ternaryfunc)tpp_call, // tp_call +#endif + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC +#if PY_VERSION_HEX >= 0x03080000 + | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_METHOD_DESCRIPTOR +#endif +#if PY_VERSION_HEX >= 0x03120000 + | Py_TPFLAGS_MANAGED_WEAKREF +#endif + , // tp_flags + (char*)"cppyy template proxy (internal)", // tp_doc + (traverseproc)tpp_traverse, // tp_traverse + (inquiry)tpp_clear, // tp_clear + (richcmpfunc)tpp_richcompare, // tp_richcompare + offsetof(TemplateProxy, fWeakrefList), // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + tpp_methods, // tp_methods + 0, // tp_members + tpp_getset, // tp_getset + 0, // tp_base + 0, // tp_dict + (descrgetfunc)tpp_descr_get, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + (newfunc)tpp_new, // tp_new + 0, // tp_free + 0, // tp_is_gc + 0, // tp_bases + 0, // tp_mro + 0, // tp_cache + 0, // tp_subclasses + 0 // tp_weaklist #if PY_VERSION_HEX >= 0x02030000 - , 0 // tp_del + , 0 // tp_del #endif #if PY_VERSION_HEX >= 0x02060000 - , 0 // tp_version_tag + , 0 // tp_version_tag #endif #if PY_VERSION_HEX >= 0x03040000 - , 0 // tp_finalize + , 0 // tp_finalize +#endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall #endif }; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.h b/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.h index 5e31ebedd6476..a286fe7147e1f 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/TemplateProxy.h @@ -32,17 +32,14 @@ class TemplateInfo { ~TemplateInfo(); public: - PyObject* fCppName; - PyObject* fPyName; - PyObject* fPyClass; + std::string fCppName; + PyObject* fPyClass; CPPOverload* fNonTemplated; // holder for non-template overloads CPPOverload* fTemplated; // holder for templated overloads CPPOverload* fLowPriority; // low priority overloads such as void*/void** - PyObject* fWeakrefList; TP_DispatchMap_t fDispatchMap; - - uint64_t fFlags; // collective for all methods + PyObject* fDoc; }; typedef std::shared_ptr TP_TInfo_t; @@ -58,6 +55,9 @@ class TemplateProxy { PyObject* fSelf; // must be first (same layout as CPPOverload) PyObject* fTemplateArgs; PyObject* fWeakrefList; +#if PY_VERSION_HEX >= 0x03080000 + vectorcallfunc fVectorCall; +#endif TP_TInfo_t fTI; public: @@ -65,7 +65,7 @@ class TemplateProxy { void AdoptMethod(PyCallable* pc); void AdoptTemplate(PyCallable* pc); PyObject* Instantiate(const std::string& fname, - PyObject* tmplArgs, Utility::ArgPreference, int* pcnt = nullptr); + CPyCppyy_PyArgs_t tmplArgs, size_t nargsf, Utility::ArgPreference, int* pcnt = nullptr); private: // private, as the python C-API will handle creation TemplateProxy() = delete; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.cxx index ba6f807bc5f41..2f8e6c935c317 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.cxx @@ -16,7 +16,7 @@ typedef struct { } ia_iterobject; static PyObject* ia_iternext(ia_iterobject* ia) { - if (ia->ia_len != -1 && ia->ia_pos >= ia->ia_len) { + if (ia->ia_len != (Py_ssize_t)-1 && ia->ia_pos >= ia->ia_len) { ia->ia_pos = 0; // debatable, but since the iterator is cached, this return nullptr; // allows for multiple conversions to e.g. a tuple } else if (ia->ia_stride == 0 && ia->ia_pos != 0) { @@ -51,6 +51,36 @@ static PyGetSetDef ia_getset[] = { {(char*)nullptr, nullptr, nullptr, nullptr, nullptr} }; + +static Py_ssize_t ia_length(ia_iterobject* ia) +{ + return ia->ia_len; +} + +static PyObject* ia_subscript(ia_iterobject* ia, PyObject* pyidx) +{ +// Subscripting the iterator allows direct access through indexing on arrays +// that do not have a defined length. This way, the return from accessing such +// an array as a data member can both be used in a loop and directly. + Py_ssize_t idx = PyInt_AsSsize_t(pyidx); + if (idx == (Py_ssize_t)-1 && PyErr_Occurred()) + return nullptr; + + if (ia->ia_len != (Py_ssize_t)-1 && (idx < 0 || ia->ia_len <= idx)) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return nullptr; + } + + return CPyCppyy::BindCppObjectNoCast( + (char*)ia->ia_array_start + ia->ia_pos*ia->ia_stride, ia->ia_klass); +} + +static PyMappingMethods ia_as_mapping = { + (lenfunc) ia_length, // mp_length + (binaryfunc) ia_subscript, // mp_subscript + (objobjargproc)nullptr, // mp_ass_subscript +}; + } // unnamed namespace @@ -62,7 +92,9 @@ PyTypeObject InstanceArrayIter_Type = { sizeof(ia_iterobject), // tp_basicsize 0, (destructor)PyObject_GC_Del, // tp_dealloc - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + &ia_as_mapping, // tp_as_mapping + 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags 0, @@ -70,9 +102,9 @@ PyTypeObject InstanceArrayIter_Type = { 0, 0, 0, PyObject_SelfIter, // tp_iter (iternextfunc)ia_iternext, // tp_iternext - 0, 0, ia_getset, 0, 0, 0, 0, - 0, // tp_getset - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, + ia_getset, // tp_getset + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 #if PY_VERSION_HEX >= 0x02030000 , 0 // tp_del #endif @@ -82,15 +114,18 @@ PyTypeObject InstanceArrayIter_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; //= support for C-style arrays of objects ==================================== PyObject* TupleOfInstances_New( - Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, dim_t ndims, dims_t dims) + Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, cdims_t dims) { // recursively set up tuples of instances on all dimensions - if (ndims == -1 /* unknown shape */ || dims[0] == -1 /* unknown size */) { + if (dims.ndim() == UNKNOWN_SIZE || dims[0] == UNKNOWN_SIZE /* unknown shape or size */) { // no known length ... return an iterable object and let the user figure it out ia_iterobject* ia = PyObject_GC_New(ia_iterobject, &InstanceArrayIter_Type); if (!ia) return nullptr; @@ -103,17 +138,17 @@ PyObject* TupleOfInstances_New( PyObject_GC_Track(ia); return (PyObject*)ia; - } else if (1 < ndims) { + } else if (1 < dims.ndim()) { // not the innermost dimension, descend one level - int nelems = (int)dims[0]; size_t block_size = 0; - for (int i = 1; i < (int)ndims; ++i) block_size += (size_t)dims[i]; + for (Py_ssize_t i = 1; i < dims.ndim(); ++i) block_size += (size_t)dims[i]; block_size *= Cppyy::SizeOf(klass); + Py_ssize_t nelems = dims[0]; PyObject* tup = PyTuple_New(nelems); - for (int i = 0; i < nelems; ++i) { + for (Py_ssize_t i = 0; i < nelems; ++i) { PyTuple_SetItem(tup, i, TupleOfInstances_New( - (char*)address + i*block_size, klass, ndims-1, dims+1)); + (char*)address + i*block_size, klass, dims.sub())); } return tup; } else { @@ -142,7 +177,6 @@ PyObject* TupleOfInstances_New( PyObject* args = PyTuple_New(1); Py_INCREF(tup); PyTuple_SET_ITEM(args, 0, tup); PyObject* arr = PyTuple_Type.tp_new(&TupleOfInstances_Type, args, nullptr); - if (PyErr_Occurred()) PyErr_Print(); Py_DECREF(args); // tup ref eaten by SET_ITEM on args @@ -161,10 +195,10 @@ PyTypeObject TupleOfInstances_Type = { 0, // tp_basicsize 0, // tp_itemsize 0, // tp_dealloc - 0, // tp_print + 0, // tp_vectorcall_offset / tp_print 0, // tp_getattr 0, // tp_setattr - 0, // tp_compare + 0, // tp_as_async / tp_compare 0, // tp_repr 0, // tp_as_number 0, // tp_as_sequence @@ -211,6 +245,9 @@ PyTypeObject TupleOfInstances_Type = { #if PY_VERSION_HEX >= 0x03040000 , 0 // tp_finalize #endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif }; } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.h b/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.h index 0a66533377f3c..de95f29b2f8ba 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/TupleOfInstances.h @@ -1,6 +1,10 @@ #ifndef CPYCPPYY_TUPLEOFINSTANCES_H #define CPYCPPYY_TUPLEOFINSTANCES_H +// Bindings +#include "Dimensions.h" + + namespace CPyCppyy { /** Representation of C-style array of instances @@ -26,7 +30,7 @@ inline bool TupleOfInstances_CheckExact(T* object) } PyObject* TupleOfInstances_New( - Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, dim_t ndims, dims_t dims); + Cppyy::TCppObject_t address, Cppyy::TCppType_t klass, cdims_t dims); } // namespace CPyCppyy diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.cxx index 68411af530881..3a979ddc7bc9c 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.cxx @@ -3,13 +3,15 @@ #include "TypeManip.h" // Standard +#include +#include #include //- helpers ------------------------------------------------------------------ static inline bool is_varchar(char c) { - return isalnum((int)c) || c == '_' || c == ')' || c == '(' /* for (anonymous) */; + return isalnum((int)c) || c == '_' || c == ')' || c == '(' /* for (anonymous)/(unnamed) */; } static inline @@ -17,13 +19,18 @@ std::string::size_type find_qualifier_index(const std::string& name) { // Find the first location that is not part of the class name proper. std::string::size_type i = name.size() - 1; + bool arr_open = false; for ( ; 0 < i; --i) { std::string::value_type c = name[i]; - if (is_varchar(c) || c == '>') { + if (!arr_open && (is_varchar(c) || c == '>')) { if (c == 't' && 6 < i && !is_varchar(name[i-5]) && name.substr(i-4, 5) == "const") i -= 4; // this skips 'const' on a pointer type else break; + } else if (c == ']') { + arr_open = true; + } else if (c == '[') { + arr_open = false; } } @@ -33,6 +40,9 @@ std::string::size_type find_qualifier_index(const std::string& name) static inline void erase_const(std::string& name) { // Find and remove all occurrence of 'const'. + if (name.empty()) + return; + std::string::size_type spos = std::string::npos; std::string::size_type start = 0; while ((spos = name.find("const", start)) != std::string::npos) { @@ -58,7 +68,7 @@ static inline void rstrip(std::string& name) // Remove space from the right side of name. std::string::size_type i = name.size(); for ( ; 0 < i; --i) { - if (!isspace(name[i])) + if (!isspace(name[i-1])) break; } @@ -66,7 +76,6 @@ static inline void rstrip(std::string& name) name = name.substr(0, i); } - //---------------------------------------------------------------------------- std::string CPyCppyy::TypeManip::remove_const(const std::string& cppname) { @@ -75,18 +84,17 @@ std::string CPyCppyy::TypeManip::remove_const(const std::string& cppname) std::string::size_type type_stop = cppname.rfind('>'); if (cppname.find("::", type_stop+1) != std::string::npos) // e.g. klass::some_typedef type_stop = cppname.find(' ', type_stop+1); - if (tmplt_start != std::string::npos) { + if (tmplt_start != std::string::npos && cppname[tmplt_start+1] != '<') { // only replace const qualifying cppname, not in template parameters std::string pre = cppname.substr(0, tmplt_start); erase_const(pre); std::string post = ""; - if (type_stop != std::string::npos) { + if (type_stop != std::string::npos && type_stop != cppname.size()-1) { post = cppname.substr(type_stop+1, std::string::npos); erase_const(post); } - type_stop = type_stop == std::string::npos ? std::string::npos : type_stop+1; - return pre + cppname.substr(tmplt_start, type_stop) + post; + return pre + cppname.substr(tmplt_start, type_stop+1-tmplt_start) + post; } std::string clean_name = cppname; @@ -133,9 +141,9 @@ std::string CPyCppyy::TypeManip::template_base(const std::string& cppname) // count '<' and '>' to be able to skip template contents if (c == '>') - ++tpl_open; - else if (c == '<') --tpl_open; + else if (c == '<' && cppname[pos+1] != '<') + ++tpl_open; if (tpl_open == 0) return cppname.substr(0, pos); @@ -144,6 +152,28 @@ std::string CPyCppyy::TypeManip::template_base(const std::string& cppname) return cppname; } +//---------------------------------------------------------------------------- +std::string CPyCppyy::TypeManip::compound(const std::string& name) +{ +// Break down the compound of a fully qualified type name. + std::string cleanName = remove_const(name); + auto idx = find_qualifier_index(cleanName); + + const std::string& cpd = cleanName.substr(idx, std::string::npos); + +// for easy identification of fixed size arrays + if (!cpd.empty() && cpd.back() == ']') { + if (cpd.front() == '[') + return "[]"; // fixed array any; dimensions handled separately + + std::ostringstream scpd; + scpd << cpd.substr(0, cpd.find('[')) << "[]"; + return scpd.str(); + } + + return cpd; +} + //---------------------------------------------------------------------------- void CPyCppyy::TypeManip::cppscope_to_pyscope(std::string& cppscope) { @@ -155,6 +185,16 @@ void CPyCppyy::TypeManip::cppscope_to_pyscope(std::string& cppscope) } } +//---------------------------------------------------------------------------- +void CPyCppyy::TypeManip::cppscope_to_legalname(std::string& cppscope) +{ +// Change characters illegal in a variable name into '_' to form a legal name. + for (char& c : cppscope) { + for (char needle : {':', '>', '<', ' ', ',', '&', '='}) + if (c == needle) c = '_'; + } +} + //---------------------------------------------------------------------------- std::string CPyCppyy::TypeManip::extract_namespace(const std::string& name) { @@ -168,9 +208,9 @@ std::string CPyCppyy::TypeManip::extract_namespace(const std::string& name) // count '<' and '>' to be able to skip template contents if (c == '>') - ++tpl_open; - else if (c == '<') --tpl_open; + else if (c == '<' && name[pos+1] != '<') + ++tpl_open; // collect name up to "::" else if (tpl_open == 0 && c == ':' && name[pos-1] == ':') { @@ -199,9 +239,9 @@ std::vector CPyCppyy::TypeManip::extract_arg_types(const std::strin // count '<' and '>' to be able to skip template contents if (c == '>') - ++tpl_open; - else if (c == '<') --tpl_open; + else if (c == '<' && sig[pos+1] != '<') + ++tpl_open; // collect type name up to ',' or end ')' else if (tpl_open == 0 && c == ',') { @@ -217,3 +257,18 @@ std::vector CPyCppyy::TypeManip::extract_arg_types(const std::strin return result; } +//---------------------------------------------------------------------------- +Py_ssize_t CPyCppyy::TypeManip::array_size(const std::string& name) +{ +// Extract the array size from a given type name (assumes 1D arrays) + std::string cleanName = remove_const(name); + if (cleanName[cleanName.size()-1] == ']') { + std::string::size_type idx = cleanName.rfind('['); + if (idx != std::string::npos) { + const std::string asize = cleanName.substr(idx+1, cleanName.size()-2); + return strtoul(asize.c_str(), nullptr, 0); + } + } + + return -1; +} diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.h b/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.h index 852804aefbd31..4fb8bd0cd6c57 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/TypeManip.h @@ -13,11 +13,14 @@ namespace TypeManip { std::string clean_type(const std::string& cppname, bool template_strip = true, bool const_strip = true); std::string template_base(const std::string& cppname); + std::string compound(const std::string& name); void cppscope_to_pyscope(std::string& cppscope); + void cppscope_to_legalname(std::string& cppscope); std::string extract_namespace(const std::string& name); std::vector extract_arg_types(const std::string& sig); + Py_ssize_t array_size(const std::string& name); } // namespace TypeManip diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx index 39b73434335d9..49415f109aa01 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Utility.cxx @@ -10,7 +10,6 @@ #include "CustomPyTypes.h" #include "TemplateProxy.h" #include "TypeManip.h" -#include "RConfig.h" // Standard #include @@ -24,14 +23,22 @@ //- data _____________________________________________________________________ +#if PY_VERSION_HEX < 0x030b0000 dict_lookup_func CPyCppyy::gDictLookupOrg = 0; bool CPyCppyy::gDictLookupActive = false; +#endif typedef std::map TC2POperatorMapping_t; static TC2POperatorMapping_t gC2POperatorMapping; static std::set gOpSkip; static std::set gOpRemove; +namespace CPyCppyy { +// special objects + extern PyObject* gNullPtrObject; + extern PyObject* gDefaultObject; +} + namespace { using namespace CPyCppyy::Utility; @@ -55,7 +62,6 @@ namespace { gC2POperatorMapping["[]"] = "__getitem__"; gC2POperatorMapping["()"] = "__call__"; - gC2POperatorMapping["/"] = CPPYY__div__; gC2POperatorMapping["%"] = "__mod__"; gC2POperatorMapping["**"] = "__pow__"; gC2POperatorMapping["<<"] = "__lshift__"; @@ -109,21 +115,13 @@ namespace { gC2POperatorMapping["="] = "__assign__"; // id. #if PY_VERSION_HEX < 0x03000000 - gC2POperatorMapping["bool"] = "__nonzero__"; + gC2POperatorMapping["bool"] = "__cpp_nonzero__"; #else - gC2POperatorMapping["bool"] = "__bool__"; + gC2POperatorMapping["bool"] = "__cpp_bool__"; #endif } } initOperatorMapping_; -// TODO: this should live with Helpers - inline void RemoveConst(std::string& cleanName) { - std::string::size_type spos = std::string::npos; - while ((spos = cleanName.find("const")) != std::string::npos) { - cleanName.swap(cleanName.erase(spos, 5)); - } - } - } // unnamed namespace @@ -131,6 +129,13 @@ namespace { unsigned long CPyCppyy::PyLongOrInt_AsULong(PyObject* pyobject) { // Convert to C++ unsigned long, with bounds checking, allow int -> ulong. + if (PyFloat_Check(pyobject)) { + PyErr_SetString(PyExc_TypeError, "can\'t convert float to unsigned long"); + return (unsigned long)-1; + } else if (pyobject == CPyCppyy::gDefaultObject) { + return (unsigned long)0; + } + unsigned long ul = PyLong_AsUnsignedLong(pyobject); if (PyErr_Occurred() && PyInt_Check(pyobject)) { PyErr_Clear(); @@ -148,15 +153,22 @@ unsigned long CPyCppyy::PyLongOrInt_AsULong(PyObject* pyobject) } //---------------------------------------------------------------------------- -ULong64_t CPyCppyy::PyLongOrInt_AsULong64(PyObject* pyobject) +PY_ULONG_LONG CPyCppyy::PyLongOrInt_AsULong64(PyObject* pyobject) { // Convert to C++ unsigned long long, with bounds checking. - ULong64_t ull = PyLong_AsUnsignedLongLong(pyobject); + if (PyFloat_Check(pyobject)) { + PyErr_SetString(PyExc_TypeError, "can\'t convert float to unsigned long long"); + return -1; + } else if (pyobject == CPyCppyy::gDefaultObject) { + return (unsigned long)0; + } + + PY_ULONG_LONG ull = PyLong_AsUnsignedLongLong(pyobject); if (PyErr_Occurred() && PyInt_Check(pyobject)) { PyErr_Clear(); long i = PyInt_AS_LONG(pyobject); if (0 <= i) { - ull = (ULong64_t)i; + ull = (PY_ULONG_LONG)i; } else { PyErr_SetString(PyExc_ValueError, "can\'t convert negative value to unsigned long long"); @@ -268,6 +280,7 @@ CPyCppyy::PyCallable* CPyCppyy::Utility::FindUnaryOperator(PyObject* pyclass, co { // Find a callable matching named operator (op) and klass arguments in the global // namespace or the klass' namespace. + if (!CPPScope_Check(pyclass)) return nullptr; @@ -312,8 +325,13 @@ CPyCppyy::PyCallable* CPyCppyy::Utility::FindBinaryOperator( PyCallable* pyfunc = 0; - const std::string& lnsname = TypeManip::extract_namespace(lcname); - if (!scope) scope = Cppyy::GetScope(lnsname); + if (!scope) { + // TODO: the following should remain sync with what clingwrapper does in its + // type remapper; there must be a better way? + if (lcname == "str" || lcname == "unicode" || lcname == "complex") + scope = Cppyy::GetScope("std"); + else scope = Cppyy::GetScope(TypeManip::extract_namespace(lcname)); + } if (scope) pyfunc = BuildOperator(lcname, rcname, op, scope, reverse); @@ -365,6 +383,22 @@ CPyCppyy::PyCallable* CPyCppyy::Utility::FindBinaryOperator( } //---------------------------------------------------------------------------- +static inline std::string AnnotationAsText(PyObject* pyobj) +{ + if (!CPyCppyy_PyText_Check(pyobj)) { + PyObject* pystr = PyObject_GetAttr(pyobj, CPyCppyy::PyStrings::gName); + if (!pystr) { + PyErr_Clear(); + pystr = PyObject_Str(pyobj); + } + + std::string str = CPyCppyy_PyText_AsString(pystr); + Py_DECREF(pystr); + return str; + } + return CPyCppyy_PyText_AsString(pyobj); +} + static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, CPyCppyy::Utility::ArgPreference pref, int* pcnt = nullptr) { @@ -379,48 +413,62 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, long l = PyInt_AS_LONG(arg); tmpl_name.append((l < INT_MIN || INT_MAX < l) ? "long" : "int"); #else - Long64_t ll = PyLong_AsLongLong(arg); - if (ll == (Long64_t)-1 && PyErr_Occurred()) { + PY_LONG_LONG ll = PyLong_AsLongLong(arg); + if (ll == (PY_LONG_LONG)-1 && PyErr_Occurred()) { PyErr_Clear(); - ULong64_t ull = PyLong_AsUnsignedLongLong(arg); - if (ull == (ULong64_t)-1 && PyErr_Occurred()) { + PY_ULONG_LONG ull = PyLong_AsUnsignedLongLong(arg); + if (ull == (PY_ULONG_LONG)-1 && PyErr_Occurred()) { PyErr_Clear(); tmpl_name.append("int"); // still out of range, will fail later } else - tmpl_name.append("ULong64_t"); // since already failed long long + tmpl_name.append("unsigned long long"); // since already failed long long } else tmpl_name.append((ll < INT_MIN || INT_MAX < ll) ? \ - ((ll < LONG_MIN || LONG_MAX < ll) ? "Long64_t" : "long") : "int"); + ((ll < LONG_MIN || LONG_MAX < ll) ? "long long" : "long") : "int"); #endif } else tmpl_name.append("int"); + + return true; + } + #if PY_VERSION_HEX < 0x03000000 - } else if (tn == (PyObject*)&PyLong_Type) { + if (tn == (PyObject*)&PyLong_Type) { if (arg) { - Long64_t ll = PyLong_AsLongLong(arg); - if (ll == (Long64_t)-1 && PyErr_Occurred()) { + PY_LONG_LONG ll = PyLong_AsLongLong(arg); + if (ll == (PY_LONG_LONG)-1 && PyErr_Occurred()) { PyErr_Clear(); - ULong64_t ull = PyLong_AsUnsignedLongLong(arg); - if (ull == (ULong64_t)-1 && PyErr_Occurred()) { + PY_ULONG_LONG ull = PyLong_AsUnsignedLongLong(arg); + if (ull == (PY_ULONG_LONG)-1 && PyErr_Occurred()) { PyErr_Clear(); tmpl_name.append("long"); // still out of range, will fail later } else - tmpl_name.append("ULong64_t"); // since already failed long long + tmpl_name.append("unsigned long long"); // since already failed long long } else - tmpl_name.append((ll < LONG_MIN || LONG_MAX < ll) ? "Long64_t" : "long"); + tmpl_name.append((ll < LONG_MIN || LONG_MAX < ll) ? "long long" : "long"); } else tmpl_name.append("long"); + + return true; + } #endif - } else if (tn == (PyObject*)&PyFloat_Type) { + + if (tn == (PyObject*)&PyFloat_Type) { // special case for floats (Python-speak for double) if from argument (only) tmpl_name.append(arg ? "double" : "float"); + return true; + } + #if PY_VERSION_HEX < 0x03000000 - } else if (tn == (PyObject*)&PyString_Type) { + if (tn == (PyObject*)&PyString_Type) { #else - } else if (tn == (PyObject*)&PyUnicode_Type) { + if (tn == (PyObject*)&PyUnicode_Type) { #endif tmpl_name.append("std::string"); - } else if (tn == (PyObject*)&PyList_Type || tn == (PyObject*)&PyTuple_Type) { + return true; + } + + if (tn == (PyObject*)&PyList_Type || tn == (PyObject*)&PyTuple_Type) { if (arg && PySequence_Size(arg)) { std::string subtype{"std::initializer_list<"}; PyObject* item = PySequence_GetItem(arg, 0); @@ -432,7 +480,10 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, Py_DECREF(item); } - } else if (CPPScope_Check(tn)) { + return true; + } + + if (CPPScope_Check(tn)) { tmpl_name.append(Cppyy::GetScopedFinalName(((CPPClass*)tn)->fCppType)); if (arg) { // try to specialize the type match for the given object @@ -449,26 +500,81 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, } } } - } else if (PyObject_HasAttr(tn, PyStrings::gCppName)) { - PyObject* tpName = PyObject_GetAttr(tn, PyStrings::gCppName); - tmpl_name.append(CPyCppyy_PyText_AsString(tpName)); - Py_DECREF(tpName); - } else if (PyObject_HasAttr(tn, PyStrings::gName)) { - PyObject* tpName = PyObject_GetAttr(tn, PyStrings::gName); + + return true; + } + + if (tn == (PyObject*)&CPPOverload_Type) { + PyObject* tpName = arg ? \ + PyObject_GetAttr(arg, PyStrings::gCppName) : \ + CPyCppyy_PyText_FromString("void* (*)(...)"); tmpl_name.append(CPyCppyy_PyText_AsString(tpName)); Py_DECREF(tpName); - } else if (PyInt_Check(tn) || PyLong_Check(tn) || PyFloat_Check(tn)) { + + return true; + } + + if (arg && PyCallable_Check(arg)) { + PyObject* annot = PyObject_GetAttr(arg, PyStrings::gAnnotations); + if (annot) { + if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) { + PyObject* ret = PyDict_GetItemString(annot, "return"); + if (ret) { + // dict is ordered, with the last value being the return type + std::ostringstream tpn; + tpn << (CPPScope_Check(ret) ? ClassName(ret) : AnnotationAsText(ret)) + << " (*)("; + + PyObject* values = PyDict_Values(annot); + for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values)-1); ++i) { + if (i) tpn << ", "; + PyObject* item = PyList_GET_ITEM(values, i); + tpn << (CPPScope_Check(item) ? ClassName(item) : AnnotationAsText(item)); + } + Py_DECREF(values); + + tpn << ')'; + tmpl_name.append(tpn.str()); + + return true; + + } else + PyErr_Clear(); + } + Py_DECREF(annot); + } else + PyErr_Clear(); + + PyObject* tpName = PyObject_GetAttr(arg, PyStrings::gCppName); + if (tpName) { + tmpl_name.append(CPyCppyy_PyText_AsString(tpName)); + Py_DECREF(tpName); + return true; + } + PyErr_Clear(); + } + + for (auto nn : {PyStrings::gCppName, PyStrings::gName}) { + PyObject* tpName = PyObject_GetAttr(tn, nn); + if (tpName) { + tmpl_name.append(CPyCppyy_PyText_AsString(tpName)); + Py_DECREF(tpName); + return true; + } + PyErr_Clear(); + } + + if (PyInt_Check(tn) || PyLong_Check(tn) || PyFloat_Check(tn)) { // last ditch attempt, works for things like int values; since this is a // source of errors otherwise, it is limited to specific types and not // generally used (str(obj) can print anything ...) PyObject* pystr = PyObject_Str(tn); tmpl_name.append(CPyCppyy_PyText_AsString(pystr)); Py_DECREF(pystr); - } else { - return false; + return true; } - return true; + return false; } std::string CPyCppyy::Utility::ConstructTemplateArgs( @@ -493,7 +599,7 @@ std::string CPyCppyy::Utility::ConstructTemplateArgs( PyObject* tn = justOne ? tpArgs : PyTuple_GET_ITEM(tpArgs, i); if (CPyCppyy_PyText_Check(tn)) { tmpl_name.append(CPyCppyy_PyText_AsString(tn)); - // some commmon numeric types (separated out for performance: checking for + // some common numeric types (separated out for performance: checking for // __cpp_name__ and/or __name__ is rather expensive) } else { if (!AddTypeName(tmpl_name, tn, (args ? PyTuple_GET_ITEM(args, i) : nullptr), pref, pcnt)) { @@ -515,6 +621,11 @@ std::string CPyCppyy::Utility::ConstructTemplateArgs( } //---------------------------------------------------------------------------- +static inline bool check_scope(const std::string& name) +{ + return (bool)Cppyy::GetScope(CPyCppyy::TypeManip::clean_type(name)); +} + void CPyCppyy::Utility::ConstructCallbackPreamble(const std::string& retType, const std::vector& argtypes, std::ostringstream& code) { @@ -527,12 +638,32 @@ void CPyCppyy::Utility::ConstructCallbackPreamble(const std::string& retType, code << " CPYCPPYY_STATIC std::unique_ptr> " "retconv{CPyCppyy::CreateConverter(\"" << retType << "\"), CPyCppyy::DestroyConverter};\n"; + std::vector arg_is_ptr; if (nArgs) { + arg_is_ptr.reserve(nArgs); code << " CPYCPPYY_STATIC std::vector>> argcvs;\n" << " if (argcvs.empty()) {\n" << " argcvs.reserve(" << nArgs << ");\n"; - for (int i = 0; i < nArgs; ++i) - code << " argcvs.emplace_back(CPyCppyy::CreateConverter(\"" << argtypes[i] << "\"), CPyCppyy::DestroyConverter);\n"; + for (int i = 0; i < nArgs; ++i) { + arg_is_ptr[i] = false; + code << " argcvs.emplace_back(CPyCppyy::CreateConverter(\""; + const std::string& at = argtypes[i]; + const std::string& res_at = Cppyy::ResolveName(at); + const std::string& cpd = TypeManip::compound(res_at); + if (!cpd.empty() && check_scope(res_at)) { + // in case of a pointer, the original argument needs to be used to ensure + // the pointer-value remains comparable + // + // in case of a reference, there is no extra indirection on the C++ side as + // would be when converting a data member, so adjust the converter + arg_is_ptr[i] = cpd.back() == '*'; + if (arg_is_ptr[i] || cpd.back() == '&') { + code << res_at.substr(0, res_at.size()-1); + } else code << at; + } else + code << at; + code << "\"), CPyCppyy::DestroyConverter);\n"; + } code << " }\n"; } @@ -549,12 +680,14 @@ void CPyCppyy::Utility::ConstructCallbackPreamble(const std::string& retType, code << " pyargs.reserve(" << nArgs << ");\n" << " try {\n"; for (int i = 0; i < nArgs; ++i) { - code << " pyargs.emplace_back(argcvs[" << i << "]->FromMemory((void*)&arg" << i << "));\n" + code << " pyargs.emplace_back(argcvs[" << i << "]->FromMemory((void*)"; + if (!arg_is_ptr[i]) code << '&'; + code << "arg" << i << "));\n" << " if (!pyargs.back()) throw " << i << ";\n"; } code << " } catch(int) {\n" << " for (auto pyarg : pyargs) Py_XDECREF(pyarg);\n" - << " PyGILState_Release(state); throw CPyCppyy::PyException{};\n" + << " CPyCppyy::PyException pyexc; PyGILState_Release(state); throw pyexc;\n" << " }\n"; } } @@ -564,6 +697,7 @@ void CPyCppyy::Utility::ConstructCallbackReturn(const std::string& retType, int // Generate code for return value conversion and error handling. bool isVoid = retType == "void"; bool isPtr = Cppyy::ResolveName(retType).back() == '*'; + if (nArgs) code << " for (auto pyarg : pyargs) Py_DECREF(pyarg);\n"; code << " bool cOk = (bool)pyresult;\n" @@ -575,7 +709,7 @@ void CPyCppyy::Utility::ConstructCallbackReturn(const std::string& retType, int " ret = nullptr;\n" " else {\n"; } - code << (isVoid ? "" : " cOk = retconv->ToMemory(pyresult, &ret);\n") + code << (isVoid ? "" : " cOk = retconv->ToMemory(pyresult, (void*)&ret);\n") << " Py_DECREF(pyresult);\n }\n"; if (isPtr) code << " }\n"; code << " if (!cOk) {" // assume error set when converter failed @@ -584,7 +718,7 @@ void CPyCppyy::Utility::ConstructCallbackReturn(const std::string& retType, int #ifdef _WIN32 " /* do nothing */ }\n" #else - " PyGILState_Release(state); throw CPyCppyy::PyException{}; }\n" + " CPyCppyy::PyException pyexc; PyGILState_Release(state); throw pyexc; }\n" #endif " PyGILState_Release(state);\n" " return"; @@ -601,11 +735,6 @@ PyObject* CPyCppyy::Utility::FuncPtr2StdFunction( // Convert a function pointer to an equivalent std::function<> object. static int maker_count = 0; - if (!address) { - PyErr_SetString(PyExc_TypeError, "can not convert null function pointer"); - return nullptr; - } - auto pf = sStdFuncLookup.find(address); if (pf != sStdFuncLookup.end()) { Py_INCREF(pf->second); @@ -681,7 +810,7 @@ Py_ssize_t CPyCppyy::Utility::GetBuffer(PyObject* pyobject, char tc, int size, v // Retrieve a linear buffer pointer from the given pyobject. // special case: don't handle character strings here (yes, they're buffers, but not quite) - if (PyBytes_Check(pyobject)) + if (PyBytes_Check(pyobject) || PyUnicode_Check(pyobject)) return 0; // special case: bytes array @@ -696,30 +825,41 @@ Py_ssize_t CPyCppyy::Utility::GetBuffer(PyObject* pyobject, char tc, int size, v memset(&bufinfo, 0, sizeof(Py_buffer)); if (PyObject_GetBuffer(pyobject, &bufinfo, PyBUF_FORMAT) == 0) { if (tc == '*' || strchr(bufinfo.format, tc) -#if defined(_WIN32) || ( defined(R__LINUX) && !defined(R__B64) ) - // ctypes is inconsistent in format on Windows; either way these types are the same size - // observed also in 32-bit Linux for a NumPy array - || (tc == 'I' && strchr(bufinfo.format, 'L')) || (tc == 'i' && strchr(bufinfo.format, 'l')) -#endif + // if `long int` and `int` are the same size (on Windows and 32bit Linux, + // for example), `ctypes` isn't too picky about the type format, so make + // sure both integer types pass the type check + || (sizeof(long int) == sizeof(int) && ((tc == 'I' && strchr(bufinfo.format, 'L')) || + (tc == 'i' && strchr(bufinfo.format, 'l')))) + // complex float is 'Zf' in bufinfo.format, but 'z' in single char + || (tc == 'z' && strstr(bufinfo.format, "Zf")) // allow 'signed char' ('b') from array to pass through '?' (bool as from struct) || (tc == '?' && strchr(bufinfo.format, 'b')) ) { buf = bufinfo.buf; - if (buf && bufinfo.ndim == 0) { - PyBuffer_Release(&bufinfo); - return bufinfo.len/bufinfo.itemsize; - } else if (buf && bufinfo.ndim == 1) { - Py_ssize_t size1d = bufinfo.shape ? bufinfo.shape[0] : bufinfo.len/bufinfo.itemsize; - PyBuffer_Release(&bufinfo); - return size1d; + + if (check && bufinfo.itemsize != size) { + PyErr_Format(PyExc_TypeError, + "buffer itemsize (%ld) does not match expected size (%d)", bufinfo.itemsize, size); + CPyCppyy_PyBuffer_Release(pyobject, &bufinfo); + return 0; } + + Py_ssize_t buflen = 0; + if (buf && bufinfo.ndim == 0) + buflen = bufinfo.len/bufinfo.itemsize; + else if (buf && bufinfo.ndim == 1) + buflen = bufinfo.shape ? bufinfo.shape[0] : bufinfo.len/bufinfo.itemsize; + CPyCppyy_PyBuffer_Release(pyobject, &bufinfo); + if (buflen) + return buflen; } else { // have buf, but format mismatch: bail out now, otherwise the old // code will return based on itemsize match - PyBuffer_Release(&bufinfo); + CPyCppyy_PyBuffer_Release(pyobject, &bufinfo); return 0; } - } + } else if (bufinfo.obj) + CPyCppyy_PyBuffer_Release(pyobject, &bufinfo); PyErr_Clear(); } @@ -728,7 +868,7 @@ Py_ssize_t CPyCppyy::Utility::GetBuffer(PyObject* pyobject, char tc, int size, v PySequenceMethods* seqmeths = Py_TYPE(pyobject)->tp_as_sequence; if (seqmeths != 0 && bufprocs != 0 -#if PY_VERSION_HEX < 0x03000000 +#if PY_VERSION_HEX < 0x03000000 && bufprocs->bf_getwritebuffer != 0 && (*(bufprocs->bf_getsegcount))(pyobject, 0) == 1 #else @@ -744,16 +884,12 @@ Py_ssize_t CPyCppyy::Utility::GetBuffer(PyObject* pyobject, char tc, int size, v (*(bufprocs->bf_getbuffer))(pyobject, &bufinfo, PyBUF_WRITABLE); buf = (char*)bufinfo.buf; Py_ssize_t buflen = bufinfo.len; -#if PY_VERSION_HEX < 0x03010000 - PyBuffer_Release(pyobject, &bufinfo); -#else - PyBuffer_Release(&bufinfo); -#endif + CPyCppyy_PyBuffer_Release(pyobject, &bufinfo); #endif if (buf && check == true) { // determine buffer compatibility (use "buf" as a status flag) - PyObject* pytc = PyObject_GetAttr(pyobject, PyStrings::gTypeCode); + PyObject* pytc = tc != '*' ? PyObject_GetAttr(pyobject, PyStrings::gTypeCode) : nullptr; if (pytc != 0) { // for array objects char cpytc = CPyCppyy_PyText_AsString(pytc)[0]; if (!(cpytc == tc || (tc == '?' && cpytc == 'b'))) @@ -775,7 +911,7 @@ Py_ssize_t CPyCppyy::Utility::GetBuffer(PyObject* pyobject, char tc, int size, v PyObject* pyvalue2 = CPyCppyy_PyText_FromFormat( (char*)"%s and given element size (%ld) do not match needed (%d)", CPyCppyy_PyText_AsString(pyvalue), - seqmeths->sq_length ? (Long_t)(buflen/(*(seqmeths->sq_length))(pyobject)) : (Long_t)buflen, + seqmeths->sq_length ? (long)(buflen/(*(seqmeths->sq_length))(pyobject)) : (long)buflen, size); Py_DECREF(pyvalue); PyErr_Restore(pytype, pyvalue2, pytrace); @@ -790,7 +926,7 @@ Py_ssize_t CPyCppyy::Utility::GetBuffer(PyObject* pyobject, char tc, int size, v } //---------------------------------------------------------------------------- -std::string CPyCppyy::Utility::MapOperatorName(const std::string& name, bool bTakesParams) +std::string CPyCppyy::Utility::MapOperatorName(const std::string& name, bool bTakesParams, bool* stubbed) { // Map the given C++ operator name on the python equivalent. if (8 < name.size() && name.substr(0, 8) == "operator") { @@ -819,15 +955,25 @@ std::string CPyCppyy::Utility::MapOperatorName(const std::string& name, bool bTa } else if (op == "*") { // dereference v.s. multiplication of two instances - return bTakesParams ? "__mul__" : "__deref__"; + if (!bTakesParams) return "__deref__"; + if (stubbed) *stubbed = true; + return "__mul__"; + + } else if (op == "/") { + // no unary, but is stubbed + return CPPYY__div__; } else if (op == "+") { // unary positive v.s. addition of two instances - return bTakesParams ? "__add__" : "__pos__"; + if (!bTakesParams) return "__pos__"; + if (stubbed) *stubbed = true; + return "__add__"; } else if (op == "-") { // unary negative v.s. subtraction of two instances - return bTakesParams ? "__sub__" : "__neg__"; + if (!bTakesParams) return "__neg__"; + if (stubbed) *stubbed = true; + return "__sub__"; } else if (op == "++") { // prefix v.s. postfix increment @@ -844,49 +990,6 @@ std::string CPyCppyy::Utility::MapOperatorName(const std::string& name, bool bTa return name; } -//---------------------------------------------------------------------------- -const std::string CPyCppyy::Utility::Compound(const std::string& name) -{ -// TODO: consolidate with other string manipulations in TypeManip.cxx -// Break down the compound of a fully qualified type name. - std::string cleanName = name; - RemoveConst(cleanName); - - std::string compound = ""; - for (int ipos = (int)cleanName.size()-1; 0 <= ipos; --ipos) { - char c = cleanName[ipos]; - if (isspace(c)) continue; - if (isalnum(c) || c == '_' || c == '>' || c == ')') break; - - compound = c + compound; - } - -// for arrays (TODO: deal with the actual size) - if (compound == "]") - return "[]"; - - return compound; -} - -//---------------------------------------------------------------------------- -Py_ssize_t CPyCppyy::Utility::ArraySize(const std::string& name) -{ -// TODO: consolidate with other string manipulations in Helpers.cxx -// Extract size from an array type, if available. - std::string cleanName = name; - RemoveConst(cleanName); - - if (cleanName[cleanName.size()-1] == ']') { - std::string::size_type idx = cleanName.rfind('['); - if (idx != std::string::npos) { - const std::string asize = cleanName.substr(idx+1, cleanName.size()-2); - return strtoul(asize.c_str(), nullptr, 0); - } - } - - return -1; -} - //---------------------------------------------------------------------------- std::string CPyCppyy::Utility::ClassName(PyObject* pyobj) { @@ -907,6 +1010,29 @@ std::string CPyCppyy::Utility::ClassName(PyObject* pyobj) return clname; } +//---------------------------------------------------------------------------- +static std::set sIteratorTypes; +bool CPyCppyy::Utility::IsSTLIterator(const std::string& classname) +{ +// attempt to recognize STL iterators (TODO: probably belongs in the backend) + if (sIteratorTypes.empty()) { + std::string tt = "::"; + for (auto c : {"std::vector", "std::list", "std::deque"}) { + for (auto i : {"iterator", "const_iterator"}) { + const std::string& itname = Cppyy::ResolveName(c+tt+i); + auto pos = itname.find('<'); + if (pos != std::string::npos) + sIteratorTypes.insert(itname.substr(0, pos)); + } + } + } + + auto pos = classname.find('<'); + if (pos != std::string::npos) + return sIteratorTypes.find(classname.substr(0, pos)) != sIteratorTypes.end(); + return false; +} + //---------------------------------------------------------------------------- CPyCppyy::Utility::PyOperators::~PyOperators() @@ -942,11 +1068,11 @@ PyObject* CPyCppyy::Utility::PyErr_Occurred_WithGIL() //---------------------------------------------------------------------------- -size_t CPyCppyy::Utility::FetchError(std::vector& errors) +size_t CPyCppyy::Utility::FetchError(std::vector& errors, bool is_cpp) { // Fetch the current python error, if any, and store it for future use. if (PyErr_Occurred()) { - PyError_t e; + PyError_t e{is_cpp}; PyErr_Fetch(&e.fType, &e.fValue, &e.fTrace); errors.push_back(e); } @@ -964,34 +1090,63 @@ void CPyCppyy::Utility::SetDetailedException(std::vector& errors, PyO return; } -// add the details to the topmsg - PyObject* separator = CPyCppyy_PyText_FromString("\n "); - +// if a _single_ exception was from C++, assume it has priority PyObject* exc_type = nullptr; + PyError_t* unique_from_cpp = nullptr; for (auto& e : errors) { - if (!exc_type) exc_type = e.fType; - else if (exc_type != e.fType) exc_type = defexc; - CPyCppyy_PyText_Append(&topmsg, separator); - if (CPyCppyy_PyText_Check(e.fValue)) { - CPyCppyy_PyText_Append(&topmsg, e.fValue); - } else if (e.fValue) { - PyObject* excstr = PyObject_Str(e.fValue); - if (!excstr) { - PyErr_Clear(); - excstr = PyObject_Str((PyObject*)Py_TYPE(e.fValue)); + if (e.fIsCpp) { + if (!unique_from_cpp) + unique_from_cpp = &e; + else { + // two C++ exceptions, resort to default + unique_from_cpp = nullptr; + exc_type = defexc; + break; } - CPyCppyy_PyText_AppendAndDel(&topmsg, excstr); - } else { - CPyCppyy_PyText_AppendAndDel(&topmsg, - CPyCppyy_PyText_FromString("unknown exception")); + } else if (!unique_from_cpp) { + // try to consolidate Python exceptions, otherwise select default + if (!exc_type) exc_type = e.fType; + else if (exc_type != e.fType) exc_type = defexc; } } - Py_DECREF(separator); - std::for_each(errors.begin(), errors.end(), PyError_t::Clear); + if (unique_from_cpp) { + // report only this error; the idea here is that all other errors come from + // the bindings (e.g. argument conversion errors), while the exception from + // C++ means that it originated from an otherwise successful call -// set the python exception - PyErr_SetString(exc_type, CPyCppyy_PyText_AsString(topmsg)); + // bind the original C++ object, rather than constructing from topmsg, as it + // is expected to have informative state + Py_INCREF(unique_from_cpp->fType); Py_INCREF(unique_from_cpp->fValue); Py_XINCREF(unique_from_cpp->fTrace); + PyErr_Restore(unique_from_cpp->fType, unique_from_cpp->fValue, unique_from_cpp->fTrace); + } else { + // add the details to the topmsg + PyObject* separator = CPyCppyy_PyText_FromString("\n "); + for (auto& e : errors) { + CPyCppyy_PyText_Append(&topmsg, separator); + if (CPyCppyy_PyText_Check(e.fValue)) { + CPyCppyy_PyText_Append(&topmsg, e.fValue); + } else if (e.fValue) { + PyObject* excstr = PyObject_Str(e.fValue); + if (!excstr) { + PyErr_Clear(); + excstr = PyObject_Str((PyObject*)Py_TYPE(e.fValue)); + } + CPyCppyy_PyText_AppendAndDel(&topmsg, excstr); + } else { + CPyCppyy_PyText_AppendAndDel(&topmsg, + CPyCppyy_PyText_FromString("unknown exception")); + } + } + + Py_DECREF(separator); + + // set the python exception + PyErr_SetString(exc_type, CPyCppyy_PyText_AsString(topmsg)); + } + +// cleanup stored errors and done with topmsg (whether used or not) + std::for_each(errors.begin(), errors.end(), PyError_t::Clear); Py_DECREF(topmsg); } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Utility.h b/bindings/pyroot/cppyy/CPyCppyy/src/Utility.h index 7c70d9bae1bd3..d92d07fc9e67d 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Utility.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Utility.h @@ -2,6 +2,7 @@ #define CPYCPPYY_UTILITY_H // Standard +#include #include #include @@ -10,12 +11,14 @@ namespace CPyCppyy { class PyCallable; +#if PY_VERSION_HEX < 0x030b0000 extern dict_lookup_func gDictLookupOrg; extern bool gDictLookupActive; +#endif // additional converter functions unsigned long PyLongOrInt_AsULong(PyObject* pyobject); -ULong64_t PyLongOrInt_AsULong64(PyObject* pyobject); +PY_ULONG_LONG PyLongOrInt_AsULong64(PyObject* pyobject); namespace Utility { @@ -54,15 +57,18 @@ bool InitProxy(PyObject* module, PyTypeObject* pytype, const char* name); Py_ssize_t GetBuffer(PyObject* pyobject, char tc, int size, void*& buf, bool check = true); // data/operator mappings -std::string MapOperatorName(const std::string& name, bool bTakesParames); +std::string MapOperatorName(const std::string& name, bool bTakesParames, bool* stubbed = nullptr); struct PyOperators { - PyOperators() : fEq(nullptr), fNe(nullptr), fLAdd(nullptr), fRAdd(nullptr), - fSub(nullptr), fLMul(nullptr), fRMul(nullptr), fDiv(nullptr), fHash(nullptr) {} + PyOperators() : fEq(nullptr), fNe(nullptr), fLt(nullptr), fLe(nullptr), fGt(nullptr), fGe(nullptr), + fLAdd(nullptr), fRAdd(nullptr), fSub(nullptr), fLMul(nullptr), fRMul(nullptr), fDiv(nullptr), + fHash(nullptr) {} ~PyOperators(); PyObject* fEq; PyObject* fNe; + PyObject *fLt, *fLe; + PyObject *fGt, *fGe; PyObject *fLAdd, *fRAdd; PyObject* fSub; PyObject *fLMul, *fRMul; @@ -71,16 +77,15 @@ struct PyOperators { }; // meta information -const std::string Compound(const std::string& name); -Py_ssize_t ArraySize(const std::string& name); std::string ClassName(PyObject* pyobj); +bool IsSTLIterator(const std::string& classname); // for threading: save call to PyErr_Occurred() PyObject* PyErr_Occurred_WithGIL(); // helpers for collecting/maintaining python exception data struct PyError_t { - PyError_t() { fType = fValue = fTrace = 0; } + PyError_t(bool is_cpp = false) : fIsCpp(is_cpp) { fType = fValue = fTrace = 0; } static void Clear(PyError_t& e) { @@ -90,9 +95,10 @@ struct PyError_t { } PyObject *fType, *fValue, *fTrace; + bool fIsCpp; }; -size_t FetchError(std::vector&); +size_t FetchError(std::vector&, bool is_cpp = false); void SetDetailedException( std::vector& errors /* clears */, PyObject* topmsg /* steals ref */, PyObject* defexc); diff --git a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx index b9e57e8433dc2..4e7442e43dd8a 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx +++ b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx @@ -53,6 +53,12 @@ typedef CPyCppyy::Parameter Parameter; // small number that allows use of stack for argument passing const int SMALL_ARGS_N = 8; +// convention to pass flag for direct calls (similar to Python's vector calls) +#define DIRECT_CALL ((size_t)1 << (8 * sizeof(size_t) - 1)) +static inline size_t CALL_NARGS(size_t nargs) { + return nargs & ~DIRECT_CALL; +} + // data for life time management --------------------------------------------- typedef std::vector ClassRefs_t; static ClassRefs_t g_classrefs(1); @@ -378,11 +384,18 @@ bool is_missclassified_stl(const std::string& name) // direct interpreter access ------------------------------------------------- -bool Cppyy::Compile(const std::string& code) +bool Cppyy::Compile(const std::string& code, bool /*silent*/) { return gInterpreter->Declare(code.c_str()); } +std::string Cppyy::ToString(TCppType_t klass, TCppObject_t obj) +{ + if (klass && obj && !IsNamespace((TCppScope_t)klass)) + return gInterpreter->ToString(GetScopedFinalName(klass).c_str(), (void*)obj); + return ""; +} + // name to opaque C++ scope representation ----------------------------------- std::string Cppyy::ResolveName(const std::string& cppitem_name) @@ -677,10 +690,12 @@ void Cppyy::Deallocate(TCppType_t /* type */, TCppObject_t instance) ::operator delete(instance); } -Cppyy::TCppObject_t Cppyy::Construct(TCppType_t type) +Cppyy::TCppObject_t Cppyy::Construct(TCppType_t type, void* arena) { TClassRef& cr = type_from_handle(type); - return (TCppObject_t)cr->New(); + if (arena) + return (TCppObject_t)cr->New(arena, TClass::kRealNew); + return (TCppObject_t)cr->New(TClass::kRealNew); } static std::map sHasOperatorDelete; @@ -771,6 +786,8 @@ void release_args(Parameter* args, size_t nargs) { static inline bool WrapperCall(Cppyy::TCppMethod_t method, size_t nargs, void* args_, void* self, void* result) { Parameter* args = (Parameter*)args_; + //bool is_direct = nargs & DIRECT_CALL; + nargs = CALL_NARGS(nargs); CallWrapper* wrap = (CallWrapper*)method; const TInterpreter::CallFuncIFacePtr_t& faceptr = wrap->fFaceptr.fGeneric ? wrap->fFaceptr : GetCallFunc(method); @@ -1002,6 +1019,15 @@ bool Cppyy::IsAggregate(TCppType_t klass) return false; } +bool Cppyy::IsDefaultConstructable(TCppType_t type) +{ +// Test if this type has a default constructor or is a "plain old data" type + TClassRef& cr = type_from_handle(type); + if (cr.GetClass()) + return cr->HasDefaultConstructor() || (cr->ClassProperty() & kClassIsAggregate); + return true; +} + // helpers for stripping scope names static std::string outer_with_template(const std::string& name) @@ -1404,6 +1430,13 @@ void Cppyy::AddSmartPtrType(const std::string& type_name) gSmartPtrTypes.insert(ResolveName(type_name)); } +void Cppyy::AddTypeReducer(const std::string& /*reducable*/, const std::string& /*reduced*/) +{ + // This function is deliberately left empty, because it is not used in + // PyROOT, and synchronizing it with cppyy-backend upstream would require + // patches to ROOT meta. +} + // type offsets -------------------------------------------------------------- ptrdiff_t Cppyy::GetBaseOffset(TCppType_t derived, TCppType_t base, @@ -1447,11 +1480,14 @@ ptrdiff_t Cppyy::GetBaseOffset(TCppType_t derived, TCppType_t base, // method/function reflection information ------------------------------------ -Cppyy::TCppIndex_t Cppyy::GetNumMethods(TCppScope_t scope) +Cppyy::TCppIndex_t Cppyy::GetNumMethods(TCppScope_t scope, bool accept_namespace) { - if (IsNamespace(scope)) + if (!accept_namespace && IsNamespace(scope)) return (TCppIndex_t)0; // enforce lazy + if (scope == GLOBAL_HANDLE) + return gROOT->GetListOfGlobalFunctions(true)->GetSize(); + TClassRef& cr = type_from_handle(scope); if (cr.GetClass() && cr->GetListOfMethods(true)) { Cppyy::TCppIndex_t nMethods = (TCppIndex_t)cr->GetListOfMethods(false)->GetSize(); @@ -1460,14 +1496,9 @@ Cppyy::TCppIndex_t Cppyy::GetNumMethods(TCppScope_t scope) if (clName.find('<') != std::string::npos) { // chicken-and-egg problem: TClass does not know about methods until // instantiation, so force it - if (clName.find("std::", 0, 5) == std::string::npos && \ - is_missclassified_stl(clName)) { - // TODO: this is too simplistic for template arguments missing std:: - clName = "std::" + clName; - } std::ostringstream stmt; stmt << "template class " << clName << ";"; - gInterpreter->Declare(stmt.str().c_str()); + gInterpreter->Declare(stmt.str().c_str()/*, silent = true*/); // now reload the methods return (TCppIndex_t)cr->GetListOfMethods(true)->GetSize(); @@ -1716,9 +1747,12 @@ bool Cppyy::IsConstMethod(TCppMethod_t method) return false; } -Cppyy::TCppIndex_t Cppyy::GetNumTemplatedMethods(TCppScope_t scope) +Cppyy::TCppIndex_t Cppyy::GetNumTemplatedMethods(TCppScope_t scope, bool accept_namespace) { - if (scope == (TCppScope_t)GLOBAL_HANDLE) { + if (!accept_namespace && IsNamespace(scope)) + return (TCppIndex_t)0; // enforce lazy + + if (scope == GLOBAL_HANDLE) { TCollection* coll = gROOT->GetListOfFunctionTemplates(); if (coll) return (TCppIndex_t)coll->GetSize(); } else { @@ -1969,20 +2003,17 @@ bool Cppyy::IsStaticMethod(TCppMethod_t method) } // data member reflection information ---------------------------------------- -Cppyy::TCppIndex_t Cppyy::GetNumDatamembers(TCppScope_t scope) +Cppyy::TCppIndex_t Cppyy::GetNumDatamembers(TCppScope_t scope, bool accept_namespace) { - if (IsNamespace(scope)) + if (!accept_namespace && IsNamespace(scope)) return (TCppIndex_t)0; // enforce lazy + if (scope == GLOBAL_HANDLE) + return gROOT->GetListOfGlobals(true)->GetSize(); + TClassRef& cr = type_from_handle(scope); - if (cr.GetClass()) { - Cppyy::TCppIndex_t sum = 0; - if (cr->GetListOfDataMembers()) - sum = cr->GetListOfDataMembers()->GetSize(); - if (cr->GetListOfUsingDataMembers()) - sum += cr->GetListOfUsingDataMembers()->GetSize(); - return sum; - } + if (cr.GetClass() && cr->GetListOfDataMembers()) + return cr->GetListOfDataMembers()->GetSize(); return (TCppIndex_t)0; // unknown class? } @@ -2016,11 +2047,10 @@ std::string Cppyy::GetDatamemberType(TCppScope_t scope, TCppIndex_t idata) TGlobal* gbl = g_globalvars[idata]; std::string fullType = gbl->GetFullTypeName(); - if ((int)gbl->GetArrayDim() > 1) - fullType.append("*"); - else if ((int)gbl->GetArrayDim() == 1) { + if ((int)gbl->GetArrayDim()) { std::ostringstream s; - s << '[' << gbl->GetMaxIndex(0) << ']' << std::ends; + for (int i = 0; i < (int)gbl->GetArrayDim(); ++i) + s << '[' << gbl->GetMaxIndex(i) << ']'; fullType.append(s.str()); } return fullType; @@ -2040,11 +2070,10 @@ std::string Cppyy::GetDatamemberType(TCppScope_t scope, TCppIndex_t idata) fullType = trueName; } - if ((int)m->GetArrayDim() > 1 || (!m->IsBasic() && m->IsaPointer())) - fullType.append("*"); - else if ((int)m->GetArrayDim() == 1) { + if ((int)m->GetArrayDim()) { std::ostringstream s; - s << '[' << m->GetMaxIndex(0) << ']' << std::ends; + for (int i = 0; i < (int)m->GetArrayDim(); ++i) + s << '[' << m->GetMaxIndex(i) << ']'; fullType.append(s.str()); } return fullType; diff --git a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h index d835aad39c795..dd5e2d886e146 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h +++ b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h @@ -26,7 +26,9 @@ namespace Cppyy { // direct interpreter access ------------------------------------------------- RPY_EXPORTED - bool Compile(const std::string& code); + bool Compile(const std::string& code, bool silent = false); + RPY_EXPORTED + std::string ToString(TCppType_t klass, TCppObject_t obj); // name to opaque C++ scope representation ----------------------------------- RPY_EXPORTED @@ -56,7 +58,7 @@ namespace Cppyy { RPY_EXPORTED void Deallocate(TCppType_t type, TCppObject_t instance); RPY_EXPORTED - TCppObject_t Construct(TCppType_t type); + TCppObject_t Construct(TCppType_t type, void* arena = nullptr); RPY_EXPORTED void Destruct(TCppType_t type, TCppObject_t instance); @@ -116,6 +118,8 @@ namespace Cppyy { bool IsEnum(const std::string& type_name); RPY_EXPORTED bool IsAggregate(TCppType_t type); + RPY_EXPORTED + bool IsDefaultConstructable(TCppType_t type); RPY_EXPORTED void GetAllCppNames(TCppScope_t scope, std::set& cppnames); @@ -148,6 +152,9 @@ namespace Cppyy { RPY_EXPORTED void AddSmartPtrType(const std::string&); + RPY_EXPORTED + void AddTypeReducer(const std::string& reducable, const std::string& reduced); + // calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 RPY_EXPORTED ptrdiff_t GetBaseOffset( @@ -155,7 +162,7 @@ namespace Cppyy { // method/function reflection information ------------------------------------ RPY_EXPORTED - TCppIndex_t GetNumMethods(TCppScope_t scope); + TCppIndex_t GetNumMethods(TCppScope_t scope, bool accept_namespace = false); RPY_EXPORTED std::vector GetMethodIndicesFromName(TCppScope_t scope, const std::string& name); @@ -190,7 +197,7 @@ namespace Cppyy { bool IsConstMethod(TCppMethod_t); RPY_EXPORTED - TCppIndex_t GetNumTemplatedMethods(TCppScope_t scope); + TCppIndex_t GetNumTemplatedMethods(TCppScope_t scope, bool accept_namespace = false); RPY_EXPORTED std::string GetTemplatedMethodName(TCppScope_t scope, TCppIndex_t imeth); RPY_EXPORTED @@ -221,7 +228,7 @@ namespace Cppyy { // data member reflection information ---------------------------------------- RPY_EXPORTED - TCppIndex_t GetNumDatamembers(TCppScope_t scope); + TCppIndex_t GetNumDatamembers(TCppScope_t scope, bool accept_namespace = false); RPY_EXPORTED std::string GetDatamemberName(TCppScope_t scope, TCppIndex_t idata); RPY_EXPORTED diff --git a/bindings/pyroot/cppyy/cppyy/.gitignore b/bindings/pyroot/cppyy/cppyy/.gitignore index a85020ea8e88b..7cda4c971bdbe 100644 --- a/bindings/pyroot/cppyy/cppyy/.gitignore +++ b/bindings/pyroot/cppyy/cppyy/.gitignore @@ -4,6 +4,10 @@ # dictionary products *.so +*.dll +*.lib +*.def +*.exp *.pcm *.rootmap *_rflx.cpp @@ -36,6 +40,7 @@ doc/tutorial/gsl_selection.xml # saved log files of benches bench/logs +bench/*.log bench/swig_*.py bench/*_wrap.cxx bench/*_main diff --git a/bindings/pyroot/cppyy/cppyy/.readthedocs.yaml b/bindings/pyroot/cppyy/cppyy/.readthedocs.yaml new file mode 100644 index 0000000000000..49dafa0fa724d --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: doc/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: doc/requirements.txt diff --git a/bindings/pyroot/cppyy/cppyy/LICENSE.txt b/bindings/pyroot/cppyy/cppyy/LICENSE.txt index 720e274e48cc2..60d53483fb73f 100644 --- a/bindings/pyroot/cppyy/cppyy/LICENSE.txt +++ b/bindings/pyroot/cppyy/cppyy/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2002-2019, The Regents of the University of California, +Copyright (c) 2002-2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or @@ -47,8 +47,11 @@ one or more of the following people and organizations, and licensed under the same conditions (except for some compatible licenses as retained in the source code): + Lucio Asnaghi + Simone Bacchio Aditi Dutta Shaheed Haque + Aaron Jomy Toby StClere-Smithe Stefan Wunsch diff --git a/bindings/pyroot/cppyy/cppyy/MANIFEST.in b/bindings/pyroot/cppyy/cppyy/MANIFEST.in index face07acf0693..ae3ddffac029c 100644 --- a/bindings/pyroot/cppyy/cppyy/MANIFEST.in +++ b/bindings/pyroot/cppyy/cppyy/MANIFEST.in @@ -1,6 +1,9 @@ # Include the license file include LICENSE.txt +# Add custom installer +recursive-include installer * + # Do not add the test or doc directories prune test prune doc diff --git a/bindings/pyroot/cppyy/cppyy/README.rst b/bindings/pyroot/cppyy/cppyy/README.rst index 4e8cc2f342d99..c0d1018742f7d 100644 --- a/bindings/pyroot/cppyy/cppyy/README.rst +++ b/bindings/pyroot/cppyy/cppyy/README.rst @@ -6,7 +6,7 @@ cppyy: Python-C++ bindings interface based on Cling/LLVM cppyy provides fully automatic, dynamic Python-C++ bindings by leveraging the Cling C++ interpreter and LLVM. It supports both PyPy (natively), CPython, and C++ language standards -through C++17. +through C++17 (and parts of C++20). Details and performance are described in `this paper `_, @@ -15,7 +15,7 @@ numbers. Full documentation: `cppyy.readthedocs.io `_. -Notebook-based tutorial: `Cppyy Tutorial `_. +Notebook-based tutorial: `Cppyy Tutorial `_. For Anaconda/miniconda, install cppyy from `conda-forge `_. @@ -25,4 +25,4 @@ Change log: https://cppyy.readthedocs.io/en/latest/changelog.html Bug reports/feedback: - https://bitbucket.org/wlav/cppyy/issues?status=new&status=open + https://github.com/wlav/cppyy/issues diff --git a/bindings/pyroot/cppyy/cppyy/bench/Makefile b/bindings/pyroot/cppyy/cppyy/bench/Makefile index dd9906af91247..57d2f05bf3190 100644 --- a/bindings/pyroot/cppyy/cppyy/bench/Makefile +++ b/bindings/pyroot/cppyy/cppyy/bench/Makefile @@ -13,18 +13,15 @@ ifeq ($(CHECK_SWIG),0) endif all : $(dicts) $(libs) $(execs) $(modules) -ifeq ($(ROOTSYS),) - clingconfig := cling-config -else - clingconfig := root-config -endif +clingconfig := cling-config cppflags:=-O3 -fPIC cppyy_cppflags=$(shell $(clingconfig) --cflags) $(cppflags) PLATFORM := $(shell uname -s) ifeq ($(PLATFORM),Darwin) - cppflags+=-dynamiclib -single_module -arch x86_64 -undefined dynamic_lookup + MACHINE := $(shell uname -m) + cppflags+=-dynamiclib -arch $(MACHINE) -undefined dynamic_lookup endif py11_%.so: py11_%.cxx lib%.so @@ -34,7 +31,7 @@ swig_%.py: %.h %.i swig -python -c++ -builtin $*.i _swig_%.so: swig_%.py lib%.so - $(CXX) $(cppflags) -shared `python-config --includes` -std=c++11 -fPIC $*_wrap.cxx -o $@ -L. -l$* + $(CXX) $(cppflags) -shared -I$(shell python -c 'import distutils.sysconfig as ds; print(ds.get_config_var("INCLUDEPY"))') -std=c++11 -fPIC $*_wrap.cxx -o $@ -L. -l$* %Dict.so: %_rflx.cpp lib%.so $(CXX) $(cppyy_cppflags) -shared -o $@ $*_rflx.cpp -L. -l$* @@ -47,7 +44,7 @@ lib%.so: %.cxx # note -O2 for cxx to make sure code actually runs instead of being optimized out of existence %_main: %_main.cxx lib%.so - $(CXX) -O2 -fPIC -o $@ $*_main.cxx -L. -l$* + $(CXX) -std=c++11 -O2 -fPIC -o $@ $*_main.cxx -L. -l$* .PHONY: bench clean @@ -55,4 +52,4 @@ bench: all pytest -s bench_runvector.py --benchmark-sort=mean clean: - -rm -f $(dicts) $(libs) $(execs) $(modules) $(wildcard *.rootmap) $(wildcard *_rdict.pcm) + -rm -f $(dicts) $(libs) $(execs) $(modules) $(wildcard *.rootmap) $(wildcard *_rdict.pcm) $(wildcard *_wrap.cxx) diff --git a/bindings/pyroot/cppyy/cppyy/bench/basic_micro.py b/bindings/pyroot/cppyy/cppyy/bench/basic_micro.py new file mode 100644 index 0000000000000..712cac138c0cf --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/bench/basic_micro.py @@ -0,0 +1,38 @@ +import cppyy, gc, math, os, psutil, time + +NLARGE = 20000000 + +process = psutil.Process(os.getpid()) + +def benchit(what, callf, N): + print("running:", what) + mpre = process.memory_info().rss/1024 + tpre = time.perf_counter() + for i in range(N): + callf() + tpost = time.perf_counter() + gc.collect() + mpost = process.memory_info().rss/1024 + if tpost - tpre < 1.: + print(" suggest increasing N by %dx" % math.ceil(1./(tpost-tpre))) + print(" time:", (tpost - tpre)/N) + if mpost == mpre: + print(" memcheck passed") + else: + print(" memcheck FAILED:", mpre, mpost) + +cppyy.cppdef(""" + void gfunc() {} + + class MyClass { + public: + void mfunc() {} + }; +""") + + +f = cppyy.gbl.gfunc +benchit("global function", f, NLARGE) + +inst = cppyy.gbl.MyClass() +benchit("member function", inst.mfunc, NLARGE) diff --git a/bindings/pyroot/cppyy/cppyy/bench/bench_functioncalls.py b/bindings/pyroot/cppyy/cppyy/bench/bench_functioncalls.py index 0777a8965bb22..23c51d96cd61b 100644 --- a/bindings/pyroot/cppyy/cppyy/bench/bench_functioncalls.py +++ b/bindings/pyroot/cppyy/cppyy/bench/bench_functioncalls.py @@ -1,14 +1,13 @@ import py, pytest, os, sys, math, warnings from support import setup_make - setup_make("functioncallsDict.so") -currpath = py.path.local(__file__).dirpath() -test_dct = str(currpath.join("functioncallsDict.so")) import cppyy -cppyy.load_reflection_info(test_dct) + +cppyy.load_library("functioncalls") +cppyy.load_library("functioncallsDict") import py_functioncalls diff --git a/bindings/pyroot/cppyy/cppyy/bench/bench_runvector.py b/bindings/pyroot/cppyy/cppyy/bench/bench_runvector.py index 8a09cd78eef52..f6444922fe42f 100644 --- a/bindings/pyroot/cppyy/cppyy/bench/bench_runvector.py +++ b/bindings/pyroot/cppyy/cppyy/bench/bench_runvector.py @@ -1,14 +1,13 @@ import py, pytest, os, sys, math, warnings from support import setup_make - setup_make("runvectorDict.so") -currpath = py.path.local(__file__).dirpath() -test_dct = str(currpath.join("runvectorDict.so")) import cppyy -cppyy.load_reflection_info(test_dct) + +cppyy.load_library("runvector") +cppyy.load_library("runvectorDict") all_configs = [('cppyy', 'cppyy.gbl')] diff --git a/bindings/pyroot/cppyy/cppyy/doc/requirements.txt b/bindings/pyroot/cppyy/cppyy/doc/requirements.txt new file mode 100644 index 0000000000000..1678b6d6f330c --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/doc/requirements.txt @@ -0,0 +1,4 @@ +# Defining the exact version will make sure things don't break +sphinx==6.2.1 +sphinx_rtd_theme==1.2.2 +readthedocs-sphinx-search==0.3.2 diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/basic_types.rst b/bindings/pyroot/cppyy/cppyy/doc/source/basic_types.rst index ce12da3ee2396..06ce060ed91ab 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/basic_types.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/basic_types.rst @@ -13,8 +13,8 @@ These are mapped to their Standard Template Library equivalents instead. The C++ code used for the examples below can be found :doc:`here `, and it is assumed that that code is -loaded at the start of any session. -Download it, save it under the name ``features.h``, and load it: +loaded before running any of the example code snippets. +Download it, save it under the name ``features.h``, and simply include it: .. code-block:: python @@ -48,8 +48,13 @@ For example, a C++ ``unsigned int`` becomes a Python2 ``long`` or Python3 ValueError: cannot convert negative integer to unsigned >>> +On some platforms, 8-bit integer types such as ``int8_t`` and ``uint8_t`` are +represented as `char` types. +For consistency, these are mapped onto Python `int`. + Some types are builtin in Python, but (STL) classes in C++. -Examples are ``str`` vs. ``std::string`` and ``complex`` vs. ``std::complex``. +Examples are ``str`` vs. ``std::string`` (see also the +:doc:`Strings ` section) and ``complex`` vs. ``std::complex``. These classes have been pythonized to behave the same wherever possible. For example, string comparison work directly, and ``std::complex`` has ``real`` and ``imag`` properties: diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/bindings_generation.rst b/bindings/pyroot/cppyy/cppyy/doc/source/bindings_generation.rst deleted file mode 100644 index d955f07f5fc22..0000000000000 --- a/bindings/pyroot/cppyy/cppyy/doc/source/bindings_generation.rst +++ /dev/null @@ -1,306 +0,0 @@ -.. _bindings_generation: - -=================== -Bindings Generation -=================== - -Binding developers have two levels of access to cling via ``cppyy``: - -* A simple command line interface. -* Automated generation of an end-user bindings package from a CMake-based - project build. - -Both are installed as part of the ``cppyy-backend`` component since they are -not needed by users of the bindings. - -Command Line Interface -====================== - -rootcling ---------- - -This provides basic access to ``cling``:: - - $ rootcling - Usage: rootcling [-v][-v0-4] [-f] [out.cxx] [opts] file1.h[+][-][!] file2.h[+][-][!] ...[LinkDef.h] - For more extensive help type: /usr/local/lib/python2.7/dist-packages/cppyy_backend/bin/rootcling -h - -The basic mode of operation is to process the header files ('fileN.h') -according to certain `#pragmas in the LinkDef.h `_ -file in order to generate bindings accessible in Python under the 'cppyy.gbl' -namespace. - -The output is - -* A .cpp file (which, when compiled to a shared library) -* A .rootmap file -* A .pcm file - -which are used at runtime by ``cling`` to expose the semantics expressed by the -header files to Python. Nominally, the compiled .cpp provides low-level Python -access to the library API defined by the header files, while ``cling`` uses the -other files to provide the rich features it supports. Thus, the shipping form -of the bindings contains: - -* A shared library (which must be compiled from the .cpp) -* A .rootmap file -* A .pcm file - -cling-config ------------- - -This is a small utility whose main purpose is to provide access to the -as-installed configuration of other components. For-example:: - - $ cling-config --help - Usage: cling-config [--cflags] [--cppflags] [--cmake] - $ cling-config --cmake - /usr/local/lib/python2.7/dist-packages/cppyy_backend/cmake - -cppyy-generator ---------------- - -This is a clang-based utility program which takes a set of C++ header files -and generate a JSON output file describing the objects found in them. This -output is intended to support more convenient access to a set of -cppyy-supported bindings:: - - $ cppyy-generator --help - usage: cppyy-generator [-h] [-v] [--flags FLAGS] [--libclang LIBCLANG] - output sources [sources ...] - ... - -See the ``Cmake interface`` for details. - -CMake interface -=============== - -The bindings generated by rootcling, are 'raw' in the sense that: - -* The .cpp file be compiled. The required compilation steps are - platform-dependent. -* The bindings are not packaged for distribution. Typically, users expect - to have a pip-compatible package. -* The binding are in the 'cppyy.gbl' namespace. This is an inconvenience at - best for users who might expect C++ code from KF5::Config to appear in - Python via "import KF5.Config". -* The bindings are loaded lazily, which limits the discoverability of the - content of the bindings. -* ``cppyy`` supports customization of the bindings via 'Pythonization' but - there is no automated way to load them. - -These issues are addressed by the CMake support. This is a blend of Python -packaging and CMake where CMake provides: - -* Platform-independent scripting of the creation of a Python 'wheel' package - for the bindings. -* An facility for CMake-based projects to automate the entire bindings - generation process, including basic automated tests. - -Python packaging of bindings ----------------------------- - -Modern Python packaging usage is based on the 'wheel'. This is places the onus -on the creation of binary artifacts in the package on the distributor. In this -case, this includes the platform-dependent steps necessary to compile the .cpp -file. - -The generated package also takes advantage of the __init__.py load-time -mechanism to enhance the bindings: - -* The bindings are rehosted in a "native" namespace so that C++ code from - KF5::Config appears in Python via "import KF5.Config". -* (TBD) Load Pythonizations. - -Both of these need/can use the output of the cppyy-generator (included in the -package) as well as other runtime support included in ``cppyy``. - -CMake usage ------------ - -The CMake usage is via two modules: - -* FindLibClang.cmake provides some bootstrap support needed to locate clang. - This is provided mostly as a temporary measure; hopefully upstream support - will allow this to be eliminated in due course. -* FindCppyy.cmake provides the interface described further here. - -Details of the usage of these modules is within the modules themselves, but -here is a summary of the usage. ``FindLibClang.cmake`` sets the following -variables: - -:: - - LibClang_FOUND - True if libclang is found. - LibClang_LIBRARY - Clang library to link against. - LibClang_VERSION - Version number as a string (e.g. "3.9"). - LibClang_PYTHON_EXECUTABLE - Compatible python version. - - -``FindCppyy.cmake`` sets the following variables: - -:: - - Cppyy_FOUND - set to true if Cppyy is found - Cppyy_DIR - the directory where Cppyy is installed - Cppyy_EXECUTABLE - the path to the Cppyy executable - Cppyy_INCLUDE_DIRS - Where to find the ROOT header files. - Cppyy_VERSION - the version number of the Cppyy backend. - -and also defines the following functions:: - - cppyy_add_bindings - Generate a set of bindings from a set of header files. - cppyy_find_pips - Return a list of available pip programs. - -cppyy_add_bindings -^^^^^^^^^^^^^^^^^^ - -Generate a set of bindings from a set of header files. Somewhat like CMake's -add_library(), the output is a compiler target. In addition ancillary files -are also generated to allow a complete set of bindings to be compiled, -packaged and installed:: - - cppyy_add_bindings( - pkg - pkg_version - author - author_email - [URL url] - [LICENSE license] - [LANGUAGE_STANDARD std] - [LINKDEFS linkdef...] - [IMPORTS pcm...] - [GENERATE_OPTIONS option...] - [COMPILE_OPTIONS option...] - [INCLUDE_DIRS dir...] - [LINK_LIBRARIES library...] - [H_DIRS H_DIRSectory] - H_FILES h_file...) - -The bindings are based on https://cppyy.readthedocs.io/en/latest/, and can be -used as per the documentation provided via the cppyy.gbl namespace. First add -the directory of the .rootmap file to the LD_LIBRARY_PATH environment -variable, then "import cppyy; from cppyy.gbl import ". - -Alternatively, use "import ". This convenience wrapper supports -"discovery" of the available C++ entities using, for example Python 3's command -line completion support. - -The bindings are complete with a setup.py, supporting Wheel-based -packaging, and a test.py supporting pytest/nosetest sanity test of the bindings. - -The bindings are generated/built/packaged using 3 environments: - -- One compatible with the header files being bound. This is used to - generate the generic C++ binding code (and some ancillary files) using - a modified C++ compiler. The needed options must be compatible with the - normal build environment of the header files. -- One to compile the generated, generic C++ binding code using a standard - C++ compiler. The resulting library code is "universal" in that it is - compatible with both Python2 and Python3. -- One to package the library and ancillary files into standard Python2/3 - wheel format. The packaging is done using native Python tooling. - -+----------------------+---------------------------------------------------------------------------------------------+ -|Arguments and options | Description | -+======================+=============================================================================================+ -|pkg | The name of the package to generate. This can be either | -| | of the form "simplename" (e.g. "Akonadi"), or of the | -| | form "namespace.simplename" (e.g. "KF5.Akonadi"). | -+----------------------+---------------------------------------------------------------------------------------------+ -|pkg_version | The version of the package. | -+----------------------+---------------------------------------------------------------------------------------------+ -|author | The name of the library author. | -+----------------------+---------------------------------------------------------------------------------------------+ -|author_email | The email address of the library author. | -+----------------------+---------------------------------------------------------------------------------------------+ -|URL url | The home page for the library. Default is | -| | "https://pypi.python.org/pypi/". | -+----------------------+---------------------------------------------------------------------------------------------+ -|LICENSE license | The license, default is "LGPL 2.0". | -+----------------------+---------------------------------------------------------------------------------------------+ -|LANGUAGE_STANDARD std | The version of C++ in use, "14" by default. | -+----------------------+---------------------------------------------------------------------------------------------+ -|IMPORTS pcm | Files which contain previously-generated bindings | -| | which pkg depends on. | -+----------------------+---------------------------------------------------------------------------------------------+ -|GENERATE_OPTIONS optio| Options which are to be passed into the rootcling | -| | command. For example, bindings which depend on Qt | -| | may need "-D__PIC__;-Wno-macro-redefined" as per | -| | https://sft.its.cern.ch/jira/browse/ROOT-8719. | -+----------------------+---------------------------------------------------------------------------------------------+ -|LINKDEFS def | Files or lines which contain extra #pragma content | -| | for the linkdef.h file used by rootcling. See | -| | https://root.cern.ch/root/html/guides/users-guide/AddingaClass.html#the-linkdef.h-file. | -| | | -| | In lines, literal semi-colons must be escaped: "\;". | -+----------------------+---------------------------------------------------------------------------------------------+ -|EXTRA_CODES code | Files which contain extra code needed by the bindings. | -| | Customization is by routines named "c13n_"; | -| | each such routine is passed the module for : | -| | | -| | :: code-block python | -| | | -| | def c13n_doit(pkg_module): | -| | print(pkg_module.__dict__) | -| | | -| | The files and individual routines within files are | -| | processed in alphabetical order. | -+----------------------+---------------------------------------------------------------------------------------------+ -|EXTRA_HEADERS hdr | Files which contain extra headers needed by the bindings. | -+----------------------+---------------------------------------------------------------------------------------------+ -|EXTRA_PYTHONS py | Files which contain extra Python code needed by the bindings. | -+----------------------+---------------------------------------------------------------------------------------------+ -|COMPILE_OPTIONS option| Options which are to be passed into the compile/link | -| | command. | -+----------------------+---------------------------------------------------------------------------------------------+ -|INCLUDE_DIRS dir | Include directories. | -+----------------------+---------------------------------------------------------------------------------------------+ -|LINK_LIBRARIES library| Libraries to link against. | -+----------------------+---------------------------------------------------------------------------------------------+ -|H_DIRS directory | Base directories for H_FILES. | -+----------------------+---------------------------------------------------------------------------------------------+ -|H_FILES h_file | Header files for which to generate bindings in pkg. | -| | Absolute filenames, or filenames relative to H_DIRS. All | -| | definitions found directly in these files will contribute | -| | to the bindings. (NOTE: This means that if "forwarding | -| | headers" are present, the real "legacy" headers must be | -| | specified as H_FILES). | -| | All header files which contribute to a given C++ namespace | -| | should be grouped into a single pkg to ensure a 1-to-1 | -| | mapping with the implementing Python class. | -+----------------------+---------------------------------------------------------------------------------------------+ - -Returns via PARENT_SCOPE variables:: - - target The CMake target used to build. - setup_py The setup.py script used to build or install pkg. - -Examples:: - - find_package(Qt5Core NO_MODULE) - find_package(KF5KDcraw NO_MODULE) - get_target_property(_H_DIRS KF5::KDcraw INTERFACE_INCLUDE_DIRECTORIES) - get_target_property(_LINK_LIBRARIES KF5::KDcraw INTERFACE_LINK_LIBRARIES) - set(_LINK_LIBRARIES KF5::KDcraw ${_LINK_LIBRARIES}) - include(${KF5KDcraw_DIR}/KF5KDcrawConfigVersion.cmake) - - cppyy_add_bindings( - "KDCRAW" "${PACKAGE_VERSION}" "Shaheed" "srhaque@theiet.org" - LANGUAGE_STANDARD "14" - LINKDEFS "../linkdef_overrides.h" - GENERATE_OPTIONS "-D__PIC__;-Wno-macro-redefined" - INCLUDE_DIRS ${Qt5Core_INCLUDE_DIRS} - LINK_LIBRARIES ${_LINK_LIBRARIES} - H_DIRS ${_H_DIRS} - H_FILES "dcrawinfocontainer.h;kdcraw.h;rawdecodingsettings.h;rawfiles.h") - -There is a fuller example of embedding the use of cppyy_add_bindings for a -large set of bindings:: - - https://cgit.kde.org/pykde5.git/plain/KF5/CMakeLists.txt?h=include_qt_binding - -cppyy_find_pips -^^^^^^^^^^^^^^^ - -Return a list of available pip programs. diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/bugs.rst b/bindings/pyroot/cppyy/cppyy/doc/source/bugs.rst index 37c9d4eb9f95d..b598a792c9f1a 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/bugs.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/bugs.rst @@ -4,7 +4,8 @@ Bugs and feedback ================= Please report bugs, ask questions, request improvements, and post general -comments on the `issue tracker`_ or on `stack overflow`_. +comments on the `issue tracker`_ or on `stack overflow`_ (marked with the +"cppyy" tag). -.. _`issue tracker`: https://bitbucket.org/wlav/cppyy/issues +.. _`issue tracker`: https://github.com/wlav/cppyy/issues .. _`stack overflow`: https://stackoverflow.com/questions/tagged/cppyy diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst b/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst index ac13e8ec19070..857d045307edb 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/changelog.rst @@ -10,16 +10,360 @@ See :doc:`packages `, for details on the package structure. PyPy support lags CPython support. -Master: 1.6.2 -------------- +2023-11-15: 3.1.2 +----------------- + +* Deprecate 3.1.1 b/c of an installation problem outside of virtualenv +* Fix installation problem when purelib and platlib differ +* Alt fix for "failed to materialize symbols" on some Linux systems + + +2023-11-13: 3.1.0 +----------------- + +* Use xcrun to find header files on Mac as a last resort +* Fix for "symbols failed to materialize" with newer gcc on Linux +* Default to C++20 on all platforms +* Add C++20 standard headers to the PCH +* Fixes for new p11 and p12 type properties +* Fix std::span compatibility +* Look for ``__cast_cpp__`` for custom converters +* Add ``macro()`` helper for evaluation of preprocessor macros +* Extended support for int8_t/uint8_t array and pointer types +* Added ``cppyy.ll.as_memoryview()`` for byte-views of arrays of PODs +* Check for ``nullptr`` as ``false`` in ``operator bool()`` +* Automatically array-ify std::vector::data() results +* Use __name__ to stringify if an annotation object provides it +* Improve consistency of ``char[]`` arrays +* Extended Numba support +* Update to latest Cling release (6.30) + + +2023-03-19: 3.0.0 +----------------- + +* Upgrade backend to Cling on top of LLVM 13 +* Improve handling of `const char*` as template argument +* Fix regression in use of unnamed but typedef'ed enums +* Report C++ warnings from ``cppdef`` as ``SyntaxWarning`` +* Add pythonizations for ``std::unordered_map`` + + +2023-01-21: 2.4.2 +----------------- + +* Added a generic ``cppyy.default`` object +* Support explicitly created initializer lists as arguments +* Pass instances by-ref in Numba traces +* Support non-POD by-value returns in Numba traces +* Nullify derived class Python proxy when the C++ object is deleted +* Add ``__cpp_template__`` back reference for instantiated templated classes +* Improved buffer checking for ``std::initializer_list`` +* Add convenience functions ``argc()`` and ``argv()`` to ``cppyy.ll`` +* Added ``nullptr`` comparisons for for typed ``nullptr`` +* Support for ``using`` pointer types as template arguments +* Walk the full inheritance tree to find the overloads +* Allow ``__destruct__`` override in Python derived class +* Allow ``NULL`` function pointers to be returned as ``std::function`` objects +* Add Python traceback to C++ exception ``what()`` + + +2022-10-03: 2.4.1 +----------------- + +* Drop Numba extension entry point + + +2022-06-29: 2.4.0 +----------------- + +* Support for free (templated) functions in Numba +* Basic support for unboxing C++ public data members in Numba +* Basic support for calling methods of C++ structs in Numba +* Added conventional `__cpp_reflex__` method for inspection in Numba +* Support for globally overloaded ordering operators +* Special cases for `__repr__`/`__str__` returning C++ stringy types +* Fix lookup of templates of function with template args +* Correct typing of int8_t/uint8_t enums +* Basic support for hidden enums +* Support function pointer returns and optimize function point variables +* Fix reuse of CPPOverload proxies in vector calls from different threads +* Use `-march=native` instead of checking the cpu for avx +* Workaround for handling exceptions from JITed code on ARM +* Drop ``from cppyy.interactive import *`` from CPython 3.11 +* Fix regression in converting `std::vector` initialization from Python `str` (copies) +* Provide access to extern "C" declared functions in namespaces +* Support for (multiple and nested) anonymous structs +* Pull forward upstream patch for PPC +* Only apply system_dirs patch (for asan) on Linux +* Add not-yet loaded classes to namespaces in dir() +* Fix lookup of templates of function with template args +* Fix lookup of templates types with << in name +* Fix regression for accessing `char16_t` data member arrays +* Add custom `__reshape__` method to CPPInstance to allow array cast +* Prioritize callee exceptions over bindings exceptions +* Prevent infinite recursion when instantiating class with no constructors + + +2021-11-14: 2.2.0 +----------------- + +* Migrated repos to github/wlav +* Properly resolve enum type of class enums +* Get proper shape of ``void*`` and enum arrays +* Fix access to (const) ref data members +* Fix sometimes PCH uninstall issue +* Fix argument passing of fixed arrays of pointers +* Include all gcc system paths (for asan) +* Initial support for Apple M1 + + +2021-07-17: 2.1.0 +----------------- + +* Support for vector calls with CPython 3.8 and newer +* Support for typed C++ literals as defaults when mixing with keywords +* Enable reshaping of multi-dim LowLevelViews +* Refactored multi-dim arrays and support for multi-dim assignment +* Support tuple-based indexing for multi-dim arrays +* Direct support for C's _Complex (_Complex_double/_float on Windows) +* sizeof() forwards to ctypes.sizeof() for ctypes' types +* Upgrade cmake fragments for Clang9 +* Prevent clash with Julia's LLVM when loading cppyy into PyCall +* Upgrade to latest Cling patch release + + +2021-05-14: 2.0.0 +----------------- + +* Upgrade to latest Cling based on Clang/LLVM 9 +* Make C++17 the default standard on Windows + + +2021-04-28: 1.9.6 +----------------- + +* Reverse operators for ``std::complex`` targeting Python's ``complex`` +* Version the precompiled header with the ``cppyy-cling`` package version +* Cover more iterator protocol use cases +* Add missing cppyy/__pyinstaller pkg to sdist +* Single-inheritance support for cross-inherited templated constructors +* Disallow ``float`` -> ``const long long&`` conversion +* Capture python exception message string in PyException from callbacks +* Thread safety in enum lookups + + +2021-03-22: 1.9.5 +----------------- + +* Do not regulate direct smart pointers (many to one can lead to double deletion) +* Use pkg_resources of ``CPyCppyy``, if available, to find the API include path + + +2021-03-17: 1.9.4 +----------------- + +* Fix for installing into a directory that has a space in the name +* Fix empty collection printing through Cling on 64b Windows +* Fix accidental shadowing of derived class typedefs by same names in base +* Streamlined templated function lookups in namespaces +* Fix edge cases when decomposing std::function template arguments +* Enable multi-cross inheritance with non-C++ python bases +* Support Bound C++ functions as template argument +* Python functions as template arguments from ``__annotations__`` or ``__cpp_name__`` +* Removed functions/apis deprecated in py3.9 +* Improved support for older pip and different installation layouts + + +2021-02-15: 1.9.3 +----------------- + +* Wheels for Linux now follow manylinux2014 +* Enable direct calls of base class' methods in Python cross-overrides +* cppyy.bind_object can now re-cast types, incl. Python cross-derived ones +* Python cross-derived objects send to (and owned by) C++ retain Python state +* Ignore, for symbol lookups, libraries that can not be reloaded +* Use PathCanonicalize when resolving paths on Windows +* Add more ways of finding the backend library +* Improve error reporting when failed to find the backend library +* Workaround for mixing std::endl in JIT-ed and compiled code on Windows 32b +* Fixed a subtle crash that arises when an invalid ``using`` is the last method +* Filter -fno-plt (coming from anaconda builds; not understood by Cling) +* Fixed memory leak in generic base ``__str__`` + + +2021-01-05: 1.9.2 +----------------- + +* Added ``cppyy.types`` module for exposing cppyy builtin types +* Improve numpy integration with custom ``__array__`` methods +* Allow operator overload resolution mixing class and global methods +* Installation fixes for PyPy when using pip + + +2020-11-23: 1.9.1 +----------------- + +* Fix custom installer in pip sdist + + +2020-11-22: 1.9.0 +----------------- + +* In-tree build resolving build/install order for PyPy with pyproject.toml +* ``std::string`` not converterd to ``str`` on function returns +* Cover more use cases where C string memory can be managed +* Automatic memory management of converted python functions +* Added pyinstaller hooks (https://stackoverflow.com/questions/64406727) +* Support for enums in pseudo-constructors of aggregates +* Fixes for overloaded/split-access protected members in cross-inheritance +* Support for deep, mixed, hierarchies for multi-cross-inheritance +* Added tp_iter method to low level views + + +2020-11-06: 1.8.6 +----------------- + +* Fix preprocessor macro of CPyCppyy header for Windows/MSVC + + +2020-10-31: 1.8.5 +----------------- + +* Fix leaks when using vector iterators on Py3/Linux + + +2020-10-10: 1.8.4 +----------------- + +* ``std::string`` globals/data members no longer automatically converted to ``str`` +* New methods for std::string to allow ``str`` interchangability +* Added a ``decode`` method to ``std::string`` +* Add pythonized ``__contains__`` to ``std::set`` +* Fix constructor generation for aggregates with static data +* Fix performance bug when using implicit conversions +* Fix memory overwrite when parsing during sorting of methods +* PyPy pip install again falls back to setup.py install + + +2020-09-21: 1.8.3 +----------------- + +* Add initializer constructors for PODs and aggregates +* Use actual underlying type for enums, where possible +* Enum values remain instances of their type +* Expose enum underlying type name as ``__underlying`` and ``__ctype__`` +* Strictly follow C++ enum scoping rules +* Same enum in transparent scope refers to same type +* More detailed enum ``repr()`` printing, where possible +* Fix for (extern) explicit template instantiations in namespaces +* Throw objects from an std::tuple a life line +* Global pythonizors now always run on all classes +* Simplified iteraton over STL-like containers defining ``begin()``/``end()`` + + +2020-09-08: 1.8.2 +----------------- + +* Add ``cppyy.set_debug()`` to enable debug output for fixing template errors +* Cover more partial template instantiation use cases +* Force template instantiation if necessary for type deduction (i.e. ``auto``) + + +2020-09-01: 1.8.1 +----------------- + +* Setup build dependencies with pyproject.toml +* Simplified flow of pointer types for callbacks and cross-derivation +* Pointer-comparing objects performs auto-cast as needed +* Add main dimension for ptr-ptr to builtin returns +* Transparant handling of ptr-ptr to instance returns +* Stricter handling of bool type in overload with int types +* Fix uint64_t template instantiation regression +* Do not filter out enum data for ``__dir__`` +* Fix lookup of interpreter-only explicit instantiations +* Fix inconsistent naming of std types with char_traits +* Further hiding of upstream code/dependencies +* Extended documentation + + +2020-07-12: 1.8.0 +----------------- + +* Support mixing of Python and C++ types in global operators +* Capture Cling error messages from cppdef and include in the Python exception +* Add a cppexec method to evalutate statements in Cling's global scope +* Support initialization of ``std::array<>`` from sequences +* Support C++17 style initialization of common STL containers +* Allow base classes with no virtual destructor (with warning) +* Support const by-value returns in Python-side method overrides +* Support for cross-language multiple inheritance of C++ bases +* Allow for pass-by-value of ``std::unique_ptr`` through move +* Reduced dependencies on upstream code +* Put remaining upstream code in CppyyLegacy namespace + + +2020-06-06: 1.7.1 +----------------- + +* Expose protected members in Python derived classes +* Support for deep Python-side derived hierarchies +* Do not generate a copy ctor in the Python derived class if private +* include, c_include, and cppdef now raise exceptions on error +* Allow mixing of keywords and default values +* Fix by-ptr return of objects in Python derived classes +* Fix for passing numpy boolean array through ``bool*`` +* Fix assignment to ``const char*`` data members +* Support ``__restrict`` and ``__restrict__`` in interfaces +* Allow passing sequence of strings through ``const char*[]`` argument + + +2020-04-27: 1.7.0 +----------------- + +* Upgrade to cppyy-cling 6.20.4 +* Pre-empt upstream's propensity of making ``std`` classes etc. global +* Allow initialization of ``std::map`` from dict with the correct types +* Allow initialization of ``std::set`` from set with the correct types +* Add optional nonst/non-const selection to ``__overload__`` +* Automatic smartification of normal object passed as smartptr by value +* Fix crash when handing a by-value object to make_shared +* Fixed a few shared/unique_ptr corner cases +* Fixed conversion of ``std::function`` taking an STL class parameter +* No longer attempt auto-cast on classes without RTTI +* Fix for ``iter()`` iteration on generic STL container + + +2020-03-15: 1.6.2 +----------------- -* Respect __len__ when using bound C++ objects in boolean expressions -* Support UTF-8 encoded unicode through std::string -* Support for std::byte +* Respect ``__len__`` when using bound C++ objects in boolean expressions +* Support UTF-8 encoded ``unicode`` through ``std::string`` +* Support for ``std::byte`` * Enable assignment to function pointer variable -* Allow passing cppyy.nullptr where are function pointer is expected +* Allow passing cppyy.nullptr where a function pointer is expected * Disable copy construction into constructed object (use ``__assign__`` instead) +* Cover more cases when to set a lifeline * Lower priority of implicit conversion to temporary with initializer_list ctor +* Add type reduction pythonization for trimming expression template type trees +* Allow mixing ``std::string`` and ``str`` as dictionary keys +* Support C-style pointer-to-struct as array +* Support C-style enum variable declarations +* Fixed const_iterator by-ref return type regression +* Resolve enums into the actual underlying type instead of int * Remove '-isystem' from makepch flags * Extended documentation @@ -37,7 +381,7 @@ Master: 1.6.2 2019-12-23: 1.6.0 ----------------- -* Classes derived from std::exception can be used as Python exceptions +* Classes derived from ``std::exception`` can be used as Python exceptions * Template handling detailing (for Eigen) * Support keyword arguments * Added add_library_path at module level @@ -57,12 +401,12 @@ Master: 1.6.2 * Added public C++ API for some CPyCppyy core functions (CPython only) * Support for char16_t/char16_t* and char32_t/char32_t* -* Respect std::hash in __hash__ +* Respect ``std::hash`` in ``__hash__`` * Fix iteration over vector of shared_ptr * Length checking on global variables of type 'signed char[N]' -* Properly support overloaded templated with non-templated __setitem__ +* Properly support overloaded templated with non-templated ``__setitem__`` * Support for array of const char* as C-strings -* Enable type resolution of clang's builtin __type_pack_element +* Enable type resolution of clang's builtin ``__type_pack_element`` * Fix for inner class type naming when it directly declares a variable @@ -98,7 +442,7 @@ Master: 1.6.2 ----------------- * Added a "low level" interface (cppyy.ll) for hard-casting and ll types -* Extened support for passing ctypes arguments through ptr, ref, ptr-ptr +* Extended support for passing ctypes arguments through ptr, ref, ptr-ptr * Fixed crash when creating an array of instances of a scoped inner struct * Extended documentation @@ -110,7 +454,7 @@ Master: 1.6.2 * Various patches to upstream's pre-compiled header generation and use * Instantiate templates with larger integer types if argument values require * Improve cppyy.interactive and partially enable it on PyPy, IPython, etc. -* Let __overload__ be more flexible in signature matching +* Let ``__overload__`` be more flexible in signature matching * Make list filtering of dir(cppyy.gbl) on Windows same as Linux/Mac * Extended documentation @@ -127,11 +471,11 @@ Master: 1.6.2 * Allow implicit conversion from a tuple of arguments * Data set on namespaces reflected on C++ even if data not yet bound * Generalized resolution of binary operators in wrapper generation -* Proper naming of arguments in namespaces for std::function<> +* Proper naming of arguments in namespaces for ``std::function<>`` * Cover more cases of STL-liker iterators -* Allow std::vector initialization with a list of constructor arguments +* Allow ``std::vector`` initialization with a list of constructor arguments * Consistent naming of ``__cppname__`` to ``__cpp_name__`` -* Added __set_lifeline__ attribute to overloads +* Added ``__set_lifeline__`` attribute to overloads * Fixes to the cmake fragments for Ubuntu * Fixes linker errors on Windows in some configurations * Support C++ naming of typedef of bool types @@ -142,10 +486,10 @@ Master: 1.6.2 2019-07-01 : 1.4.12 ------------------- -* Automatic conversion of python functions to std::function arguments +* Automatic conversion of python functions to ``std::function`` arguments * Fix for templated operators that can map to different python names * Fix on p3 crash when setting a detailed exception during exception handling -* Fix lookup of std::nullopt +* Fix lookup of ``std::nullopt`` * Fix bug that prevented certain templated constructors from being considered * Support for enum values as data members on "enum class" enums * Support for implicit conversion when passing by-value @@ -179,7 +523,7 @@ Master: 1.6.2 2019-04-22 : 1.4.8 ------------------ -* std::tuple is now iterable for return assignments w/o tie +* ``std::tuple`` is now iterable for return assignments w/o tie * Support for opaque handles and typedefs of pointers to classes * Keep unresolved enums desugared and provide generic converters * Treat int8_t and uint8_t as integers (even when they are chars) @@ -221,7 +565,7 @@ Master: 1.6.2 * Allow templated free functions to be attached as methods to classes * Allow cross-derivation from templated classes * More support for 'using' declarations (methods and inner namespaces) -* Fix overload resolution for std::set::rbegin()/rend() operator == +* Fix overload resolution for ``std::set::rbegin()``/``rend()`` ``operator==`` * Fixes for bugs #61, #67 * Several pointer truncation fixes for 64b Windows * Linker and lookup fixes for Windows @@ -234,7 +578,7 @@ Master: 1.6.2 * Improved support for alias templates * Faster template lookup * Have rootcling/genreflex respect compile-time flags (except for --std if - overridden by CLING_EXTRA_FLAGS) + overridden by CLING_EXTRA_ARGS) * Utility to build dictionarys on Windows (32/64) * Name mangling fixes in Cling for JITed global/static variables on Windows * Several pointer truncation fixes for 64b Windows @@ -247,7 +591,7 @@ Master: 1.6.2 * Preserve 'const' when overriding virtual functions * Support for by-ref (using ctypes) for function callbacks * Identity of nested typedef'd classes matches actual -* Expose function pointer variables as std::function's +* Expose function pointer variables as ``std::function``'s * More descriptive printout of global functions * Ensure that standard pch is up-to-date and that it is removed on uninstall diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/classes.rst b/bindings/pyroot/cppyy/cppyy/doc/source/classes.rst index e7b7599434eef..efbf929a2db22 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/classes.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/classes.rst @@ -84,12 +84,19 @@ will result in a Python ``ReferenceError`` exception. `Destructors` ------------- -There should not be a reason to call a destructor directly in CPython, but +There should no be reason to call a destructor directly in CPython, but e.g. PyPy uses a garbage collector and that makes it sometimes useful to destruct -a C++ object where you want it destroyed. -Destructors are accessible through the conventional ``__destruct__`` method. -Accessing an object after it has been destroyed will result in a Python -``ReferenceError`` exception. +a C++ object exactly when you want it destroyed. +Destructors are by convention accessible through the ``__destruct__`` method +(since "~" can not be part of a Python method name). +If a Python-side derived class overrides ``__destruct__``, that method will +be called when the instance gets deleted in C++. +The Python destructor, ``__del__``, gets called when the Python proxy goes +away, which will only delete the C++ instance if owned by Python. +Note that ``__del__`` is not guaranteed to be called, it may e.g. be skipped +on program shutdown or because of an outstanding exception. +Accessing an object after it has been destroyed using ``__destruct__`` will +result in a Python ``ReferenceError`` exception. `Inheritance` @@ -139,7 +146,7 @@ Example: >>> from cppyy.gbl import Abstract, call_abstract_method >>> class PyConcrete(Abstract): ... def abstract_method(self): - ... print("Hello, Python World!\n") + ... return "Hello, Python World!\n" ... def concrete_method(self): ... pass ... @@ -153,6 +160,49 @@ if you do, you *must* call the base class constructor through the ``super`` mechanism. +`Multiple cross-inheritance` +---------------------------- + +Python requires that any multiple inheritance (also in pure Python) has an +unambiguous method resolution order (mro), including for classes and thus +also for meta-classes. +In Python2, it was possible to resolve any mro conflicts automatically, but +meta-classes in Python3, although syntactically richer, have functionally +become far more limited. +In particular, the mro is checked in the builtin class builder, instead of +in the meta-class of the meta-class (which in Python3 is the builtin ``type`` +rather than the meta-class itself as in Python2, another limitation, and +which actually checks the mro a second time for no reason). +The upshot is that a helper is required (``cppyy.multi``) to resolve the mro +to support Python3. +The helper is written to also work in Python2. +Example: + + .. code-block:: python + + >>> class PyConcrete(cppyy.multi(cppyy.gbl.Abstract1, cppyy.gbl.Abstract2)): + ... def abstract_method1(self): + ... return "first message" + ... def abstract_method2(self): + ... return "second message" + ... + >>> pc = PyConcrete() + >>> cppyy.gbl.call_abstract_method1(pc) + first message + >>> cppyy.gbl/call_abstract_method2(pc) + second message + >>> + +Contrary to multiple inheritance in Python, in C++ there are no two separate +instances representing the base classes. +Thus, a single ``__init__`` call needs to construct and initialize all bases, +rather than calling ``__init__`` on each base independently. +To support this syntax, the arguments to each base class should be grouped +together in a tuple. +If there are no arguments, provide an empty tuple (or omit them altogether, +if these arguments apply to the right-most base(s)). + + .. _sec-methods-label: `Methods` @@ -230,6 +280,44 @@ behave as expected: .. _sec-operators-label: + +`Structs/Unions` +---------------- + +Structs and unions are both supported, named or anonymous. +If the latter, the field are accessible through the parent scope by their +declared name. +For example: + + .. code-block:: python + + >>> cppyy.cppdef("""\ + ... struct PointXYZ { + ... PointXYZI() : intensity(5.) {} + ... double x, y, z; + ... union { + ... int offset1; + ... struct { + ... int offset2; + ... float intensity; + ... }; + ... float data_c[4]; + ... }; + ... };""") + True + >>> p = cppyy.gbl.PointXYZI() + >>> type(p.x) + + >>> p.intensity + 5.0 + >>> type(p.data_c[1]) + + >>> p.data_c[1] = 3.0 + >>> p.intensity + 3.0 + >>> + + `Operators` ----------- diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/cmake_interface.rst b/bindings/pyroot/cppyy/cppyy/doc/source/cmake_interface.rst index d44347357fcdb..1a2efeb642d84 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/cmake_interface.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/cmake_interface.rst @@ -27,6 +27,17 @@ packaging and CMake where CMake provides: * An facility for CMake-based projects to automate the entire bindings generation process, including basic automated tests. +.. note:: + + The JIT needs to resolve linker symbols in order to call them through + generated wrappers. + Thus, any classes, functions, and data that will be used in Python need + to be exported. + This is the default behavior on Mac and Linux, but not on Windows. + On that platform, use ``__declspec(dllexport)`` to explicitly export the + classes and function you expect to call. + CMake has simple `support for exporting all`_ C++ symbols. + Python packaging ---------------- @@ -77,7 +88,7 @@ variables: Cppyy_FOUND - set to true if Cppyy is found Cppyy_DIR - the directory where Cppyy is installed Cppyy_EXECUTABLE - the path to the Cppyy executable - Cppyy_INCLUDE_DIRS - Where to find the ROOT header files. + Cppyy_INCLUDE_DIRS - Where to find the Cppyy header files. Cppyy_VERSION - the version number of the Cppyy backend. and also defines the following functions:: @@ -159,8 +170,7 @@ The bindings are generated/built/packaged using 3 environments: +----------------------+---------------------------------------------------------------------------------------------+ |GENERATE_OPTIONS optio| Options which are to be passed into the rootcling | | | command. For example, bindings which depend on Qt | -| | may need "-D__PIC__;-Wno-macro-redefined" as per | -| | https://sft.its.cern.ch/jira/browse/ROOT-8719. | +| | may need "-D__PIC__;-Wno-macro-redefined". | +----------------------+---------------------------------------------------------------------------------------------+ |LINKDEFS def | Files or lines which contain extra #pragma content | | | for the linkdef.h file used by rootcling. See | @@ -228,13 +238,11 @@ Examples:: H_DIRS ${_H_DIRS} H_FILES "dcrawinfocontainer.h;kdcraw.h;rawdecodingsettings.h;rawfiles.h") -There is a fuller example of embedding the use of cppyy_add_bindings for a -large set of bindings:: - - https://cgit.kde.org/pykde5.git/plain/KF5/CMakeLists.txt?h=include_qt_binding - cppyy_find_pips ^^^^^^^^^^^^^^^ Return a list of available pip programs. + + +.. _`support for exporting all`: https://cmake.org/cmake/help/latest/prop_tgt/WINDOWS_EXPORT_ALL_SYMBOLS.html diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/conf.py b/bindings/pyroot/cppyy/cppyy/doc/source/conf.py index 9fc9075104407..f58cacbfab58b 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/conf.py +++ b/bindings/pyroot/cppyy/cppyy/doc/source/conf.py @@ -47,7 +47,7 @@ # General information about the project. project = u'cppyy' -copyright = u'2018-19, Wim Lavrijsen' +copyright = u'2018-23, Wim Lavrijsen' author = u'Wim Lavrijsen' # The version info for the project you're documenting, acts as replacement for @@ -55,16 +55,16 @@ # built documents. # # The short X.Y version. -version = '1.6' +version = '3.0' # The full version, including alpha/beta/rc tags. -release = '1.6.0' +release = '3.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -356,4 +356,4 @@ #epub_use_index = True def setup(app): - app.add_stylesheet('css/custom.css') + app.add_css_file('css/custom.css') diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/cppyy_features_header.rst b/bindings/pyroot/cppyy/cppyy/doc/source/cppyy_features_header.rst index c745d4c7367d8..bbacee55f2658 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/cppyy_features_header.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/cppyy_features_header.rst @@ -78,6 +78,27 @@ File features.h a->abstract_method(); } + //----- + class Abstract1 { + public: + virtual ~Abstract1() {} + virtual std::string abstract_method1() = 0; + }; + + class Abstract2 { + public: + virtual ~Abstract2() {} + virtual std::string abstract_method2() = 0; + }; + + std::string call_abstract_method1(Abstract1* a) { + return a->abstract_method1(); + } + + std::string call_abstract_method2(Abstract2* a) { + return a->abstract_method2(); + } + //----- int global_function(int) { return 42; diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/cuda.rst b/bindings/pyroot/cppyy/cppyy/doc/source/cuda.rst new file mode 100644 index 0000000000000..03e11f2d905e1 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/doc/source/cuda.rst @@ -0,0 +1,42 @@ +.. _cuda: + + +CUDA support +============ + +.. warning:: + + This is an **experimental** feature, available starting with release + 2.3.0. + It is still incomplete and has only been tested on Linux on x86_64. + +CUDA is supported by passing all JITed code through two pipelines: one for the +CPU and one for the GPU. +Use of the ``__CUDA__`` pre-processor macro enables more fine-grained control +over which pipeline sees what, which is used e.g. in the pre-compiled header: +the GPU pipeline has the CUDA headers included, the CPU pipeline does not. +Building the pre-compiled header will also pick up common CUDA libraries such +as cuBLAS, if installed. + +Each version of CUDA requires specific versions of Clang and the system +compiler (e.g. gcc) for proper functioning; it's therefore best to build the +backend (``cppyy-cling``) from source for the specific combination of +interest. +The 3.x series of cppyy uses Clang13, the 2.x series Clang9, and this may +limit the CUDA versions supported (especially since CUDA has changed the APIs +for launching kernels in v11). + +There are three environment variables to control Cling's handling of CUDA: + +* ``CLING_ENABLE_CUDA`` (required): set to ``1`` to enable the CUDA + backend. + +* ``CLING_CUDA_PATH`` (optional): set to the local CUDA installation if not + in a standard location. + +* ``CLING_CUDA_ARCH`` (optional): set the architecture to target; default is + ``sm_35`` (Clang9 is limited to ``sm_75``). + +After enabling CUDA with ``CLING_ENABLE_CUDA=1`` CUDA code can be used and +kernels can be launched from JITed code by in ``cppyy.cppdef()``. +There is currently no syntax or helpers yet to launch kernels from Python. diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/dictionaries.rst b/bindings/pyroot/cppyy/cppyy/doc/source/dictionaries.rst deleted file mode 100644 index a7653b287f1cb..0000000000000 --- a/bindings/pyroot/cppyy/cppyy/doc/source/dictionaries.rst +++ /dev/null @@ -1,97 +0,0 @@ -.. _dictionaries: - -Dictionaries -============ - -Loading code directly into Cling is fine for interactive work and small -scripts, but large scale applications should take advantage of pre-packaging -code, linking in libraries, and describing other dependencies. -The necessary tools are installed as part of the backend. - - -Dictionary generation ---------------------- - -A "reflection dictionary" makes it simple to combine the necessary headers and -libraries into a single package for use and distribution. -The relevant headers are read by a tool called `genreflex`_ which generates -C++ files that are to be compiled into a shared library. -That library can further be linked with any relevant project libraries that -contain the implementation of the functionality declared in the headers. -For example, given a file called ``project_header.h`` and an implementation -residing in ``libproject.so``, the following will generate a -``libProjectDict.so`` reflection dictionary:: - - $ genreflex project_header.h - $ g++ -std=c++17 -fPIC -rdynamic -O2 -shared `genreflex --cppflags` project_header_rflx.cpp -o libProjectDict.so -L$PROJECTHOME/lib -lproject - -Instead of loading the header text into Cling, you can now load the -dictionary: - -.. code-block:: python - - >>> import cppyy - >>> cppyy.load_reflection_info('libProjectDict.so') - - >>> from cppyy.gbl import SomeClassFromProject - >>> - -and use the C++ entities from the header as before. - -.. _`genreflex`: https://linux.die.net/man/1/genreflex - - -Automatic class loader ----------------------- - -Explicitly loading dictionaries is fine if this is hidden under the hood of -a Python package and thus simply done on import. -Otherwise, the automatic class loader is more convenient, as it allows direct -use without having to manually find and load dictionaries. - -The class loader utilizes so-called rootmap files, which by convention should -live alongside the dictionaries in places reachable by LD_LIBRARY_PATH. -These are simple text files, which map C++ entities (such as classes) to the -dictionaries and other libraries that need to be loaded for their use. - -The ``genreflex`` tool can produce rootmap files automatically. -For example:: - - $ genreflex project_header.h --rootmap=libProjectDict.rootmap --rootmap-lib=libProjectDict.so - $ g++ -std=c++17 -fPIC -rdynamic -O2 -shared `genreflex --cppflags` project_header_rflx.cpp -o libProjectDict.so -L$CPPYYHOME/lib -lCling -L$PROJECTHOME/lib -lproject - -where the first option (``--rootmap``) specifies the output file name, and the -second option (``--rootmap-lib``) the name of the reflection library. -It is necessary to provide that name explicitly, since it is only in the -separate linking step where these names are fixed (if the second option is not -given, the library is assumed to be libproject_header.so). - -With the rootmap file in place, the above example can be rerun without explicit -loading of the reflection info library: - -.. code-block:: python - - >>> import cppyy - >>> from cppyy.gbl import SomeClassFromProject - >>> - - -Selection files ---------------- -.. _selection-files: - -Sometimes it is necessary to restrict or expand what genreflex will pick up -from the header files. -For example, to add or remove standard classes or to hide implementation -details. -This is where `selection files`_ come in. -These are XML specifications that allow exact or pattern matching to classes, -functions, etc. -See ``genreflex --help`` for a detailed specification and add -``--selection=project_selection.xml`` to the ``genreflex`` command line. - -With the aid of a selection file, a large project can be easily managed: -simply ``#include`` all relevant headers into a single header file that is -handed to ``genreflex``. - -.. _`selection files`: https://linux.die.net/man/1/genreflex diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/examples.rst b/bindings/pyroot/cppyy/cppyy/doc/source/examples.rst index ca6c653eaf9fa..99d11edf579ae 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/examples.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/examples.rst @@ -1,29 +1,51 @@ .. _examples: -Examples -======== +Example repos +============= The detailed feature lists have examples that work using a header file, and there is the `tutorial`_ that shows mixing of C++ and Python interactively. -More complete examples that show packaging include these repos (in -alphabetical order): +The `cookie cutter`_ repo provides a good cmake based example. +More complete examples that show packaging include these repos: + * `flatsurf`_ + * `iheartia`_ + * `ns-3`_ + * `popsicle`_ + * `ogdf-python`_ + * `python-vspline`_ + * `mupdf`_ + * `NWChemEx`_ + * `EPICpy`_ + * `bgfx-python`_ * `cppyy-bbhash`_ + * `dnpy`_ * `PyEtaler`_ + * `gco-cppyy`_ * `gmpxxyy`_ * `cppyy-knearestneighbors`_ + * `lyncs`_ * `libsemigroups_cppyy`_ - * `python-vspline`_ - -In addition, cppyy is being re-integrated in its roots, which includes -several sophisticated pythonizations. -See the "experimental" version of the `PyROOT`_ project. + * `SopraClient`_ -.. _tutorial: https://bitbucket.org/wlav/cppyy/src/master/doc/tutorial/CppyyTutorial.ipynb?viewer=nbviewer&fileviewer=notebook-viewer%3Anbviewer +.. _tutorial: https://github.com/wlav/cppyy/blob/master/doc/tutorial/CppyyTutorial.ipynb +.. _cookie cutter: https://github.com/camillescott/cookiecutter-cppyy-cmake +.. _flatsurf: https://github.com/flatsurf +.. _iheartia: https://github.com/iheartla/iheartla +.. _ns-3: https://www.nsnam.org +.. _popsicle: https://github.com/kunitoki/popsicle +.. _ogdf-python: https://github.com/N-Coder/ogdf-python +.. _python-vspline: https://bitbucket.org/kfj/python-vspline +.. _mupdf: https://mupdf.com/ +.. _NWChemEx: https://github.com/NWChemEx-Project +.. _EPICpy: https://github.com/travisseymour/EPICpy +.. _bgfx-python: https://github.com/fbertola/bgfx-python .. _cppyy-bbhash: https://github.com/camillescott/cppyy-bbhash +.. _dnpy: https://github.com/txjmb/dnpy .. _PyEtaler: https://github.com/etaler/PyEtaler +.. _gco-cppyy: https://github.com/agoose77/gco-cppyy .. _gmpxxyy: https://github.com/flatsurf/gmpxxyy .. _cppyy-knearestneighbors: https://github.com/jclay/cppyy-knearestneighbors-example +.. _lyncs: https://github.com/Lyncs-API .. _libsemigroups_cppyy: https://github.com/libsemigroups/libsemigroups_cppyy -.. _python-vspline: https://bitbucket.org/kfj/python-vspline -.. _PyROOT: https://root.cern.ch/gitweb/?p=root.git;a=tree;f=bindings/pyroot_experimental/PyROOT +.. _SopraClient: https://github.com/SoPra-Team-17/Client diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/features.rst b/bindings/pyroot/cppyy/cppyy/doc/source/features.rst deleted file mode 100644 index 94ad073940aa6..0000000000000 --- a/bindings/pyroot/cppyy/cppyy/doc/source/features.rst +++ /dev/null @@ -1,92 +0,0 @@ -.. _features: - -Miscellaneous -============= - -.. toctree:: - :hidden: - - cppyy_features_header - -This is a collection of a few more features listed that do not have a proper -place yet in the rest of the documentation. - -The C++ code used for the examples below can be found -:doc:`here `, and it is assumed that that code is -loaded at the start of any session. -Download it, save it under the name ``features.h``, and load it: - - .. code-block:: python - - >>> import cppyy - >>> cppyy.include('features.h') - >>> - - -`STL algorithms` ----------------- - -It is usually easier to use a Python equivalent or code up the effect of an -STL algorithm directly, but when operating on a large container, calling an -STL algorithm may offer better performance. -It is important to note that all STL algorithms are templates and need the -correct types to be properly instantiated. -STL containers offer typedefs to obtain those exact types and these should -be used rather than relying on the usual implicit conversions of Python types -to C++ ones. -For example, as there is no ``char`` type in Python, the ``std::remove`` call -below can not be instantiated using a Python string, but the -``std::string::value_type`` must be used instead: - - .. code-block:: python - - >>> cppstr = cppyy.gbl.std.string - >>> n = cppstr('this is a C++ string') - >>> print(n) - this is a C++ string - >>> n.erase(cppyy.gbl.std.remove(n.begin(), n.end(), cppstr.value_type(' '))) - object at 0x7fba35d1af50> - >>> print(n) - thisisaC++stringing - >>> - - -`Odds and ends` ---------------- - -* **memory**: C++ instances created by calling their constructor from python - are owned by python. - You can check/change the ownership with the __python_owns__ flag that every - bound instance carries. - Example: - - .. code-block:: python - - >>> from cppyy.gbl import Concrete - >>> c = Concrete() - >>> c.__python_owns__ # True: object created in Python - True - >>> - -* **namespaces**: Are represented as python classes. - Namespaces are more open-ended than classes, so sometimes initial access may - result in updates as data and functions are looked up and constructed - lazily. - Thus the result of ``dir()`` on a namespace shows the classes available, - even if they may not have been created yet. - It does not show classes that could potentially be loaded by the class - loader. - Once created, namespaces are registered as modules, to allow importing from - them. - Namespace currently do not work with the class loader. - Fixing these bootstrap problems is on the TODO list. - The global namespace is ``cppyy.gbl``. - -* **NULL**: Is represented as ``cppyy.nullptr``. - Starting C++11, the keyword ``nullptr`` is used to represent ``NULL``. - For clarity of intent, it is recommended to use this instead of ``None`` - (or the integer ``0``, which can serve in some cases), as ``None`` is better - understood as ``void`` in C++. - -* **unary operators**: Are supported if a python equivalent exists, and if the - operator is defined in the C++ class. diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/functions.rst b/bindings/pyroot/cppyy/cppyy/doc/source/functions.rst index bc0744dd75e59..3645e14564d45 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/functions.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/functions.rst @@ -81,13 +81,33 @@ that they are accessible either through the class or through an instance, just like Python's ``staticmethod``. -`Methods` ---------- +`Instance methods` +------------------ For class methods, see the :ref:`methods section ` under the :doc:`classes heading`. +`Lambda's` +---------- + +C++ lambda functions are supported by first binding to a ``std::function``, +then providing a proxy to that on the Python side. +Example:: + + >>> cppyy.cppdef("""\ + ... auto create_lambda(int a) { + ... return [a](int b) { return a+b; }; + ... }""") + True + >>> l = cppyy.gbl.create_lambda(4) + >>> type(l) + at 0x11505b830> + >>> l(2) + 6 + >>> + + `Operators` ----------- @@ -119,27 +139,27 @@ Examples, using multiply from :doc:`features.h `: .. code-block:: python - >>> mul = cppyy.gbl.multiply - >>> mul(1, 2) - 2 - >>> mul(1., 5) - 5.0 - >>> mul[int](1, 1) - 1 - >>> mul[int, int](1, 1) - 1 - >>> mul[int, int, float](1, 1) - 1.0 - >>> mul[int, int](1, 'a') - TypeError: Template method resolution failed: - none of the 6 overloaded methods succeeded. Full details: - int ::multiply(int a, int b) => - TypeError: could not convert argument 2 (int/long conversion expects an integer object) - ... - Failed to instantiate "multiply(int,std::string)" - >>> mul['double, double, double'](1., 5) - 5.0 - >>> + >>> mul = cppyy.gbl.multiply + >>> mul(1, 2) + 2 + >>> mul(1., 5) + 5.0 + >>> mul[int](1, 1) + 1 + >>> mul[int, int](1, 1) + 1 + >>> mul[int, int, float](1, 1) + 1.0 + >>> mul[int, int](1, 'a') + TypeError: Template method resolution failed: + none of the 6 overloaded methods succeeded. Full details: + int ::multiply(int a, int b) => + TypeError: could not convert argument 2 (int/long conversion expects an integer object) + ... + Failed to instantiate "multiply(int,std::string)" + >>> mul['double, double, double'](1., 5) + 5.0 + >>> `Overloading` @@ -147,7 +167,7 @@ Examples, using multiply from :doc:`features.h `: C++ supports overloading, whereas Python supports "duck typing", thus C++ overloads have to be selected dynamically in response to the available -"ducks". +"ducks." This may lead to additional lookups or template instantiations. However, pre-existing methods (incl. auto-instantiated methods) are always preferred over new template instantiations: @@ -161,46 +181,170 @@ preferred over new template instantiations: >>> C++ does a static dispatch at compile time based on the argument types. -The dispatch is a selection among overloads (incl. templates) visible at that -point in the *translation unit*. +The dispatch is a selection among overloads (incl. templates) visible at the +current parse location in the *translation unit*. Bound C++ in Python does a dynamic dispatch: it considers all overloads -visible *globally* at that point in the execution. -Because the dispatch is fundamentally different (albeit in line with the -expectation of the respective languages), differences can occur. -Especially if overloads live in different header files and are only an -implicit conversion apart, or if types that have no direct equivalent in +visible *globally* at the time of execution. +These two approaches, even if completely in line with the expectations of the +respective languages, are fundamentally different and there can thus be +discrepancies in overload selection. +For example, if overloads live in different header files and are only an +implicit conversion apart; or if types that have no direct equivalent in Python, such as e.g. ``unsigned short``, are used. -There are two rounds to finding an overload. -If all overloads fail argument conversion during the first round, where -implicit conversions are not allowed, _and_ at least one converter has -indicated that it can do implicit conversions, a second round is tried. -In this second round, implicit conversions are allowed, including class -instantiation of temporaries. -During some template calls, implicit conversions are not allowed at all, to -make sure new instantiations happen instead. +It is implicitly assumed that the Python code is correct as-written and there +are no warnings or errors for overloads that C++ would consider ambiguous, +but only if every possible overload fails. +For example, the following overload would be ambiguous in C++ (the value +provided is an integer, but can not be passed through a 4-byte ``int`` type), +but instead ``cppyy`` silently accepts promotion to ``double``: + + .. code-block:: python + + >>> cppyy.cppdef(r"""\ + ... void process_data(double) { std::cerr << "processing double\n"; } + ... void process_data(int32_t) { std::cerr << "processing int\n"; }""") + True + >>> cppyy.gbl.process_data(2**32) # too large for int32_t type + processing double + >>> + +There are two rounds to run-time overload resolution. +The first round considers all overloads in sorted order, with promotion but +no implicit conversion allowed. +The sorting is based on priority scores of each overload. +Higher priority is given to overloads with argument types that can be +promoted or align better with Python types. +E.g. ``int`` is preferred over ``double`` and ``double`` is preferred over +``float``. +If argument conversion fails for all overloads during this round *and* at +least one argument converter has indicated that it can do implicit +conversion, a second round is tried where implicit conversion, including +instantiation of temporaries, is allowed. +The implicit creation of temporaries, although convenient, can be costly in +terms of run-time performance. + +During some template calls, implicit conversion is not allowed, giving +preference to new instantiations (as is the case in C++). +If, however, a previously instantiated overload is available and would match +with promotion, it is preferred over a (costly) new instantiation, unless a +template overload is explicitly selected using template arguments. +For example: + + .. code-block:: python + + >>> cppyy.cppdef(r"""\ + ... template + ... T process_T(T t) { return t; }""") + True + >>> type(cppyy.gbl.process_T(1.0)) + + >>> type(cppyy.gbl.process_T(1)) # selects available "double" overload + + >>> type(cppyy.gbl.process_T[int](1)) # explicit selection of "int" overload + + >>> + +The template parameters used for instantiation can depend on the argument +values. +For example, if the type of an argument is Python ``int``, but its value is +too large for a 4-byte C++ ``int``, the template may be instantiated with, +for example, an ``int64_t`` instead (if available on the platform). +Since Python does not have unsigned types, the instantiation mechanism +strongly prefers signed types. +However, if an argument value is too large to fit in a signed integer type, +but would fit in an unsigned type, then that will be used. + +If it is important that a specific overload is selected, then use the +``__overload__`` method to match a specific function signature. +An optional boolean second parameter can be used to restrict the selected +method to be const (if ``True``) or non-const (if ``False``). +The return value of which is a first-class callable object, that can be +stored to by-pass the overload resolution: + + .. code-block:: python + + >>> gf_double = global_function.__overload__('double') + >>> gf_double(1) # int implicitly promoted + 2.718281828459045 + >>> + +The ``__overload__`` method only does a lookup; it performs no (implicit) +conversions and the types in the signature to match should be the fully +resolved ones (no typedefs). +To see all overloads available for selection, use ``help()`` on the function +or look at its ``__doc__`` string: + + .. code-block:: python + + >>> print(global_function.__doc__) + int ::global_function(int) + double ::global_function(double) + >>> + +For convenience, the ``:any:`` signature allows matching any overload, for +example to reduce a method to its ``const`` overload only, use: + + .. code-block:: python + + MyClass.some_method = MyClass.some_method.__overload__(':any:', True) + + +`Overloads and exceptions` +-------------------------- -In the rare occasion where the automatic overload selection fails, the -``__overload__`` function can be called to access a specific overload -matching a specific function signature: +Python error reporting is done using exceptions. +Failed argument conversion during overload resolution can lead to different +types of exceptions coming from respective attempted overloads. +The final error report issued if all overloads fail, is a summary of the +individual errors, but by Python language requirements it has to have a +single exception type. +If all the exception types match, that type is used, but if there is an +amalgam of types, the exception type chosen will be ``TypeError``. +For example, attempting to pass a too large value through ``uint8_t`` will +uniquely raise a ``ValueError`` .. code-block:: python - >>> global_function.__overload__('double')(1) # int implicitly converted - 2.718281828459045 - >>> + >>> cppyy.cppdef("void somefunc(uint8_t) {}") + True + >>> cppyy.gbl.somefunc(2**16) + Traceback (most recent call last): + File "", line 1, in + ValueError: void ::somefunc(uint8_t) => + ValueError: could not convert argument 1 (integer to character: value 65536 not in range [0,255]) + >>> -Note that ``__overload__`` only does a lookup; it performs no (implicit) -conversions. -To see all available overloads, use ``help()`` or look at the ``__doc__`` -string of the function: +But if other overloads are present that fail in a different way, the error +report will be a ``TypeError``: .. code-block:: python - >>> print(global_function.__doc__) - int ::global_function(int) - double ::global_function(double) - >>> + >>> cppyy.cppdef(r""" + ... void somefunc(uint8_t) {} + ... void somefunc(std::string) {}""") + True + >>> cppyy.gbl.somefunc(2**16) + Traceback (most recent call last): + File "", line 1, in + TypeError: none of the 2 overloaded methods succeeded. Full details: + void ::somefunc(std::string) => + TypeError: could not convert argument 1 + void ::somefunc(uint8_t) => + ValueError: could not convert argument 1 (integer to character: value 65536 not in range [0,255]) + >>> + +Since C++ exceptions are converted to Python ones, there is an interplay +possible between the two as part of overload resolution and ``cppyy`` +allows C++ exceptions as such, enabling detailed type disambiguation and +input validation. +(The original use case was for filling database fields, requiring an exact +field label and data type match.) + +If, however, all methods fail and there is only one C++ exception (the other +exceptions originating from argument conversion, never succeeding to call +into C++), this C++ exception will be preferentially reported and will have +the original C++ type. `Return values` @@ -218,7 +362,7 @@ Well-written APIs will have clear clues in their naming convention about the ownership rules. For example, functions called ``New...``, ``Clone...``, etc. can be expected to return freshly allocated objects. -A simple name-matching in the pythonization then makes it simple to mark all +A basic name-matching in the pythonization then makes it simple to mark all these functions as creators. The return values are :ref:`auto-casted `. @@ -285,3 +429,40 @@ Example: 21 >>> +Python functions can be used to instantiate C++ templates, assuming the +type information of the arguments and return types can be inferred. +If this can not be done directly from the template arguments, then it can +be provided through Python annotations, by explicitly adding the +``__annotations__`` special data member (e.g. for older versions of Python +that do not support annotations), or by the function having been bound by +``cppyy`` in the first place. +For example: + + .. code-block:: python + + >>> import cppyy + >>> cppyy.cppdef("""\ + ... template + ... R callT(R(*f)(U...), A&&... a) { + ... return f(a...); + ... }""") + True + >>> def f(a: 'int') -> 'double': + ... return 3.1415*a + ... + >>> cppyy.gbl.callT(f, 2) + 6.283 + >>> def f(a: 'int', b: 'int') -> 'int': + ... return 3*a*b + ... + >>> cppyy.gbl.callT(f, 6, 7) + 126 + >>> + + +`extern "C"` +------------ + +Functions with C linkage are supported and are simply represented as +overloads of a single function. +Such functions are allowed both globally as well as in namespaces. diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/history.rst b/bindings/pyroot/cppyy/cppyy/doc/source/history.rst new file mode 100644 index 0000000000000..ed166a4be33b5 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/doc/source/history.rst @@ -0,0 +1,52 @@ +.. _history: + +History +======= + +.. toctree:: + :hidden: + +What is now called `cppyy` started life as `RootPython` from `CERN`_, but +cppyy is not associated with CERN (it is still used there, however, +underpinning `PyROOT`_). + +Back in late 2002, Pere Mato of CERN, had the idea of using the `CINT`_ C++ +interpreter, which formed the interactive interface to `ROOT`_, to call from +Python into C++: this became RootPython. +This binder interfaced with Python through `boost.python`_ (v1), transpiling +Python code into C++ and interpreting the result with CINT. +In early 2003, I ported this code to boost.python v2, then recently released. +In practice, however, re-interpreting the transpiled code was unusably slow, +thus I modified the code to make direct use of CINT's internal reflection +system, gaining about 25x in performance. +I presented this work as `PyROOT` at the ROOT Users' Workshop in early 2004, +and, after removing the boost.python dependency by using the C-API directly +(gaining another factor 7 in speedup!), it was included in ROOT. +PyROOT was presented at the SciPy'06 conference, but was otherwise not +advocated outside of High Energy Physics (HEP). + +In 2010, the PyPy core developers and I held a `sprint at CERN`_ to use +`Reflex`, a standalone alternative to CINT's reflection of C++, to add +automatic C++ bindings, PyROOT-style, to `PyPy`_. +This is where the name "cppyy" originated. +Coined by Carl Friedrich Bolz, if you want to understand the meaning, just +pronounce it slowly: cpp-y-y. + +After the ROOT team replaced CINT with `Cling`_, PyROOT soon followed. +As part of Google's Summer of Code '16, Aditi Dutta moved PyPy/cppyy to Cling +as well, and packaged the code for use through `PyPI`_. +I continued this integration with the Python eco-system by forking PyROOT, +reducing its dependencies, and repackaging it as CPython/cppyy. +The combined result is the current cppyy project. +Mid 2018, version 1.0 was released. + + +.. _`CERN`: https://cern.ch/ +.. _`PyROOT`: https://root.cern.ch/root/htmldoc/guides/users-guide/ROOTUsersGuide.html#python-interface +.. _`CINT`: https://en.wikipedia.org/wiki/CINT +.. _`ROOT`: https://root.cern.ch +.. _`boost.python`: https://wiki.python.org/moin/boost.python/GettingStarted +.. _`sprint at CERN`: https://morepypy.blogspot.com/2010/07/cern-sprint-report-wrapping-c-libraries.html +.. _`PyPy`: https://www.pypy.org/ +.. _`Cling`: https://github.com/vgvassilev/cling +.. _`PyPI`: https://pypi.org/ diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/index.rst b/bindings/pyroot/cppyy/cppyy/doc/source/index.rst index 84e50af93c54e..106de46bd6d5f 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/index.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/index.rst @@ -19,13 +19,16 @@ template instantiation, automatic object downcasting, exception mapping, and interactive exploration of C++ libraries. cppyy delivers this without any language extensions, intermediate languages, or the need for boiler-plate hand-written code. -For design and performance, see this `PyHPC paper`_, albeit that the -CPython/cppyy performance has been vastly improved since. +For design and performance, see this `PyHPC'16 paper`_, albeit that the +CPython/cppyy performance has been vastly improved since, as well as this +`CAAS presentation`_. +For a quick teaser, see `Jason Turner's`_ introduction video. cppyy is based on `Cling`_, the C++ interpreter, to match Python's dynamism, interactivity, and run-time behavior. Consider this session, showing dynamic, interactive, mixing of C++ and Python -features (more examples are in the `tutorial`_): +features (there are more examples throughout the documentation and in the +`tutorial`_): .. code-block:: python @@ -73,7 +76,7 @@ and heavy use of templates: .. code-block:: python >>> import cppyy - >>> cppyy.include('boost/any.hpp') + >>> cppyy.include('boost/any.hpp') # assumes you have boost installed >>> from cppyy.gbl import std, boost >>> val = boost.any() # the capsule >>> val.__assign__(std.vector[int]()) # assign it a std::vector @@ -113,9 +116,11 @@ code and many thousands of classes. cppyy minimizes dependencies to allow its use in distributed, heterogeneous, development environments. -.. _Cling: https://root.cern.ch/cling -.. _tutorial: https://bitbucket.org/wlav/cppyy/src/master/doc/tutorial/CppyyTutorial.ipynb?viewer=nbviewer&fileviewer=notebook-viewer%3Anbviewer -.. _`PyHPC paper`: http://wlav.web.cern.ch/wlav/Cppyy_LavrijsenDutta_PyHPC16.pdf +.. _Cling: https://github.com/vgvassilev/cling +.. _tutorial: https://github.com/wlav/cppyy/blob/master/doc/tutorial/CppyyTutorial.ipynb +.. _`PyHPC'16 paper`: http://wlav.web.cern.ch/wlav/Cppyy_LavrijsenDutta_PyHPC16.pdf +.. _`CAAS presentation`: https://www.youtube.com/watch?v=stMD7VDWlVU +.. _`Jason Turner's`: https://www.youtube.com/watch?v=TL83P77vZ1k .. _`Boost`: http://www.boost.org/ .. _`CPython`: http://python.org .. _`PyPy`: http://pypy.org @@ -126,6 +131,7 @@ development environments. Contents: .. toctree:: + :maxdepth: 1 changelog license @@ -143,13 +149,17 @@ development environments. :caption: Features :maxdepth: 1 + toplevel basic_types + strings classes functions type_conversions stl exceptions python + numba + cuda lowlevel misc debugging @@ -170,10 +180,18 @@ development environments. repositories testing +.. toctree:: + :caption: Background + :maxdepth: 1 + + history + philosophy + Bugs and feedback ----------------- Please report bugs or requests for improvement on the `issue tracker`_. -.. _`issue tracker`: https://bitbucket.org/wlav/cppyy/issues + +.. _`issue tracker`: https://github.com/wlav/cppyy/issues diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/installation.rst b/bindings/pyroot/cppyy/cppyy/doc/source/installation.rst index d59d12cb9d364..6b0356df81ec2 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/installation.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/installation.rst @@ -8,9 +8,15 @@ When installing through `conda-forge`_, ``conda`` will install the compiler for you, to match the other conda-forge packages. When using ``pip`` and the wheels from `PyPI`_, you minimally need gcc5, clang5, or MSVC'17. -When installing from source, the only requirement is full support for C++11 -(e.g. minimum gcc 4.8.1 on GNU/Linux), but older compilers than the ones -listed for the wheels have not been tested. + +.. note:: + + On Windows, a command prompt from which to run Python (or Python run + directly) needs to be opened from within an environment with MSVC setup, + otherwise the compiler will not be accessible. + +When installing from source, the only requirement is a compiler that supports +C++14, this in order to build LLVM. With CPython on Linux or MacOS, probably by far the easiest way to install cppyy, is through conda-forge on `Anaconda`_ (or `miniconda`_). @@ -29,15 +35,29 @@ and install from the conda-forge channel:: (WORK) $ conda install -c conda-forge cppyy (WORK) [current compiler] $ -To install with ``pip`` through `PyPI`_, it is recommend to use -`virtualenv`_ (or module `venv`_ for modern pythons). -The use of virtualenv prevents pollution of any system directories and allows -you to wipe out the full installation simply by removing the virtualenv + +To install with ``pip`` through `PyPI`_, use `venv`. +The use of virtual environment (`venv`) prevents pollution of any system directories and allows +you to wipe out the full installation simply by removing the virtual environment (`venv`) created directory ("WORK" in this example):: + $ python -m venv WORK + $ WORK\Scripts\activate + (WORK) $ python -m pip install cppyy + (WORK) $ + +.. note:: + If you are using python version less than 3.3, you should use `virtualenv` instead of `venv`. + First install virtualenv package that allows you to create virtual environment. + + $ python -m pip install virtualenv + $ virtualenv WORK + $ source WORK/bin/activate + (WORK) $ python -m pip install cppyy + (WORK) $ If you use the ``--user`` option to ``pip`` and use ``pip`` directly on the @@ -57,15 +77,7 @@ Wheels on PyPI Wheels for the backend (``cppyy-cling``) are available on PyPI for GNU/Linux, MacOS-X, and MS Windows (both 32b and 64b). - -The Linux wheels are built on manylinux, but with gcc 5.5, not the 4.8.2 that -ships with manylinux1, since cppyy exposes C++ APIs and g++ introduced -ABI incompatibilities starting with its 5 series forward. -Using 4.8.2 would have meant that any software using cppyy would have to -be (re)compiled for the older gcc ABI, which the odds don't favor. -Note that building cppyy fully with 4.8.2 (and requiring the old ABI across -the board) does work. - +The Linux wheels are built for manylinux2014, but with the dual ABI enabled. The wheels for MS Windows were build with MSVC Community Edition 2017. There are no wheels for the ``CPyCppyy`` and ``cppyy`` packages, to allow @@ -112,13 +124,14 @@ enough, the following can be made to work:: C++ standard with pip --------------------- -The C++17 standard is the default for Mac and Linux (both PyPI and -conda-forge); but it is C++14 for MS Windows (compiler limitation). +The C++20 standard is the default on all systems as of release 3.0.1 (both +PyPI and conda-forge); it is C++17 for older releases. When installing from PyPI using ``pip``, you can control the standard -selection by setting the ``STDCXX`` envar to '17', '14', or '11' (for Linux, -the backend does not need to be recompiled). -Note that the build will lower your choice if the compiler used does not -support a newer standard. +selection by setting the ``STDCXX`` envar to '20', '17', or '14' (for Linux, +the backend does not need to be recompiled) for the 3.x releases; '17', '14', +or '11' for the 2.x releases. +Note that the build will automatically lower your choice if the compiler used +does not support a newer standard. Install from source @@ -129,7 +142,7 @@ To build an existing release from source, tell ``pip`` to not download any binary wheels. Build-time only dependencies are ``cmake`` (for general build), ``python`` (obviously, but also for LLVM), and a modern C++ compiler (one that supports -at least C++11). +at least C++14). Use the envar ``STDCXX`` to control the C++ standard version; ``MAKE`` to change the ``make`` command, ``MAKE_NPROCS`` to control the maximum number of parallel jobs allowed, and ``VERBOSE=1`` to see full build/compile commands. @@ -208,6 +221,11 @@ You can then select the appropriate PCH with the ``CLING_STANDARD_PCH`` envar:: Or disable it completely by setting that envar to "none". +.. note:: + + Without the PCH, the default C++ standard will be the one with which + ``cppyy-cling`` was built. + .. _`conda-forge`: https://anaconda.org/conda-forge/cppyy .. _`Anaconda`: https://www.anaconda.com/distribution/ diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/license.rst b/bindings/pyroot/cppyy/cppyy/doc/source/license.rst index 20dfde8c22d7f..21e880acffc49 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/license.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/license.rst @@ -1,7 +1,7 @@ License and copyright ===================== -Copyright (c) 2017-2020, The Regents of the University of California, +Copyright (c) 2017-2021, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or @@ -51,6 +51,7 @@ the same conditions (except for some compatible licenses as retained in the source code): * CERN + * Lucio Asnaghi * Simone Bacchio * Robert Bradshaw * Ellis Breen diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/lowlevel.rst b/bindings/pyroot/cppyy/cppyy/doc/source/lowlevel.rst index 9e709ac772e14..c01396f0ff853 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/lowlevel.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/lowlevel.rst @@ -29,6 +29,22 @@ It needs to be imported explicitly: >>> +`LowLevelView` +-------------- + +Python has an elaborate array interface (buffer) specification, but no +standard library array type that completely implements it; instead, the +canonical Python array type is the NumPy one. +cppyy introduces the basic ``LowLevelView`` array class to avoid having a +direct dependency on NumPy and to guarantee zero copy. +The ``LowLevelView`` type gives access to array details such as the size, +type, etc. and allows reading/writing of array elements, both for interactive +use and through the buffer interface to allow NumPy to interface with them. +For more complex operations, it's recommended to copy from the +``LowLevelView`` inta a NumPy array, or to create a NumPy view (see below, +under :ref:`NumPy Casts `). + + `C/C++ casts` ------------- @@ -116,6 +132,8 @@ is required to turn it into something usable. The syntax is "template-style", just like for the C-style cast above. +.. _npcasts: + `NumPy casts` ------------- @@ -183,7 +201,7 @@ above for cppyy) the result of at least one of the following: Takes an optional ``byref`` parameter and if set to true, returns a pointer to the address instead. -* **as_ctypes**: Takes a cppyy bound C++ object and returns its address as +* **ll.as_ctypes**: Takes a cppyy bound C++ object and returns its address as a ``ctypes.c_void_p`` object. Takes an optional ``byref`` parameter and if set to true, returns a pointer to the address instead. @@ -235,9 +253,9 @@ C++ has three ways of allocating heap memory (``malloc``, ``new``, and ``new[]``) and three corresponding ways of deallocation (``free``, ``delete``, and ``delete[]``). Direct use of ``malloc`` and ``new`` should be avoided for C++ classes, as -these may override ``operator new`` to control their allocation own. +these may override ``operator new`` to control their own allocation. However these low-level allocators can be necessary for builtin types on -occassion if the C++ side takes ownership (otherwise, prefer either +occasion if the C++ side takes ownership (otherwise, prefer either ``array`` from the builtin module ``array`` or ``ndarray`` from Numpy). The low-level module adds the following functions: @@ -292,4 +310,18 @@ The low-level module adds the following functions: >>> +`argc/argv` +----------- + +C/C++'s ``main`` function can take the number of command line arguments +(``argc``) and their values (``argv``) as function arguments. +A common idiom has these values subsequently passed on to the entry point of +e.g. a framework or library. +Since the type of ``argv`` in particular (``char*[]``) is clunky to work with +in Python, the low level module contains two convenient helper functions, +``ll.argc()`` and ``ll.argv()``, that convert the command line arguments as +provided by Python's ``sys`` module, into typed values that are can be passed +to by C/C++. + + .. _`ctypes module`: https://docs.python.org/3/library/ctypes.html diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/misc.rst b/bindings/pyroot/cppyy/cppyy/doc/source/misc.rst index 60cf32a4dfc89..9eacb6e480e27 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/misc.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/misc.rst @@ -71,10 +71,15 @@ Global Interpreter Lock (GIL) release. whether C++ signals (such as SIGABRT) should be converted into Python exceptions. -* ``__cppname__``: a string that every C++ bound class carries and contains +* ``__cpp_name__``: a string that every C++ bound class carries and contains the actual C++ name (as opposed to ``__name__`` which has the Python name). This can be useful for template instantiations, documentation, etc. +* ``__cpp_template__``: a back-reference to the template used to instantiate + a templated class. + This variable only exists if the class was dynamically instantiated from + Python at least once. + `STL algorithms` ---------------- @@ -107,6 +112,10 @@ below can not be instantiated using a Python string, but the `Reduced typing` ---------------- +Note: ``from cppyy.interactive import *`` is no longer supported for CPython +3.11 and later because the ``dict`` object features it relies on have been +removed. + Typing ``cppyy.gbl`` all the time gets old rather quickly, but the dynamic nature of ``cppyy`` makes something like ``from cppyy.gbl import *`` impossible. @@ -171,3 +180,7 @@ available, but still saves some typing in may cases. For clarity of intent, it is recommended to use this instead of ``None`` (or the integer ``0``, which can serve in some cases), as ``None`` is better understood as ``void`` in C++. + +* **default value**: Represented with The untyped ``cppyy.default``. + The generic value ``cppyy.default`` will convert to the type specific default + value (per C++ rules) when used as a function argument or in assignment. diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/numba.rst b/bindings/pyroot/cppyy/cppyy/doc/source/numba.rst new file mode 100644 index 0000000000000..4cf83e99747aa --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/doc/source/numba.rst @@ -0,0 +1,363 @@ +.. _numba: + + +Numba support +============= + +.. caution:: + + This is an **experimental** feature, available starting with release + 2.4.0. + It is still incomplete (see listing below) and has only been tested on + Linux on x86_64. + +Numba `is a JIT compiler`_ for Python functions that can be statically typed +based on their input arguments. +Since C++ objects are always statically typed and already implemented at the +machine level, they can be dynamically integrated into the Numba type tracing +and lowering by exposing type details through C++ reflection at runtime. + +JIT-compiling traces of mixed Python/bound C++ code reduces, and in some +cases removes, the overhead of boxing/unboxing native data into their Python +proxies and vice versa. +It can also reduce or remove temporaries, especially for template +expressions. +Thus, there can be significant speedups for mixed code, beyond the Numba +compilation of Python code itself. +The current implementation integrates compiled C++ through function pointers, +object pointers, and pointer offsets, into the intermediate representation +(IR) as generated by Numba. +A future version may integrate Cling-generated IR directly into Numba IR (or +vice versa), e.g. if the C++ code is exposed from (precompiled) headers. +This would allow inlining of C++ code into Numba traces, for further +expected speedups. + + +Why Numba? +---------- + +The advertised premise of Numba is that it "makes Python code fast." +However, there is a much more compelling reason: Numba allows developers to +stay in their chosen ecosystem, be it Python or C++, in mixed environments, +without paying for their choice in lost performance. +For example, a Python developer using Numba does not need to rewrite a kernel +into C++ just to run performantly in a C++ framework. +Similarly, a C++ developer can use Numba to compile and create function +pointers to Python code for easy, performant, access. +This becomes even more compelling if the deployment target is a GPU, which +would otherwise certainly require a rewrite of the Python code. +Add that Numba, as a JIT-compiler, is fully run-time just like ``cppyy``, +and the use case for integration is clear. +(Numba does not currently provide support for C++.) + + +Usage +------- + +``cppyy`` does not use Numba extension hooks to minimize accidental +dependencies. +Instead, it requires that the extensions are loaded explicitly by any code +that uses it:: + + import cppyy.numba_ext + +After that, Numba is able to trace ``cppyy`` bound code when applying the +usual ``numba.njit`` decorator. + +Numba type declarations are done lazily, with the ``numba_ext`` module only +initially registering hooks on proxy base classes, to keep overheads in +Numba's type-resolution to a minimum. +On use in a JITed trace, each C++ type or function call is refined to the +actual, concrete types and type-specific overloads, with templates +instantiated as-needed. +Where possible, lowering is kept generic to reduce the number of callbacks +in Numba's compilation chain. + + +Examples +-------- + +The following, non-exhaustive, set of examples gives an idea of the +current level of support. +More examples can be found in the `test suite`_. + +C++ free (global) functions can be called and overloads will be selected, or +a template will be instantiated, based on the provided types. +Exact type matches are fully supported, there is some support for typedefs +add implicit conversions for builtin types, there is no support for +conversions of custom types or default arguments. + +- **Basic usage**: To use ``cppyy`` in Numba JITed code, simply import + ``cppyy.numba_ext``, after which further use is transparent and the same + as when otherwise using ``cppyy`` in Python. + Example: + +.. code-block:: python + + >>> import numba + >>> import cppyy + >>> import cppyy.numba_ext # enables numba to work with cppyy + >>> import math + >>> @numba.jit(nopython=True) + ... def cpp_sqrt(x): + ... return cppyy.gbl.sqrt(x) # direct use, no extra setup required + >>> print("Sqrt of 4: ", cpp_sqrt(4.0)) + Sqrt of 4: 2.0 + >>> print("Sqrt of Pi: ", cpp_sqrt(math.pi)) + Sqrt of Pi: 1.7724538509055159 + + +- **Overload selection**: C++ overloads provide different implementations + for different argument types (not to be confused with Numba overloads, + which provide different implementations for the same argument types). + Unfortunately, mapping of Python types to C++ types is often not exact, + so a "best match" is chosen, similarly to what ``cppyy`` normally does. + However, the latter, being dynamic, is more flexible. + For example, best-match C++ integer type can be value dependent, whereas + in the Numba trace, it is by definition fixed at JIT time. + Example: + +.. code-block:: python + + >>> cppyy.cppdef(""" + ... int mul(int x) { return x * 2; } + ... float mul(float x) { return x * 3; } + ... """) + >>> @numba.jit(nopython=True) + ... def oversel(a): + ... total = type(a[0])(0) + ... for i in range(len(a)): + ... total += cppyy.gbl.mul(a[i]) + ... return total + + >>> a = np.array(range(10), dtype=np.float32) + >>> print("Array: ", a) + Array: [0. 1. 2. 3. 4. 5. 6. 7. 8. 9.] + >>> print("Overload selection output: ", oversel(a)) + Overload selection output: 135.0 + >>> a = np.array(range(10), dtype=np.int32) + >>> print("Array: ", a) + Array: [0 1 2 3 4 5 6 7 8 9] + >>> print("Overload selection output: ", oversel(a)) + Overload selection output: 90 + +- **Template instantiation**: templates are instantiated as needed as part + of the overload selection. + The best match is done for the arguments provided at the point of first + use. + If those arguments vary based on program input, it may make sense to + provide Numba with type hints and pre-compile such functions. + Example: + +.. code-block:: python + + >>> import cppyy + >>> import cppyy.numba_ext + >>> import numba + >>> import numpy as np + >>> cppyy.cppdef(""" + ... template + ... T square(T t) { return t*t; } + ... """) + >>> @numba.jit(nopython=True) + ... def tsa(a): + ... total = type(a[0])(0) + ... for i in range(len(a)): + ... total += cppyy.gbl.square(a[i]) + ... return total + >>> a = np.array(range(10), dtype=np.float32) + >>> print("Float array: ", a) + Float array: [0. 1. 2. 3. 4. 5. 6. 7. 8. 9.] + >>> print("Sum of squares: ", tsa(a)) + Sum of squares: 285.0 + >>> print() + >>> a = np.array(range(10), dtype=np.int32) + >>> print("Integer array: ", a) + Integer array: [0 1 2 3 4 5 6 7 8 9] + >>> print("Sum of squares: ", tsa(a)) + Sum of squares: 285 + + +Instances of C++ classes can be passed into Numba traces. +They can be returned from functions called *within* the trace, but cannot yet +be returned *from* the trace. +Their public data is accessible (read-only) if of built-in type and their +public methods can be called, for which overload selection works. +Example: + +.. code-block:: python + + >>> import cppyy + >>> import numba + >>> import numpy as np + >>> + >>> cppyy.cppdef("""\ + ... class MyData { + ... public: + ... MyData(int i, int j) : fField1(i), fField2(j) {} + ... + ... public: + ... int get_field1() { return fField1; } + ... int get_field2() { return fField2; } + ... + ... MyData copy() { return *this; } + ... + ... public: + ... int fField1; + ... int fField2; + ... };""") + True + >>> @numba.jit(nopython=True) + >>> def tsdf(a, d): + ... total = type(a[0])(0) + ... for i in range(len(a)): + ... total += a[i] + d.fField1 + d.fField2 + ... return total + ... + >>> d = cppyy.gbl.MyData(5, 6) + >>> a = np.array(range(10), dtype=np.int32) + >>> print(tsdf(a, d)) + 155 + >>> # example of method calls + >>> @numba.jit(nopython=True) + >>> def tsdm(a, d): + ... total = type(a[0])(0) + ... for i in range(len(a)): + ... total += a[i] + d.get_field1() + d.get_field2() + ... return total + ... + >>> print(tsdm(a, d)) + 155 + >>> # example of object return by-value + >>> @numba.jit(nopython=True) + >>> def tsdcm(a, d): + ... total = type(a[0])(0) + ... for i in range(len(a)): + ... total += a[i] + d.copy().fField1 + d.get_field2() + ... return total + ... + >>> print(tsdcm(a, d)) + 155 + >>> + + +Demo: Numba physics example +--------------------------- + +Motivating example taken from: +`numba_scalar_impl.py `_ + +.. code-block:: python + + >>> import numba + >>> import cppyy + >>> import cppyy.numba_ext + ... + >>> cppyy.cppdef(""" + ... #include + ... struct Atom { + ... float x; + ... float y; + ... float z; + ... }; + ... + ... std::vector atoms = {{1, 2, 3}, {2, 3, 4}, {3, 4, 5}, {4, 5, 6}, {5, 6, 7}}; + ... """) + ... + >>> @numba.njit + >>> def lj_numba_scalar(r): + ... sr6 = (1./r)**6 + ... pot = 4.*(sr6*sr6 - sr6) + ... return pot + + >>> @numba.njit + >>> def distance_numba_scalar(atom1, atom2): + ... dx = atom2.x - atom1.x + ... dy = atom2.y - atom1.y + ... dz = atom2.z - atom1.z + ... + ... r = (dx * dx + dy * dy + dz * dz) ** 0.5 + ... + ... return r + ... + >>> def potential_numba_scalar(cluster): + ... energy = 0.0 + ... for i in range(cluster.size() - 1): + ... for j in range(i + 1, cluster.size()): + ... r = distance_numba_scalar(cluster[i], cluster[j]) + ... e = lj_numba_scalar(r) + ... energy += e + ... + ... return energy + ... + >>> print("Total lennard jones potential =", potential_numba_scalar(cppyy.gbl.atoms)) + Total lennard jones potential = -0.5780277345740283 + + +Overhead +-------- + +The main overhead of JITing Numba traces is in the type annotation in Numba +itself, optimization of the IR and assembly by the backend less so. +(There is also a non-negligible cost to Numba initialization, which is why +``cppyy`` does not provide automatic extension hooks.) +The use of ``cppyy`` bound C++, which relies on the same Numba machinery, +does not change that, since the reflection-based lookups are in C++ and +comparatively very fast. +For example, there is no appreciable difference in wall clock time to JIT a +trace using Numba's included math functions (from module ``math`` or +``numpy``) or one that uses C++ bound ones whether from the standard library +or a templated versions from e.g. Eigen. +Use of very complex template expressions may change this balance, but in +principle, wherever it makes sense in the first place to use Numba JITing, it +is also fine, performance-wise, to use ``cppyy`` bound C++ inside the trace. + +A second important overhead is in unboxing Python proxies of C++ objects, +in particular when passed as an argument to a Numba-JITed function. +The main costs are in the lookup (types are matched at every invocation) and +to a lesser extent the subsequent copying of the instance data. +Thus, functions that take a C++ object as an argument will require more time +spent in the function body for JITing to be worth it than functions that do +not. + +The current implementation invokes C++ callables through function pointers +and accesses data through offsets calculations from the object's base +address. +A future implementation may be able to inline C++ into the Numba trace if +code is available in headers files or was JITed. + + +Further Information +------------------- + +- Numba documentation: + `numba.readthedocs.io `_. + +- "Using C++ From Numba, Fast and Automatic", presented at `PyHEP 2022 `_ + + - `PyHEP 2022 video `_ + - `PyHEP 2022 slides `_ + - `PyHEP 2022 notebook `_ + +- Presentation at CERN's ROOT Parallelism, Performance and Programming Model + (`PPP `_) Meeting + + - `PPP slides `_ + - `PPP notebook `_ + + +Acknowledgements +---------------- + +This work is supported in part by the `Compiler Research Organization`_ +(Princeton University), with contributions from +`Vassil Vassilev `_, +`Baidyanath Kundu `_, and +`Aaron Jomy `_. + + +.. _is a JIT compiler: https://numba.pydata.org/ + +.. _test suite: https://github.com/wlav/cppyy/blob/master/test/test_numba.py + +.. _Compiler Research Organization: https://compiler-research.org/ diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/packages.rst b/bindings/pyroot/cppyy/cppyy/doc/source/packages.rst index f20430213ac4d..aa8aa3259d222 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/packages.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/packages.rst @@ -43,7 +43,7 @@ following structure:: (A) _cppyy (PyPy) / \ - (1) cppyy (3) cling-backend -- (4) cppyy-cling + (1) cppyy (3) cppyy-backend -- (4) cppyy-cling \ / (2) CPyCppyy (CPython) diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/philosophy.rst b/bindings/pyroot/cppyy/cppyy/doc/source/philosophy.rst new file mode 100644 index 0000000000000..4ff9a2b7dd44b --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/doc/source/philosophy.rst @@ -0,0 +1,236 @@ +.. _philosophy: + +Philosophy +========== + +.. toctree:: + :hidden: + +As a Python-C++ language binder, cppyy has several unique features: it fills +gaps and covers use cases not available through other binders. +This document explains some of the design choices made and the thinking +behind the implementations of those features. +It's categorized as "philosophy" because a lot of it is open to +interpretation. +Its main purpose is simply to help you decide whether cppyy covers your use +cases and binding requirements, before committing any time to +:ref:`trying it out `. + + +Run-time v.s. compile-time +-------------------------- + +What performs better, run-time or compile-time? +The obvious answer is compile-time: see the performance differences between +C++ and Python, for example. +Obvious, but completely wrong, however. +In fact, when it comes to Python, it is even the `wrong question.` + +Everything in Python is run-time: modules, classes, functions, etc. are all +run-time constructs. +A Python module that defines a class is a set of instructions to the Python +interpreter that lead to the construction of the desired class object. +A C/C++ extension module that defines a class does the same thing by calling +a succession of Python interpreter Application Programming Interfaces (APIs; +the exact same that Python uses itself internally). +If you use a compile-time binder such as `SWIG`_ or `pybind11`_ to bind a C++ +class, then what gets compiled is the series of API calls necessary to +construct a Python-side equivalent at `run-time` (when the module gets +loaded), not the Python class object. +In short, whether a binding is created at "compile-time" or at run-time has +no measurable bearing on performance. + +What does affect performance is the overhead to cross the language barrier. +This consists of unboxing Python objects to extract or convert the underlying +objects or data to something that matches what C++ expects; overload +resolution based on the unboxed arguments; offset calculations; and finally +the actual dispatch. +As a practical matter, overload resolution is the most costly part, followed +by the unboxing and conversion. +Best performance is achieved by specialization of the paths through the +run-time: recognize early the case at hand and select an optimized path. +For that reason, `PyPy`_ is so fast: JIT-ed traces operate on unboxed objects +and resolved overloads are baked into the trace, incurring no further cost. +Similarly, this is why pybind11 is so slow: its code generation is the C++ +compiler's template engine, so complex path selection and specialization is +very hard to do in a performance-portable way. + +In cppyy, a great deal of attention has gone into built-in specialization +paths, which drives its performance. +For example, basic inheritance sequentially lines up classes, whereas +multiple (virtual) inheritance usually requires thunks. +Thus, when calling base class methods on a derived instance, the latter +requires offset calculations that depend on that instance, whereas the former +has fixed offsets fully determined by the class definitions themselves. +By labeling classes appropriately, single inheritance classes (by far the +most common case) do not incur the overhead in PyPy's JIT-ed traces that is +otherwise unavoidable for multiple virtual inheritance. +As another example, consider that the C++ standard does not allow modifying +a ``std::vector`` while looping over it, whereas Python has no such +restriction, complicating loops. +Thus, cppyy has specialized ``std::vector`` iteration for both PyPy and +CPython, easily outperforming looping over an equivalent numpy array. + +In CPython, the performance of `non-overloaded` function calls depends +greatly on the Python interpreter's internal specializations; and Python3 +has many specializations specific to basic extension modules (C function +pointer calls), gaining a performance boost of more than 30% over Python2. +Only since Python3.8 is there also better support for closure objects (vector +calls) as cppyy uses, to short-cut through the interpreter's own overhead. + +As a practical consideration, whether a binder performs well on code that you +care about, depends `entirely` on whether it has the relevant specializations +for your most performance-sensitive use cases. +The only way to know for sure is to write a test application and measure, but +a binder that provides more specializations, or makes it easy to add your +own, is more likely to deliver. + + +`Manual v.s. automatic` +----------------------- + +Python is, today, one of the most popular programming languages and has a +rich and mature eco-system around it. +But when the project that became cppyy started in the field of High Energy +Physics (HEP), Python usage was non-existent there. +As a Python user to work in this predominantly C++ environment, you had to +bring your own bindings, thus automatic was the only way to go. +Binders such as SWIG, SIP (or even boost.python with Pyste) all had the fatal +assumption that you were providing Python bindings to your `own` C++ code, +and that you were thus able to modify those (many) areas of the C++ codes +that their parsers could not handle. +The `CINT`_ interpreter was already well established in HEP, however, and +although it, too, had many limitations, C++ developers took care not to write +code that it could not parse. +In particular, since CINT drove automatic I/O, all data classes as needed for +analysis were parsable by CINT and consequently, by using CINT for the +bindings, at the very least one could run any analysis in Python. +This was key. + +Besides not being able to parse some code (a problem that's history for cppyy +since moving to Cling), all automatic parsers suffer from the problem that +the bindings produced have a strong "C++ look-and-feel" and that choices need +to be made in cases that can be bound in different, equally valid, ways. +As an example of the latter, consider the return of an ``std::vector``: +should this be automatically converted to a Python ``list``? +Doing so is more "pythonic", but incurs a significant overhead, and no +automatic choice will satisfy all cases: user input is needed. + +The typical way to solve these issues, is to provide an intermediate language +where corner cases can be brushed up, code can be made more Python friendly, +and design choices can be resolved. +Unfortunately, learning an intermediate language is quite an investment in +time and effort. +With cppyy, however, no such extra language is needed: using Cling, C++ code +can be embedded and JIT-ed for the same purpose. +In particular, cppyy can handle `boxed` Python objects and the full Python +C-API is available through Cling, allowing complete manual control where +necessary, and all within a single code base. +Similarly, a more pythonistic look-and-feel can be achieved in Python itself. +As a rule, Python is always the best place, far more so than any intermediate +language, to do Python-thingies. +Since all bound proxies are normal Python classes, functions, etc., Python's +introspection (and regular expressions engine) can be used to provide rule +based improvements in a way similar to the use of directives in an +intermediate language. + +On a practical note, it's often said that an automatic binder can provide +bindings to 95% of your code out-of-the-box, with only the remaining part +needing manual intervention. +This is broadly true, but realize that that 5% contains the most difficult +cases and is where 20-30% of the effort would have gone in case the bindings +were done fully manually. +It is therefore important to consider what manual tools an automatic binder +offers and to make sure they fit your work style and needs, because you are +going to spend a significant amount of time with them. + + +`LLVM dependency` +----------------- + +cppyy depends on `LLVM`_, through Cling. +LLVM is properly internalized, so that it doesn't conflict with other uses; +and in particular it is fine to mix `Numba`_ and cppyy code. +It does mean a download cost of about 20MB for the binary wheel (exact size +differs per platform) on installation, and additional `primarily initial` +memory overheads at run-time. +Whether this is onerous depends strongly not only on the application, but +also on the rest of the software stack. + +The initial cost of loading cppyy, and thus starting the Cling interpreter, +is about 45MB (platform dependent). +Initial uses of standard (e.g. STL) C++ results in deserialization of the +precompiled header at another eventual total cost of about 25MB (again, +platform dependent). +The actual bindings of course also carry overheads. +As a rule of thumb, you should budget for ~100MB all-in for the overhead +caused by the bindings. + +Other binders do not have this initial memory overhead, but do of course +occur an overhead per module, class, function, etc. +At scale, however, cppyy has some advantages: all binding is lazy (including +the option of automatic loading), standard classes are never duplicated, and +there is no additional "per-module" overhead. +Thus, eventually (depending on the number of classes bound, across how many +modules, what use fraction, etc.), this initial cost is recouped when +compared to other binders. +As a rule of thumb, if about 10% of classes are used, it takes several +hundreds of bound classes before the cppyy-approach is beneficial. +In High Energy Physics, from which it originated, cppyy is regularly used in +software stacks of many thousands of classes, where this advantage is very +important. + + +`Distributing headers` +---------------------- + +cppyy requires C/C++ headers to be available at run-time, which was never a +problem in the developer-centric world from which it originated: software +always had supported C++ APIs already, made available through header files, +and Python simply piggy-backed onto those. +JIT-ing code in those headers, which potentially picked up system headers +that were configured differently, was thus also never a problem. +Or rather, the same problem exists for C++, and configuration for C++ to +resolve potential issues translates transparently to Python. + +There are only two alternatives: precompile headers into LLVM bitcode and +distribute those or provide a restricted set of headers. +Precompiled headers (and modules) were never designed to be portable and +relocatable, however, thus that may not be the panacea it seems. +A restricted set of headers is some work, but cppyy can operate on abstract +interface classes just fine (including Python-side cross-inheritance). + + +`Large deployment` +------------------ + +The single biggest headache in maintaining an installation of Python +extension modules is that Python patch releases can break them. +The two typical solutions are to either restrict the choice of Python +interpreter and version that are supported (common in HPC) or to provide +binaries (wheels) for a large range of different interpreters and versions +(as e.g. done for conda). + +In the case of cppyy, only CPython/CPyCppyy and PyPy/_cppyy (an internal +module) depend on the Python interpreter (see: +:ref:`Package Structure `). +The user-facing ``cppyy`` module is pure Python and the backend (Cling) is +Python-independent. +Most importantly, since all bindings are generated at run-time, there are no +extension modules to regenerate and/or recompile. + +Thus, the end-user only needs to rebuild/reinstall CPyCppyy for each relevant +version of Python (and nothing extra is needed for PyPy) to switch Python +versions and/or interpreter. +The rest of the software stack remains completely unchanged. +Only if Cling in cppyy's backend is updated, which happens infrequently, and +non-standard precompiled headers or modules are used, do these need to be +rebuild in full. + + +.. _`SWIG`: http://swig.org/ +.. _`pybind11`: https://pybind11.readthedocs.io/en/stable/ +.. _`PyPy`: https://www.pypy.org/ +.. _`CINT`: https://en.wikipedia.org/wiki/CINT +.. _`LLVM`: https://llvm.org/ +.. _`Numba`: http://numba.pydata.org/ diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/pythonizations.rst b/bindings/pyroot/cppyy/cppyy/doc/source/pythonizations.rst index effb89902d672..71049e505e96c 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/pythonizations.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/pythonizations.rst @@ -67,31 +67,40 @@ function ``GetLength`` and replaces it with Python's ``__len__``: .. code-block:: python - import cppyy - - def replace_getlength(klass, name): - try: - klass.__len__ = klass.__dict__['GetLength'] - except KeyError: - pass - - cppyy.py.add_pythonization(replace_getlength, 'MyNamespace') - - cppyy.cppdef(""" - namespace MyNamespace { - class MyClass { - public: - MyClass(int i) : fInt(i) {} - int GetLength() { return fInt; } - - private: - int fInt; - }; - }""") - - m = cppyy.gbl.MyNamespace.MyClass(42) - assert len(m) == 42 - + >>> import cppyy + >>> + >>> def replace_getlength(klass, name): + ... try: + ... klass.__len__ = klass.__dict__['GetLength'] + ... del klass.GetLength + ... except KeyError: + ... pass + ... + >>> cppyy.py.add_pythonization(replace_getlength, 'MyNamespace') + >>> + >>> cppyy.cppdef(""" + ... namespace MyNamespace { + ... class MyClass { + ... public: + ... MyClass(int i) : fInt(i) {} + ... int GetLength() { return fInt; } + ... + ... private: + ... int fInt; + ... }; + ... }""") + True + >>> m = cppyy.gbl.MyNamespace.MyClass(42) + >>> len(m) + 42 + >>> m.GetLength() + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'MyClass' object has no attribute 'GetLength' + >>> + +The deletion of ``GetLength`` method with ``del`` can be omitted +if both ``MyClass.GetLength`` and ``MyClass.__len__`` should be valid. C++ callbacks ------------- @@ -118,5 +127,46 @@ which is also called for all derived classes. Just as with the Python callbacks, the first argument will be the Python class proxy, the second the C++ name, for easy filtering. When called, cppyy will be completely finished with the class proxy, so any -and all changes, including such low-level ones such as the replacement of -iteration or buffer protocols, are fair game. +and all changes are fair game, including the low-level ones such as the replacement of +iteration or buffer protocols. + +An example pythonization replacing ``MyClass.GetLength`` method with Python's ``__len__`` +done with the C++ callbacks: + +.. code-block:: python + + >>> import cppyy + >>> + >>> cppyy.cppdef(""" + ... #include + ... + ... namespace MyNamespace { + ... class MyClass { + ... public: + ... MyClass(int i) : fInt(i) {} + ... int GetLength() { return fInt; } + ... + ... private: + ... int fInt; + ... + ... // pythonizations + ... public: + ... static void __cppyy_pythonize__(PyObject* klass, const std::string&){ + ... auto cppName = "GetLength"; + ... auto pythonizationName = "__len__"; + ... auto* methodObject = PyObject_GetAttrString(klass, cppName); + ... PyObject_SetAttrString(klass, pythonizationName, methodObject); + ... Py_DECREF(methodObject); + ... PyObject_DelAttrString(klass, cppName); + ... } + ... }; + ... }""") + True + >>> m = cppyy.gbl.MyNamespace.MyClass(42) + >>> len(m) + 42 + >>> m.GetLength() + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'MyClass' object has no attribute 'GetLength' + >>> diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/repositories.rst b/bindings/pyroot/cppyy/cppyy/doc/source/repositories.rst index 87662e8bc6e17..37951760ea9e8 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/repositories.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/repositories.rst @@ -10,10 +10,10 @@ Because of this layering and because it leverages several existing packages through reuse, the relevant codes are contained across a number of repositories. -* Frontend, cppyy: https://bitbucket.org/wlav/cppyy -* CPython (v2/v3) intermediate: https://bitbucket.org/wlav/cpycppyy -* PyPy intermediate (module _cppyy): https://bitbucket.org/pypy/pypy/ -* Backend, cppyy: https://bitbucket.org/wlav/cppyy-backend +* Frontend, cppyy: https://github.com/wlav/cppyy +* CPython (v2/v3) intermediate: https://github.com/wlav/CPyCppyy +* PyPy intermediate (module _cppyy): https://foss.heptapod.net/pypy +* Backend, cppyy: https://github.com/wlav/cppyy-backend The backend repo contains both the cppyy-cling (under "cling") and cppyy-backend (under "clingwrapper") packages. @@ -53,15 +53,28 @@ For example:: > set TMP=C:\TMP > set TEMP=C:\TMP -Start with the ``cppyy-cling`` package (cppyy-backend repo, subdirectory -"cling"), which requires source to be pulled in from upstream, and thus takes -a few extra steps:: +The first package to build is ``cppyy-cling``. +It may take a long time, especially on a laptop (Mac ARM being a notable +exception), since Cling comes with a builtin version of LLVM/Clang. +Consider therefore for a moment your reasons for building from source: there +being no pre-built wheel for the platform that you're interested in or simply +needing the latest version from the repository; or perhaps you are planning +to develop/modify the sources. - $ git clone https://bitbucket.org/wlav/cppyy-backend.git +If the former, clone the repository, check out a specific tagged release as +needed, then run the following steps to add Cling and build a wheel. +Once built, install the wheel as appropriate:: + + $ git clone https://github.com/wlav/cppyy-backend.git $ cd cppyy-backend/cling $ python setup.py egg_info $ python create_src_directory.py - $ python -m pip install . --upgrade + $ python setup.py bdist_wheel + $ python -m pip install dist/cppyy_cling-* --upgrade + +.. note:: + ``cppyy-cling`` wheels do not depend on the Python interpreter and can + thus be re-used for any version of Python or PyPy. The ``egg_info`` setup command is needed for ``create_src_directory.py`` to find the right version. @@ -69,47 +82,80 @@ That script in turn downloads the proper release from `upstream`_, trims and patches it, and installs the result in the "src" directory. When done, the structure of ``cppyy-cling`` looks again like a PyPA package -and can be used/installed as expected, here using ``pip``. - -The ``cppyy-cling`` package, because it contains Cling/Clang/LLVM, is rather -large to build, so by default the setup script will use all cores (x2 if -hyperthreading is enabled). -You can change this behavior with the ``MAKE_NPROCS`` envar. -The wheel of ``cppyy-cling`` is reused by pip for all versions of CPython and -PyPy, thus the long compilation is needed only once for all different -versions of Python on the same machine. - -Unless you build on the manylinux1 docker images, wheels for ``cppyy``, -``CPyCppyy``, and ``cppyy-backend`` are disabled, because ``setuptools`` (as -used by ``pip``) does not properly resolve dependencies for wheels. -You will see a harmless "error" message to that effect fly by in the (verbose) -output. -You can force manual build of those wheels, as long as you make sure that you -have the proper dependencies *installed*, using ``--force-bdist``, when -building from the repository. +and can be used/installed as expected, here done with ``pip``. + +By default, the setup script will use all cores (x2 if hyperthreading is +enabled). +You can change this behavior by setting the ``MAKE_NPROCS`` envar to the +desired number of allowable sub jobs. + +If on the other hand you are building from source to develop/modify +``cppyy-cling``, consider using the ``cmake`` interface. +The first installation will still be just as slow, but subsequent builds can +be incremental and thus much faster. +For this use, first install the latest version from a pre-built wheel, which +will setup the proper directory structure, then use cmake to build and +install the latest or modified version of ``cppyy-cling`` into that:: + + $ python -m pip install cppyy-cling + $ git clone https://github.com/wlav/cppyy-backend.git + $ cd cppyy-backend/cling + $ python setup.py egg_info + $ python create_src_directory.py + $ mkdir dev + $ cd dev + $ cmake ../src -Wno-dev -DCMAKE_CXX_STANDARD=17 -DLLVM_ENABLE_EH=0 -DLLVM_ENABLE_RTTI=0 -DLLVM_ENABLE_TERMINFO=0 -DLLVM_ENABLE_ASSERTIONS=0 -Dminimal=ON -Druntime_cxxmodules=OFF -Dbuiltin_zlib=ON -Dbuiltin_cling=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX= + $ make -j install + +where the ``cmake`` command needs to be given the full path to +`site-packages/cppyy_backend` in the virtual environment or other +installation location. +Adjust other options (esp. ``CMAKE_CXX_STANDARD``) as needed. +For the build command, adjust the ``cmake`` command as appropriate for your +favorite, or platform-specific, build system and/or use ``cmake --build`` +instead of ``make`` directly. +See the `cmake documentation`_ for details. Next up is ``cppyy-backend`` (cppyy-backend, subdirectory "clingwrapper"; omit the first step if you already cloned the repo for ``cppyy-cling``):: - $ git clone https://bitbucket.org/wlav/cppyy-backend.git + $ git clone https://github.com/wlav/cppyy-backend.git $ cd cppyy-backend/clingwrapper - $ python -m pip install . --upgrade + $ python -m pip install . --upgrade --no-use-pep517 --no-deps + +Note the use of ``--no-use-pep517``, which prevents ``pip`` from needlessly +going out to pypi.org and creating a local "clean" build environment from the +cached or remote wheels. +Instead, by skipping PEP 517, the local installation will be used. +This is imperative if there was a change in public headers or if the version +of ``cppyy-cling`` was locally updated and is thus not available on PyPI. Upgrading ``CPyCppyy`` (if on CPython; it's not needed for PyPy) and ``cppyy`` is very similar:: - $ git clone https://bitbucket.org/wlav/CPyCppyy.git + $ git clone https://github.com/wlav/CPyCppyy.git $ cd CPyCppyy - $ python -m pip install . --upgrade + $ python -m pip install . --upgrade --no-use-pep517 --no-deps + +Just like ``cppyy-cling``, ``CPyCppyy`` has ``cmake`` scripts which are the +recommended way for development, as incremental builds are faster:: + + $ mkdir build + $ cmake ../CPyCppyy + $ make -j + +then simply point the ``PYTHONPATH`` envar to the `build` directory above to +pick up the local `cppyy.so` module. Finally, the top-level package ``cppyy``:: - $ git clone https://bitbucket.org/wlav/cppyy.git + $ git clone https://github.com/wlav/cppyy.git $ cd cppyy - $ python -m pip install . --upgrade + $ python -m pip install . --upgrade --no-deps Please see the `pip documentation`_ for more options, such as developer mode. .. _`setuptools`: https://setuptools.readthedocs.io/ .. _`upstream`: https://root.cern.ch/download/ +.. _`cmake documentation`: https://cmake.org/ .. _`pip documentation`: https://pip.pypa.io/ diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/stl.rst b/bindings/pyroot/cppyy/cppyy/doc/source/stl.rst index 376eb33e5603e..8db0710441289 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/stl.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/stl.rst @@ -40,8 +40,8 @@ The rest of this section shows examples of how STL containers can be used in a natural, pythonistic, way. -`vector` --------- +`std::vector` +------------- A ``std::vector`` is the most commonly used C++ container type because it is more efficient and performant than specialized types such as ``list`` and @@ -55,12 +55,19 @@ In practice, it can interplay well with all these containers, but e.g. efficiency and performance can differ significantly. A vector can be instantiated from any sequence, including generators, and -vectors of objects can be recursively constructed: +vectors of objects can be recursively constructed. +If the template type is to be inferred from the argument to the constructor, +the first element needs to be accessible, which precludes generators. .. code-block:: python >>> from cppyy.gbl.std import vector, pair - >>> v = vector[int](range(10)) + >>> v = vector[int](range(10)) # from generator + >>> len(v) + 10 + >>> v = vector([x for x in range(10)]) # type inferred + >>> type(v) + at 0x12d226f00> >>> len(v) 10 >>> vp = vector[pair[int, int]](((1, 2), (3, 4))) @@ -71,7 +78,7 @@ vectors of objects can be recursively constructed: >>> To extend a vector in-place with another sequence object, use ``+=``, just as -would work for Python's list: +for Python's ``list``: .. code-block:: python @@ -79,10 +86,9 @@ would work for Python's list: >>> len(v) 20 >>> - -The easiest way to print the full contents of a vector, is by using a list -and printing that instead. -Indexing and slicing of a vector follows the normal Python slicing rules: + +Indexing and slicing of a vector follows the normal Python slicing rules; +printing a vector prints all its elements: .. code-block:: python @@ -92,12 +98,12 @@ Indexing and slicing of a vector follows the normal Python slicing rules: 19 >>> v[-4:] object at 0x7f9051057650> - >>> list(v[-4:]) - [16, 17, 18, 19] + >>> print(v[-4:]) + { 6, 7, 8, 9 } >>> The usual iteration operations work on vector, but the C++ rules still apply, -so a vector that is being iterated over can `not` be modified in the loop +so a vector that is being iterated over can *not* be modified in the loop body. (On the plus side, this makes it much faster to iterate over a vector than, say, a numpy ndarray.) @@ -208,6 +214,69 @@ To be sure, the code is `too` strict in the simplistic example above, and with a future version of Cling it should be possible to lift some of these restrictions without causing incorrect results. + +`std::map` +---------- + +C++'s ``map`` is an associative container similar to Python's ``dict``, +albeit one that has stronger type constraints. +A ``map`` can be instantiated from a ``dict`` (and types can be inferred) or +from a collection of ``pair`` mappings. + + .. code-block:: python + + >>> from cppyy.gbl.std import map + >>> m = map[str, int](*("one", 1), ("two", 2))) # type explicit, from pairs + >>> print(m) + { "one" => 1, "two" => 2 } + >>> m = map({1: "one", 2: "two"}) # type implicit, from dict + >>> type(m) + at 0x12d068d60> + >>> print(m) + { 1 => "one", 2 => "two" } + >>> + + +`std::string` +------------- + +Python's `str` is a unicode type since Python3, whereas ``std::string`` is +single-byte char-based. +Having the two correctly interact therefore deserves it's own +:doc:`chapter `. + + +`std::tuple` +------------ + +C++ ``tuple`` is supported but it should be noted that its use, and in +particular instantiating (heavily overloaded) ``get<>`` functions for member +access is inefficient. +They are really only meant for use when you have to pass a ``tuple`` to C++ +code; and if returned from a C++ function, it is easier to simply unpack them. +In all other cases, prefer Python's builtin ``tuple``. +Example usage: + + .. code-block:: python + + >>> from cppyy.gbl.std import make_tuple, get + >>> t = make_tuple(1, '2', 5.) + >>> print(t) + object at 0x12033ee70> + >>> len(t) + 3 + >>> get[0](t) # access with templated std::get<> + 1 + >>> get[1](t) + b'2' + >>> get[2](t) + 5.0 + >>> a, b, c = t # unpack through iteration + >>> print(a, b, c) + 1 2 5.0 + >>> + + .. rubric:: Footnotes .. [#f1] The meaning of "temporary" differs between Python and C++: in a statement such as ``func(std.vector[int]((1, 2, 3)))``, there is no temporary as far as Python is concerned, even as there clearly is in the case of a similar statement in C++. Thus that call will succeed even if ``func`` takes a non-const reference. diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/strings.rst b/bindings/pyroot/cppyy/cppyy/doc/source/strings.rst new file mode 100644 index 0000000000000..a986a8e465d99 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/doc/source/strings.rst @@ -0,0 +1,115 @@ +.. _strings: + + +Strings/Unicode +=============== + +Both Python and C++ have core types to represent text and these are expected +to be freely interchangeable. +``cppyy`` makes it easy to do just that for the most common cases, while +allowing customization where necessary to cover the full range of diverse use +cases (such as different codecs). +In addition to these core types, there is a range of other character types, +from ``const char*`` and ``std::wstring`` to ``bytes``, that see much less +use, but are also fully supported. + + +`std::string` +""""""""""""" + +The C++ core type ``std::string`` is considered the equivalent of Python's +``str``, even as purely implementation-wise, it is more akin to ``bytes``: +as a practical matter, a C++ programmer would use ``std::string`` where a +Python developer would use ``str`` (and vice versa), not ``bytes``. + +A Python ``str`` is unicode, however, whereas an ``std::string`` is character +based, thus conversions require encoding or decoding. +To allow for different encodings, ``cppyy`` defers implicit conversions +between the two types until forced, at which point it will default to seeing +``std::string`` as ASCII based and ``str`` to use the UTF-8 codec. +To support this, the bound ``std::string`` has been pythonized to allow it to +be a drop-in for a range of uses as appropriate within the local context. + +In particular, it is sometimes necessary (e.g. for function arguments that +take a non-const reference or a pointer to non-const ``std::string`` +variables), to use an actual ``std::string`` instance to allow in-place +modifications. +The pythonizations then allow their use where ``str`` is expected. +For example: + + .. code-block:: python + + >>> cppyy.cppexec("std::string gs;") + True + >>> cppyy.gbl.gs = "hello" + >>> type(cppyy.gbl.gs) # C++ std::string type + + >>> d = {"hello": 42} # dict filled with str + >>> d[cppyy.gbl.gs] # drop-in use of std::string -> str + 42 + >>> + +To handle codecs other than UTF-8, the ``std::string`` pythonization adds a +``decode`` method, with the same signature as the equivalent method of +``bytes``. +If it is known that a specific C++ function always returns an ``std::string`` +representing unicode with a codec other than UTF-8, it can in turn be +explicitly pythonized to do the conversion with that codec. + + +`std::string_view` +"""""""""""""""""" + +It is possible to construct a (char-based) ``std::string_view`` from a Python +``str``, but it requires the unicode object to be encoded and by default, +UTF-8 is chosen. +This will give the expected result if all characters in the ``str`` are from +the ASCII set, but otherwise it is recommend to encode on the Python side and +pass the resulting ``bytes`` object instead. + + +`std::wstring` +"""""""""""""" + +C++'s "wide" string, ``std::wstring``, is based on ``wchar_t``, a character +type that is not particularly portable as it can be 2 or 4 bytes in size, +depending on the platform. +cppyy supports ``std::wstring`` directly, using the ``wchar_t`` array +conversions provided by Python's C-API. + + +`const char*` +""""""""""""" + +The C representation of text, ``const char*``, is problematic for two +reasons: it does not express ownership; and its length is implicit, namely up +to the first occurrence of ``'\0'``. +The first can, up to an extent, be ameliorated: there are a range of cases +where ownership can be inferred. +In particular, if the C string is set from a Python ``str``, it is the latter +that owns the memory and the bound proxy of the former that in turn owns the +(unconverted) ``str`` instance. +However, if the ``const char*``'s memory is allocated in C/C++, memory +management is by necessity fully manual. +Length, on the other hand, can only be known in the case of a fixed array. +However even then, the more common case is to use the fixed array as a +buffer, with the actual string still only extending up to the ``'\0'`` char, +so that is assumed. +(C++'s ``std::string`` suffers from none of these issues and should always be +preferred when you have a choice.) + + +`char*` +""""""" + +The C representation of a character array, ``char*``, has all the problems of +``const char*``, but in addition is often used as "data array of 8-bit int". + + +`character types` +""""""""""""""""" + +cppyy directly supports the following character types, both as single +variables and in array form: ``char``, ``signed char``, ``unsigned char``, +``wchar_t``, ``char16_t``, and ``char32_t``. + diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/testing.rst b/bindings/pyroot/cppyy/cppyy/doc/source/testing.rst index cafa97c473238..d485bba103348 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/testing.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/testing.rst @@ -12,7 +12,7 @@ in the template tests (see file ``test_templates.py``). To run the tests, first install cppyy by any usual means, then clone the cppyy repo, and enter the ``test`` directory:: - $ git clone https://bitbucket.org/wlav/cppyy.git + $ git clone https://github.com/wlav/cppyy.git $ cd cppyy/test Next, build the dictionaries, the manner of which depends on your platform. diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/toplevel.rst b/bindings/pyroot/cppyy/cppyy/doc/source/toplevel.rst new file mode 100644 index 0000000000000..740d419a1d9bf --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/doc/source/toplevel.rst @@ -0,0 +1,144 @@ +.. _toplevel: + + +Top Level +========= + +cppyy provides a couple of helper functions at the module level that provide +(direct) access to the Cling interpreter (any C++ code is always accessed +through the global namespace ``cppyy.gbl``). +The documentation makes use of these helpers throughout, so they are listed +here first, but their documentation is more conveniently accessible through +the Python interpreter itself, using the ``help()`` function:: + + $ python + >>> import cppyy + >>> help(cppyy) + + +`Loading C++` +------------- + +C++ code can be loaded as text to be JITed, or be compiled ahead of time and +supplied in the form of a shared library. +In the latter case, C++ headers need to be loaded as well to declare +classes, functions, and variables to Cling. +Instead of headers, pre-compiled code can be used; in particular all of the +standard C++ headers and several system headers are pre-compiled at startup. +cppyy provides the following helpers to load C++ code: + +* ``cppdef``: direct access to the interpreter. + This function accepts C++ declarations as a string and JITs them (bindings + are not created until actual use). + The code is loaded into the global scope, thus any previously loaded code + is available from one ``cppdef`` call to the next, as are all standard + C++ headers that have been loaded through pre-compiled headers. + Example:: + + >>> cppyy.cppdef(r"""\ + ... void hello() { + ... std::cout << "Hello, World!" << std::endl; + ... }""") + True + >>> cppyy.gbl.hello() + Hello, World! + >>> + +* ``cppexec``: direct access to the interpreter. + This function accepts C++ statements as a string, JITs and executes them. + Just like ``cppdef``, execution is in the global scope and all previously + loaded code is available. + If the statements are declarations, the effect is the same as ``cppdef``, + but ``cppexec`` also accepts executable lines. + Example:: + + >>> cppyy.cppexec(r"""std::string hello = "Hello, World!";""") + True + >>> cppyy.cppexec("std::cout << hello << std::endl;") + Hello, World! + True + >>> + +* ``include``: load declarations into the interpreter. + This function accepts C++ declarations from a file, typically a header. + Files are located through include paths given to the Cling. + Example:: + + >>> cppyy.include("vector") # equivalent to "#include " + True + >>> + +* ``c_include``: load declarations into the interpreter. + This function accepts C++ declarations from a file, typically a header. + Name mangling is an important difference between C and C++ code. + The use of ``c_include`` instead of ``include`` prevents mangling. + +* ``load_library``: load compiled C++ into the interpreter. + This function takes the name of a shared library and loads it into current + process, exposing all external symbols to Cling. + Libraries are located through load paths given to Cling, either through the + "-L" compiler flag or the dynamic search path environment variable (system + dependent). + Any method that brings symbols into the process (including normal linking, + e.g. when embedding Python in a C++ application) is suitable to expose + symbols. + An alternative for ``load_library`` is for example ``ctypes.CDLL``, but + that function does not respect dynamic load paths on all platforms. + +If a compilation error occurs during JITing of C++ code in any of the above +helpers, a Python ``SyntaxError`` exception is raised. +If a compilation warning occurs, a Python warning is issued. + + +`Configuring Cling` +------------------- + +It is often convenient to add additional search paths for Cling to find +headers and libraries when loading a module (Python does not have standard +locations to place headers and libraries, but their locations can usually +be inferred from the location of the module, i.e. it's ``__file__`` +attribute). +cppyy provides the following two helpers: + +* ``add_include_path``: add additional paths for Cling to look for headers. + +* ``add_library_path``: add additional paths for Cling to look for libraries. + +Both functions accept either a string (a single path) or a list (for adding +multiple paths). +Paths are allowed to be relative, but absolute paths are recommended. + + +`C++ language` +-------------- + +Some C++ compilation-time features have no Python equivalent. +Instead, convenience functions are provided: + +* ``sizeof``: takes a proxied C++ type or its name as a string and returns + the storage size (in units of ``char``). + +* ``typeid``: takes a proxied C++ type or its name as a string and returns + the the C++ runtime type information (RTTI). + +* ``nullptr``: C++ ``NULL``. + + +`Preprocessor` +-------------- + +Preprocessor macro's (``#define``) are not available on the Python side, +because there is no type information available for them. +They are, however, often used for constant data (e.g. flags or numbers; note +that modern C++ recommends the use of ``const`` and ``constexpr`` instead). +Within limits, macro's representing constant data are accessible through the +``macro`` helper function. +Example:: + + >>> import cppyy + >>> cppyy.cppdef('#define HELLO "Hello, World!"') + True + >>> cppyy.macro("HELLO") + 'Hello, World!' + >>> + diff --git a/bindings/pyroot/cppyy/cppyy/doc/source/utilities.rst b/bindings/pyroot/cppyy/cppyy/doc/source/utilities.rst index cd2faca817fb5..77135000b3519 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/source/utilities.rst +++ b/bindings/pyroot/cppyy/cppyy/doc/source/utilities.rst @@ -52,6 +52,17 @@ to locate those library dependencies. Alternatively, you can add the additional libraries to load to the mapping files of the class loader (see below). +.. note:: + + The JIT needs to resolve linker symbols in order to call them through + generated wrappers. + Thus, any classes, functions, and data that will be used in Python need + to be exported. + This is the default behavior on Mac and Linux, but not on Windows. + On that platform, use ``__declspec(dllexport)`` to explicitly export the + classes and function you expect to call. + CMake has simple `support for exporting all`_ C++ symbols. + In tandem with any dictionary, a pre-compiled module (.pcm) file will be generated. C++ modules are still on track for inclusion in the C++20 standard and most @@ -211,7 +222,7 @@ Once support for C++ modules is fully fleshed out, access to the header file will no longer be needed. .. _`rootcling manual`: https://root.cern.ch/root/html/guides/users-guide/AddingaClass.html#the-linkdef.h-file -.. _`helper script`: https://bitbucket.org/wlav/cppyy/src/master/test/make_dict_win32.py +.. _`helper script`: https://github.com/wlav/cppyy/blob/master/test/make_dict_win32.py Class loader @@ -267,3 +278,6 @@ cppyy-supported bindings:: This utility is mainly used as part of the :doc:`CMake interface `. + + +.. _`support for exporting all`: https://cmake.org/cmake/help/latest/prop_tgt/WINDOWS_EXPORT_ALL_SYMBOLS.html diff --git a/bindings/pyroot/cppyy/cppyy/doc/tutorial/CppyyTutorial.ipynb b/bindings/pyroot/cppyy/cppyy/doc/tutorial/CppyyTutorial.ipynb index 34b31fe318d24..1fe0fdba2246a 100644 --- a/bindings/pyroot/cppyy/cppyy/doc/tutorial/CppyyTutorial.ipynb +++ b/bindings/pyroot/cppyy/cppyy/doc/tutorial/CppyyTutorial.ipynb @@ -91,7 +91,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Namespaces have simularities to modules, so we could have imported the class as well.\n", + "Namespaces have similarities to modules, so we could have imported the class as well.\n", "\n", "Bound C++ classes are first-class Python object. We can instantiate them, use normal Python introspection tools, call `help()`, they raise Python exceptions on failure, manage memory through Python's ref-counting and garbage collection, etc., etc. Furthermore, we can use them in conjunction with other C++ classes." ] diff --git a/bindings/pyroot/cppyy/cppyy/installer/cppyy_monkey_patch.py b/bindings/pyroot/cppyy/cppyy/installer/cppyy_monkey_patch.py new file mode 100644 index 0000000000000..206822082c8d3 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/installer/cppyy_monkey_patch.py @@ -0,0 +1,39 @@ +# monkey patch to be able to select a specific backend based on PyPy's version, +# which is not possible in the pyproject.toml file as there is currently no +# marker for it (this may change, after which this file can be removed) + +try: + # _BACKEND is the primary, __legacy__ the backwards compatible backend + from setuptools.build_meta import _BACKEND + main = _BACKEND +except (NameError, ImportError): + # fallback as the name __legacy__ is actually documented (and part of __all__) + main = __legacy__ + +# the following ensures proper build/installation order, after which the normal +# install through setup.py picks up their wheels from the cache (TODO: note the +# duplication here with setup.py; find a better way) +_get_requires_for_build_wheel = main.get_requires_for_build_wheel +def get_requires_for_build_wheel(*args, **kwds): + try: + import __pypy__, sys + version = sys.pypy_version_info + requirements = ['cppyy-cling==6.30.0'] + if version[0] == 5: + if version[1] <= 9: + requirements = ['cppyy-cling<6.12'] + elif version[1] <= 10: + requirements = ['cppyy-cling<=6.15'] + elif version[0] == 6: + if version[1] <= 0: + requirements = ['cppyy-cling<=6.15'] + elif version[0] == 7: + if version[1] <= 3 and version[2] <= 3: + requirements = ['cppyy-cling<=6.18.2.3'] + except ImportError: + # CPython + requirements = ['cppyy-backend==1.15.2', 'cppyy-cling==6.30.0'] + + return requirements + _get_requires_for_build_wheel(*args, **kwds) + +main.get_requires_for_build_wheel = get_requires_for_build_wheel diff --git a/bindings/pyroot/cppyy/cppyy/pyproject.toml b/bindings/pyroot/cppyy/cppyy/pyproject.toml new file mode 100644 index 0000000000000..91c1695fbc051 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/pyproject.toml @@ -0,0 +1,4 @@ +[build-system] +requires = ["setuptools", "wheel"] +backend-path = ["installer"] +build-backend = "cppyy_monkey_patch:main" diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py index b3b1b84251362..957443289d7c0 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py @@ -35,26 +35,36 @@ __all__ = [ 'cppdef', # declare C++ source to Cling + 'cppexec', # execute a C++ statement + 'macro', # attempt to evaluate a cpp macro 'include', # load and jit a header file 'c_include', # load and jit a C header file 'load_library', # load a shared library 'nullptr', # unique pointer representing NULL 'sizeof', # size of a C++ type 'typeid', # typeid of a C++ type + 'multi', # helper for multiple inheritance 'add_include_path', # add a path to search for headers 'add_library_path', # add a path to search for headers 'add_autoload_map', # explicitly include an autoload map + 'set_debug', # enable/disable debug output ] from ._version import __version__ -import os, sys, sysconfig, warnings +import ctypes, os, sys, sysconfig, warnings if not 'CLING_STANDARD_PCH' in os.environ: - local_pch = os.path.join(os.path.dirname(__file__), 'allDict.cxx.pch') - if os.path.exists(local_pch): - os.putenv('CLING_STANDARD_PCH', local_pch) - os.environ['CLING_STANDARD_PCH'] = local_pch + def _set_pch(): + try: + import cppyy_backend as cpb + local_pch = os.path.join(os.path.dirname(__file__), 'allDict.cxx.pch.'+str(cpb.__version__)) + if os.path.exists(local_pch): + os.putenv('CLING_STANDARD_PCH', local_pch) + os.environ['CLING_STANDARD_PCH'] = local_pch + except (ImportError, AttributeError): + pass + _set_pch(); del _set_pch try: import __pypy__ @@ -78,7 +88,13 @@ #- external typemap ---------------------------------------------------------- from . import _typemap -_typemap.initialize(_backend) +_typemap.initialize(_backend) # also creates (u)int8_t mapper + +try: + gbl.std.int8_t = gbl.int8_t # ensures same _integer_ type + gbl.std.uint8_t = gbl.uint8_t +except (AttributeError, TypeError): + pass #- pythonization factories --------------------------------------------------- @@ -86,8 +102,9 @@ py._set_backend(_backend) def _standard_pythonizations(pyclass, name): - # pythonization of tuple; TODO: placed here for convenience, but verify that decision - if name.find('std::tuple<', 0, 11) == 0 or name.find('tuple<', 0, 6) == 0: + # pythonization of tuple; TODO: placed here for convenience, but a custom case + # for tuples on each platform can be made much more performant ... + if name.find('tuple<', 0, 6) == 0: import cppyy pyclass._tuple_len = cppyy.gbl.std.tuple_size(pyclass).value def tuple_len(self): @@ -95,74 +112,176 @@ def tuple_len(self): pyclass.__len__ = tuple_len def tuple_getitem(self, idx, get=cppyy.gbl.std.get): if idx < self.__class__._tuple_len: - return get[idx](self) + res = get[idx](self) + try: + res.__life_line = self + except Exception: + pass + return res raise IndexError(idx) pyclass.__getitem__ = tuple_getitem + # pythonization of std::string; placed here because it's simpler to write the + # custom "npos" object (to allow easy result checking of find/rfind) in Python + elif pyclass.__cpp_name__ == "std::string": + class NPOS(0x3000000 <= sys.hexversion and int or long): + def __eq__(self, other): + return other == -1 or int(self) == other + def __ne__(self, other): + return other != -1 and int(self) != other + del pyclass.__class__.npos # drop b/c is const data + pyclass.npos = NPOS(pyclass.npos) + + return True + if not ispypy: - py.add_pythonization(_standard_pythonizations) # should live on std only + py.add_pythonization(_standard_pythonizations, "std") # TODO: PyPy still has the old-style pythonizations, which require the full # class name (not possible for std::tuple ...) -# std::make_shared creates needless templates: rely on Python's introspection +# std::make_shared/unique create needless templates: rely on Python's introspection # instead. This also allows Python derived classes to be handled correctly. -class py_make_shared(object): - def __init__(self, cls): - self.cls = cls +class py_make_smartptr(object): + __slots__ = ['cls', 'ptrcls'] + def __init__(self, cls, ptrcls): + self.cls = cls + self.ptrcls = ptrcls def __call__(self, *args): if len(args) == 1 and type(args[0]) == self.cls: obj = args[0] else: obj = self.cls(*args) - obj.__python_owns__ = False # C++ to take ownership - return gbl.std.shared_ptr[self.cls](obj) - -class make_shared(object): + return self.ptrcls[self.cls](obj) # C++ takes ownership + +class make_smartptr(object): + __slots__ = ['ptrcls', 'maker'] + def __init__(self, ptrcls, maker): + self.ptrcls = ptrcls + self.maker = maker + def __call__(self, ptr): + return py_make_smartptr(type(ptr), self.ptrcls)(ptr) def __getitem__(self, cls): - return py_make_shared(cls) - -gbl.std.make_shared = make_shared() -del make_shared + try: + if not cls.__module__ == int.__module__: + return py_make_smartptr(cls, self.ptrcls) + except AttributeError: + pass + if type(cls) == str and not cls in ('int', 'float'): + return py_make_smartptr(getattr(gbl, cls), self.ptrcls) + return self.maker[cls] + +gbl.std.make_shared = make_smartptr(gbl.std.shared_ptr, gbl.std.make_shared) +gbl.std.make_unique = make_smartptr(gbl.std.unique_ptr, gbl.std.make_unique) +del make_smartptr + + +#--- interface to Cling ------------------------------------------------------ +class _stderr_capture(object): + def __init__(self): + self._capture = not gbl.gDebug and True or False + self.err = "" + + def __enter__(self): + if self._capture: + _begin_capture_stderr() + return self + + def __exit__(self, tp, val, trace): + if self._capture: + self.err = _end_capture_stderr() - -#--- CFFI style interface ---------------------------------------------------- def cppdef(src): """Declare C++ source to Cling.""" - if not gbl.gInterpreter.Declare(src): - return False + with _stderr_capture() as err: + errcode = gbl.gInterpreter.Declare(src) + if not errcode or err.err: + if 'warning' in err.err.lower() and not 'error' in err.err.lower(): + warnings.warn(err.err, SyntaxWarning) + return True + raise SyntaxError('Failed to parse the given C++ code%s' % err.err) return True +def cppexec(stmt): + """Execute C++ statement in Cling's global scope.""" + if stmt and stmt[-1] != ';': + stmt += ';' + + # capture stderr, but note that ProcessLine could legitimately be writing to + # std::cerr, in which case the captured output needs to be printed as normal + with _stderr_capture() as err: + errcode = ctypes.c_int(0) + try: + gbl.gInterpreter.ProcessLine(stmt, ctypes.pointer(errcode)) + except Exception as e: + sys.stderr.write("%s\n\n" % str(e)) + if not errcode.value: errcode.value = 1 + + if errcode.value: + raise SyntaxError('Failed to parse the given C++ code%s' % err.err) + elif err.err and err.err[1:] != '\n': + sys.stderr.write(err.err[1:]) + + return True + +def macro(cppm): + """Attempt to evalute a C/C++ pre-processor macro as a constant""" + + try: + macro_val = getattr(getattr(gbl, '__cppyy_macros', None), cppm+'_', None) + if macro_val is None: + cppdef("namespace __cppyy_macros { auto %s_ = %s; }" % (cppm, cppm)) + return getattr(getattr(gbl, '__cppyy_macros'), cppm+'_') + except Exception: + pass + + raise ValueError('Failed to evaluate macro %s', cppm) + + def load_library(name): """Explicitly load a shared library.""" - gSystem = gbl.gSystem - if name[:3] != 'lib': - if not gSystem.FindDynamicLibrary(gbl.TString(name), True) and\ - gSystem.FindDynamicLibrary(gbl.TString('lib'+name), True): - name = 'lib'+name - sc = gSystem.Load(name) + with _stderr_capture() as err: + gSystem = gbl.gSystem + if name[:3] != 'lib': + if not gSystem.FindDynamicLibrary(gbl.TString(name), True) and\ + gSystem.FindDynamicLibrary(gbl.TString('lib'+name), True): + name = 'lib'+name + sc = gSystem.Load(name) if sc == -1: - raise RuntimeError("Unable to load library "+name) + # special case for Windows as of python3.8: use winmode=0, otherwise the default + # will not consider regular search paths (such as $PATH) + if 0x3080000 <= sys.hexversion and 'win32' in sys.platform and os.path.isabs(name): + return ctypes.CDLL(name, ctypes.RTLD_GLOBAL, winmode=0) # raises on error + raise RuntimeError('Unable to load library "%s"%s' % (name, err.err)) + return True def include(header): """Load (and JIT) header file
into Cling.""" - gbl.gInterpreter.Declare('#include "%s"' % header) + with _stderr_capture() as err: + errcode = gbl.gInterpreter.Declare('#include "%s"' % header) + if not errcode: + raise ImportError('Failed to load header file "%s"%s' % (header, err.err)) + return True def c_include(header): """Load (and JIT) header file
into Cling.""" - gbl.gInterpreter.Declare("""extern "C" { + with _stderr_capture() as err: + errcode = gbl.gInterpreter.Declare("""extern "C" { #include "%s" }""" % header) + if not errcode: + raise ImportError('Failed to load header file "%s"%s' % (header, err.err)) + return True def add_include_path(path): """Add a path to the include paths available to Cling.""" if not os.path.isdir(path): - raise OSError("no such directory: %s" % path) + raise OSError('No such directory: %s' % path) gbl.gInterpreter.AddIncludePath(path) def add_library_path(path): """Add a path to the library search paths available to Cling.""" if not os.path.isdir(path): - raise OSError("no such directory: %s" % path) + raise OSError('No such directory: %s' % path) gbl.gSystem.AddDynamicPath(path) # add access to Python C-API headers @@ -177,34 +296,64 @@ def add_library_path(path): # add access to extra headers for dispatcher (CPyCppyy only (?)) if not ispypy: - if 'CPPYY_API_PATH' in os.environ: + try: apipath_extra = os.environ['CPPYY_API_PATH'] - else: - apipath_extra = os.path.join(os.path.dirname(apipath), 'site', 'python'+sys.version[:3]) + if os.path.basename(apipath_extra) == 'CPyCppyy': + apipath_extra = os.path.dirname(apipath_extra) + except KeyError: + apipath_extra = None + + if apipath_extra is None: + try: + import pkg_resources as pr + + d = pr.get_distribution('CPyCppyy') + for line in d.get_metadata_lines('RECORD'): + if 'API.h' in line: + part = line[0:line.find(',')] + + ape = os.path.join(d.location, part) + if os.path.exists(ape): + apipath_extra = os.path.dirname(os.path.dirname(ape)) + + del part, d, pr + except Exception: + pass + + if apipath_extra is None: + ldversion = sysconfig.get_config_var('LDVERSION') + if not ldversion: ldversion = sys.version[:3] + + apipath_extra = os.path.join(os.path.dirname(apipath), 'site', 'python'+ldversion) if not os.path.exists(os.path.join(apipath_extra, 'CPyCppyy')): import glob, libcppyy - apipath_extra = os.path.dirname(libcppyy.__file__) - # a "normal" structure finds the include directory 3 levels up, - # ie. from lib/pythonx.y/site-packages + ape = os.path.dirname(libcppyy.__file__) + # a "normal" structure finds the include directory up to 3 levels up, + # ie. dropping lib/pythonx.y[md]/site-packages for i in range(3): - if not os.path.exists(os.path.join(apipath_extra, 'include')): - apipath_extra = os.path.dirname(apipath_extra) - - apipath_extra = os.path.join(apipath_extra, 'include') - # add back pythonx.y or site/pythonx.y if available - for p in glob.glob(os.path.join(apipath_extra, 'python'+sys.version[:3]+'*'))+\ - glob.glob(os.path.join(apipath_extra, '*', 'python'+sys.version[:3]+'*')): - if os.path.exists(os.path.join(p, 'CPyCppyy')): - apipath_extra = p + if os.path.exists(os.path.join(ape, 'include')): break + ape = os.path.dirname(ape) + + ape = os.path.join(ape, 'include') + if os.path.exists(os.path.join(ape, 'CPyCppyy')): + apipath_extra = ape + else: + # add back pythonx.y or site/pythonx.y if present + for p in glob.glob(os.path.join(ape, 'python'+sys.version[:3]+'*'))+\ + glob.glob(os.path.join(ape, '*', 'python'+sys.version[:3]+'*')): + if os.path.exists(os.path.join(p, 'CPyCppyy')): + apipath_extra = p + break - cpycppyy_path = os.path.join(apipath_extra, 'CPyCppyy') if apipath_extra.lower() != 'none': - if not os.path.exists(cpycppyy_path): - warnings.warn("CPyCppyy API path not found (tried: %s); set CPPYY_API_PATH to fix" % os.path.dirname(cpycppyy_path)) + if not os.path.exists(os.path.join(apipath_extra, 'CPyCppyy')): + warnings.warn("CPyCppyy API not found (tried: %s); set CPPYY_API_PATH envar to the 'CPyCppyy' API directory to fix" % apipath_extra) else: add_include_path(apipath_extra) + del apipath_extra + if os.getenv('CONDA_PREFIX'): # MacOS, Linux include_path = os.path.join(os.getenv('CONDA_PREFIX'), 'include') @@ -215,7 +364,7 @@ def add_library_path(path): if os.path.exists(include_path): add_include_path(include_path) # assuming that we are in PREFIX/lib/python/site-packages/cppyy, add PREFIX/include to the search path -include_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, os.path.pardir, os.path.pardir, 'include')) +include_path = os.path.abspath(os.path.join(os.path.dirname(__file__), *(4*[os.path.pardir]+['include']))) if os.path.exists(include_path): add_include_path(include_path) del include_path, apipath, ispypy @@ -226,6 +375,13 @@ def add_autoload_map(fname): raise OSError("no such file: %s" % fname) gbl.gInterpreter.LoadLibraryMap(fname) +def set_debug(enable=True): + """Enable/disable debug output.""" + if enable: + gbl.gDebug = 10 + else: + gbl.gDebug = 0 + def _get_name(tt): if type(tt) == str: return tt @@ -243,7 +399,10 @@ def sizeof(tt): try: return _sizes[tt] except KeyError: - sz = gbl.gInterpreter.ProcessLine("sizeof(%s);" % (_get_name(tt),)) + try: + sz = ctypes.sizeof(tt) + except TypeError: + sz = gbl.gInterpreter.ProcessLine("sizeof(%s);" % (_get_name(tt),)) _sizes[tt] = sz return sz @@ -262,3 +421,12 @@ def typeid(tt): tid = getattr(gbl._cppyy_internal, tidname) _typeids[tt] = tid return tid + +def multi(*bases): # after six, see also _typemap.py + """Resolve metaclasses for multiple inheritance.""" + # contruct a "no conflict" meta class; the '_meta' is needed by convention + nc_meta = type.__new__(type, 'cppyy_nc_meta', tuple(type(b) for b in bases if type(b) is not type), {}) + class faux_meta(type): + def __new__(cls, name, this_bases, d): + return nc_meta(name, bases, d) + return type.__new__(faux_meta, 'faux_meta', (), {}) diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/__init__.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/__init__.py new file mode 100644 index 0000000000000..84cb94ecc0979 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/__init__.py @@ -0,0 +1,6 @@ +import os + +__all__ = ['get_hook_dirs'] + +def get_hook_dirs(): + return [os.path.dirname(__file__)] diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/hook-cppyy.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/hook-cppyy.py new file mode 100644 index 0000000000000..2e0a98b1f53b0 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__pyinstaller/hook-cppyy.py @@ -0,0 +1,34 @@ +# PyInstaller hooks to declare the "data files" (libraries, headers, etc.) of +# cppyy-backend. Placed here rather then in cppyy-backend to guarantee that any +# packaging of top-level "cppyy" picks up the backend as well. +# +# See also setup.cfg. + +__all__ = ['data'] + + +def _backend_files(): + import cppyy_backend, glob, os + + all_files = glob.glob(os.path.join( + os.path.dirname(cppyy_backend.__file__), '*')) + + def datafile(path): + return path, os.path.join('cppyy_backend', os.path.basename(path)) + + return [datafile(filename) for filename in all_files if os.path.isdir(filename)] + +def _api_files(): + import cppyy, os + + paths = str(cppyy.gbl.gInterpreter.GetIncludePath()).split('-I') + for p in paths: + if not p: continue + + apipath = os.path.join(p.strip()[1:-1], 'CPyCppyy') + if os.path.exists(apipath): + return [(apipath, os.path.join('include', 'CPyCppyy'))] + + return [] + +datas = _backend_files()+_api_files() diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py index b5939faa4f148..8d6f1e0f2c0fd 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_cpython_cppyy.py @@ -10,7 +10,10 @@ 'addressof', 'bind_object', 'nullptr', + 'default', '_backend', + '_begin_capture_stderr', + '_end_capture_stderr' ] # first load the dependency libraries of the backend, then pull in the @@ -30,7 +33,7 @@ # TODO: this reliese on CPPOverload cooking up a func_code object, which atm # is simply not implemented for p3 :/ - # convince inspect that PyROOT method proxies are possible drop-ins for python + # convince inspect that cppyy method proxies are possible drop-ins for python # methods and classes for pydoc import inspect @@ -38,7 +41,7 @@ def isfunction(object): if type(object) == _backend.CPPOverload and not object.im_class: return True - return inspect._old_isfunction( object ) + return inspect._old_isfunction(object) inspect.isfunction = isfunction inspect._old_ismethod = inspect.ismethod @@ -52,13 +55,31 @@ def ismethod(object): ### template support --------------------------------------------------------- class Template(object): # expected/used by ProxyWrappers.cxx in CPyCppyy + stl_sequence_types = ['std::vector', 'std::list', 'std::set', 'std::deque'] + stl_unrolled_types = ['std::pair'] + stl_fixed_size_types = ['std::array'] + stl_mapping_types = ['std::map', 'std::unordered_map'] + def __init__(self, name): - self.__name__ = name + self.__name__ = name + self.__cpp_name__ = name + self._instantiations = dict() def __repr__(self): return "" % (self.__name__, hex(id(self))) - def __call__(self, *args): + def __getitem__(self, *args): + # multi-argument to [] becomes a single tuple argument + if args and type(args[0]) is tuple: + args = args[0] + + # if already instantiated, return the existing class + try: + return self._instantiations[args] + except KeyError: + pass + + # construct the type name from the types or their string representation newargs = [self.__name__] for arg in args: if type(arg) == str: @@ -66,6 +87,9 @@ def __call__(self, *args): newargs.append(arg) pyclass = _backend.MakeCppTemplateClass(*newargs) + # memoize the class to prevent spurious lookups/re-pythonizations + self._instantiations[args] = pyclass + # special case pythonization (builtin_map is not available from the C-API) if 'push_back' in pyclass.__dict__ and not '__iadd__' in pyclass.__dict__: if 'reserve' in pyclass.__dict__: @@ -79,12 +103,44 @@ def iadd(self, ll): return self pyclass.__iadd__ = iadd + # back-pointer for reflection + pyclass.__cpp_template__ = self + return pyclass - def __getitem__(self, *args): - if args and type(args[0]) == tuple: - return self.__call__(*(args[0])) - return self.__call__(*args) + def __call__(self, *args): + # for C++17, we're required to derive the type when using initializer syntax + # (i.e. a tuple or list); not sure how to do that in general, but below the + # most common cases are covered + if args: + args0 = args[0] + if args0 and (type(args0) is tuple or type(args0) is list): + t = type(args0[0]) + if t is float: t = 'double' + + if self.__name__ in self.stl_sequence_types: + return self[t](*args) + if self.__name__ in self.stl_fixed_size_types: + return self[t, len(args0)](*args) + if self.__name__ in self.stl_unrolled_types: + return self[tuple(type(a) for a in args0)](*args0) + + if args0 and type(args0) is dict: + if self.__name__ in self.stl_mapping_types: + try: + pair = args0.items().__iter__().__next__() + except AttributeError: + pair = args0.items()[0] + t1 = type(pair[0]) + if t1 is float: t1 = 'double' + t2 = type(pair[1]) + if t2 is float: t2 = 'double' + return self[t1, t2](*args) + + return self.__getitem__(*(type(a) for a in args0))(*args) + + # old 'metaclass-style' template instantiations + return self.__getitem__(*args) _backend.Template = Template @@ -130,8 +186,26 @@ def add_default_paths(): addressof = _backend.addressof bind_object = _backend.bind_object nullptr = _backend.nullptr +default = _backend.default def load_reflection_info(name): sc = gbl.gSystem.Load(name) if sc == -1: raise RuntimeError("Unable to load reflection library "+name) + +def _begin_capture_stderr(): + _backend._begin_capture_stderr() + +def _end_capture_stderr(): + err = _backend._end_capture_stderr() + if err: + try: + return "\n%s" % err + except UnicodeDecodeError as e: + original_error = e + try: + return "\n%s" % err.decode('gbk') # not guaranteed, but common + except UnicodeDecodeError: + pass + return "C++ issued an error message that could not be decoded (%s)" % str(original_error) + return "" diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py index dc47c9e750b65..78ad08a188ab6 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pypy_cppyy.py @@ -15,7 +15,12 @@ # first load the dependency libraries of the backend, then # pull in the built-in low-level cppyy -c = loader.load_cpp_backend() +try: + c = loader.load_cpp_backend() +except RuntimeError: + import sysconfig + os.environ['CPPYY_BACKEND_LIBRARY'] = "libcppyy_backend"+sysconfig.get_config_var("SO") + c = loader.load_cpp_backend() os.environ['CPPYY_BACKEND_LIBRARY'] = c._name # some older versions can be fixed up through a compatibility @@ -32,12 +37,22 @@ pass _backend._cpp_backend = c +def fixup_legacy(): + version = sys.pypy_version_info + if version[0] < 7 or (version[0] == 7 and version[1] <= 3 and version[2] <= 3): + _backend.gbl.CppyyLegacy = _backend.gbl +fixup_legacy() +del fixup_legacy + #- exports ------------------------------------------------------------------- import sys _thismodule = sys.modules[__name__] for name in __all__: - setattr(_thismodule, name, getattr(_backend, name)) + try: + setattr(_thismodule, name, getattr(_backend, name)) + except AttributeError: + pass del name, sys nullptr = _backend.nullptr @@ -46,6 +61,14 @@ def load_reflection_info(name): if sc == -1: raise RuntimeError("Unable to load reflection library "+name) +def _begin_capture_stderr(): + pass + +def _end_capture_stderr(): + return "" + # add other exports to all __all__.append('load_reflection_info') __all__.append('_backend') +__all__.append('_begin_capture_stderr') +__all__.append('_end_capture_stderr') diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py index 79863c5ffefb0..d49b90f8a06f8 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_pythonization.py @@ -5,6 +5,7 @@ 'add_pythonization', 'remove_pythonization', 'pin_type', + 'add_type_reducer', ] def _set_backend(backend): @@ -30,6 +31,14 @@ def remove_pythonization(pythonizor, scope = ''): def pin_type(klass): return _backend._pin_type(klass) + +# mapper to reduce template expression trees +def add_type_reducer(reducable, reduced): + """Reduce to type on returns from function calls. + """ + return _backend._add_type_reducer(reducable, reduced) + + # exception pythonizations def add_exception_mapping(cpp_exception, py_exception): _backend.UserExceptions[cpp_exception] = py_exception @@ -277,7 +286,7 @@ def __call__(self, obj, name): named_setters[name] = k else: fset = self.make_set_proxy(k) - break + break if self.match_del: for k in dir(obj): #.__dict__: diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py index c89ed3ac0d1fa..3aefbf4eee797 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_typemap.py @@ -83,7 +83,7 @@ def initialize(backend): # integer types int_tm = _create_mapper(int) - for tp in ['short', 'unsigned short', 'int']: + for tp in ['int8_t', 'uint8_t', 'short', 'unsigned short', 'int']: tm[tp] = int_tm if sys.hexversion < 0x3000000: diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py index 4a9b9788449b8..f71b21a5dd538 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/_version.py @@ -1 +1 @@ -__version__ = '1.6.2' +__version__ = '3.1.2' diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py index 8c078ca7064d9..49f5f04f04edb 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/ll.py @@ -2,6 +2,9 @@ """ import cppyy +import ctypes +import sys +import warnings try: import __pypy__ @@ -11,6 +14,8 @@ ispypy = False __all__ = [ + 'argv', + 'argc', 'cast', 'static_cast', 'reinterpret_cast', @@ -18,10 +23,10 @@ 'malloc', 'free', 'array_new', - 'array_detele', + 'array_delete', 'signals_as_exception', - 'set_signals_as_exception' - 'FatalError' + 'set_signals_as_exception', + 'FatalError', 'BusError', 'SegmentationViolation', 'IllegalInstruction', @@ -29,8 +34,17 @@ ] +# convenience functions to create C-style argv/argc +def argv(): + argc = len(sys.argv) + cargsv = (ctypes.c_char_p * len(sys.argv))(*(x.encode() for x in sys.argv)) + return ctypes.POINTER(ctypes.c_char_p)(cargsv) + +def argc(): + return len(sys.argv) + # import low-level python converters -for _name in ['addressof', 'as_cobject', 'as_capsule', 'as_ctypes']: +for _name in ['addressof', 'as_cobject', 'as_capsule', 'as_ctypes', 'as_memoryview']: try: exec('%s = cppyy._backend.%s' % (_name, _name)) __all__.append(_name) @@ -73,11 +87,24 @@ def __init__(self, func): def __getitem__(self, t): self.array_type = t return self - def __call__(self, size): + def __call__(self, size, managed=False): res = self.func[self.array_type](size) - res.reshape((size,)) + try: + res.reshape((size,)+res.shape[1:]) + if managed: res.__python_owns__ = True + except AttributeError: + res.__reshape__((size,)) + if managed: + warnings.warn("managed low-level arrays of instances not supported") return res +class CArraySizer(ArraySizer): + def __call__(self, size, managed=False): + res = ArraySizer.__call__(self, size, managed) + res.__cpp_array__ = False + return res + + # import casting helpers cast = cppyy.gbl.__cppyy_internal.cppyy_cast static_cast = cppyy.gbl.__cppyy_internal.cppyy_static_cast @@ -85,7 +112,7 @@ def __call__(self, size): dynamic_cast = cppyy.gbl.__cppyy_internal.cppyy_dynamic_cast # import memory allocation/free-ing helpers -malloc = ArraySizer(cppyy.gbl.__cppyy_internal.cppyy_malloc) +malloc = CArraySizer(cppyy.gbl.__cppyy_internal.cppyy_malloc) free = cppyy.gbl.free # for symmetry array_new = ArraySizer(cppyy.gbl.__cppyy_internal.cppyy_array_new) array_delete = cppyy.gbl.__cppyy_internal.cppyy_array_delete diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py index e76dad20cbba9..980dbdae9125d 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/numba_ext.py @@ -16,7 +16,10 @@ import numba.core.typing as nb_typing from llvmlite import ir - +from numba.extending import make_attribute_wrapper +import itertools +import re +import inspect # setuptools entry point for Numba def _init_extension(): @@ -26,10 +29,12 @@ def _init_extension(): class Qualified: default = 0 value = 1 + instance = 2 -ir_voidptr = ir.PointerType(ir.IntType(8)) # by convention -ir_byteptr = ir_voidptr -ir_intptr_t = ir.IntType(cppyy.sizeof('void*')*8) # use MACHINE_BITS? +ir_byte = ir.IntType(8) +ir_voidptr = ir.PointerType(ir_byte) # by convention +ir_byteptr = ir_voidptr # for clarity +ir_intptr_t = ir.IntType(cppyy.sizeof('void*')*8) # special case access to unboxing/boxing APIs cppyy_as_voidptr = cppyy.addressof('Instance_AsVoidPtr') @@ -43,7 +48,6 @@ class Qualified: 'uint8_t' : nb_types.uint8, 'short' : nb_types.short, 'unsigned short' : nb_types.ushort, - 'internal_enum_type_t' : nb_types.intc, 'int' : nb_types.intc, 'unsigned int' : nb_types.uintc, 'int32_t' : nb_types.int32, @@ -52,28 +56,81 @@ class Qualified: 'uint64_t' : nb_types.uint64, 'long' : nb_types.long_, 'unsigned long' : nb_types.ulong, - 'Long64_t' : nb_types.longlong, # Note: placed above long long as the last value is used in numba2cpp - 'long long' : nb_types.longlong, # this value will be used in numba2cpp + 'long long' : nb_types.longlong, 'unsigned long long' : nb_types.ulonglong, 'float' : nb_types.float32, - 'long double' : nb_types.float64, # Note: see Long64_t - 'double' : nb_types.float64, # this value will be used in numba2cpp + 'double' : nb_types.float64, + 'char' : nb_types.char, + 'unsigned char' : nb_types.uchar, + 'char*' : nb_types.unicode_type } +def resolve_std_vector(val): + return re.match(r'std::vector<(.+?)>', val).group(1) + +def resolve_const_types(val): + return re.match(r'const\s+(.+)\s*\*', val).group(1) + def cpp2numba(val): if type(val) != str: # TODO: distinguish ptr/ref/byval + # TODO: Only metaclasses/proxies end up here since + # ref cases makes the RETURN_TYPE from reflex a string return typeof_scope(val, nb_typing.typeof.Purpose.argument, Qualified.value) + elif val.startswith("std::vector"): + type_arr = getattr(numba, str(cpp2numba(resolve_std_vector(val))))[:] + return type_arr + elif val[-1] == '*' or val[-1] == '&': + if val.startswith('const'): + return nb_types.CPointer(cpp2numba(resolve_const_types(val))) + return nb_types.CPointer(_cpp2numba[val[:-1]]) return _cpp2numba[val] _numba2cpp = dict() for key, value in _cpp2numba.items(): _numba2cpp[value] = key +# prefer "int" in the case of intc over "int32_t" +_numba2cpp[nb_types.intc] = 'int' def numba2cpp(val): if hasattr(val, 'literal_type'): val = val.literal_type - return _numba2cpp[val] + if val == nb_types.int64: # Python int + # TODO: this is only necessary until "best matching" is in place + val = nb_types.intc # more likely match candidate + elif isinstance(val, numba.types.CPointer): + return _numba2cpp[val.dtype] + elif isinstance(val, numba.types.RawPointer): + return _numba2cpp[nb_types.voidptr] + elif isinstance(val, numba.types.Array): + return "std::vector<" + _numba2cpp[val.dtype] + ">" + elif isinstance(val, CppClassNumbaType): + return val._scope.__cpp_name__ + else: + try: + return _numba2cpp[val] + except: + raise RuntimeError("Type mapping failed from Numba to C++ for ", val) + +def numba_arg_convertor(args): + args_cpp = [] + for i, arg in enumerate(list(args)): + # If the user explicitly passes an argument using numba CPointer, the regex match is used + # to detect the pass by reference since the dispatcher always returns typeref[val*] + match = re.search(r"typeref\[(.*?)\*\]", str(arg)) + if match: + literal_val = match.group(1) + arg_type = numba.typeof(eval(literal_val)) + args_cpp.append(to_ref(numba2cpp(arg_type))) + else: + args_cpp.append(numba2cpp(arg)) + return tuple(args_cpp) + +def to_ref(type_list): + ref_list = [] + for i, l in enumerate(type_list): + ref_list.append(l + '&') + return ref_list # TODO: looks like Numba treats unsigned types as signed when lowering, # which seems to work as they're just reinterpret_casts @@ -83,7 +140,6 @@ def numba2cpp(val): 'uint8_t' : ir.IntType(8), 'short' : ir.IntType(nb_types.short.bitwidth), 'unsigned short' : ir.IntType(nb_types.ushort.bitwidth), - 'internal_enum_type_t' : ir.IntType(nb_types.intc.bitwidth), 'int' : ir.IntType(nb_types.intc.bitwidth), 'unsigned int' : ir.IntType(nb_types.uintc.bitwidth), 'int32_t' : ir.IntType(32), @@ -99,8 +155,19 @@ def numba2cpp(val): } def cpp2ir(val): - return _cpp2ir[val] - + try: + return _cpp2ir[val] + except KeyError: + if val.startswith("std::vector"): + ## TODO should be possible to obtain the vector length from the CPPDataMember val + type_arr = ir.VectorType(cpp2ir(resolve_std_vector(val)), 3) + return type_arr + elif val != "char*" and val[-1] == "*": + if val.startswith('const'): + return ir.PointerType(cpp2ir(resolve_const_types(val))) + else: + type_2 = _cpp2ir[val[:-1]] + return ir.PointerType(type_2) # # C++ function pointer -> Numba @@ -118,6 +185,8 @@ def __init__(self, func, is_method=False): self._signatures = list() self._impl_keys = dict() + self._arg_set_matched = tuple() + self.ret_type = None def is_precise(self): return True # by definition @@ -128,28 +197,36 @@ def get_call_type(self, context, args, kwds): except KeyError: pass - ol = CppFunctionNumbaType(self._func.__overload__(tuple(numba2cpp(x) for x in args)), self._is_method) + ol = CppFunctionNumbaType(self._func.__overload__(numba_arg_convertor(args)), self._is_method) + thistype = None if self._is_method: - args = (nb_types.voidptr, *args) + thistype = nb_types.voidptr + self.ret_type = cpp2numba(ol._func.__cpp_reflex__(cpp_refl.RETURN_TYPE)) ol.sig = nb_typing.Signature( - return_type=cpp2numba(ol._func.__cpp_reflex__(cpp_refl.RETURN_TYPE)), + return_type=self.ret_type, args=args, - recvr=None) # this pointer + recvr=thistype) + + extsig = ol.sig + if self._is_method: + self.ret_type = ol.sig.return_type + args = (nb_types.voidptr, *args) + extsig = nb_typing.Signature( + return_type=ol.sig.return_type, args=args, recvr=None) self._impl_keys[args] = ol + self._arg_set_matched = numba_arg_convertor(args) + @nb_iutils.lower_builtin(ol, *args) def lower_external_call(context, builder, sig, args, - ty=nb_types.ExternalFunctionPointer(ol.sig, ol.get_pointer), pyval=self._func): + ty=nb_types.ExternalFunctionPointer(extsig, ol.get_pointer), pyval=self._func, is_method=self._is_method): ptrty = context.get_function_pointer_type(ty) ptrval = context.add_dynamic_addr( builder, ty.get_pointer(pyval), info=str(pyval)) fptr = builder.bitcast(ptrval, ptrty) - if hasattr(context, 'cppyy_currentcall_this'): - args = [context.cppyy_currentcall_this]+args - del context.cppyy_currentcall_this return context.call_function_pointer(builder, fptr, args) return ol.sig @@ -160,9 +237,12 @@ def get_call_signatures(self): def get_impl_key(self, sig): return self._impl_keys[sig.args] + #TODO : Remove the redundancy of __overload__ matching and use this function to only obtain the address given the matched overload def get_pointer(self, func): if func is None: func = self._func - ol = func.__overload__(tuple(numba2cpp(x) for x in self.sig.args[int(self._is_method):])) + + ol = func.__overload__(numba_arg_convertor(self.sig.args)) + address = cppyy.addressof(ol) if not address: raise RuntimeError("unresolved address for %s" % str(ol)) @@ -216,10 +296,19 @@ def __init__(self, name, offset, cpptype): # class CppClassNumbaType(CppFunctionNumbaType): def __init__(self, scope, qualifier): + addr = None + cppinstance_val = None + if qualifier == Qualified.instance: + addr = cppyy.addressof(scope) + cppinstance_val = scope + scope = type(scope) + qualifier = Qualified.default super(CppClassNumbaType, self).__init__(scope.__init__) self.name = 'CppClass(%s)' % scope.__cpp_name__ # overrides value in Type self._scope = scope self._qualifier = qualifier + self._cppinstanceval = cppinstance_val + self._addr = addr def get_scope(self): return self._scope @@ -239,6 +328,7 @@ def is_precise(self): def key(self): return (self._scope, self._qualifier) + @nb_tmpl.infer_getattr class CppClassFieldResolver(nb_tmpl.AttributeTemplate): key = CppClassNumbaType @@ -286,7 +376,6 @@ def cppclass_getattr_impl(context, builder, typ, val, attr): return builder.load(pf) elif q == Qualified.value: - # TODO: access members of by value returns model = nb_dm.default_manager.lookup(typ) return model.get(builder, val, attr) @@ -302,22 +391,70 @@ def cppclass_getattr_impl(context, builder, typ, val, attr): # assume this is a method q = typ.get_qualifier() if q == Qualified.default: - context.cppyy_currentcall_this = builder.bitcast(val, ir_voidptr) + return builder.bitcast(val, ir_voidptr) elif q == Qualified.value: - # TODO: take address of by value returns - context.cppyy_currentcall_this = None + return None - else: - assert not "unknown qualified type" + assert not "unknown qualified type" + return None + + +class ImplAggregateValueModel(nb_dm.models.StructModel): + def get(self, builder, val, pos): + """Get a field at the given position/field name""" + + if isinstance(pos, str): + pos = self.get_field_position(pos) + + # Use the offsets for direct addressing, rather than getting the elements + # from the struct type. + dmi = self._data_members[pos] + + stack = nb_cgu.alloca_once(builder, self.get_data_type()) + builder.store(val, stack) + + llval = builder.bitcast(stack, ir_byteptr) + pfc = builder.gep(llval, [ir.Constant(ir_intptr_t, dmi.f_offset)]) + pf = builder.bitcast(pfc, ir.PointerType(dmi.f_irtype)) + + return builder.load(pf) - return context.cppyy_currentcall_this +class ImplClassValueModel(ImplAggregateValueModel): + # TODO : Should the address have to be passed here and stored in meminfo + # value: representation inside function body. Maybe stored in stack. + # The representation here are flexible. + def get_value_type(self): + return self.get_data_type() + + # data: representation used when storing into containers (e.g. arrays). + def get_data_type(self): + # The struct model relies on data being a POD, but for C++ objects, there + # can be hidden data (e.g. vtable, thunks, or simply private members), and + # the alignment of Cling and Numba also need not be the same. Therefore, the + # struct is split in a series of byte members to get the total size right + # and to allow addressing at the correct offsets. + if self._data_type is None: + self._data_type = ir.LiteralStructType([ir_byte for i in range(self._sizeof)], packed=True) + return self._data_type + + # return: representation used for return argument. + def get_return_type(self): + return self.get_data_type() scope_numbatypes = (dict(), dict()) @nb_ext.typeof_impl.register(cpp_types.Scope) def typeof_scope(val, c, q = Qualified.default): + is_instance = False + cppinstance_val = None + if q == Qualified.instance: + cppinstance_val = val + val = type(val) + q = Qualified.default + is_instance = True + global scope_numbatypes try: @@ -333,16 +470,24 @@ def typeof_scope(val, c, q = Qualified.default): class ImplClassType(CppClassNumbaType): pass - cnt = ImplClassType(val, q) + if is_instance: + cnt = ImplClassType(cppinstance_val, Qualified.instance) + else: + cnt = ImplClassType(val, q) + scope_numbatypes[q][val] = cnt # declare data members to Numba data_members = list() + member_methods = dict() + for name, field in val.__dict__.items(): if type(field) == cpp_types.DataMember: data_members.append(CppDataMemberInfo( name, field.__cpp_reflex__(cpp_refl.OFFSET), field.__cpp_reflex__(cpp_refl.TYPE)) ) + elif type(field) == cpp_types.Function: + member_methods[name] = field.__cpp_reflex__(cpp_refl.RETURN_TYPE) # TODO: this refresh is needed b/c the scope type is registered as a # callable after the tracing started; no idea of the side-effects ... @@ -354,6 +499,7 @@ class ImplClassType(CppClassNumbaType): class ImplClassModel(nb_dm.models.StructModel): def __init__(self, dmm, fe_type): self._data_members = data_members + self._member_methods = member_methods # TODO: eventually we need not derive from StructModel members = [(dmi.f_name, dmi.f_nbtype) for dmi in data_members] @@ -379,10 +525,13 @@ def get_value_type(self): # as a pointer to POD to allow indexing by Numba for data member type checking, but the # address offsetting for loading data member values is independent (see get(), below), # so the exact layout need not match a POD + + # TODO: this doesn't work for real PODs, b/c those are unpacked into their elements and + # passed through registers return ir.PointerType(super(ImplClassModel, self).get_value_type()) # argument: representation used for function argument. Needs to be builtin type, - # but unlike other Numba composites, C++ proxies are no flattened. + # but unlike other Numba composites, C++ proxies are not flattened. def get_argument_type(self): return self.get_value_type() @@ -398,20 +547,39 @@ def from_argument(self, builder, value): # access to public data members def get(self, builder, val, pos): """Get a field at the given position/field name""" + if isinstance(pos, str): pos = self.get_field_position(pos) + dmi = self._data_members[pos] + llval = builder.bitcast(val, ir_byteptr) pfc = builder.gep(llval, [ir.Constant(ir_intptr_t, dmi.f_offset)]) pf = builder.bitcast(pfc, ir.PointerType(dmi.f_irtype)) + return builder.load(pf) elif q == Qualified.value: - @nb_ext.register_model(ImplClassType) - class ImplClassModel(nb_dm.models.StructModel): - def __init__(self, dmm, fe_type): - members = [(dmi.f_name, dmi.f_nbtype) for dmi in data_members] - nb_dm.models.StructModel.__init__(self, dmm, fe_type, members) + if val.__cpp_reflex__(cpp_refl.IS_AGGREGATE): + @nb_ext.register_model(ImplClassType) + class ImplClassModel(ImplAggregateValueModel): + pass + else: + @nb_ext.register_model(ImplClassType) + class ImplClassModel(ImplClassValueModel): + pass + + def init(self, dmm, fe_type, sz = cppyy.sizeof(val)): + self._data_members = data_members + self._member_methods = member_methods + self._sizeof = sz + + # TODO: this code exists purely to be able to use the indexing and hierarchy + # of the base class StructModel, which isn't much of a reason + members = [(dmi.f_name, dmi.f_nbtype) for dmi in data_members] + nb_dm.models.StructModel.__init__(self, dmm, fe_type, members) + + ImplClassModel.__init__ = init else: assert not "unknown qualified type" @@ -431,27 +599,44 @@ def unbox_instance(typ, obj, c): return nb_ext.NativeValue(pobj, is_error=None, cleanup=None) + def make_implclass(context, builder, typ, **kwargs): + return nb_cgu.create_struct_proxy(typ)(context, builder, **kwargs) + # C++ object to Python proxy wrapping for returns from Numba trace @nb_ext.box(ImplClassType) def box_instance(typ, val, c): assert not "requires object model and passing of intact object, not memberwise copy" + global cppyy_from_voidptr - ir_pyobj = c.context.get_argument_type(nb_types.pyobject) - ir_int = cpp2ir('int') + if type(val) == ir.Constant: + if val.constant == ir.Undefined: + assert not "Value passed to instance boxing is undefined" + return NULL - ptrty = ir.PointerType(ir.FunctionType(ir_pyobj, [ir_voidptr, cpp2ir('char*'), ir_int])) - ptrval = c.context.add_dynamic_addr(c.builder, cppyy_from_voidptr, info='Instance_FromVoidPtr') - fp = c.builder.bitcast(ptrval, ptrty) + implclass = make_implclass(c.context, c.builder, typ) + classobj = c.pyapi.unserialize(c.pyapi.serialize_object(cpp_types.Instance)) + pyobj = c.context.get_argument_type(nb_types.pyobject) - module = c.builder.basic_block.function.module - clname = c.context.insert_const_string(module, typ._scope.__cpp_name__) + box_list = [] - NULL = c.context.get_constant_null(nb_types.voidptr) # TODO: get the real thing - return c.context.call_function_pointer(c.builder, fp, [NULL, clname, ir_int(0)]) + model = implclass._datamodel + cfr = CppClassFieldResolver(c.context) - return cnt + for i in typ._scope.__dict__: + if isinstance(cfr.generic_resolve(typ, i), nb_types.Type): + box_list.append(c.box(cfr.generic_resolve(typ, i), getattr(implclass, i))) + + box_res = c.pyapi.call_function_objargs( + classobj, tuple(box_list) + ) + # Required for nopython mode, numba nrt requres each member box call to decref since it steals the reference + for i in box_list: + c.pyapi.decref(i) + return box_res + + return cnt # # C++ instance -> Numba @@ -464,5 +649,5 @@ def typeof_instance(val, c): return scope_numbatypes[Qualified.default][type(val)] except KeyError: pass - - return typeof_scope(type(val), c, Qualified.default) + # Pass the val itself to obtain Cling address of the CPPInstance for reference to C++ objects + return typeof_scope(val, c, Qualified.instance) \ No newline at end of file diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/reflex.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/reflex.py index efa1d5fc8ef57..35730cbb47c38 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/reflex.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/reflex.py @@ -17,6 +17,7 @@ cppyy.include("CPyCppyy/Reflex.h") IS_NAMESPACE = cppyy.gbl.Cppyy.Reflex.IS_NAMESPACE + IS_AGGREGATE = cppyy.gbl.Cppyy.Reflex.IS_AGGREGATE OFFSET = cppyy.gbl.Cppyy.Reflex.OFFSET RETURN_TYPE = cppyy.gbl.Cppyy.Reflex.RETURN_TYPE diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/types.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/types.py index eb6e32be0f9f0..4a979ac59c285 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/types.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/types.py @@ -20,6 +20,7 @@ 'Method', 'Scope', 'InstanceArray', + 'LowLevelView', 'Template' ] @@ -27,7 +28,8 @@ Function = bck.CPPOverload Method = bck.CPPOverload Scope = bck.CPPScope - InstanceArray = bck.InstancesArray + InstanceArray = bck.InstanceArray + LowLevelView = bck.LowLevelView Template = bck.TemplateProxy del bck diff --git a/bindings/pyroot/cppyy/cppyy/setup.cfg b/bindings/pyroot/cppyy/cppyy/setup.cfg index 8debd01371e4f..cc470d8cba241 100644 --- a/bindings/pyroot/cppyy/cppyy/setup.cfg +++ b/bindings/pyroot/cppyy/cppyy/setup.cfg @@ -2,4 +2,8 @@ universal=0 [metadata] -license_file = LICENSE.txt +license_files = LICENSE.txt + +[options.entry_points] +pyinstaller40 = + hook-dirs = cppyy.__pyinstaller:get_hook_dirs diff --git a/bindings/pyroot/cppyy/cppyy/setup.py b/bindings/pyroot/cppyy/cppyy/setup.py index df5fa6ffc7438..ef9398a94ed21 100755 --- a/bindings/pyroot/cppyy/cppyy/setup.py +++ b/bindings/pyroot/cppyy/cppyy/setup.py @@ -2,36 +2,28 @@ from setuptools import setup, find_packages, Extension from distutils import log -from setuptools.dist import Distribution from setuptools.command.install import install as _install -force_bdist = False -if '--force-bdist' in sys.argv: - force_bdist = True - sys.argv.remove('--force-bdist') - -add_pkg = ['cppyy'] +add_pkg = ['cppyy', 'cppyy.__pyinstaller'] try: import __pypy__, sys version = sys.pypy_version_info - requirements = ['cppyy-backend'] + requirements = ['cppyy-backend==1.15.2', 'cppyy-cling==6.30.0'] if version[0] == 5: if version[1] <= 9: - requirements = ['cppyy-cling<6.12', 'cppyy-backend<0.3'] + requirements = ['cppyy-backend<0.3', 'cppyy-cling<6.12'] add_pkg += ['cppyy_compat'] elif version[1] <= 10: - requirements = ['cppyy-cling<=6.15', 'cppyy-backend<0.4'] + requirements = ['cppyy-backend<0.4', 'cppyy-cling<=6.15'] elif version[0] == 6: if version[1] <= 0: - requirements = ['cppyy-cling<=6.15', 'cppyy-backend<1.1'] + requirements = ['cppyy-backend<1.1', 'cppyy-cling<=6.15'] elif version[0] == 7: - if version[1] <= 2: - requirements = ['cppyy-cling<=6.18.2.3', 'cppyy-backend<=1.10'] - else: - requirements = ['cppyy-cling<=6.18.2.7', 'cppyy-backend<=1.10'] + if version[1] <= 3 and version[2] <= 3: + requirements = ['cppyy-backend<=1.10', 'cppyy-cling<=6.18.2.3'] except ImportError: # CPython - requirements = ['cppyy-cling==6.18.2.7', 'cppyy-backend==1.10.8', 'CPyCppyy==1.10.2'] + requirements = ['CPyCppyy==1.12.16', 'cppyy-backend==1.15.2', 'cppyy-cling==6.30.0'] setup_requirements = ['wheel'] if 'build' in sys.argv or 'install' in sys.argv: @@ -55,23 +47,23 @@ def find_version(*file_paths): raise RuntimeError("Unable to find version string.") -# -# platform-dependent helpers -# -def is_manylinux(): - try: - for line in open('/etc/redhat-release').readlines(): - if 'CentOS release 6.10 (Final)' in line: - return True - except (OSError, IOError): - pass - return False - - # # customized commands # class my_install(_install): + def __init__(self, *args, **kwds): + if 0x3000000 <= sys.hexversion: + super(_install, self).__init__(*args, **kwds) + else: + # b/c _install is a classobj, not type + _install.__init__(self, *args, **kwds) + + try: + import cppyy_backend as cpb + self._pchname = 'allDict.cxx.pch.' + str(cpb.__version__) + except (ImportError, AttributeError): + self._pchname = None + def run(self): # base install _install.run(self) @@ -79,22 +71,24 @@ def run(self): # force build of the .pch underneath the cppyy package if not available yet install_path = os.path.join(os.getcwd(), self.install_libbase, 'cppyy') - try: - import cppyy_backend as cpb - if not os.path.exists(os.path.join(cpb.__file__, 'etc', 'allDict.cxx.pch')): - log.info("installing pre-compiled header in %s", install_path) - cpb.loader.set_cling_compile_options(True) - cpb.loader.ensure_precompiled_header(install_path, 'allDict.cxx.pch') - except (ImportError, AttributeError): - # ImportError may occur with wrong pip requirements resolution (unlikely) - # AttributeError will occur with (older) PyPy as it relies on older backends - pass + if self._pchname: + try: + import cppyy_backend as cpb + if not os.path.exists(os.path.join(cpb.__file__, 'etc', self._pchname)): + log.info("installing pre-compiled header in %s", install_path) + cpb.loader.set_cling_compile_options(True) + cpb.loader.ensure_precompiled_header(install_path, self._pchname) + except (ImportError, AttributeError): + # ImportError may occur with wrong pip requirements resolution (unlikely) + # AttributeError will occur with (older) PyPy as it relies on older backends + self._pchname = None def get_outputs(self): outputs = _install.get_outputs(self) - # pre-emptively add allDict.cxx.pch, which may or may not be created; need full - # path to make sure the final relative path is correct - outputs.append(os.path.join(os.getcwd(), self.install_libbase, 'cppyy', 'allDict.cxx.pch')) + if self._pchname: + # pre-emptively add allDict.cxx.pch, which may or may not be created; need full + # path to make sure the final relative path is correct + outputs.append(os.path.join(os.getcwd(), self.install_libbase, 'cppyy', self._pchname)) return outputs @@ -102,29 +96,6 @@ def get_outputs(self): 'install': my_install } -# -# customized distribition to disable binaries -# -class MyDistribution(Distribution): - def run_commands(self): - # pip does not resolve dependencies before building binaries, so unless - # packages are installed one-by-one, on old install is used or the build - # will simply fail hard. The following is not completely quiet, but at - # least a lot less conspicuous. - if not is_manylinux() and not force_bdist: - disabled = set(( - 'bdist_wheel', 'bdist_egg', 'bdist_wininst', 'bdist_rpm')) - for cmd in self.commands: - if not cmd in disabled: - self.run_command(cmd) - else: - log.info('Command "%s" is disabled', cmd) - cmd_obj = self.get_command_obj(cmd) - cmd_obj.get_outputs = lambda: None - else: - return Distribution.run_commands(self) - - setup( name='cppyy', version=find_version('python', 'cppyy', '_version.py'), @@ -149,12 +120,13 @@ def run_commands(self): 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: Implementation :: PyPy', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: C', 'Programming Language :: C++', @@ -166,11 +138,23 @@ def run_commands(self): keywords='C++ bindings data science calling language integration', + include_package_data=True, + package_data={'': ['installer/cppyy_monkey_patch.py']}, + package_dir={'': 'python'}, packages=find_packages('python', include=add_pkg), + # TODO: numba_extensions will load all extensions even if the package + # itself is not otherwise imported, just installed; in the case of cppyy, + # that is currently too heavy (and breaks on conda) + + #entry_points={ + # 'numba_extensions': [ + # 'init = cppyy.numba_ext:_init_extension', + # ], + #}, + cmdclass=cmdclass, - distclass=MyDistribution, zip_safe=False, ) diff --git a/bindings/pyroot/cppyy/cppyy/test/Makefile b/bindings/pyroot/cppyy/cppyy/test/Makefile index 73123699dd053..da0b71045e385 100644 --- a/bindings/pyroot/cppyy/cppyy/test/Makefile +++ b/bindings/pyroot/cppyy/cppyy/test/Makefile @@ -13,14 +13,15 @@ dicts = advancedcppDict.so \ std_streamsDict.so \ stltypesDict.so \ templatesDict.so + all : $(dicts) genreflex_flags := $(shell genreflex --cppflags) -cppflags=$(shell cling-config --cppflags) $(genreflex_flags) -O3 -fPIC -I$(shell python -c 'import distutils.sysconfig as ds; print(ds.get_config_var("INCLUDEPY"))') -Wno-register +cppflags=$(shell cling-config --cppflags) $(genreflex_flags) -O3 -fPIC -I$(shell python -c 'import sysconfig as sc; print(sc.get_config_var("INCLUDEPY"))') -Wno-register PLATFORM := $(shell uname -s) ifeq ($(PLATFORM),Darwin) - cppflags+=-dynamiclib -single_module -arch x86_64 -undefined dynamic_lookup -Wno-delete-non-virtual-dtor + cppflags+=-dynamiclib -single_module -undefined dynamic_lookup -Wno-delete-non-virtual-dtor endif %Dict.so: %_rflx.cpp %.cxx diff --git a/bindings/pyroot/cppyy/cppyy/test/advancedcpp.cxx b/bindings/pyroot/cppyy/cppyy/test/advancedcpp.cxx index f7e09a7b87d46..3daa11c14f7d1 100644 --- a/bindings/pyroot/cppyy/cppyy/test/advancedcpp.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/advancedcpp.cxx @@ -95,6 +95,9 @@ some_int_holder my_global_int_holders[5] = { some_int_holder(13), some_int_holder(42), some_int_holder(88), some_int_holder(-1), some_int_holder(17) }; +some_abstract_class* g_abstract_ptr = nullptr; + + // for life-line and identity testing int some_class_with_data::some_data::s_num_data = 0; diff --git a/bindings/pyroot/cppyy/cppyy/test/advancedcpp.h b/bindings/pyroot/cppyy/cppyy/test/advancedcpp.h index e267989dbef0d..d09e799f47d6d 100644 --- a/bindings/pyroot/cppyy/cppyy/test/advancedcpp.h +++ b/bindings/pyroot/cppyy/cppyy/test/advancedcpp.h @@ -292,6 +292,8 @@ class some_int_holder { }; extern some_int_holder my_global_int_holders[5]; +extern some_abstract_class* g_abstract_ptr; + //=========================================================================== class some_class_with_data { // for life-line and identity testing diff --git a/bindings/pyroot/cppyy/cppyy/test/advancedcpp.xml b/bindings/pyroot/cppyy/cppyy/test/advancedcpp.xml index 1ce87cd8b1323..e414fe9400123 100644 --- a/bindings/pyroot/cppyy/cppyy/test/advancedcpp.xml +++ b/bindings/pyroot/cppyy/cppyy/test/advancedcpp.xml @@ -43,6 +43,7 @@ + diff --git a/bindings/pyroot/cppyy/cppyy/test/bindexplib.py b/bindings/pyroot/cppyy/cppyy/test/bindexplib.py index af2adccbc7578..838dca4d50df0 100755 --- a/bindings/pyroot/cppyy/cppyy/test/bindexplib.py +++ b/bindings/pyroot/cppyy/cppyy/test/bindexplib.py @@ -1,6 +1,6 @@ from __future__ import print_function -import sys, subprocess +import os, sys, subprocess target = sys.argv[1] output = sys.argv[2] @@ -29,3 +29,4 @@ def isokay(name): elif parts[4] == '()' and parts[5] == 'External': if isokay(parts[7]): outf.write('\t%s\n' % parts[7]) + diff --git a/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx b/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx index c1d47b4016504..79fcb7fc65a2b 100644 --- a/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx @@ -1,25 +1,47 @@ +#if __cplusplus >= 201103L + #include "cpp11features.h" -// for std::shared_ptr<> testing -int TestSharedPtr::s_counter = 0; +// for std::shared/unique_ptr<> testing +int TestSmartPtr::s_counter = 0; + +std::shared_ptr create_shared_ptr_instance() { + return std::shared_ptr(new TestSmartPtr); +} -std::shared_ptr create_shared_ptr_instance() { - return std::shared_ptr(new TestSharedPtr); +std::unique_ptr create_unique_ptr_instance() { + return std::unique_ptr(new TestSmartPtr); } -int TestSharedPtr::get_value() { +int TestSmartPtr::get_value() { return 17; } -int DerivedTestSharedPtr::get_value() { +int DerivedTestSmartPtr::get_value() { return m_int + 76; } -int pass_shared_ptr(std::shared_ptr p) { +int pass_shared_ptr(std::shared_ptr p) { + return p->get_value(); +} + +int move_shared_ptr(std::shared_ptr&& p) { return p->get_value(); } +int move_unique_ptr(std::unique_ptr&& p) { + return p->get_value(); +} + +int move_unique_ptr_derived(std::unique_ptr&& p) { + return p->get_value(); +} + +TestSmartPtr create_TestSmartPtr_by_value() { + return TestSmartPtr{}; +} + // for move ctors etc. int TestMoving1::s_move_counter = 0; @@ -35,3 +57,5 @@ void implicit_converion_move(TestMoving2&&) { // for std::function testing std::function FNCreateTestStructFunc() { return [](const FNTestStruct& t) { return t.t; }; } std::function FunctionNS::FNCreateTestStructFunc() { return [](const FNTestStruct& t) { return t.t; }; } + +#endif // c++11 and later diff --git a/bindings/pyroot/cppyy/cppyy/test/cpp11features.h b/bindings/pyroot/cppyy/cppyy/test/cpp11features.h index f0eba340766b0..aae0cc6ac5d3d 100644 --- a/bindings/pyroot/cppyy/cppyy/test/cpp11features.h +++ b/bindings/pyroot/cppyy/cppyy/test/cpp11features.h @@ -1,34 +1,42 @@ +#if __cplusplus >= 201103L + #include #include #include //=========================================================================== -class TestSharedPtr { // for std::shared_ptr<> testing +class TestSmartPtr { // for std::shared/unique_ptr<> testing public: static int s_counter; public: - TestSharedPtr() { ++s_counter; } - TestSharedPtr(const TestSharedPtr&) { ++s_counter; } - virtual ~TestSharedPtr() { --s_counter; } + TestSmartPtr() { ++s_counter; } + TestSmartPtr(const TestSmartPtr&) { ++s_counter; } + virtual ~TestSmartPtr() { --s_counter; } public: virtual int get_value(); }; -std::shared_ptr create_shared_ptr_instance(); +std::shared_ptr create_shared_ptr_instance(); +std::unique_ptr create_unique_ptr_instance(); -class DerivedTestSharedPtr : TestSharedPtr { +class DerivedTestSmartPtr : TestSmartPtr { public: - DerivedTestSharedPtr(int i) : m_int(i) {} + DerivedTestSmartPtr(int i) : m_int(i) {} virtual int get_value(); public: int m_int; }; -int pass_shared_ptr(std::shared_ptr p); +int pass_shared_ptr(std::shared_ptr p); +int move_shared_ptr(std::shared_ptr&& p); +int move_unique_ptr(std::unique_ptr&& p); +int move_unique_ptr_derived(std::unique_ptr&& p); + +TestSmartPtr create_TestSmartPtr_by_value(); //=========================================================================== @@ -66,12 +74,12 @@ void implicit_converion_move(TestMoving2&&); //=========================================================================== struct TestData { // for initializer list construction - TestData(int i) : m_int(i) {} + TestData(int i=0) : m_int(i) {} int m_int; }; struct TestData2 { - TestData2(int i) : m_int(i) {} + TestData2(int i=0) : m_int(i) {} virtual ~TestData2() {} int m_int; }; @@ -112,3 +120,5 @@ namespace std { size_t operator()(const StructWithHash&) const { return 17; } }; } // namespace std + +#endif // c++11 and later diff --git a/bindings/pyroot/cppyy/cppyy/test/cpp11features.xml b/bindings/pyroot/cppyy/cppyy/test/cpp11features.xml index c7685110cc8f5..e6cf2188bd803 100644 --- a/bindings/pyroot/cppyy/cppyy/test/cpp11features.xml +++ b/bindings/pyroot/cppyy/cppyy/test/cpp11features.xml @@ -1,9 +1,14 @@ - - + + + + + + + @@ -12,7 +17,7 @@ - + diff --git a/bindings/pyroot/cppyy/cppyy/test/crossinheritance.cxx b/bindings/pyroot/cppyy/cppyy/test/crossinheritance.cxx index 79b90f7155e26..8dacb1609e8ff 100644 --- a/bindings/pyroot/cppyy/cppyy/test/crossinheritance.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/crossinheritance.cxx @@ -76,3 +76,55 @@ CrossInheritance::CountableBase::~CountableBase() { int CrossInheritance::CountableBase::call() { return -1; } + +CrossInheritance::Component::Component() { + ++s_count; +} + +CrossInheritance::Component::~Component() { + --s_count; +} + +int CrossInheritance::Component::get_count() { + return s_count; +} + +int CrossInheritance::Component::s_count = 0; + +namespace { + +class ComponentWithValue : public CrossInheritance::Component { +public: + ComponentWithValue(int value) : m_value(value) {} + int getValue() { return m_value; } + +protected: + int m_value; +}; + +} // unnamed namespace + +CrossInheritance::Component* CrossInheritance::build_component(int value) { + return new ComponentWithValue(value); +} + +CrossInheritance::Component* CrossInheritance::cycle_component(Component* c) { + return c; +} + +// for protected member testing +AccessProtected::MyBase::MyBase() : my_data(101) { + /* empty */ +} + +AccessProtected::MyBase::~MyBase() { + /* empty */ +} + +int AccessProtected::MyBase::get_data_v() { + return my_data; +} + +int AccessProtected::MyBase::get_data() { + return my_data; +} diff --git a/bindings/pyroot/cppyy/cppyy/test/crossinheritance.h b/bindings/pyroot/cppyy/cppyy/test/crossinheritance.h index f030c9fce3409..a8c32002941bb 100644 --- a/bindings/pyroot/cppyy/cppyy/test/crossinheritance.h +++ b/bindings/pyroot/cppyy/cppyy/test/crossinheritance.h @@ -95,6 +95,41 @@ class CountableBase { static int s_count; }; +class Component { +public: + Component(); + Component(const Component&) = delete; + Component& operator=(const Component&) = delete; + virtual ~Component(); + + static int get_count(); + +private: + static int s_count; +}; + +Component* build_component(int value); +Component* cycle_component(Component* c); + } // namespace CrossInheritance + +//=========================================================================== +namespace AccessProtected { // for protected member testing + +class MyBase { +public: + MyBase(); + virtual ~MyBase(); + +protected: + virtual int get_data_v(); + int get_data(); + +protected: + int my_data; +}; + +} // AccessProtected + #endif // !CPPYY_TEST_CROSSINHERITANCE_H diff --git a/bindings/pyroot/cppyy/cppyy/test/crossinheritance.xml b/bindings/pyroot/cppyy/cppyy/test/crossinheritance.xml index 63d4fa1d00e67..12db5fcaa3b07 100644 --- a/bindings/pyroot/cppyy/cppyy/test/crossinheritance.xml +++ b/bindings/pyroot/cppyy/cppyy/test/crossinheritance.xml @@ -2,5 +2,9 @@ + + + + diff --git a/bindings/pyroot/cppyy/cppyy/test/datatypes.cxx b/bindings/pyroot/cppyy/cppyy/test/datatypes.cxx index 1d4ca138ff77e..b78b6b7d3112b 100644 --- a/bindings/pyroot/cppyy/cppyy/test/datatypes.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/datatypes.cxx @@ -35,6 +35,7 @@ CppyyTestData::CppyyTestData() : m_const_int(17), m_owns_arrays(false) m_ldouble = -88.l; m_complex = {99., 101.}; m_icomplex = {121, 141}; + m_ccomplex = {151., 161.}; m_enum = kNothing; m_voidp = (void*)0; @@ -44,6 +45,8 @@ CppyyTestData::CppyyTestData() : m_const_int(17), m_owns_arrays(false) #if __cplusplus > 201402L m_byte_array2 = new std::byte[N]; #endif + m_int8_array2 = new int8_t[N]; + m_uint8_array2 = new uint8_t[N]; m_short_array2 = new short[N]; m_ushort_array2 = new unsigned short[N]; m_int_array2 = new int[N]; @@ -54,6 +57,7 @@ CppyyTestData::CppyyTestData() : m_const_int(17), m_owns_arrays(false) m_float_array2 = new float[N]; m_double_array2 = new double[N]; m_complex_array2 = new complex_t[N]; + m_ccomplex_array2 = new ccomplex_t[N]; for (int i = 0; i < N; ++i) { m_bool_array[i] = bool(i%2); @@ -64,24 +68,29 @@ CppyyTestData::CppyyTestData() : m_const_int(17), m_owns_arrays(false) m_byte_array[i] = (std::byte)(3u*i); m_byte_array2[i] = (std::byte)(4u*i); #endif - m_short_array[i] = -1*i; - m_short_array2[i] = -2*i; - m_ushort_array[i] = 3u*i; - m_ushort_array2[i] = 4u*i; - m_int_array[i] = -5*i; - m_int_array2[i] = -6*i; - m_uint_array[i] = 7u*i; - m_uint_array2[i] = 8u*i; - m_long_array[i] = -9l*i; - m_long_array2[i] = -10l*i; - m_ulong_array[i] = 11ul*i; - m_ulong_array2[i] = 12ul*i; + m_int8_array[i] = - 1*i; + m_int8_array2[i] = - 2*i; + m_uint8_array[i] = 3u*i; + m_uint8_array2[i] = 4u*i; + m_short_array[i] = - 5*i; + m_short_array2[i] = - 6*i; + m_ushort_array[i] = 7u*i; + m_ushort_array2[i] = 8u*i; + m_int_array[i] = - 9*i; + m_int_array2[i] = -10*i; + m_uint_array[i] = 11u*i; + m_uint_array2[i] = 23u*i; + m_long_array[i] = -13l*i; + m_long_array2[i] = -14l*i; + m_ulong_array[i] = 15ul*i; + m_ulong_array2[i] = 18ul*i; m_float_array[i] = -13.f*i; m_float_array2[i] = -14.f*i; m_double_array[i] = -15.*i; m_double_array2[i] = -16.*i; m_complex_array2[i] = {17.*i, 18.*i}; + m_ccomplex_array2[i] = {19.*i, 20.*i}; } m_owns_arrays = true; @@ -104,6 +113,8 @@ void CppyyTestData::destroy_arrays() { #if __cplusplus > 201402L delete[] m_byte_array2; #endif + delete[] m_int8_array2; + delete[] m_uint8_array2; delete[] m_short_array2; delete[] m_ushort_array2; delete[] m_int_array2; @@ -114,6 +125,7 @@ void CppyyTestData::destroy_arrays() { delete[] m_float_array2; delete[] m_double_array2; delete[] m_complex_array2; + delete[] m_ccomplex_array2; m_owns_arrays = false; } @@ -148,6 +160,7 @@ long double CppyyTestData::get_ldouble() { return m_ldouble; } long double CppyyTestData::get_ldouble_def(long double ld) { return ld; } complex_t CppyyTestData::get_complex() { return m_complex; } icomplex_t CppyyTestData::get_icomplex() { return m_icomplex; } +ccomplex_t CppyyTestData::get_ccomplex() { return m_ccomplex; } CppyyTestData::EWhat CppyyTestData::get_enum() { return m_enum; } void* CppyyTestData::get_voidp() { return m_voidp; } @@ -161,6 +174,10 @@ unsigned char* CppyyTestData::get_uchar_array2() { return m_uchar_array2; } std::byte* CppyyTestData::get_byte_array() { return m_byte_array; } std::byte* CppyyTestData::get_byte_array2() { return m_byte_array2; } #endif +int8_t* CppyyTestData::get_int8_array() { return m_int8_array; } +int8_t* CppyyTestData::get_int8_array2() { return m_int8_array2; } +uint8_t* CppyyTestData::get_uint8_array() { return m_uint8_array; } +uint8_t* CppyyTestData::get_uint8_array2() { return m_uint8_array2; } short* CppyyTestData::get_short_array() { return m_short_array; } short* CppyyTestData::get_short_array2() { return m_short_array2; } unsigned short* CppyyTestData::get_ushort_array() { return m_ushort_array; } @@ -180,6 +197,8 @@ double* CppyyTestData::get_double_array() { return m_double_array; } double* CppyyTestData::get_double_array2() { return m_double_array2; } complex_t* CppyyTestData::get_complex_array() { return m_complex_array; } complex_t* CppyyTestData::get_complex_array2() { return m_complex_array2; } +ccomplex_t* CppyyTestData::get_ccomplex_array() { return m_ccomplex_array; } +ccomplex_t* CppyyTestData::get_ccomplex_array2() { return m_ccomplex_array2; } CppyyTestPod CppyyTestData::get_pod_val() { return m_pod; } CppyyTestPod* CppyyTestData::get_pod_val_ptr() { return &m_pod; } @@ -216,6 +235,7 @@ const double& CppyyTestData::get_double_cr() { return m_double; const long double& CppyyTestData::get_ldouble_cr() { return m_ldouble; } const complex_t& CppyyTestData::get_complex_cr() { return m_complex; } const icomplex_t& CppyyTestData::get_icomplex_cr() { return m_icomplex; } +const ccomplex_t& CppyyTestData::get_ccomplex_cr() { return m_ccomplex; } const CppyyTestData::EWhat& CppyyTestData::get_enum_cr() { return m_enum; } //- getters ref ------------------------------------------------------------- @@ -246,6 +266,7 @@ double& CppyyTestData::get_double_r() { return m_double; } long double& CppyyTestData::get_ldouble_r() { return m_ldouble; } complex_t& CppyyTestData::get_complex_r() { return m_complex; } icomplex_t& CppyyTestData::get_icomplex_r() { return m_icomplex; } +ccomplex_t& CppyyTestData::get_ccomplex_r() { return m_ccomplex; } CppyyTestData::EWhat& CppyyTestData::get_enum_r() { return m_enum; } //- setters ----------------------------------------------------------------- @@ -276,6 +297,7 @@ void CppyyTestData::set_double(double d) { m_double = d; } void CppyyTestData::set_ldouble(long double ld) { m_ldouble = ld; } void CppyyTestData::set_complex(complex_t cd) { m_complex = cd; } void CppyyTestData::set_icomplex(icomplex_t ci) { m_icomplex = ci; } +void CppyyTestData::set_ccomplex(ccomplex_t cd) { m_ccomplex = cd; } void CppyyTestData::set_enum(EWhat w) { m_enum = w; } void CppyyTestData::set_voidp(void* p) { m_voidp = p; } @@ -319,6 +341,7 @@ void CppyyTestData::set_double_cr(const double& d) { m_double = void CppyyTestData::set_ldouble_cr(const long double& ld) { m_ldouble = ld; } void CppyyTestData::set_complex_cr(const complex_t& cd) { m_complex = cd; } void CppyyTestData::set_icomplex_cr(const icomplex_t& ci) { m_icomplex = ci; } +void CppyyTestData::set_ccomplex_cr(const ccomplex_t& cd) { m_ccomplex = cd; } void CppyyTestData::set_enum_cr(const EWhat& w) { m_enum = w; } //- setters ref ------------------------------------------------------------- @@ -355,6 +378,8 @@ void CppyyTestData::set_uchar_p(unsigned char* uc) { *uc = 'd'; } #if __cplusplus > 201402L void CppyyTestData::set_byte_p(std::byte* b) { *b = (std::byte)'e'; } #endif +void CppyyTestData::set_int8_p(int8_t* i8) { *i8 = -27; } +void CppyyTestData::set_uint8_p(uint8_t* ui8) { *ui8 = 28; } void CppyyTestData::set_short_p(short* s) { *s = -1; } void CppyyTestData::set_ushort_p(unsigned short* us) { *us = 2; } void CppyyTestData::set_int_p(int* i) { *i = -3; } @@ -402,6 +427,14 @@ void CppyyTestData::set_byte_ppa(std::byte** b) { (*b)[0] = (std::byte)'n'; (*b)[1] = (std::byte)'o'; (*b)[2] = (std::byte)'p'; } #endif +void CppyyTestData::set_int8_ppa(int8_t** i8) { + (*i8) = new int8_t[3]; + (*i8)[0] = -27; (*i8)[1] = -28; (*i8)[2] = -29; +} +void CppyyTestData::set_uint8_ppa(uint8_t** ui8) { + (*ui8) = new uint8_t[3]; + (*ui8)[0] = 28; (*ui8)[1] = 29; (*ui8)[2] = 30; +} void CppyyTestData::set_short_ppa(short** s) { (*s) = new short[3]; (*s)[0] = -1; (*s)[1] = -2; (*s)[2] = -3; @@ -518,6 +551,7 @@ void CppyyTestData::set_double_rv(double&& d) { m_double = d; } void CppyyTestData::set_ldouble_rv(long double&& ld) { m_ldouble = ld; } void CppyyTestData::set_complex_rv(complex_t&& cd) { m_complex = cd; } void CppyyTestData::set_icomplex_rv(icomplex_t&& ci) { m_icomplex = ci; } +void CppyyTestData::set_ccomplex_rv(ccomplex_t&& cd) { m_ccomplex = cd; } void CppyyTestData::set_enum_rv(EWhat&& w) { m_enum = w; } //- passers ----------------------------------------------------------------- @@ -531,6 +565,7 @@ unsigned long* CppyyTestData::pass_array(unsigned long* a) { return a; } float* CppyyTestData::pass_array(float* a) { return a; } double* CppyyTestData::pass_array(double* a) { return a; } complex_t* CppyyTestData::pass_array(complex_t* a) { return a; } +ccomplex_t* CppyyTestData::pass_array(ccomplex_t* a) { return a; } //- static data members ----------------------------------------------------- bool CppyyTestData::s_bool = false; @@ -560,8 +595,11 @@ double CppyyTestData::s_double = -707.; long double CppyyTestData::s_ldouble = -808.l; complex_t CppyyTestData::s_complex = {909., -909.}; icomplex_t CppyyTestData::s_icomplex = {979, -979}; +ccomplex_t CppyyTestData::s_ccomplex = {919., -919.}; CppyyTestData::EWhat CppyyTestData::s_enum = CppyyTestData::kNothing; void* CppyyTestData::s_voidp = (void*)0; +std::string CppyyTestData::s_strv = "Hello"; +std::string* CppyyTestData::s_strp = nullptr; //- strings ----------------------------------------------------------------- const char* CppyyTestData::get_valid_string(const char* in) { return in; } @@ -619,6 +657,7 @@ double g_double = -688.; long double g_ldouble = -788.l; complex_t g_complex = {808., -808.}; icomplex_t g_icomplex = {909, -909}; +ccomplex_t g_ccomplex = {858., -858.}; EFruit g_enum = kBanana; void* g_voidp = nullptr; @@ -780,3 +819,228 @@ void StoreCallable_sf::set_callable(const std::function& double StoreCallable_sf::operator()(double d1, double d2) { return fF(d1, d2); } + + +//= array of C strings passing ============================================== +std::vector ArrayOfCStrings::takes_array_of_cstrings(const char* args[], int len) +{ + std::vector v; + v.reserve(len); + for (int i = 0; i < len; ++i) + v.emplace_back(args[i]); + + return v; +} + + +//= aggregate testing ====================================================== +int AggregateTest::Aggregate1::sInt = 17; +int AggregateTest::Aggregate2::sInt = 27; + + +//= multi-dim arrays ======================================================= +namespace MultiDimArrays { + +template +static inline T** allocate_2d(size_t N, size_t M) { + T** arr = (T**)malloc(sizeof(void*)*N); + for (size_t i = 0; i < N; ++i) + arr[i] = (T*)malloc(sizeof(T)*M); + return arr; +} + +static inline void free_2d(void** arr, size_t N) { + if (arr) { + for (size_t i = 0; i < N; ++i) + free(arr[i]); + free(arr); + } +} + +template +static inline T*** allocate_3d(size_t N, size_t M, size_t K) { + T*** arr = (T***)malloc(sizeof(void*)*N); + for (size_t i = 0; i < N; ++i) { + arr[i] = (T**)malloc(sizeof(void*)*M); + for (size_t j = 0; j < M; ++j) + arr[i][j] = (T*)malloc(sizeof(T)*K); + } + return arr; +} + +static inline void free_3d(void*** arr, size_t N, size_t M) { + if (arr) { + for (size_t i = 0; i < N; ++i) { + for (size_t j = 0; j < M; ++j) + free(arr[i][j]); + free(arr[i]); + } + free(arr); + } +} + +} // namespace MultiDimArrays + +MultiDimArrays::DataHolder::DataHolder() { + m_short2a = allocate_2d(5, 7); + m_unsigned_short2a = allocate_2d(5, 7); + m_int2a = allocate_2d(5, 7); + m_unsigned_int2a = allocate_2d(5, 7); + m_long2a = allocate_2d(5, 7); + m_unsigned_long2a = allocate_2d(5, 7); + m_long_long2a = allocate_2d(5, 7); + m_unsigned_long_long2a = allocate_2d(5, 7); + m_float2a = allocate_2d(5, 7); + m_double2a = allocate_2d(5, 7); + + for (size_t i = 0; i < 5; ++i) { + for (size_t j = 0; j < 7; ++j) { + size_t val = 5*i+j; + m_short2a[i][j] = (short)val; + m_unsigned_short2a[i][j] = (unsigned short)val; + m_int2a[i][j] = (int)val; + m_unsigned_int2a[i][j] = (unsigned int)val; + m_long2a[i][j] = (long)val; + m_unsigned_long2a[i][j] = (unsigned long)val; + m_long_long2a[i][j] = (long long)val; + m_unsigned_long_long2a[i][j] = (unsigned long long)val; + m_float2a[i][j] = (float)val; + m_double2a[i][j] = (double)val; + } + } + + m_short2b = nullptr; + m_unsigned_short2b = nullptr; + m_int2b = nullptr; + m_unsigned_int2b = nullptr; + m_long2b = nullptr; + m_unsigned_long2b = nullptr; + m_long_long2b = nullptr; + m_unsigned_long_long2b = nullptr; + m_float2b = nullptr; + m_double2b = nullptr; + + for (size_t i = 0; i < 3; ++i) { + for (size_t j = 0; j < 5; ++j) { + size_t val = 3*i+j; + m_short2c[i][j] = (short)val; + m_unsigned_short2c[i][j] = (unsigned short)val; + m_int2c[i][j] = (int)val; + m_unsigned_int2c[i][j] = (unsigned int)val; + m_long2c[i][j] = (long)val; + m_unsigned_long2c[i][j] = (unsigned long)val; + m_long_long2c[i][j] = (long long)val; + m_unsigned_long_long2c[i][j] = (unsigned long long)val; + m_float2c[i][j] = (float)val; + m_double2c[i][j] = (double)val; + + for (size_t k = 0; k < 7; ++k) { + val = 3*i+2*j+k; + m_short3c[i][j][k] = (short)val; + m_unsigned_short3c[i][j][k] = (unsigned short)val; + m_int3c[i][j][k] = (int)val; + m_unsigned_int3c[i][j][k] = (unsigned int)val; + m_long3c[i][j][k] = (long)val; + m_unsigned_long3c[i][j][k] = (unsigned long)val; + m_long_long3c[i][j][k] = (long long)val; + m_unsigned_long_long3c[i][j][k] = (unsigned long long)val; + m_float3c[i][j][k] = (float)val; + m_double3c[i][j][k] = (double)val; + } + } + } + + m_short3a = allocate_3d(5, 7, 11); + m_unsigned_short3a = allocate_3d(5, 7, 11); + m_int3a = allocate_3d(5, 7, 11); + m_unsigned_int3a = allocate_3d(5, 7, 11); + m_long3a = allocate_3d(5, 7, 11); + m_unsigned_long3a = allocate_3d(5, 7, 11); + m_long_long3a = allocate_3d(5, 7, 11); + m_unsigned_long_long3a = allocate_3d(5, 7, 11); + m_float3a = allocate_3d(5, 7, 11); + m_double3a = allocate_3d(5, 7, 11); + + for (size_t i = 0; i < 5; ++i) { + for (size_t j = 0; j < 7; ++j) { + for (size_t k = 0; k < 11; ++k) { + size_t val = 7*i+3*j+k; + m_short3a[i][j][k] = (short)val; + m_unsigned_short3a[i][j][k] = (unsigned short)val; + m_int3a[i][j][k] = (int)val; + m_unsigned_int3a[i][j][k] = (unsigned int)val; + m_long3a[i][j][k] = (long)val; + m_unsigned_long3a[i][j][k] = (unsigned long)val; + m_long_long3a[i][j][k] = (long long)val; + m_unsigned_long_long3a[i][j][k] = (unsigned long long)val; + m_float3a[i][j][k] = (float)val; + m_double3a[i][j][k] = (double)val; + } + } + } +} + +MultiDimArrays::DataHolder::~DataHolder() { + free_2d((void**)m_short2a, 5); + free_2d((void**)m_unsigned_short2a, 5); + free_2d((void**)m_int2a, 5); + free_2d((void**)m_unsigned_int2a, 5); + free_2d((void**)m_long2a, 5); + free_2d((void**)m_unsigned_long2a, 5); + free_2d((void**)m_long_long2a, 5); + free_2d((void**)m_unsigned_long_long2a, 5); + free_2d((void**)m_float2a, 5); + free_2d((void**)m_double2a, 5); + + free_2d((void**)m_short2b, 5); + free_2d((void**)m_unsigned_short2b, 5); + free_2d((void**)m_int2b, 5); + free_2d((void**)m_unsigned_int2b, 5); + free_2d((void**)m_long2b, 5); + free_2d((void**)m_unsigned_long2b, 5); + free_2d((void**)m_long_long2b, 5); + free_2d((void**)m_unsigned_long_long2b, 5); + free_2d((void**)m_float2b, 5); + free_2d((void**)m_double2b, 5); + + free_3d((void***)m_short3a, 5, 7); + free_3d((void***)m_unsigned_short3a, 5, 7); + free_3d((void***)m_int3a, 5, 7); + free_3d((void***)m_unsigned_int3a, 5, 7); + free_3d((void***)m_long3a, 5, 7); + free_3d((void***)m_unsigned_long3a, 5, 7); + free_3d((void***)m_long_long3a, 5, 7); + free_3d((void***)m_unsigned_long_long3a, 5, 7); + free_3d((void***)m_float3a, 5, 7); + free_3d((void***)m_double3a, 5, 7); +} + +#define MULTIDIM_ARRAYS_NEW2D(type, name) \ +type** MultiDimArrays::DataHolder::new_##name##2d(int N, int M) { \ + type** arr = allocate_2d(N, M); \ + for (size_t i = 0; i < N; ++i) { \ + for (size_t j = 0; j < M; ++j) { \ + size_t val = 7*i+j; \ + arr [i][j] = (type)val; \ + } \ + } \ + return arr; \ +} + +MULTIDIM_ARRAYS_NEW2D(short, short) +MULTIDIM_ARRAYS_NEW2D(unsigned short, ushort) +MULTIDIM_ARRAYS_NEW2D(int, int) +MULTIDIM_ARRAYS_NEW2D(unsigned int, uint) +MULTIDIM_ARRAYS_NEW2D(long, long) +MULTIDIM_ARRAYS_NEW2D(unsigned long, ulong) +MULTIDIM_ARRAYS_NEW2D(long long, llong) +MULTIDIM_ARRAYS_NEW2D(unsigned long long, ullong) +MULTIDIM_ARRAYS_NEW2D(float, float) +MULTIDIM_ARRAYS_NEW2D(double, double) + + +//=========================================================================== +namespace Int8_Uint8_Arrays { + int8_t test[6] = {-0x12, -0x34, -0x56, -0x78}; + uint8_t utest[6] = { 0x12, 0x34, 0x56, 0x78}; +} diff --git a/bindings/pyroot/cppyy/cppyy/test/datatypes.h b/bindings/pyroot/cppyy/cppyy/test/datatypes.h index 97bbe9682060a..e15fd9b8a4a3f 100644 --- a/bindings/pyroot/cppyy/cppyy/test/datatypes.h +++ b/bindings/pyroot/cppyy/cppyy/test/datatypes.h @@ -1,22 +1,18 @@ #ifndef CPPYY_TEST_DATATYPES_H #define CPPYY_TEST_DATATYPES_H -#ifndef CPPYY_DUMMY_BACKEND -#include "RtypesCore.h" +#ifdef _WIN32 +typedef __int64 Long64_t; +typedef unsigned __int64 ULong64_t; #else -// copied from RtypesCore.h ... -#if defined(R__WIN32) -typedef __int64 Long64_t; //Portable signed long integer 8 bytes -typedef unsigned __int64 ULong64_t; //Portable unsigned long integer 8 bytes -#else -typedef long long Long64_t; //Portable signed long integer 8 bytes -typedef unsigned long long ULong64_t;//Portable unsigned long integer 8 bytes -#endif +typedef long long Long64_t; +typedef unsigned long long ULong64_t; #endif #include #include #include #include +#include #include #include #include @@ -85,6 +81,11 @@ class FourVector { //=========================================================================== typedef std::complex complex_t; // maps to Py_complex typedef std::complex icomplex_t; // no equivalent +#ifndef _WIN32 +typedef _Complex double ccomplex_t; // C-style complex, maps to Py_complex +#else +typedef _C_double_complex ccomplex_t; // id. +#endif class CppyyTestData { public: @@ -126,6 +127,7 @@ class CppyyTestData { long double get_ldouble_def(long double ld = 1); complex_t get_complex(); icomplex_t get_icomplex(); + ccomplex_t get_ccomplex(); EWhat get_enum(); void* get_voidp(); @@ -139,6 +141,10 @@ class CppyyTestData { std::byte* get_byte_array(); std::byte* get_byte_array2(); #endif + int8_t* get_int8_array(); + int8_t* get_int8_array2(); + uint8_t* get_uint8_array(); + uint8_t* get_uint8_array2(); short* get_short_array(); short* get_short_array2(); unsigned short* get_ushort_array(); @@ -158,6 +164,8 @@ class CppyyTestData { double* get_double_array2(); complex_t* get_complex_array(); complex_t* get_complex_array2(); + ccomplex_t* get_ccomplex_array(); + ccomplex_t* get_ccomplex_array2(); CppyyTestPod get_pod_val(); // for m_pod CppyyTestPod* get_pod_val_ptr(); @@ -194,6 +202,7 @@ class CppyyTestData { const long double& get_ldouble_cr(); const complex_t& get_complex_cr(); const icomplex_t& get_icomplex_cr(); + const ccomplex_t& get_ccomplex_cr(); const EWhat& get_enum_cr(); // getters ref @@ -224,6 +233,7 @@ class CppyyTestData { long double& get_ldouble_r(); complex_t& get_complex_r(); icomplex_t& get_icomplex_r(); + ccomplex_t& get_ccomplex_r(); EWhat& get_enum_r(); // setters @@ -254,6 +264,7 @@ class CppyyTestData { void set_ldouble(long double); void set_complex(complex_t); void set_icomplex(icomplex_t); + void set_ccomplex(ccomplex_t); void set_enum(EWhat); void set_voidp(void*); @@ -296,6 +307,7 @@ class CppyyTestData { void set_ldouble_cr(const long double&); void set_complex_cr(const complex_t&); void set_icomplex_cr(const icomplex_t&); + void set_ccomplex_cr(const ccomplex_t&); void set_enum_cr(const EWhat&); // setters ref @@ -332,6 +344,8 @@ class CppyyTestData { #if __cplusplus > 201402L void set_byte_p(std::byte*); #endif + void set_int8_p(int8_t*); + void set_uint8_p(uint8_t*); void set_short_p(short*); void set_ushort_p(unsigned short*); void set_int_p(int*); @@ -355,6 +369,8 @@ class CppyyTestData { #if __cplusplus > 201402L void set_byte_ppa(std::byte**); #endif + void set_int8_ppa(int8_t**); + void set_uint8_ppa(uint8_t**); void set_short_ppa(short**); void set_ushort_ppa(unsigned short**); void set_int_ppa(int**); @@ -407,6 +423,7 @@ class CppyyTestData { void set_ldouble_rv(long double&&); void set_complex_rv(complex_t&&); void set_icomplex_rv(icomplex_t&&); + void set_ccomplex_rv(ccomplex_t&&); void set_enum_rv(EWhat&&); // passers @@ -420,6 +437,7 @@ class CppyyTestData { float* pass_array(float*); double* pass_array(double*); complex_t* pass_array(complex_t*); + ccomplex_t* pass_array(ccomplex_t*); unsigned char* pass_void_array_B(void* a) { return pass_array((unsigned char*)a); } short* pass_void_array_h(void* a) { return pass_array((short*)a); } @@ -431,6 +449,7 @@ class CppyyTestData { float* pass_void_array_f(void* a) { return pass_array((float*)a); } double* pass_void_array_d(void* a) { return pass_array((double*)a); } complex_t* pass_void_array_Z(void* a) { return pass_array((complex_t*)a); } + ccomplex_t* pass_void_array_cZ(void* a) { return pass_array((ccomplex_t*)a); } // strings const char* get_valid_string(const char* in); @@ -472,6 +491,7 @@ class CppyyTestData { long double m_ldouble; complex_t m_complex; icomplex_t m_icomplex; + ccomplex_t m_ccomplex; EWhat m_enum; void* m_voidp; @@ -486,6 +506,10 @@ class CppyyTestData { std::byte m_byte_array[N]; std::byte* m_byte_array2; #endif + int8_t m_int8_array[N]; + int8_t* m_int8_array2; + uint8_t m_uint8_array[N]; + uint8_t* m_uint8_array2; short m_short_array[N]; short* m_short_array2; unsigned short m_ushort_array[N]; @@ -507,6 +531,8 @@ class CppyyTestData { complex_t* m_complex_array2; icomplex_t m_icomplex_array[N]; icomplex_t* m_icomplex_array2; + ccomplex_t m_ccomplex_array[N]; + ccomplex_t* m_ccomplex_array2; // object types CppyyTestPod m_pod; @@ -540,8 +566,11 @@ class CppyyTestData { static long double s_ldouble; static complex_t s_complex; static icomplex_t s_icomplex; + static ccomplex_t s_ccomplex; static EWhat s_enum; static void* s_voidp; + static std::string s_strv; + static std::string* s_strp; private: bool m_owns_arrays; @@ -582,6 +611,7 @@ extern double g_double; extern long double g_ldouble; extern complex_t g_complex; extern icomplex_t g_icomplex; +extern ccomplex_t g_ccomplex; extern EFruit g_enum; extern void* g_voidp; @@ -612,6 +642,7 @@ static const double g_c_double = -699.; static const long double g_c_ldouble = -799.l; static const complex_t g_c_complex = {1., 2.}; static const icomplex_t g_c_icomplex = {3, 4}; +static const ccomplex_t g_c_ccomplex = {5., 6.}; static const EFruit g_c_enum = kApple; static const void* g_c_voidp = nullptr; @@ -702,4 +733,133 @@ class StoreCallable_sf { double operator()(double, double); }; + +//= array of struct variants ================================================ +namespace ArrayOfStruct { + +struct Foo { + int fVal; +}; + +struct Bar1 { + Bar1() : fArr(new Foo[2]) { fArr[0].fVal = 42; fArr[1].fVal = 13; } + Bar1(const Bar1&) = delete; + Bar1& operator=(const Bar1&) = delete; + ~Bar1() { delete[] fArr; } + Foo* fArr; +}; + +struct Bar2 { + Bar2(int num_foo) : fArr(std::unique_ptr{new Foo[num_foo]}) { + for (int i = 0; i < num_foo; ++i) fArr[i].fVal = 2*i; + } + std::unique_ptr fArr; +}; + +} // namespace ArrayOfStruct + + +//= array of C strings passing ============================================== +namespace ArrayOfCStrings { + std::vector takes_array_of_cstrings(const char* args[], int len); +} + + +//= aggregate testing ====================================================== +namespace AggregateTest { + +struct Aggregate1 { + static int sInt; +}; + +struct Aggregate2 { + static int sInt; + int fInt = 42; +}; + +} // namespace AggregateTest + + +//= multi-dim arrays ======================================================= +namespace MultiDimArrays { + +struct DataHolder { + DataHolder(); + ~DataHolder(); + + short** m_short2a; + unsigned short** m_unsigned_short2a; + int** m_int2a; + unsigned int** m_unsigned_int2a; + long** m_long2a; + unsigned long** m_unsigned_long2a; + long long** m_long_long2a; + unsigned long long** m_unsigned_long_long2a; + float** m_float2a; + double** m_double2a; + + short** m_short2b; + short** new_short2d(int N, int M); + unsigned short** m_unsigned_short2b; + unsigned short** new_ushort2d(int N, int M); + int** m_int2b; + int** new_int2d(int N, int M); + unsigned int** m_unsigned_int2b; + unsigned int** new_uint2d(int N, int M); + long** m_long2b; + long** new_long2d(int N, int M); + unsigned long** m_unsigned_long2b; + unsigned long** new_ulong2d(int N, int M); + long long** m_long_long2b; + long long** new_llong2d(int N, int M); + unsigned long long** m_unsigned_long_long2b; + unsigned long long** new_ullong2d(int N, int M); + float** m_float2b; + float** new_float2d(int N, int M); + double** m_double2b; + double** new_double2d(int N, int M); + + short m_short2c[3][5]; + unsigned short m_unsigned_short2c[3][5]; + int m_int2c[3][5]; + unsigned int m_unsigned_int2c[3][5]; + long m_long2c[3][5]; + unsigned long m_unsigned_long2c[3][5]; + long long m_long_long2c[3][5]; + unsigned long long m_unsigned_long_long2c[3][5]; + float m_float2c[3][5]; + double m_double2c[3][5]; + + short*** m_short3a; + unsigned short*** m_unsigned_short3a; + int*** m_int3a; + unsigned int*** m_unsigned_int3a; + long*** m_long3a; + unsigned long*** m_unsigned_long3a; + long long*** m_long_long3a; + unsigned long long*** m_unsigned_long_long3a; + float*** m_float3a; + double*** m_double3a; + + short m_short3c[3][5][7]; + unsigned short m_unsigned_short3c[3][5][7]; + int m_int3c[3][5][7]; + unsigned int m_unsigned_int3c[3][5][7]; + long m_long3c[3][5][7]; + unsigned long m_unsigned_long3c[3][5][7]; + long long m_long_long3c[3][5][7]; + unsigned long long m_unsigned_long_long3c[3][5][7]; + float m_float3c[3][5][7]; + double m_double3c[3][5][7]; +}; + +} // namespace MultiDimArrays + + +//= int8_t/uint8_t arrays =================================================== +namespace Int8_Uint8_Arrays { + extern int8_t test[6]; + extern uint8_t utest[6]; +} + #endif // !CPPYY_TEST_DATATYPES_H diff --git a/bindings/pyroot/cppyy/cppyy/test/datatypes.xml b/bindings/pyroot/cppyy/cppyy/test/datatypes.xml index 37521acfc37d9..9281b656ad725 100644 --- a/bindings/pyroot/cppyy/cppyy/test/datatypes.xml +++ b/bindings/pyroot/cppyy/cppyy/test/datatypes.xml @@ -55,4 +55,16 @@ + + + + + + + + + + + + diff --git a/bindings/pyroot/cppyy/cppyy/test/doc_args_funcs.py b/bindings/pyroot/cppyy/cppyy/test/doc_args_funcs.py new file mode 100644 index 0000000000000..481c3a84334a0 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/test/doc_args_funcs.py @@ -0,0 +1,5 @@ +def ann_f1(arg: 'int') -> 'double': + return 3.1415*arg + +def ann_f2(arg1: 'int', arg2: 'int') -> 'int': + return 3*arg1*arg2 diff --git a/bindings/pyroot/cppyy/cppyy/test/fragile.cxx b/bindings/pyroot/cppyy/cppyy/test/fragile.cxx index b3706174f6926..8ca3cf02cc6b3 100644 --- a/bindings/pyroot/cppyy/cppyy/test/fragile.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/fragile.cxx @@ -62,3 +62,8 @@ void fragile::sigabort() { assert(0); } + +// for duplicate testing +int fragile::add42(int i) { + return i + 42; +} diff --git a/bindings/pyroot/cppyy/cppyy/test/fragile.h b/bindings/pyroot/cppyy/cppyy/test/fragile.h index e9f51b10eb71b..76c9b3a2ad26b 100644 --- a/bindings/pyroot/cppyy/cppyy/test/fragile.h +++ b/bindings/pyroot/cppyy/cppyy/test/fragile.h @@ -135,4 +135,8 @@ int destroy_handle(OpaqueHandle_t handle, intptr_t addr); void segfault(); void sigabort(); + +// for duplicate testing +int add42(int i); + } // namespace fragile diff --git a/bindings/pyroot/cppyy/cppyy/test/fragile.xml b/bindings/pyroot/cppyy/cppyy/test/fragile.xml index 0f2865ce8f9a2..652b56a35aaa2 100644 --- a/bindings/pyroot/cppyy/cppyy/test/fragile.xml +++ b/bindings/pyroot/cppyy/cppyy/test/fragile.xml @@ -39,4 +39,6 @@ + + diff --git a/bindings/pyroot/cppyy/cppyy/test/make_dict_win32.py b/bindings/pyroot/cppyy/cppyy/test/make_dict_win32.py index accd03327e55c..6837f6b068c9c 100755 --- a/bindings/pyroot/cppyy/cppyy/test/make_dict_win32.py +++ b/bindings/pyroot/cppyy/cppyy/test/make_dict_win32.py @@ -39,7 +39,7 @@ os.remove(fg) def _get_config_exec(): - return [sys.executable, '-m', 'cppyy_backend._cling_config'] + return [sys.executable, '-m', 'cppyy_backend._cling_config'] def get_config(what): config_exec_args = _get_config_exec() @@ -78,7 +78,7 @@ def get_python_lib_dir(): cppflags = get_config('cppflags') if uses_python_capi: cppflags += ' -I"' + get_python_include_dir() + '"' -BUILDOBJ_CMD_PART = "cl -O2 -nologo -TP -c -nologo " + cppflags + " -FIsehmap.h -Zc:__cplusplus -MD -GR -D_WINDOWS -DWIN32 " + PLATFORMFLAG + " -EHsc- -W3 -wd4141 -wd4291 -wd4244 -wd4049 -D_XKEYCHECK_H -D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER -DNOMINMAX -D_CRT_SECURE_NO_WARNINGS {fn}.cxx -Fo{fn}.obj" +BUILDOBJ_CMD_PART = "cl -O2 -nologo -TP -c -nologo " + cppflags + " -FIsehmap.h -MD -GR -D_WINDOWS -DWIN32 " + PLATFORMFLAG + " -EHsc- -W3 -wd4141 -wd4291 -wd4244 -wd4049 -D_XKEYCHECK_H -D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER -DNOMINMAX -D_CRT_SECURE_NO_WARNINGS {fn}.cxx -Fo{fn}.obj" BUILDOBJ_CMD = BUILDOBJ_CMD_PART.format(fn=fn) if os.system(BUILDOBJ_CMD): sys.exit(1) diff --git a/bindings/pyroot/cppyy/cppyy/test/overloads.cxx b/bindings/pyroot/cppyy/cppyy/test/overloads.cxx index c463b941b4bc0..eeab7dde49719 100644 --- a/bindings/pyroot/cppyy/cppyy/test/overloads.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/overloads.cxx @@ -26,6 +26,10 @@ int d_overload::get_int(ns_b_overload::a_overload* a) { return a->i1; } int d_overload::get_int(ns_a_overload::a_overload* a) { return a->i1; } int d_overload::get_int(a_overload* a) { return a->i1; } +class bb_ol {}; +bb_ol* get_bb_ol() { return new bb_ol{}; } +class dd_ol {}; +dd_ol* get_dd_ol() { return new dd_ol{}; } more_overloads::more_overloads() {} std::string more_overloads::call(const aa_ol&) { return "aa_ol"; } @@ -54,3 +58,10 @@ double calc_mean(long n, const double* a) { return calc_mean(n, a); } double calc_mean(long n, const int* a) { return calc_mean(n, a); } double calc_mean(long n, const short* a) { return calc_mean(n, a); } double calc_mean(long n, const long* a) { return calc_mean(n, a); } + + +std::string more_overloads3::slice(size_t) const { return "const"; } +std::string more_overloads3::slice(size_t) { return "non-const"; } +std::string more_overloads3::slice(size_t, size_t) const { return "const"; } +std::string more_overloads3::slice(size_t, size_t) { return "non-const"; } + diff --git a/bindings/pyroot/cppyy/cppyy/test/overloads.h b/bindings/pyroot/cppyy/cppyy/test/overloads.h index 23cfddc27cb12..96b5da1b54d46 100644 --- a/bindings/pyroot/cppyy/cppyy/test/overloads.h +++ b/bindings/pyroot/cppyy/cppyy/test/overloads.h @@ -60,8 +60,11 @@ class d_overload { class aa_ol {}; class bb_ol; +bb_ol* get_bb_ol(); class cc_ol {}; class dd_ol; +dd_ol* get_dd_ol(); + class more_overloads { public: @@ -106,3 +109,11 @@ double calc_mean(long n, const double* a); double calc_mean(long n, const int* a); double calc_mean(long n, const short* a); double calc_mean(long n, const long* a); + +class more_overloads3 { +public: + std::string slice(size_t) const; + std::string slice(size_t); + std::string slice(size_t, size_t); + std::string slice(size_t, size_t) const; +}; diff --git a/bindings/pyroot/cppyy/cppyy/test/overloads.xml b/bindings/pyroot/cppyy/cppyy/test/overloads.xml index 131296e72d41d..418344baf8f56 100644 --- a/bindings/pyroot/cppyy/cppyy/test/overloads.xml +++ b/bindings/pyroot/cppyy/cppyy/test/overloads.xml @@ -11,8 +11,11 @@ + + + diff --git a/bindings/pyroot/cppyy/cppyy/test/std_streams.cxx b/bindings/pyroot/cppyy/cppyy/test/std_streams.cxx index 94e924b7b2bde..30a7940ea69eb 100644 --- a/bindings/pyroot/cppyy/cppyy/test/std_streams.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/std_streams.cxx @@ -1,3 +1,5 @@ #include "std_streams.h" +#ifndef _WIN32 template class std::basic_ios >; +#endif diff --git a/bindings/pyroot/cppyy/cppyy/test/std_streams.h b/bindings/pyroot/cppyy/cppyy/test/std_streams.h index a81ce490f576e..e67622373c7f6 100644 --- a/bindings/pyroot/cppyy/cppyy/test/std_streams.h +++ b/bindings/pyroot/cppyy/cppyy/test/std_streams.h @@ -1,9 +1,15 @@ #ifndef STD_STREAMS_H #define STD_STREAMS_H 1 +#ifndef __CLING__ #include +#endif #include +#ifndef __CLING__ +#ifndef _WIN32 extern template class std::basic_ios >; +#endif +#endif #endif // STD_STREAMS_H diff --git a/bindings/pyroot/cppyy/cppyy/test/std_streams.xml b/bindings/pyroot/cppyy/cppyy/test/std_streams.xml index 01eb8c73b5427..b14aa884e1e48 100644 --- a/bindings/pyroot/cppyy/cppyy/test/std_streams.xml +++ b/bindings/pyroot/cppyy/cppyy/test/std_streams.xml @@ -6,7 +6,7 @@ - + diff --git a/bindings/pyroot/cppyy/cppyy/test/stltypes.h b/bindings/pyroot/cppyy/cppyy/test/stltypes.h index a75355f676394..c7d8fa8974f3e 100644 --- a/bindings/pyroot/cppyy/cppyy/test/stltypes.h +++ b/bindings/pyroot/cppyy/cppyy/test/stltypes.h @@ -57,17 +57,17 @@ typedef stringy_class wstringy_class_t; //- class that has an STL-like interface class no_dict_available; - + template class stl_like_class { -public: +public: no_dict_available* begin() { return 0; } no_dict_available* end() { return (no_dict_available*)1; } int size() { return 4; } int operator[](int i) { return i; } std::string operator[](double) { return "double"; } std::string operator[](const std::string&) { return "string"; } -}; +}; namespace { stl_like_class stlc_1; @@ -148,6 +148,28 @@ class stl_like_class7 { }; +// similar, but now with a base/derives split +class stl_like_class_base { +protected: + std::vector fVec; + +public: + void push_back(double d) { fVec.push_back(d); } + std::vector::size_type size() { return fVec.size(); } +}; + +class stl_like_class8 : public stl_like_class_base { +public: + double& operator[](int idx) { return fVec[idx]; } +}; + +class stl_like_class9 : public stl_like_class_base { +public: + double* begin() { return fVec.data(); } + double* end() { return fVec.data() + fVec.size(); } +}; + + //- helpers for testing array namespace ArrayTest { @@ -205,9 +227,6 @@ class MyError : public std::exception { public: explicit MyError(const std::string& msg); MyError(const MyError&); -#ifndef WIN32 - MyError(const MyError&&) = delete; -#endif virtual ~MyError(); MyError& operator=(const MyError&) = default; const char* what() const throw() override; diff --git a/bindings/pyroot/cppyy/cppyy/test/support.py b/bindings/pyroot/cppyy/cppyy/test/support.py index 21b58482f3054..fe84bde9807ed 100644 --- a/bindings/pyroot/cppyy/cppyy/test/support.py +++ b/bindings/pyroot/cppyy/cppyy/test/support.py @@ -1,5 +1,5 @@ from __future__ import print_function -import py, sys, subprocess +import os, py, sys, subprocess currpath = py.path.local(__file__).dirpath() @@ -16,19 +16,32 @@ def setup_make(targetname): raise OSError("'make' failed:\n%s" % (stdout,)) if sys.hexversion >= 0x3000000: - pylong = int - pyunicode = str - maxvalue = sys.maxsize + pylong = int + pyunicode = str + maxvalue = sys.maxsize else: - pylong = long - pyunicode = unicode - maxvalue = sys.maxint + pylong = long + pyunicode = unicode + maxvalue = sys.maxint IS_WINDOWS = 0 if 'win32' in sys.platform: - import platform - if '64' in platform.architecture()[0]: - IS_WINDOWS = 64 - maxvalue = 2**31-1 - else: - IS_WINDOWS = 32 + import platform + if '64' in platform.architecture()[0]: + IS_WINDOWS = 64 + maxvalue = 2**31-1 + else: + IS_WINDOWS = 32 + +IS_MAC_ARM = 0 +if 'darwin' in sys.platform: + import platform + if 'arm64' in platform.machine(): + IS_MAC_ARM = 64 + os.environ["CPPYY_UNCAUGHT_QUIET"] = "1" + +try: + import __pypy__ + ispypy = True +except ImportError: + ispypy = False diff --git a/bindings/pyroot/cppyy/cppyy/test/templ_args_funcs.py b/bindings/pyroot/cppyy/cppyy/test/templ_args_funcs.py new file mode 100644 index 0000000000000..b8fc6178a2e9e --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/test/templ_args_funcs.py @@ -0,0 +1,9 @@ +import cppyy + +def ann_adapt(node: 'FPTA::Node&') -> cppyy.gbl.FPTA.EventId: + return cppyy.gbl.FPTA.EventId(node.fData) + +def ann_ref_mod(node: 'FPTA::Node&') -> cppyy.gbl.FPTA.EventId: + ev_id = cppyy.gbl.FPTA.EventId(node.fData) + node.fData = 81 + return ev_id diff --git a/bindings/pyroot/cppyy/cppyy/test/templates.cxx b/bindings/pyroot/cppyy/cppyy/test/templates.cxx index bc5ff8bc2b48f..90c3ec48e6da5 100644 --- a/bindings/pyroot/cppyy/cppyy/test/templates.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/templates.cxx @@ -35,3 +35,22 @@ void some_empty() { template void some_empty(); } // namespace T_WithRValue + + +// The following is hidden from the Cling interpreter, but available to the +// linker; it allows for testing whether a function return is picked up from +// the compiled instantation or from the interpreter. + +namespace FailedTypeDeducer { + +template +class A { +public: + T result() { return T{42}; } +}; + +template class A; + +template class B; + +} // namespace FailedTypeDeducer diff --git a/bindings/pyroot/cppyy/cppyy/test/templates.h b/bindings/pyroot/cppyy/cppyy/test/templates.h index 827a616d24e20..8d6f0d8ae3748 100644 --- a/bindings/pyroot/cppyy/cppyy/test/templates.h +++ b/bindings/pyroot/cppyy/cppyy/test/templates.h @@ -6,7 +6,7 @@ #include #include -#ifndef WIN32 +#ifndef _WIN32 #include inline std::string demangle_it(const char* name, const char* errmsg) { int status; @@ -83,8 +83,13 @@ int global_some_foo(T) { return 42; } +template +int global_some_bar() { + return T; +} + template -int global_some_bar(T) { +int global_some_bar_var(T) { return 13; } @@ -128,6 +133,7 @@ struct select_template_arg {}; template struct select_template_arg<0, T0, T...> { typedef T0 type; + typedef type argument; }; template @@ -150,6 +156,13 @@ int some_bar() { return T; } +struct SomeStruct { + template + static int some_bar() { + return T; + } +}; + inline std::string tuplify(std::ostringstream& out) { out << "NULL)"; return out.str(); @@ -291,6 +304,12 @@ class Derived : public Base { using _Mybase::get3; }; +template class Foo { T x; }; + +namespace Bar { + using ::using_problem::Foo; +} + } // namespace using_problem @@ -310,9 +329,13 @@ bool is_valid(T&& new_value) { // variadic templates namespace some_variadic { -#ifdef WIN32 +#ifdef _WIN32 +#ifdef __CLING__ extern __declspec(dllimport) std::string gTypeName; #else +extern __declspec(dllexport) std::string gTypeName; +#endif +#else extern std::string gTypeName; #endif @@ -409,9 +432,13 @@ T fn_T(Args&&... args) { // template with empty body namespace T_WithEmptyBody { -#ifdef WIN32 +#ifdef _WIN32 +#ifdef __CLING__ extern __declspec(dllimport) std::string side_effect; #else +extern __declspec(dllexport) std::string side_effect; +#endif +#else extern std::string side_effect; #endif @@ -478,4 +505,45 @@ class MyVec { } // namespace TemplateWithSetItem + +//=========================================================================== +// type reduction examples on gmpxx-like template expressions +namespace TypeReduction { + +template +struct BinaryExpr; + +template +struct Expr { + Expr() {} + Expr(const BinaryExpr&) {} +}; + +template +struct BinaryExpr { + BinaryExpr(const Expr&, const Expr&) {} +}; + +template +BinaryExpr operator+(const Expr& e1, const Expr& e2) { + return BinaryExpr(e1, e2); +} + +} // namespace TypeReduction + + +//=========================================================================== +// type deduction examples +namespace FailedTypeDeducer { + +template +class B { +public: + auto result() { return 5.; } +}; + +extern template class B; + +} + #endif // !CPPYY_TEST_TEMPLATES_H diff --git a/bindings/pyroot/cppyy/cppyy/test/templates.xml b/bindings/pyroot/cppyy/cppyy/test/templates.xml index 63d9829332435..32f56da76f3a0 100644 --- a/bindings/pyroot/cppyy/cppyy/test/templates.xml +++ b/bindings/pyroot/cppyy/cppyy/test/templates.xml @@ -6,6 +6,7 @@ + @@ -15,6 +16,7 @@ + @@ -33,4 +35,11 @@ + + + + + + + diff --git a/bindings/pyroot/cppyy/cppyy/test/test_advancedcpp.py b/bindings/pyroot/cppyy/cppyy/test/test_advancedcpp.py index 6861348186455..2c75124817e39 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_advancedcpp.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_advancedcpp.py @@ -1,6 +1,6 @@ import py -from pytest import raises -from .support import setup_make, pylong, IS_WINDOWS +from pytest import raises, skip +from .support import setup_make, pylong, IS_WINDOWS, ispypy currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("advancedcppDict")) @@ -417,6 +417,13 @@ def test08_void_pointer_passing(self): assert cppyy.addressof(ptr) == 0 pp.set_address_ptr_ref(ptr) assert cppyy.addressof(ptr) == 0x1234 + + # alternate path through static method handling, which does NOT + # provide 'pp' as argument (and thus need no removing) + sf = pp.set_address_ptr_ref + sf(ptr) + assert cppyy.addressof(ptr) == 0x1234 + pp.set_address_ptr_ptr(ptr) assert cppyy.addressof(ptr) == 0x4321 @@ -543,8 +550,10 @@ def test12_actual_type(self): assert isinstance(b.cycle(d), derived_class) assert isinstance(d.cycle(d), derived_class) - assert isinstance(b.clone(), base_class) # TODO: clone() leaks - assert isinstance(d.clone(), derived_class) # TODO: clone() leaks + base_class.clone.__creates__ = True + assert isinstance(b.clone(), base_class) + derived_class.clone.__creates__ = True + assert isinstance(d.clone(), derived_class) # special case when round-tripping through a void* ptr voidp = b.mask(d) @@ -619,6 +628,9 @@ def test16_template_global_functions(self): def test17_assign_to_return_byref(self): """Test assignment to an instance returned by reference""" + if ispypy: + skip('segfaults in pypy') + from cppyy import gbl a = gbl.std.vector(gbl.ref_tester)() @@ -627,10 +639,9 @@ def test17_assign_to_return_byref(self): assert len(a) == 1 assert a[0].m_i == 42 - # TODO: - # a[0] = gbl.ref_tester(33) - # assert len(a) == 1 - # assert a[0].m_i == 33 + a[0] = gbl.ref_tester(33) + assert len(a) == 1 + assert a[0].m_i == 33 def test18_math_converters(self): """Test operator int/long/double incl. typedef""" @@ -684,8 +695,7 @@ def test21_access_to_global_variables(self): assert cppyy.gbl.my_global_string1 == "aap noot mies" assert cppyy.gbl.my_global_string2 == "zus jet teun" assert list(cppyy.gbl.my_global_string3) == ["aap", "noot", "mies"] - # TODO: currently fails b/c double** not understood as &double* - #assert cppyy.gbl.my_global_ptr[0] == 1234. + assert cppyy.gbl.my_global_ptr[0] == 1234. v = cppyy.gbl.my_global_int_holders assert len(v) == 5 @@ -693,6 +703,21 @@ def test21_access_to_global_variables(self): for i in range(len(v)): assert v[i].m_val == expected_vals[i] + o = cppyy.gbl.some_concrete_class() + assert type(o) != type(cppyy.gbl.g_abstract_ptr) + cppyy.gbl.g_abstract_ptr = o + assert type(o) != type(cppyy.gbl.g_abstract_ptr) + assert cppyy.gbl.g_abstract_ptr == o + cppyy.gbl.g_abstract_ptr = cppyy.nullptr + + cppyy.cppexec("std::vector* gtestv1 = nullptr;") + pyv = cppyy.gbl.std.vector[int]() + cppyy.gbl.gtestv1 = pyv + cppyy.cppexec("auto* gtestv2 = gtestv1;") + pyv.push_back(42) + assert len(cppyy.gbl.gtestv1) == 1 + assert len(cppyy.gbl.gtestv2) == 1 + def test22_exceptions(self): """Catching of C++ exceptions""" @@ -768,18 +793,39 @@ def test25_ostream_printing(self): (cppyy.gbl.Printable4, "::operator<<(4)"), (cppyy.gbl.Printable6, "Printable6")]: assert str(tst[0]()) == tst[1] - assert '__lshiftc__' in tst[0].__dict__ - assert tst[0].__lshiftc__ - del tst[0].__lshiftc__ - assert str(tst[0]()) == tst[1] - assert tst[0].__lshiftc__ - s = cppyy.gbl.std.ostringstream() - tst[0].__lshiftc__(s, tst[0]()) - assert s.str() == tst[1] + if '__lshiftc__' in tst[0].__dict__: + # only cached for global functions and in principle should + # not be needed anymore ... + assert tst[0].__lshiftc__ + del tst[0].__lshiftc__ + assert str(tst[0]()) == tst[1] + assert tst[0].__lshiftc__ + s = cppyy.gbl.std.ostringstream() + tst[0].__lshiftc__(s, tst[0]()) + assert s.str() == tst[1] # print through base class (used to fail with compilation error) assert str(cppyy.gbl.Printable5()) == "Ok." + # print through friend + cppyy.cppdef("""\ + namespace PrintingNS { + class X { + friend std::ostream& operator<<(std::ostream& os, const X&) { return os << "X"; } + }; + + class Y { + }; + + std::ostream& operator<<(std::ostream& os, const Y&) { return os << "Y"; } + } """) + + x = cppyy.gbl.PrintingNS.X() + assert str(x) == 'X' + + y = cppyy.gbl.PrintingNS.Y() + assert str(y) == 'Y' + def test26_using_directive(self): """Test using directive in namespaces""" @@ -788,3 +834,127 @@ def test26_using_directive(self): assert cppyy.gbl.UserDirs.foo1() == cppyy.gbl.UsedSpace1.foo1() assert cppyy.gbl.UserDirs.bar() == cppyy.gbl.UsedSpace2.bar() assert cppyy.gbl.UserDirs.foo2() == cppyy.gbl.UsedSpace1.inner.foo2() + + def test27_shadowed_typedef(self): + """Test that typedefs are not shadowed""" + + import cppyy + + cppyy.cppdef(""" + namespace ShadowedTypedef { + struct A { + typedef std::shared_ptr Ptr; + typedef int Val; + }; + + struct B : public A { + typedef std::shared_ptr Ptr; + typedef double Val; + }; + + struct C : public A { + /* empty */ + }; }""") + + ns = cppyy.gbl.ShadowedTypedef + + ns.A.Ptr # pull A::Ptr first + ns.A.Val # id. A::Val + ns.B.Ptr # used to be A.Ptr through python-side dict lookup + ns.B.Val # id. B::Val + ns.C.Ptr # is A.Ptr + ns.C.Val # is A.Val + + assert ns.A.Ptr == ns.A.Ptr + assert ns.B.Ptr == ns.B.Ptr + assert ns.A.Ptr != ns.B.Ptr + assert ns.A.Ptr == ns.C.Ptr + assert ns.A.Val == ns.C.Val + + # TODO: currently only classes are checked; typedefs of builtin types are + # mapped through the type mapper and as such can be anything + #assert ns.A.Val != ns.B.Val + #assert type(ns.A.Val(1)) == int + #assert type(ns.B.Val(1)) == float + + def test28_extern_C_in_namespace(self): + """Access to extern "C" declared functions in namespaces""" + + import cppyy + + cppyy.cppdef("""\ + namespace extern_c_in_ns { + extern "C" int some_func_xc(void) { return 21; } + int some_func() { return some_func_xc(); } + + namespace deeper { + extern "C" int some_other_func_xc(void) { return 42; } + } }""") + + ns = cppyy.gbl.extern_c_in_ns + + assert ns.some_func() == 21 + assert ns.some_func_xc() == 21 + + assert ns.deeper.some_other_func_xc() == 42 + + def test29_castcpp(self): + """Allow casting a Python class to a C++ one""" + + import cppyy + import math + + cppyy.cppdef("""\ + namespace castcpp { + struct MyPoint { + double x, y; + }; + + double norm_cr(const MyPoint& p) { + return std::sqrt(p.x*p.x + p.y*p.y); + } + + double norm_r(MyPoint& p) { + return std::sqrt(p.x*p.x + p.y*p.y); + } + + double norm_m(MyPoint&& p) { + return std::sqrt(p.x*p.x + p.y*p.y); + } + + double norm_v(MyPoint p) { + return std::sqrt(p.x*p.x + p.y*p.y); + } + + double norm_p(MyPoint* p) { + return std::sqrt(p->x*p->x + p->y*p->y); + } }""") + + ns = cppyy.gbl.castcpp + + class MyPyPoint1: + def __init__(self, x, y): + self.x = x + self.y = y + + class MyPyPoint2(MyPyPoint1): + def __cast_cpp__(self): + return ns.MyPoint(self.x, self.y) + + class MyPyPoint3(MyPyPoint1): + def __cast_cpp__(self): + return (self.x, self.y) + + p1 = MyPyPoint1(5, 10) + p2 = MyPyPoint2(p1.x, p1.y) + p3 = MyPyPoint3(p1.x, p1.y) + pynorm = math.sqrt(p2.x**2+p2.y**2) + + for norm in [ns.norm_cr, ns.norm_r, ns.norm_v, ns.norm_p]: + with raises(TypeError): + norm(p1) + + assert round(norm(p2) - pynorm, 8) == 0 + + for norm in [ns.norm_cr, ns.norm_m, ns.norm_v]: + assert round(norm(p3) - pynorm, 8) == 0 diff --git a/bindings/pyroot/cppyy/cppyy/test/test_api.py b/bindings/pyroot/cppyy/cppyy/test/test_api.py index 746d34fecc743..6d4c60c118c70 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_api.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_api.py @@ -1,17 +1,11 @@ -import py -from pytest import raises - -try: - import __pypy__ - is_pypy = True -except ImportError: - is_pypy = False +from pytest import raises, skip +from .support import ispypy class TestAPI: def setup_class(cls): - if is_pypy: - py.test.skip('C++ API only available on CPython') + if ispypy: + skip('C++ API only available on CPython') import cppyy cppyy.include('CPyCppyy/API.h') @@ -111,21 +105,12 @@ class APICheck3Converter : public CPyCppyy::Converter { *a3 = *(APICheck3*)CPyCppyy::Instance_AsVoidPtr(value); return true; } - #ifdef WIN32 - virtual bool HasState() { return true; } // see register_a3 for reason - #endif }; typedef CPyCppyy::ConverterFactory_t cf_t; void register_a3() { - #ifndef WIN32 - CPyCppyy::RegisterConverter("APICheck3", (cf_t)+[](Py_ssize_t*) { static APICheck3Converter c{}; return &c; }); - CPyCppyy::RegisterConverter("APICheck3&", (cf_t)+[](Py_ssize_t*) { static APICheck3Converter c{}; return &c; }); - #else - // Clang's JIT does not support relocation of the static variable on Windows - CPyCppyy::RegisterConverter("APICheck3", (cf_t)+[](Py_ssize_t*) { return new APICheck3Converter{}; }); - CPyCppyy::RegisterConverter("APICheck3&", (cf_t)+[](Py_ssize_t*) { return new APICheck3Converter{}; }); - #endif + CPyCppyy::RegisterConverter("APICheck3", (cf_t)+[](CPyCppyy::cdims_t) { static APICheck3Converter c{}; return &c; }); + CPyCppyy::RegisterConverter("APICheck3&", (cf_t)+[](CPyCppyy::cdims_t) { static APICheck3Converter c{}; return &c; }); } void unregister_a3() { CPyCppyy::UnregisterConverter("APICheck3"); @@ -179,19 +164,11 @@ class APICheck4Executor : public CPyCppyy::Executor { a4->setExecutorCalled(); return CPyCppyy::Instance_FromVoidPtr(a4, "APICheck4", true); } - #ifdef WIN32 - virtual bool HasState() { return true; } // see register_a4 for reason - #endif }; typedef CPyCppyy::ExecutorFactory_t ef_t; void register_a4() { - #ifndef WIN32 - CPyCppyy::RegisterExecutor("APICheck4*", (ef_t)+[]() { static APICheck4Executor c{}; return &c; }); - #else - // Clang's JIT does not support relocation of the static variable on Windows - CPyCppyy::RegisterExecutor("APICheck4*", (ef_t)+[]() { return new APICheck4Executor{}; }); - #endif + CPyCppyy::RegisterExecutor("APICheck4*", (ef_t)+[](CPyCppyy::cdims_t) { static APICheck4Executor c{}; return &c; }); } void unregister_a4() { CPyCppyy::UnregisterExecutor("APICheck4*"); diff --git a/bindings/pyroot/cppyy/cppyy/test/test_boost.py b/bindings/pyroot/cppyy/cppyy/test/test_boost.py index d3fc3c6509b5c..c3c82041bff63 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_boost.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_boost.py @@ -1,5 +1,5 @@ import os -from pytest import mark, raises +from pytest import mark, raises, skip from .support import setup_make noboost = False @@ -12,23 +12,26 @@ class TestBOOSTANY: def setup_class(cls): import cppyy + cppyy.include('boost/any.hpp') def test01_any_class(self): """Availability of boost::any""" import cppyy + assert cppyy.gbl.boost.any from cppyy.gbl import std from cppyy.gbl.boost import any - assert std.list(any) + assert std.list[any] def test02_any_usage(self): """boost::any assignment and casting""" import cppyy + assert cppyy.gbl.boost from cppyy.gbl import std, boost @@ -67,6 +70,7 @@ def test02_any_usage(self): class TestBOOSTOPERATORS: def setup_class(cls): import cppyy + cppyy.include('boost/operators.hpp') def test01_ordered(self): @@ -74,7 +78,10 @@ def test01_ordered(self): import cppyy - cppyy.include("gmpxx.h") + try: + cppyy.include("gmpxx.h") + except ImportError: + skip("gmpxx not installed") cppyy.cppdef(""" namespace boost_test { class Derived : boost::ordered_field_operators, boost::ordered_field_operators {}; @@ -88,6 +95,7 @@ class Derived : boost::ordered_field_operators, boost::ordered_field_op class TestBOOSTVARIANT: def setup_class(cls): import cppyy + cppyy.include("boost/variant/variant.hpp") cppyy.include("boost/variant/get.hpp") @@ -122,3 +130,38 @@ class C { }; } """) raises(Exception, boost.get['BV::B'], v[0]) assert type(boost.get['BV::B'](v[1])) == cpp.BV.B assert type(boost.get['BV::C'](v[2])) == cpp.BV.C + + +@mark.skipif(noboost == True, reason="boost not found") +class TestBOOSTERASURE: + def setup_class(cls): + import cppyy + + cppyy.include("boost/type_erasure/any.hpp") + cppyy.include("boost/type_erasure/member.hpp") + cppyy.include("boost/mpl/vector.hpp") + + def test01_erasure_usage(self): + """boost::type_erasure usage""" + + import cppyy + + cppyy.cppdef(""" + BOOST_TYPE_ERASURE_MEMBER((has_member_f), f, 0) + + using LengthsInterface = boost::mpl::vector< + boost::type_erasure::copy_constructible<>, + has_member_f() const>>; + + using Lengths = boost::type_erasure::any; + + struct Unerased { + std::vector f() const { return std::vector{}; } + }; + + Lengths lengths() { + return Unerased{}; + } + """) + + assert cppyy.gbl.lengths() is not None diff --git a/bindings/pyroot/cppyy/cppyy/test/test_boost_any.py b/bindings/pyroot/cppyy/cppyy/test/test_boost_any.py deleted file mode 100644 index 029ec181da093..0000000000000 --- a/bindings/pyroot/cppyy/cppyy/test/test_boost_any.py +++ /dev/null @@ -1,71 +0,0 @@ -import py -from pytest import raises -from .support import setup_make - - -class TestBOOSTANY: - def setup_class(cls): - cls.disable = False - import cppyy - # TODO: better error handling - cppyy.include('boost/any.hpp') - if not hasattr(cppyy.gbl, 'boost'): - cls.disable = True - - def test01_any_class(self): - """Availability of boost::any""" - - if self.disable: - import warnings - warnings.warn("no boost/any.hpp found, skipping test01_any_class") - return - - import cppyy - assert cppyy.gbl.boost.any - - from cppyy.gbl import std - from cppyy.gbl.boost import any - - assert std.list(any) - - def test02_any_usage(self): - """boost::any assignment and casting""" - - if self.disable: - import warnings - warnings.warn("no boost/any.hpp found, skipping test02_any_usage") - return - - import cppyy - assert cppyy.gbl.boost - - from cppyy.gbl import std, boost - - val = boost.any() - # test both by-ref and by rvalue - v = std.vector[int]() - val.__assign__(v) - val.__assign__(std.move(std.vector[int](range(100)))) - assert val.type() == cppyy.typeid(std.vector[int]) - - extract = boost.any_cast[std.vector[int]](val) - assert type(extract) is std.vector[int] - assert len(extract) == 100 - extract += range(100) - assert len(extract) == 200 - - val.__assign__(std.move(extract)) # move forced - #assert len(extract) == 0 # not guaranteed by the standard - - # TODO: we hit boost::any_cast(boost::any* operand) instead - # of the reference version which raises - boost.any_cast.__useffi__ = False - try: - # raises(Exception, boost.any_cast[int], val) - assert not boost.any_cast[int](val) - except Exception: - # getting here is good, too ... - pass - - extract = boost.any_cast[std.vector[int]](val) - assert len(extract) == 200 diff --git a/bindings/pyroot/cppyy/cppyy/test/test_concurrent.py b/bindings/pyroot/cppyy/cppyy/test/test_concurrent.py new file mode 100644 index 0000000000000..15e51e3f68524 --- /dev/null +++ b/bindings/pyroot/cppyy/cppyy/test/test_concurrent.py @@ -0,0 +1,301 @@ +from pytest import raises, skip +from .support import IS_MAC_ARM + + +class TestCONCURRENT: + + def setup_class(cls): + import cppyy + + cls.data = [3.1415, 2.7183, 1.4142, 1.3807, -9.2848] + + cppyy.cppdef("""\ + namespace Workers { + double calc(double d) { return d*42.; } + }""") + + cppyy.gbl.Workers.calc.__release_gil__ = True + + def test01_simple_threads(self): + """Run basic Python threads""" + + import cppyy + import threading + + cppyy.gbl.Workers.calc.__release_gil__ = True + + threads = [] + for d in self.data: + threads.append(threading.Thread(target=cppyy.gbl.Workers.calc, args=(d,))) + + for t in threads: + t.start() + + for t in threads: + t.join() + + def test02_futures(self): + """Run with Python futures""" + + import cppyy + + try: + import concurrent.futures + except ImportError: + skip("module concurrent is not installed") + + total = 0. + with concurrent.futures.ThreadPoolExecutor(max_workers=len(self.data)) as executor: + futures = [executor.submit(cppyy.gbl.Workers.calc, d) for d in self.data] + for f in concurrent.futures.as_completed(futures): + total += f.result() + assert round(total+26.4642, 8) == 0.0 + + def test03_timeout(self): + """Time-out with threads""" + + return + import cppyy + import threading, time + + cppyy.cppdef("""\ + namespace test12_timeout { + bool _islive = false; volatile bool* islive = &_islive; + bool _stopit = false; volatile bool* stopit = &_stopit; + }""") + + cppyy.gbl.gInterpreter.ProcessLine.__release_gil__ = True + cmd = r"""\ + *test12_timeout::islive = true; + while (!*test12_timeout::stopit); + """ + + t = threading.Thread(target=cppyy.gbl.gInterpreter.ProcessLine, args=(cmd,)) + t.start() + + # have to give ProcessLine() time to actually start doing work + while not cppyy.gbl.test12_timeout.islive: + time.sleep(0.1) # in seconds + + # join the thread with a timeout after 0.1s + t.join(0.1) # id. + + if t.is_alive(): # was timed-out + cppyy.gbl.test12_timeout.stopit[0] = True + + def test04_cpp_threading_with_exceptions(self): + """Threads and Python exceptions""" + + if IS_MAC_ARM: + skip("JIT exceptions can not be caught in JITed code on Mac ARM") + + import cppyy + + cppyy.include("CPyCppyy/PyException.h") + + cppyy.cppdef("""namespace thread_test { + #include + + struct consumer { + virtual ~consumer() {} + virtual void process(int) = 0; + }; + + struct worker { + worker(consumer* c) : cons(c) { } + ~worker() { wait(); } + + void start() { + t = std::thread([this] { + int counter = 0; + while (counter++ < 10) + try { + cons->process(counter); + } catch (CPyCppyy::PyException& e) { + err_msg = e.what(); + return; + } + }); + } + + void wait() { + if (t.joinable()) + t.join(); + } + + std::thread t; + consumer* cons = nullptr; + std::string err_msg; + }; }""") + + ns = cppyy.gbl.thread_test + consumer = ns.consumer + worker = ns.worker + worker.wait.__release_gil__ = True + + class C(consumer): + count = 0 + def process(self, c): + self.count += 1 + + c = C() + assert c.count == 0 + + w = worker(c) + w.start() + w.wait() + + assert c.count == 10 + + class C(consumer): + count = 0 + def process(self, c): + raise RuntimeError("all wrong") + + c = C() + + w = worker(c) + w.start() + w.wait() + + assert "RuntimeError" in w.err_msg + assert "all wrong" in w.err_msg + + def test05_float2d_callback(self): + """Passing of 2-dim float arguments""" + + import cppyy + + cppyy.cppdef("""\ + namespace FloatDim2 { + #include + + struct Buffer { + Buffer() = default; + + void setData(float** newData) { + data = newData; + } + + void setSample(int newChannel, int newSample, float value) { + data[newChannel][newSample] = value; + } + + float** data = nullptr; + }; + + struct Processor { + virtual ~Processor() = default; + virtual void process(float** data, int channels, int samples) = 0; + }; + + void callback(Processor& p) { + std::thread t([&p] { + int channels = 2; + int samples = 32; + + float** data = new float*[channels]; + for (int i = 0; i < channels; ++i) + data[i] = new float[samples]; + + p.process(data, channels, samples); + + for (int i = 0; i < channels; ++i) + delete[] data[i]; + + delete[] data; + }); + + t.join(); + } }""") + + cppyy.gbl.FloatDim2.callback.__release_gil__ = True + + class Processor(cppyy.gbl.FloatDim2.Processor): + buffer = cppyy.gbl.FloatDim2.Buffer() + + def process(self, data, channels, samples): + self.buffer.setData(data) + + try: + for c in range(channels): + for s in range(samples): + self.buffer.setSample(c, s, 0.0) # < used to crash here + except Exception as e: + print(e) + + p = Processor() + cppyy.gbl.FloatDim2.callback(p) + + def test06_overload_reuse_in_threads(self): + """Threads reuse overload objects; check for clashes""" + + import cppyy + import threading + + cppyy.cppdef("""\ + namespace CPPOverloadReuse { + class Simulation1 { + public: + virtual void set_something(std::map, std::string) {} + }; }""") + + def test(): + o = {"a": "b"} # causes a temporary to be created + simulation = cppyy.gbl.CPPOverloadReuse.Simulation1() + simulation.set_something(o, ".") + + threads = [threading.Thread(target=test) for i in range(0, 100)] + + for t in threads: + t.start() + + for t in threads: + t.join() + + def test07_overload_reuse_in_threads_wo_gil(self): + """Threads reuse overload objects; check for clashes if no GIL""" + + if IS_MAC_ARM: + # the culprit here is occasional std::system_error if a thread can not be joined + skip("JIT exceptions can not be caught in JITed code on Mac ARM") + + import cppyy + import threading + + cppyy.cppdef("""\ + namespace CPPOverloadReuse { + class Simulation2 { + int fCount; + public: + Simulation2(int count) : fCount(count) {} + virtual int do_something() { return fCount; } + }; }""") + + cppyy.gbl.CPPOverloadReuse.Simulation2.do_something.__release_gil__ = True + + class State(object): + lock = threading.Lock() + c1, c2, c3 = 0, 0, 0 + + def test(i, state=State): + #global c1, c2, c3, lock + + simulation = cppyy.gbl.CPPOverloadReuse.Simulation2(i) + res = simulation.do_something() + + with state.lock: + state.c3 += state.c1 + state.c2 += res + state.c1 += 1 + + threads = [threading.Thread(target=test, args=[i]) for i in range(0, 1000)] + + for t in threads: + t.start() + + for t in threads: + t.join() + + assert State.c1 == 1000 + assert State.c2 == State.c3 diff --git a/bindings/pyroot/cppyy/cppyy/test/test_conversions.py b/bindings/pyroot/cppyy/cppyy/test/test_conversions.py index bab91102f30dc..4abaadb4ffa07 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_conversions.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_conversions.py @@ -1,6 +1,6 @@ import py from pytest import raises -from .support import setup_make, pylong, pyunicode +from .support import setup_make currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("conversionsDict")) @@ -47,6 +47,7 @@ def test02_memory_handling_of_temporaries(self): assert CC.s_count == 0 c = CC() + assert c.__python_owns__ assert CC.s_count == 1 del c; gc.collect() assert CC.s_count == 0 @@ -89,9 +90,38 @@ def test03_error_handling(self): def test04_implicit_conversion_from_tuple(self): """Allow implicit conversions from tuples as arguments {}-like""" + # Note: fails on windows b/c the assignment operator for strings is + # template, which ("operator=(std::string)") doesn't instantiate import cppyy m = cppyy.gbl.std.map[str, str]() m.insert(('a', 'b')) # implicit conversion to std::pair assert m['a'] == 'b' + + def test05_bool_conversions(self): + """Test operator bool() and null pointer behavior""" + + import cppyy + + cppyy.cppdef("""\ + namespace BoolConversions { + struct Test1 {}; + struct Test2 { + Test2(bool b) : m_b(b) {} + explicit operator bool() const { return m_b; } + bool m_b; + }; + + Test1* CreateNullTest1() { return nullptr; } + Test2* CreateNullTest2() { return nullptr; } + }""") + + ns = cppyy.gbl.BoolConversions + + for t in [ns.CreateNullTest1(), ns.CreateNullTest2()]: + assert not t + + assert ns.Test1() + assert ns.Test2(True) + assert not ns.Test2(False) diff --git a/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py b/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py index 8a12e2dd785a0..986360c5a861e 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py @@ -1,12 +1,6 @@ import py, sys from pytest import raises -from .support import setup_make - -try: - import __pypy__ - is_pypy = True -except ImportError: - is_pypy = False +from .support import setup_make, ispypy currpath = py.path.local(__file__).dirpath() @@ -21,112 +15,184 @@ def setup_class(cls): import cppyy cls.cpp11features = cppyy.load_reflection_info(cls.test_dct) - def test01_shared_ptr(self): - """Usage and access of std::shared_ptr<>""" - - from cppyy.gbl import TestSharedPtr, create_shared_ptr_instance + def test01_smart_ptr(self): + """Usage and access of std::shared/unique_ptr<>""" - # proper memory accounting - assert TestSharedPtr.s_counter == 0 + from cppyy.gbl import TestSmartPtr + from cppyy.gbl import create_shared_ptr_instance, create_unique_ptr_instance + import gc - ptr1 = create_shared_ptr_instance() - assert ptr1 - assert not not ptr1 - assert TestSharedPtr.s_counter == 1 + for cf in [create_shared_ptr_instance, create_unique_ptr_instance]: + assert TestSmartPtr.s_counter == 0 - ptr2 = create_shared_ptr_instance() - assert ptr2 - assert not not ptr2 - assert TestSharedPtr.s_counter == 2 + ptr1 = cf() + assert ptr1 + assert not not ptr1 + assert TestSmartPtr.s_counter == 1 - del ptr2 - import gc; gc.collect() - assert TestSharedPtr.s_counter == 1 + ptr2 = cf() + assert ptr2 + assert not not ptr2 + assert TestSmartPtr.s_counter == 2 - del ptr1 - gc.collect() - assert TestSharedPtr.s_counter == 0 + del ptr2 + gc.collect() + assert TestSmartPtr.s_counter == 1 - def test02_shared_ptr_construction(self): - """Shared pointer ctor is templated, taking special care""" + del ptr1 + gc.collect() + assert TestSmartPtr.s_counter == 0 - from cppyy.gbl import std, TestSharedPtr + def test02_smart_ptr_construction(self): + """Shared/Unique pointer ctor is templated, requiring special care""" - # proper memory accounting - assert TestSharedPtr.s_counter == 0 + from cppyy.gbl import std, TestSmartPtr + import gc - class C(TestSharedPtr): + class C(TestSmartPtr): pass - c = C() - assert TestSharedPtr.s_counter == 1 - c.__python_owns__ = False - cc = std.shared_ptr[TestSharedPtr](c) - assert cc.__python_owns__ + for cls in [std.shared_ptr, std.unique_ptr]: + assert TestSmartPtr.s_counter == 0 - del c + c = C() + assert TestSmartPtr.s_counter == 1 + c.__python_owns__ = False + cc = cls[TestSmartPtr](c) + assert cc.__python_owns__ - assert cc - assert TestSharedPtr.s_counter == 1 + del c - del cc + assert cc + assert TestSmartPtr.s_counter == 1 - import gc - gc.collect() - assert TestSharedPtr.s_counter == 0 + del cc + gc.collect() + assert TestSmartPtr.s_counter == 0 - def test03_shared_ptr_memory_handling(self): - """Test shared pointer memory ownership""" + def test03_smart_ptr_memory_handling(self): + """Test shared/unique pointer memory ownership""" - from cppyy.gbl import std, TestSharedPtr + from cppyy.gbl import std, TestSmartPtr + import gc - # proper memory accounting - assert TestSharedPtr.s_counter == 0 + class C(TestSmartPtr): + pass - t = TestSharedPtr() - assert TestSharedPtr.s_counter == 1 - assert t.__python_owns__ + for cls in [std.shared_ptr, std.unique_ptr]: + assert TestSmartPtr.s_counter == 0 - tt = std.shared_ptr[TestSharedPtr](t) - assert not t.__python_owns__ + t = TestSmartPtr() + assert TestSmartPtr.s_counter == 1 + assert t.__python_owns__ - class C(TestSharedPtr): - pass + tt = std.shared_ptr[TestSmartPtr](t) + assert not t.__python_owns__ - c = C() - assert TestSharedPtr.s_counter == 2 - assert c.__python_owns__ + c = C() + assert TestSmartPtr.s_counter == 2 + assert c.__python_owns__ - cc = std.shared_ptr[TestSharedPtr](c) - assert not c.__python_owns__ + cc = std.shared_ptr[TestSmartPtr](c) + assert not c.__python_owns__ - del cc, tt + del cc, tt + gc.collect() + assert TestSmartPtr.s_counter == 0 + def test04_shared_ptr_passing(self): + """Ability to pass shared_ptr through shared_ptr""" + + from cppyy.gbl import std, TestSmartPtr, DerivedTestSmartPtr + from cppyy.gbl import pass_shared_ptr, move_shared_ptr, create_TestSmartPtr_by_value import gc + + for ff, mv in [(pass_shared_ptr, lambda x: x), (move_shared_ptr, std.move)]: + assert TestSmartPtr.s_counter == 0 + + dd = std.make_shared[DerivedTestSmartPtr](DerivedTestSmartPtr(24)) + assert TestSmartPtr.s_counter == 1 + assert ff(mv(dd)) == 100 + + del dd + gc.collect() + assert TestSmartPtr.s_counter == 0 + + # ability to take over by-value python-owned objects + tsp = create_TestSmartPtr_by_value() + assert TestSmartPtr.s_counter == 1 + assert tsp.__python_owns__ + + shared_stp = std.make_shared[TestSmartPtr](tsp) + assert TestSmartPtr.s_counter == 1 + assert not tsp.__python_owns__ + + del shared_stp gc.collect() - assert TestSharedPtr.s_counter == 0 + assert TestSmartPtr.s_counter == 0 - def test04_shared_ptr_passing(self): - """Ability to pass shared_ptr through shared_ptr""" + # alternative make_shared with type taken from pointer + tsp = create_TestSmartPtr_by_value() + shared_stp = std.make_shared(tsp) + assert TestSmartPtr.s_counter == 1 + del shared_stp + gc.collect() + assert TestSmartPtr.s_counter == 0 - from cppyy.gbl import std, TestSharedPtr, DerivedTestSharedPtr - from cppyy.gbl import pass_shared_ptr + def test05_unique_ptr_passing(self): + """Ability to pass unique_ptr through unique_ptr""" - # proper memory accounting - assert TestSharedPtr.s_counter == 0 + from cppyy.gbl import std, TestSmartPtr, DerivedTestSmartPtr + from cppyy.gbl import move_unique_ptr, move_unique_ptr_derived + from cppyy.gbl import create_TestSmartPtr_by_value + import gc - dd = std.make_shared[DerivedTestSharedPtr](DerivedTestSharedPtr(24)) - assert TestSharedPtr.s_counter == 1 + assert TestSmartPtr.s_counter == 0 - assert pass_shared_ptr(dd) == 100 + # move matching unique_ptr + dd = std.make_unique[DerivedTestSmartPtr](DerivedTestSmartPtr(24)) + assert TestSmartPtr.s_counter == 1 + assert move_unique_ptr_derived(std.move(dd)) == 100 + assert dd.__python_owns__ del dd + gc.collect() + assert TestSmartPtr.s_counter == 0 - import gc + # move with conversion + dd = std.make_unique[DerivedTestSmartPtr](DerivedTestSmartPtr(24)) + assert TestSmartPtr.s_counter == 1 + # TODO: why does the following fail, but succeed for shared_ptr?? + # assert move_unique_ptr(std.move(dd)) == 100 + assert dd.__python_owns__ + + del dd gc.collect() - assert TestSharedPtr.s_counter == 0 + assert TestSmartPtr.s_counter == 0 + + # ability to take over by-value python-owned objects + tsp = create_TestSmartPtr_by_value() + assert TestSmartPtr.s_counter == 1 + assert tsp.__python_owns__ + + unique_stp = std.make_unique[TestSmartPtr](tsp) + assert TestSmartPtr.s_counter == 1 + assert not tsp.__python_owns__ + + del unique_stp + gc.collect() + assert TestSmartPtr.s_counter == 0 + + # alternative make_unique with type taken from pointer + tsp = create_TestSmartPtr_by_value() + unique_stp = std.make_unique(tsp) + assert TestSmartPtr.s_counter == 1 - def test05_nullptr(self): + del unique_stp + gc.collect() + assert TestSmartPtr.s_counter == 0 + + def test06_nullptr(self): """Allow the programmer to pass NULL in certain cases""" import cppyy @@ -135,9 +201,12 @@ def test05_nullptr(self): nullptr = cppyy.nullptr # assert not hasattr(cppyy.gbl, 'nullptr') - # usage is tested in datatypes.py:test15_nullptr_passing + assert cppyy.bind_object(cppyy.nullptr, 'std::vector') == cppyy.nullptr + assert not cppyy.bind_object(cppyy.nullptr, 'std::vector') != cppyy.nullptr + + # further usage is tested in datatypes.py:test15_nullptr_passing - def test06_move(self): + def test07_move(self): """Move construction, assignment, and methods""" import cppyy, gc @@ -154,7 +223,7 @@ def moveit(T): i2 = T(i1) # cctor assert T.s_move_counter == 0 - if is_pypy or 0x3000000 <= sys.hexversion: + if ispypy or 0x3000000 <= sys.hexversion: i3 = T(std.move(T())) # can't check ref-count else: i3 = T(T()) # should call move, not memoized cctor @@ -170,7 +239,7 @@ def moveit(T): i4.__assign__(i2) assert T.s_move_counter == 3 - if is_pypy or 0x3000000 <= sys.hexversion: + if ispypy or 0x3000000 <= sys.hexversion: i4.__assign__(std.move(T())) # can't check ref-count else: i4.__assign__(T()) @@ -197,7 +266,7 @@ def moveit(T): assert cppyy.gbl.TestMoving1.s_instance_counter == 0 assert cppyy.gbl.TestMoving2.s_instance_counter == 0 - def test07_initializer_list(self): + def test08_initializer_list(self): """Initializer list construction""" from cppyy.gbl import std, TestData, TestData2, WithInitList @@ -221,7 +290,20 @@ def test07_initializer_list(self): for i in range(len(l)): assert v[i].m_int == l[i].m_int - def test08_lambda_calls(self): + import cppyy + + cppyy.cppdef(r""" + namespace InitializerListTest { + std::vector foo(const std::initializer_list& vals) { + return std::vector{vals}; + } }""") + + ns = cppyy.gbl.InitializerListTest + + for l in (['x'], ['x', 'y', 'z']): + assert ns.foo(l) == std.vector['std::string'](l) + + def test09_lambda_calls(self): """Call (global) lambdas""" import cppyy @@ -248,7 +330,7 @@ def test08_lambda_calls(self): assert l3 assert l3(2) == 48 - def test09_optional(self): + def test10_optional(self): """Use of optional and nullopt""" import cppyy @@ -258,17 +340,17 @@ def test09_optional(self): assert cppyy.gbl.std.nullopt cppyy.cppdef(""" - enum Enum { A = -1 }; - bool callopt(std::optional) { return true; } + enum Enum { A = -1 }; + bool callopt(std::optional) { return true; } """) a = cppyy.gbl.std.optional[cppyy.gbl.Enum]() assert cppyy.gbl.callopt(a) - c = cppyy.gbl.nullopt + c = cppyy.gbl.std.nullopt assert cppyy.gbl.callopt(c) - def test10_chrono(self): + def test11_chrono(self): """Use of chrono and overloaded operator+""" import cppyy @@ -278,8 +360,7 @@ def test10_chrono(self): # following used to fail with compilation error t = std.chrono.system_clock.now() + std.chrono.seconds(1) - - def test11_stdfunction(self): + def test12_stdfunction(self): """Use of std::function with arguments in a namespace""" import cppyy @@ -295,8 +376,8 @@ def test11_stdfunction(self): # and for good measure, inline cppyy.cppdef("""namespace FunctionNS2 { - struct FNTestStruct { FNTestStruct(int i) : t(i) {} int t; }; - std::function FNCreateTestStructFunc() { return [](const FNTestStruct& t) { return t.t; }; } + struct FNTestStruct { FNTestStruct(int i) : t(i) {} int t; }; + std::function FNCreateTestStructFunc() { return [](const FNTestStruct& t) { return t.t; }; } }""") from cppyy.gbl import FunctionNS2 @@ -305,7 +386,7 @@ def test11_stdfunction(self): f = FunctionNS.FNCreateTestStructFunc() assert f(t) == 27 - def test12_stdhash(self): + def test13_stdhash(self): """Use of std::hash""" import cppyy @@ -319,3 +400,93 @@ def test12_stdhash(self): sw = StructWithHash() assert hash(sw) == 17 assert hash(sw) == 17 + + def test14_shared_ptr_passing(self): + """Ability to pass normal pointers through shared_ptr by value""" + + from cppyy.gbl import std, TestSmartPtr, DerivedTestSmartPtr + from cppyy.gbl import pass_shared_ptr + import gc + + for cls, val in [(lambda: TestSmartPtr(), 17), (lambda: DerivedTestSmartPtr(24), 100)]: + assert TestSmartPtr.s_counter == 0 + + obj = cls() + + assert TestSmartPtr.s_counter == 1 + assert not obj.__smartptr__() + assert pass_shared_ptr(obj) == val + assert obj.__smartptr__() + assert obj.__python_owns__ + assert TestSmartPtr.s_counter == 1 + + assert not not obj # pass was by shared copy + + del obj + gc.collect() + assert TestSmartPtr.s_counter == 0 + + def test15_unique_ptr_template_deduction(self): + """Argument type deduction with std::unique_ptr""" + + import cppyy + + cppyy.cppdef("""namespace UniqueTempl { + template + std::unique_ptr returnptr(std::unique_ptr&& a) { + return std::move(a); + } }""") + + uptr_in = cppyy.gbl.std.make_unique[int]() + uptr_out = cppyy.gbl.UniqueTempl.returnptr["int"](cppyy.gbl.std.move(uptr_in)) + assert not not uptr_out + + uptr_in = cppyy.gbl.std.make_unique['int']() + with raises(ValueError): # not an RValue + cppyy.gbl.UniqueTempl.returnptr[int](uptr_in) + + def test16_unique_ptr_moves(self): + """std::unique_ptr requires moves""" + + import cppyy + cppyy.cppdef("""namespace unique_ptr_moves { + template + std::unique_ptr returnptr_value(std::unique_ptr a) { + return std::move(a); + } + template + std::unique_ptr returnptr_move(std::unique_ptr&& a) { + return std::move(a); + } }""") + + up = cppyy.gbl.std.make_unique[int](42) + + ns = cppyy.gbl.unique_ptr_moves + up = ns.returnptr_value(up) ; assert up and up.get()[0] == 42 + up = ns.returnptr_value(cppyy.gbl.std.move(up)); assert up and up.get()[0] == 42 + up = ns.returnptr_move(cppyy.gbl.std.move(up)) ; assert up and up.get()[0] == 42 + + with raises(TypeError): + ns.returnptr_move(up) + + def test17_unique_ptr_data(self): + """std::unique_ptr as data means implicitly no copy ctor""" + + import cppyy + + cppyy.cppdef("""namespace unique_ptr_data{ + class Example { + private: + std::unique_ptr x; + public: + Example() {} + virtual ~Example() = default; + double y = 66.; + }; }""") + + class Inherit(cppyy.gbl.unique_ptr_data.Example): + pass + + a = Inherit() + # Test whether this attribute was inherited + assert a.y == 66. diff --git a/bindings/pyroot/cppyy/cppyy/test/test_crossinheritance.py b/bindings/pyroot/cppyy/cppyy/test/test_crossinheritance.py index 49d1f52ec5135..b90c33ad60bff 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_crossinheritance.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_crossinheritance.py @@ -1,6 +1,7 @@ -import py -from pytest import raises -from .support import setup_make, pylong +import py, os +from pytest import raises, skip +from .support import setup_make, pylong, IS_MAC_ARM + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("crossinheritanceDict")) @@ -29,6 +30,9 @@ def get_value(self): assert Derived().get_value() == 13 + assert 'Derived' in str(Derived()) + assert 'Derived' in repr(Derived()) + assert Base1.call_get_value(Base1()) == 42 assert Base1.call_get_value(Derived()) == 13 @@ -179,7 +183,7 @@ def sum_all(self, *args): assert d.sum_all(-7, -5) == 1 assert Base1.call_sum_all(d, -7, -5) == 1 - def test07_const_methods(self): + def test06_const_methods(self): """Declared const methods should keep that qualifier""" import cppyy @@ -201,7 +205,7 @@ def __init__(self): assert CX.IBase4.call_get_value(c1) == 17 assert CX.IBase4.call_get_value(c2) == 27 - def test08_templated_base(self): + def test07_templated_base(self): """Derive from a base class that is instantiated from a template""" import cppyy @@ -225,7 +229,7 @@ def get_value(self): p1 = TPyDerived1() assert p1.get_value() == 13 - def test09_error_handling(self): + def test08_error_handling(self): """Python errors should propagate through wrapper""" import cppyy @@ -241,13 +245,33 @@ def sum_value(self, val): d = Derived() raises(ValueError, Base1.call_sum_value, d, -7) + if IS_MAC_ARM: + skip("JIT exceptions from signals not supported on Mac ARM") + try: Base1.call_sum_value(d, -7) assert not "should not get here" except ValueError as e: assert errmsg in str(e) - def test10_interface_checking(self): + cppyy.cppdef("""\ + namespace CrossInheritance { + std::string call_base1(Base1* b) { + try { + b->sum_value(-7); + } catch (CPyCppyy::PyException& e) { + e.clear(); + return e.what(); + } + return ""; + } }""") + + res = cppyy.gbl.CrossInheritance.call_base1(d) + + assert 'ValueError' in res + assert os.path.basename(__file__) in res + + def test09_interface_checking(self): """Conversion errors should be Python exceptions""" import cppyy @@ -261,9 +285,10 @@ def get_value(self): self.m_int*2 # missing return d = Derived(4) + assert raises(TypeError, Base1.call_get_value, d) - def test11_python_in_templates(self): + def test10_python_in_templates(self): """Usage of Python derived objects in std::vector""" import cppyy, gc @@ -308,25 +333,23 @@ def call(self): gc.collect() assert CB.s_count == 0 + start_count - def test12_python_in_make_shared(self): + def test11_python_in_make_shared(self): """Usage of Python derived objects with std::make_shared""" import cppyy - cppyy.cppdef(""" - namespace MakeSharedTest { + cppyy.cppdef("""namespace MakeSharedTest { class Abstract { public: - virtual ~Abstract() = 0; - virtual int some_imp() = 0; + virtual ~Abstract() = 0; + virtual int some_imp() = 0; }; Abstract::~Abstract() {} int call_shared(std::shared_ptr& ptr) { - return ptr->some_imp(); - } } - """) + return ptr->some_imp(); + } }""") from cppyy.gbl import std, MakeSharedTest from cppyy.gbl.MakeSharedTest import Abstract, call_shared @@ -348,7 +371,32 @@ def some_imp(self): assert call_shared(v) == 13 assert v.some_imp() == 13 - def test13_python_shared_ptr_memory(self): + def test12a_counter_test(self): + """Test countable base counting""" + + import cppyy, gc + + std = cppyy.gbl.std + CB = cppyy.gbl.CrossInheritance.CountableBase + + class PyCountable(CB): + def call(self): + try: + return self.extra + 42 + except AttributeError: + return 42 + + start_count = CB.s_count + + # test counter + pyc = PyCountable() + assert CB.s_count == 1 + start_count + + del pyc + gc.collect() + assert CB.s_count == 0 + start_count + + def test12_python_shared_ptr_memory(self): """Usage of Python derived objects with std::shared_ptr""" import cppyy, gc @@ -375,18 +423,17 @@ def call(self): gc.collect() assert CB.s_count == 0 + start_count - def test14_virtual_dtors_and_del(self): + def test13_virtual_dtors_and_del(self): """Usage of virtual destructors and Python-side del.""" - import cppyy + import cppyy, warnings - cppyy.cppdef(""" - namespace VirtualDtor { + cppyy.cppdef("""namespace VirtualDtor { class MyClass1 {}; // no virtual dtor ... class MyClass2 { public: - virtual ~MyClass2() {} + virtual ~MyClass2() {} }; class MyClass3 : public MyClass2 {}; @@ -394,19 +441,23 @@ class MyClass3 : public MyClass2 {}; template class MyClass4 { public: - virtual ~MyClass4() {} - }; } - """) + virtual ~MyClass4() {} + }; }""") VD = cppyy.gbl.VirtualDtor - try: + # rethought this: just issue a warning if there is no virtual destructor + # as the C++ side now carries the type of the dispatcher, not the type of + # the direct base class + with warnings.catch_warnings(record=True) as w: class MyPyDerived1(VD.MyClass1): - pass - except TypeError: - pass - else: - assert not "should have failed" + pass # TODO: verify warning is given + assert len(w) == 1 + assert issubclass(w[-1].category, RuntimeWarning) + assert "has no virtual destructor" in str(w[-1].message) + + d = MyPyDerived1() + del d # used to crash class MyPyDerived2(VD.MyClass2): pass @@ -420,3 +471,1300 @@ class MyPyDerived3(VD.MyClass3): class MyPyDerived4(VD.MyClass4[int]): pass + + def test14_protected_access(self): + """Derived classes should have access to protected members""" + + import cppyy + + ns = cppyy.gbl.AccessProtected + + assert not 'my_data' in ns.MyBase.__dict__ + assert not hasattr(ns.MyBase(), 'my_data') + + class MyPyDerived(ns.MyBase): + pass + + assert 'my_data' in MyPyDerived.__dict__ + assert MyPyDerived().my_data == 101 + + class MyPyDerived(ns.MyBase): + def __init__(self): + super(MyPyDerived, self).__init__() + assert self.my_data == 101 + self.py_data = 13 + self.my_data = 42 + + m = MyPyDerived() + assert m.py_data == 13 + assert m.my_data == 42 + assert m.get_data() == 42 + assert m.get_data_v() == 42 + + def test15_object_returns(self): + """Return of C++ objects from overridden functions""" + + import cppyy + + # Part 1: return of a new C++ object + cppyy.cppdef("""namespace object_returns { + class Base { + public: + virtual Base* foo() { return new Base(); } + virtual ~Base() {} + virtual std::string whoami() { return "Base"; } + }; + + class CppDerived : public Base { + CppDerived* foo() { return new CppDerived(); } + ~CppDerived() {} + virtual std::string whoami() { return "CppDerived"; } + }; + + Base* call_foo(Base& obj) { return obj.foo(); } }""") + + ns = cppyy.gbl.object_returns + + class PyDerived1(ns.Base): + def foo(self): + return ns.CppDerived() + + obj = PyDerived1() + assert not ns.call_foo(obj) + + class PyDerived2(ns.Base): + def foo(self): + x = ns.CppDerived() + x.__python_owns__ = False + return x + + obj = PyDerived2() + assert not not ns.call_foo(obj) + + # Part 2: return of a new Python derived object + class PyDerived3(ns.Base): + def foo(self): + return PyDerived3() + + def whoami(self): + return "PyDerived3" + + obj = PyDerived3() + newobj = ns.call_foo(obj) + assert not newobj + + class PyDerived4(ns.Base): + def foo(self): + d = PyDerived4() + d.__python_owns__ = False + d.alpha = 2 + return d + + def whoami(self): + return "PyDerived4" + + obj = PyDerived4() + new_obj = ns.call_foo(obj) + assert not not new_obj + assert new_obj.whoami() == "PyDerived4" + + def test16_cctor_access_controlled(self): + """Python derived class of C++ class with access controlled cctor""" + + import cppyy + + cppyy.cppdef("""namespace cctor_access_controlled { + class CommonBase { + public: + virtual ~CommonBase() {} + virtual std::string whoami() = 0; + }; + + class Base1 : public CommonBase { + Base1(const Base1&) {} + public: + Base1() {} + virtual ~Base1() {} + virtual std::string whoami() { return "Base1"; } + }; + + class Base2 : public CommonBase { + protected: + Base2(const Base2&) {} + public: + Base2() {} + virtual ~Base2() {} + virtual std::string whoami() { return "Base2"; } + }; + + std::string callit(CommonBase& obj) { return obj.whoami(); } }""") + + ns = cppyy.gbl.cctor_access_controlled + + for base in (ns.Base1, ns.Base2): + class PyDerived(base): + def whoami(self): + return "PyDerived" + + obj = PyDerived() + assert ns.callit(obj) == "PyDerived" + + def test17_deep_hierarchy(self): + """Test a deep Python hierarchy with pure virtual functions""" + + import cppyy + + cppyy.cppdef("""namespace deep_hierarchy { + class Base { + public: + virtual ~Base() {} + virtual std::string whoami() = 0; + }; + + std::string callit(Base& obj) { return obj.whoami(); } }""") + + ns = cppyy.gbl.deep_hierarchy + + class PyDerived1(ns.Base): + def whoami(self): + return "PyDerived1" + + obj = PyDerived1() + assert ns.callit(obj) == "PyDerived1" + + class PyDerived2(PyDerived1): + pass + + obj = PyDerived2() + assert obj.whoami() == "PyDerived1" + assert ns.callit(obj) == "PyDerived1" + + class PyDerived3(PyDerived1): + def whoami(self): + return "PyDerived3" + + obj = PyDerived3() + assert obj.whoami() == "PyDerived3" + assert ns.callit(obj) == "PyDerived3" + + class PyDerived4(PyDerived2): + def whoami(self): + return "PyDerived4" + + obj = PyDerived4() + assert obj.whoami() == "PyDerived4" + assert ns.callit(obj) == "PyDerived4" + + def test18_abstract_hierarchy(self): + """Hierarchy with abstract classes""" + + + import cppyy + + cppyy.cppdef("""namespace abstract_classes { + class Base { + public: + virtual ~Base() {} + virtual std::string whoami() = 0; + virtual std::string message() = 0; + }; + + std::string whois(Base& obj) { return obj.whoami(); } + std::string saywot(Base& obj) { return obj.message(); } }""") + + ns = cppyy.gbl.abstract_classes + + class PyDerived1(ns.Base): + def __init__(self): + super(PyDerived1, self).__init__() + self._name = "PyDerived1" + + def whoami(self): + return self._name + + class PyDerived2(PyDerived1): + def __init__(self): + super(PyDerived2, self).__init__() + self._message = "Hello, World!" + + def message(self): + return self._message + + obj = PyDerived2() + assert obj.whoami() == "PyDerived1" + assert ns.whois(obj) == "PyDerived1" + + assert obj.message() == "Hello, World!" + assert ns.saywot(obj) == "Hello, World!" + + def test19_cpp_side_multiple_inheritance(self): + """Hierarchy with multiple inheritance on the C++ side""" + + import cppyy + + cppyy.cppdef(""" namespace cpp_side_multiple_inheritance { + struct Result { + Result() : result(1337) {} + Result(int r) : result(r) {} + int result; + }; + + class Base1 { + public: + virtual ~Base1() {} + virtual Result abstract1() = 0; + }; + + class Base2 { + public: + virtual ~Base2() {} + virtual Result abstract2() = 0; + }; + + class Base : public Base1, public Base2 { + public: + Result abstract2() override { return Result(999); } + }; } """) + + ns = cppyy.gbl.cpp_side_multiple_inheritance + + class Derived(ns.Base): + def abstract1(self): + return ns.Result(1) + + def test20_basic_multiple_inheritance(self): + """Basic multiple inheritance""" + + import cppyy + + cppyy.cppdef("""namespace basic_multiple_inheritance { + class MyClass1 { + public: + MyClass1() : m_1(13) {} + virtual ~MyClass1() {} + virtual int x() = 0; + + public: + int m_1; + }; + int callx(MyClass1& m) { return m.x(); } + + class MyClass2 { + public: + MyClass2() : m_2(42) {} + virtual ~MyClass2() {} + virtual int y() = 0; + + public: + int m_2; + }; + int cally(MyClass2& m) { return m.y(); } + + class MyClass3 { + public: + MyClass3() : m_3(67) {} + virtual ~MyClass3() {} + virtual int z() = 0; + + public: + int m_3; + }; + int callz(MyClass3& m) { return m.z(); } }""") + + ns = cppyy.gbl.basic_multiple_inheritance + + class MyPyDerived(cppyy.multi(ns.MyClass1, ns.MyClass2)): + def x(self): + return 16 + + def y(self): + return 32 + + assert len(MyPyDerived.__bases__) == 2 + + a = MyPyDerived() + assert a.x() == ns.callx(a) + assert a.y() == ns.cally(a) + + assert a.m_1 == 13 + assert a.m_2 == 42 + + class MyPyDerived2(cppyy.multi(ns.MyClass1, ns.MyClass2, ns.MyClass3)): + def x(self): + return 16 + + def y(self): + return 32 + + def z(self): + return 64 + + assert len(MyPyDerived2.__bases__) == 3 + + a = MyPyDerived2() + assert a.x() == ns.callx(a) + assert a.y() == ns.cally(a) + assert a.z() == ns.callz(a) + + assert a.m_1 == 13 + assert a.m_2 == 42 + assert a.m_3 == 67 + + def test21_multiple_inheritance_with_constructors(self): + """Multiple inheritance with constructors""" + + import cppyy + + cppyy.cppdef("""namespace multiple_inheritance_with_constructors { + class MyClass1 { + public: + MyClass1() : m_1(13) {} + MyClass1(int i) : m_1(i) {} + virtual ~MyClass1() {} + virtual int x() = 0; + + public: + int m_1; + }; + int callx(MyClass1& m) { return m.x(); } + + class MyClass2 { + public: + MyClass2() : m_2(42) {} + MyClass2(int i) : m_2(i) {} + virtual ~MyClass2() {} + virtual int y() = 0; + + public: + int m_2; + }; + int cally(MyClass2& m) { return m.y(); } + + class MyClass3 { + public: + MyClass3() : m_3(67) {} + MyClass3(int i) : m_3(i) {} + virtual ~MyClass3() {} + virtual int z() = 0; + + public: + int m_3; + }; + int callz(MyClass3& m) { return m.z(); } }""") + + ns = cppyy.gbl.multiple_inheritance_with_constructors + + class MyPyDerived(cppyy.multi(ns.MyClass1, ns.MyClass2)): + def __init__(self, val1, val2): + super(MyPyDerived, self).__init__((val1,), (val2,)) + + def x(self): + return 16 + + def y(self): + return 32 + + assert len(MyPyDerived.__bases__) == 2 + + a = MyPyDerived(27, 88) + assert a.x() == ns.callx(a) + assert a.y() == ns.cally(a) + + assert a.m_1 == 27 + assert a.m_2 == 88 + + class MyPyDerived2(cppyy.multi(ns.MyClass1, ns.MyClass2, ns.MyClass3)): + def __init__(self, val1, val2, val3): + super(MyPyDerived2, self).__init__((val1,), (val2,), (val3,)) + + def x(self): + return 16 + + def y(self): + return 32 + + def z(self): + return 64 + + assert len(MyPyDerived2.__bases__) == 3 + + a = MyPyDerived2(27, 88, -11) + assert a.x() == ns.callx(a) + assert a.y() == ns.cally(a) + assert a.z() == ns.callz(a) + + assert a.m_1 == 27 + assert a.m_2 == 88 + assert a.m_3 == -11 + + def test22_multiple_inheritance_with_defaults(self): + """Multiple inheritance with defaults""" + + import cppyy + + cppyy.cppdef("""namespace multiple_inheritance_with_defaults { + class MyClass1 { + public: + MyClass1(int i=13) : m_1(i) {} + virtual ~MyClass1() {} + virtual int x() = 0; + + public: + int m_1; + }; + int callx(MyClass1& m) { return m.x(); } + + class MyClass2 { + public: + MyClass2(int i=42) : m_2(i) {} + virtual ~MyClass2() {} + virtual int y() = 0; + + public: + int m_2; + }; + int cally(MyClass2& m) { return m.y(); } + + class MyClass3 { + public: + MyClass3(int i=67) : m_3(i) {} + virtual ~MyClass3() {} + virtual int z() = 0; + + public: + int m_3; + }; + int callz(MyClass3& m) { return m.z(); } }""") + + ns = cppyy.gbl.multiple_inheritance_with_defaults + + class MyPyDerived(cppyy.multi(ns.MyClass1, ns.MyClass2, ns.MyClass3)): + def __init__(self, val1=None, val2=None, val3=None, nArgs=3): + a1 = val1 is not None and (val1,) or () + a2 = val2 is not None and (val2,) or () + a3 = val3 is not None and (val3,) or () + if nArgs == 3: + super(MyPyDerived, self).__init__(a1, a2, a3) + elif nArgs == 0: + super(MyPyDerived, self).__init__() + elif nArgs == 1: + super(MyPyDerived, self).__init__(a1) + elif nArgs == 2: + super(MyPyDerived, self).__init__(a1, a2) + + def x(self): + return 16 + + def y(self): + return 32 + + def z(self): + return 64 + + assert len(MyPyDerived.__bases__) == 3 + + def verify(a, n1, n2, n3): + assert a.m_1 == n1 + assert a.m_2 == n2 + assert a.m_3 == n3 + + a = MyPyDerived(27, 88, -11) + assert a.x() == ns.callx(a) + assert a.y() == ns.cally(a) + assert a.z() == ns.callz(a) + + verify(a, 27, 88, -11) + + a = MyPyDerived(val2=27) + verify(a, 13, 27, 67) + + a = MyPyDerived(nArgs=0) + verify(a, 13, 42, 67) + + a = MyPyDerived(27, nArgs=1) + verify(a, 27, 42, 67) + + a = MyPyDerived(27, 55, nArgs=2) + verify(a, 27, 55, 67) + + def test23_const_byvalue_return(self): + """Const by-value return in overridden method""" + + import cppyy + + cppyy.cppdef("""namespace const_byvalue_return { + struct Const { + Const() = default; + explicit Const(const std::string& s) { m_value = s; } + std::string m_value; + }; + + struct Abstract { + virtual ~Abstract() {} + virtual const Const return_const() = 0; + }; + + const Const callit(Abstract* a) { return a->return_const(); } }""") + + ns = cppyy.gbl.const_byvalue_return + + class ReturnConstByValue(ns.Abstract): + def return_const(self): + return ns.Const("abcdef") + + a = ReturnConstByValue() + assert a.return_const().m_value == "abcdef" + assert ns.callit(a).m_value == "abcdef" + + def test24_non_copyable(self): + """Inheriting from a non-copyable base class""" + + import cppyy + + cppyy.cppdef("""\ + namespace non_copyable { + struct Copyable { + Copyable() = default; + virtual ~Copyable() {} + + Copyable(const Copyable&) = default; + Copyable& operator=(const Copyable&) = default; + }; + + struct Movable { + Movable() = default; + virtual ~Movable() {} + + Movable(const Movable&) = delete; + Movable& operator=(const Movable&) = delete; + Movable(Movable&&) = default; + Movable& operator=(Movable&&) = default; + }; + + class NoCopyNoMove { + public: + NoCopyNoMove() = delete; + NoCopyNoMove(const NoCopyNoMove&) = delete; + NoCopyNoMove(NoCopyNoMove&&) = delete; + NoCopyNoMove& operator=(const NoCopyNoMove&) = delete; + NoCopyNoMove& operator=(NoCopyNoMove&&) = delete; + virtual ~NoCopyNoMove() = default; + + template + explicit NoCopyNoMove(DerivedType* ptr) : fActual(ptr) {} + + std::string callme() { + if (!fActual) return "failed!"; + return fActual->callme_imp(); + } + + private: + virtual std::string callme_imp() = 0; + + protected: + NoCopyNoMove* fActual; + }; }""") + + ns = cppyy.gbl.non_copyable + + Copyable = ns.Copyable + Movable = ns.Movable + NoCopyNoMove = ns.NoCopyNoMove + + class DerivedCopyable(Copyable): + pass + + # used to fail with compilation error + class DerivedMovable(Movable): + pass + + # used to fail with compilation error + class DerivedMulti(cppyy.multi(Movable, Copyable)): + pass + + # used to fail with compilation error + class DerivedNoCopyNoMove(NoCopyNoMove): + def __init__(self): + super(DerivedNoCopyNoMove, self).__init__(self) + # TODO: chicken-and-egg situation here, 'this' from 'self' is + # nullptr until the constructor has been called, so it can't + # be passed as an argument to the same constructor + self.fActual = self + + def callme_imp(self): + return "Hello, World!" + + assert DerivedNoCopyNoMove().callme() == "Hello, World!" + + def test25_default_ctor_and_multiple_inheritance(self): + """Regression test: default ctor did not get added""" + + import cppyy + + cppyy.cppdef("""namespace default_ctor_and_multiple { + struct Copyable { + Copyable() = default; + virtual ~Copyable() {} + + Copyable(const Copyable&) = default; + Copyable& operator=(const Copyable&) = default; + }; + + struct Movable { + Movable() = default; + virtual ~Movable() {} + + Movable(const Movable&) = delete; + Movable& operator=(const Movable&) = delete; + Movable(Movable&&) = default; + Movable& operator=(Movable&&) = default; + }; + + struct SomeClass { + virtual ~SomeClass() {} + }; }""") + + ns = cppyy.gbl.default_ctor_and_multiple + Copyable = ns.Copyable + Movable = ns.Movable + SomeClass = ns.SomeClass + + class DerivedMulti(cppyy.multi(Movable, Copyable, SomeClass)): + def __init__(self): + super(DerivedMulti, self).__init__() + + d = DerivedMulti() + assert d + + def test26_no_default_ctor(self): + """Make sure no default ctor is created if not viable""" + + import cppyy, warnings + + cppyy.cppdef("""namespace no_default_ctor { + struct NoDefCtor1 { + NoDefCtor1(int) {} + virtual ~NoDefCtor1() {} + }; + + struct NoDefCtor2 { + NoDefCtor2() = delete; + virtual ~NoDefCtor2() {} + }; + + class NoDefCtor3 { + NoDefCtor3() = default; + public: + virtual ~NoDefCtor3() {} + }; + + class Simple {}; }""") + + ns = cppyy.gbl.no_default_ctor + + for kls in (ns.NoDefCtor1, ns.NoDefCtor2, ns.NoDefCtor3): + class PyDerived(kls): + def __init__(self): + super(PyDerived, self).__init__() + + with raises(TypeError): + PyDerived() + + with warnings.catch_warnings(record=True) as w: + class PyDerived(cppyy.multi(kls, ns.Simple)): + def __init__(self): + super(PyDerived, self).__init__() + + with raises(TypeError): + PyDerived() + + with warnings.catch_warnings(record=True) as w: + class PyDerived(cppyy.multi(ns.Simple, kls)): + def __init__(self): + super(PyDerived, self).__init__() + + with raises(TypeError): + PyDerived() + + def test27_interfaces(self): + """Inherit from base with non-standard offset""" + + import cppyy + + cppyy.gbl.gInterpreter.Declare("""\ + namespace NonStandardOffset { + struct Calc1 { + virtual int calc1() = 0; + virtual ~Calc1() = default; + }; + + struct Calc2 { + virtual int calc2() = 0; + virtual ~Calc2() = default; + }; + + struct Base : virtual public Calc1, virtual public Calc2 { + Base() {} + }; + + struct Derived : Base, virtual public Calc2 { + int calc1() override { return 1; } + int calc2() override { return 2; } + }; + + int callback1(Calc1* c1) { return c1->calc1(); } + int callback2(Calc2* c2) { return c2->calc2(); } + }""") + + ns = cppyy.gbl.NonStandardOffset + + class MyPyDerived(ns.Derived): + pass + + obj = MyPyDerived() + + assert obj.calc1() == 1 + assert ns.callback1(obj) == 1 + + assert obj.calc2() == 2 + assert ns.callback2(obj) == 2 + + def test28_cross_deep(self): + """Deep inheritance hierarchy""" + + import cppyy + + cppyy.cppdef("""\ + namespace CrossDeep { + + class A { + public: + A(const std::string& /* name */) {} + virtual ~A() {} + virtual int fun1() const { return 0; } + virtual int fun2() const { return fun1(); } + }; }""") + + A = cppyy.gbl.CrossDeep.A + + class B(A): + def __init__ (self, name = 'b'): + super(B, self).__init__(name) + + def fun1(self): + return 1 + + class C(B): + def fun1(self): + return -1 + + class D(B): + pass + + for inst, val1 in [(A('a'), 0), (B('b'), 1), (C('c'), -1), (D('d'), 1)]: + assert inst.fun1() == val1 + assert inst.fun2() == inst.fun1() + + def test29_cross_deep_multi(self): + """Deep multi-inheritance hierarchy""" + + import cppyy + + cppyy.cppdef("""\ + namespace CrossMultiDeep { + + class A { + public: + virtual ~A() {} + virtual int calc_a() { return 17; } + }; + + int calc_a(A* a) { return a->calc_a(); } + + class B { + public: + virtual ~B() {} + virtual int calc_b() { return 42; } + }; + + int calc_b(B* b) { return b->calc_b(); } }""") + + ns = cppyy.gbl.CrossMultiDeep + + class C(cppyy.multi(ns.A, ns.B)): + def calc_a(self): + return 18 + + def calc_b(self): + return 43 + + c = C() + assert ns.calc_a(c) == 18 + assert ns.calc_b(c) == 43 + + class D(ns.B): + def calc_b(self): + return 44 + + d = D() + assert ns.calc_b(d) == 44 + + class E(cppyy.multi(ns.A, D)): + def calc_a(self): + return 19 + + e = E() + assert ns.calc_a(e) == 19 + assert ns.calc_b(e) == 44 + + class F(ns.A): + def calc_a(self): + return 20 + + f = F() + assert ns.calc_a(f) == 20 + + class G(cppyy.multi(F, ns.B)): + def calc_b(self): + return 45 + + g = G() + assert ns.calc_a(g) == 20 + assert ns.calc_b(g) == 45 + + class H(object): + def calc_a(self): + return 66 + + class I(cppyy.multi(ns.A, H)): + def calc_a(self): + return 77 + + i = I() + assert ns.calc_a(i) == 77 + + class J(cppyy.multi(H, ns.A)): + def calc_a(self): + return 88 + + j = J() + assert ns.calc_a(j) == 88 + + + def test30_access_and_overload(self): + """Inheritance with access and overload complications""" + + import cppyy + + cppyy.cppdef("""\ + namespace AccessAndOverload { + class Base { + public: + virtual ~Base() {} + + protected: + virtual int call1(int i) { return i; } + virtual int call1(int i, int j) { return i+j; } + + virtual void call2(int) { return; } + virtual void call2(int, int) { return; } + + int call3(int i) { return i; } + + private: + int call3(int i, int j) { return i+j; } + }; }""") + + ns = cppyy.gbl.AccessAndOverload + + # used to produce uncompilable code + class PyDerived(ns.Base): + pass + + def test31_object_rebind(self): + """Usage of bind_object to cast with Python derived objects""" + + import cppyy, gc + + ns = cppyy.gbl.CrossInheritance + ns.build_component.__creates__ = True + + assert ns.Component.get_count() == 0 + + cmp1 = ns.build_component(42) + assert cmp1.__python_owns__ + assert type(cmp1) == ns.Component + with raises(AttributeError): + cmp1.getValue() + + assert ns.Component.get_count() == 1 + + # introduce the actual component type; would have been a header, + # but this simply has to match what is in crossinheritance.cxx + cppyy.cppdef("""namespace CrossInheritance { + class ComponentWithValue : public Component { + public: + ComponentWithValue(int value) : m_value(value) {} + int getValue() { return m_value; } + + protected: + int m_value; + }; }""") + + # rebind cmp1 to its actual C++ class + act_cmp1 = cppyy.bind_object(cmp1, ns.ComponentWithValue) + assert not cmp1.__python_owns__ # b/c transferred + assert act_cmp1.__python_owns__ + assert act_cmp1.getValue() == 42 + + del act_cmp1, cmp1 + gc.collect() + assert ns.Component.get_count() == 0 + + # introduce a Python derived class + ns.ComponentWithValue.__init__.__creates__ = False + class PyComponentWithValue(ns.ComponentWithValue): + def getValue(self): + return self.m_value + 12 + + # wipe the python-side connection + pycmp2a = PyComponentWithValue(27) + assert not pycmp2a.__python_owns__ + pycmp2a.__python_owns__ = True + assert ns.Component.get_count() == 1 + + pycmp2b = ns.cycle_component(pycmp2a) + assert ns.Component.get_count() == 1 + assert pycmp2b is pycmp2a + + del pycmp2b, pycmp2a + gc.collect() + assert ns.Component.get_count() == 0 + + cmp2 = cppyy.bind_object(cppyy.addressof(PyComponentWithValue(13)), ns.Component) + assert ns.Component.get_count() == 1 + + cmp2 = ns.cycle_component(cmp2) # causes auto down-cast + assert ns.Component.get_count() == 1 + #assert type(cmp2) != PyComponentWithValue + + # rebind cmp2 to the python type + act_cmp2 = cppyy.bind_object(cmp2, PyComponentWithValue) + act_cmp2.__python_owns__ = True + assert act_cmp2.getValue() == 13+12 + + del cmp2, act_cmp2 + gc.collect() + assert ns.Component.get_count() == 0 + + # introduce a Python derived class with initialization + ns.ComponentWithValue.__init__.__creates__ = True + class PyComponentWithInit(ns.ComponentWithValue): + def __init__(self, cppvalue): + super(PyComponentWithInit, self).__init__(cppvalue) + self.m_pyvalue = 11 + + def getValue(self): + return self.m_value + self.m_pyvalue + + cmp3 = cppyy.bind_object(PyComponentWithInit(77), PyComponentWithInit) + assert type(cmp3) == PyComponentWithInit + assert ns.Component.get_count() == 1 + + assert cmp3.getValue() == 77+11 + + del cmp3 + gc.collect() + assert ns.Component.get_count() == 0 + + pyc = PyComponentWithInit(88) + cmp4 = cppyy.bind_object(cppyy.addressof(pyc), ns.Component) + assert type(cmp4) == ns.Component + assert ns.Component.get_count() == 1 + + # rebind cmp4 to the python type + act_cmp4 = cppyy.bind_object(cmp4, PyComponentWithInit) + assert act_cmp4.getValue() == 88+11 + + del cmp4, act_cmp4, pyc + gc.collect() + assert ns.Component.get_count() == 0 + + ns.ComponentWithValue.__init__.__creates__ = False + cmp5 = cppyy.bind_object(cppyy.addressof(PyComponentWithInit(22)), ns.Component) + cmp5.__python_owns__ = True + assert type(cmp5) == ns.Component + assert ns.Component.get_count() == 1 + + # rebind cmp5 to the python type + act_cmp5 = cppyy.bind_object(cmp5, PyComponentWithInit) + assert not cmp5.__python_owns__ + assert act_cmp5.__python_owns__ + assert act_cmp5.getValue() == 22+11 + + del cmp5, act_cmp5 + gc.collect() + assert ns.Component.get_count() == 0 + + def test32_by_value_arguments(self): + """Override base function taking by-value arguments""" + + import cppyy + + cppyy.cppdef("""\ + namespace CrossCallWithValue { + struct Data { + int value; + }; + + struct CppBase { + virtual ~CppBase() {} + + int func(Data d) { + return d.value + extra_func(d); + } + + virtual int extra_func(Data d) = 0; + }; }""") + + ns = cppyy.gbl.CrossCallWithValue + + class PyDerived(ns.CppBase): + def extra_func(self, d): + return 42 + d.value + + d = ns.Data(13) + p = PyDerived() + + assert p.func(d) == 42 + 2 * d.value + + def test33_direct_base_methods(self): + """Call base class methods directly""" + + import cppyy + + cppyy.cppdef("""\ + namespace DirectCalls { + struct A { + virtual ~A() {} + virtual int func() { + return 1; + } + }; + + struct B : public A { + virtual int func() { + return 2; + } + }; }""") + + ns = cppyy.gbl.DirectCalls + + a = ns.A() + assert a.func() == 1 + assert ns.A.func(a) == 1 + + b = ns.B() + assert b.func() == 2 + assert ns.B.func(b) == 2 + assert ns.A.func(b) == 1 + + with raises(TypeError): + ns.B.func(a) + + class C(ns.A): + def func(self): + from_a = ns.A.func(self) + return from_a + 2 + + c = C() + assert c.func() == 3 + + def test34_no_ctors_in_base(self): + """Base classes with no constructors""" + + import cppyy + + cppyy.cppdef("""\ + namespace BlockedCtors { + class Z { + public: + virtual ~Z() {} + }; + + class X : Z { + X(); + X(const X&&) = delete; + }; + + class Y: Z { + protected: + Y() {} + Y(const Y&&) = delete; + }; }""") + + ns = cppyy.gbl.BlockedCtors + + with raises(TypeError): + ns.X() + + with raises(TypeError): + ns.Y() + + class PyY1(ns.Y): + pass + + with raises(TypeError): + PyY1() + + class PyY2(ns.Y): + def __init__(self): + super(ns.Y, self).__init__() + + assert PyY2() + + def test35_deletion(self): + """C++ side deletion should propagate to the Python side""" + + import cppyy + + cppyy.cppdef("""\ + namespace DeletionTest1 { + class Base { + public: + virtual ~Base() {} + void do_work() {} + }; + + void delete_it(Base *p) { delete p; } + }""") + + ns = cppyy.gbl.DeletionTest1 + + class Derived(ns.Base): + was_deleted = False + def __del__(self): + Derived.was_deleted = True + + o1 = Derived() + o1.do_work() + ns.delete_it(o1) + + with raises(ReferenceError): + o1.do_work() + + assert Derived.was_deleted == False + del o1 + assert Derived.was_deleted == True + + def test36_custom_destruct(self): + """C++ side deletion calls __destruct__""" + + import cppyy + + cppyy.cppdef("""\ + namespace DeletionTest2 { + class Base { + public: + virtual ~Base() {} + void do_work() {} + }; + + void delete_it(Base *p) { delete p; } + }""") + + ns = cppyy.gbl.DeletionTest2 + + class Derived(ns.Base): + was_cpp_deleted = False + was_py_deleted = False + + def __destruct__(self): + Derived.was_cpp_deleted = True + + def __del__(self): + Derived.was_py_deleted = True + + o1 = Derived() + o1.do_work() + ns.delete_it(o1) + + with raises(ReferenceError): + o1.do_work() + + assert Derived.was_cpp_deleted == True + assert Derived.was_py_deleted == False + del o1 + assert Derived.was_py_deleted == True + + def test37_deep_tree(self): + """Find overridable methods deep in the tree""" + + import cppyy + + cppyy.cppdef("""\ + namespace DeepTree { + + class Base { + public: + virtual ~Base() {} + + virtual std::string f1() { return "C++: Base::f1()"; } + virtual std::string f2() { return "C++: Base::f2()"; } + virtual std::string f3() { return "C++: Base::f3()"; } + }; + + class Intermediate: public Base { + public: + virtual ~Intermediate() {} + + using Base::f2; + }; + + class Sub: public Intermediate { + public: + virtual ~Sub() {} + + using Intermediate::f3; // `using Base::f3;` would also work + }; + + class CppSub: public Sub { + public: + virtual ~CppSub() {} + + std::string f1() { return "C++: CppSub::f1()"; } + std::string f2() { return "C++: CppSub::f2()"; } + std::string f3() { return "C++: CppSub::f3()"; } + }; + + std::string call_fs(Base *b) { + std::string res; + res += b->f1(); + res += b->f2(); + res += b->f3(); + return res; + } }""") + + ns = cppyy.gbl.DeepTree + + cppsub = ns.CppSub() + assert cppsub.f1() == "C++: CppSub::f1()" + assert cppsub.f2() == "C++: CppSub::f2()" + assert cppsub.f3() == "C++: CppSub::f3()" + assert ns.call_fs(cppsub) == cppsub.f1() + cppsub.f2() + cppsub.f3() + + class PySub(ns.Sub): + def f1(self): + return "Python: PySub::f1()" + + def f2(self): + return "Python: PySub::f2()" + + def f3(self): + return "Python: PySub::f3()" + + pysub = PySub() + assert pysub.f1() == "Python: PySub::f1()" + assert pysub.f2() == "Python: PySub::f2()" + assert pysub.f3() == "Python: PySub::f3()" + assert ns.call_fs(pysub) == pysub.f1() + pysub.f2() + pysub.f3() diff --git a/bindings/pyroot/cppyy/cppyy/test/test_datatypes.py b/bindings/pyroot/cppyy/cppyy/test/test_datatypes.py index cf8950003d738..710ff5eb1e982 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_datatypes.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_datatypes.py @@ -1,5 +1,5 @@ -import py -from pytest import raises +import py, sys +from pytest import raises, skip from .support import setup_make, pylong, pyunicode currpath = py.path.local(__file__).dirpath() @@ -11,11 +11,15 @@ def setup_module(mod): class TestDATATYPES: def setup_class(cls): - cls.test_dct = test_dct import cppyy + + cls.test_dct = test_dct cls.datatypes = cppyy.load_reflection_info(cls.test_dct) cls.N = cppyy.gbl.N - cls.has_byte = 201402 < cppyy.gbl.gInterpreter.ProcessLine("__cplusplus;") + + at_least_17 = 201402 < cppyy.gbl.gInterpreter.ProcessLine("__cplusplus;") + cls.has_byte = at_least_17 + cls.has_optional = at_least_17 def test01_instance_data_read_access(self): """Read access to instance public data and verify values""" @@ -36,9 +40,9 @@ def test01_instance_data_read_access(self): assert c.m_uchar == 'c' assert type(c.m_wchar) == pyunicode assert c.m_wchar == u'D' - assert type(c.m_char32) == pyunicode - assert c.m_char16 == u'\u00df' assert type(c.m_char16) == pyunicode + assert c.m_char16 == u'\u00df' + assert type(c.m_char32) == pyunicode assert c.m_char32 == u'\u00df' # reading integer types @@ -95,6 +99,47 @@ def test01_instance_data_read_access(self): assert round(c.get_icomplex_r().imag - 141., 11) == 0 assert complex(cppyy.gbl.std.complex['int'](1, 2)) == complex(1, 2) + # _Complex double type + assert type(c.get_ccomplex()) == complex + assert round(c.get_ccomplex().real - 151., 11) == 0 + assert round(c.get_ccomplex().imag - 161., 11) == 0 + assert repr(c.get_ccomplex()) == '(151+161j)' + assert round(c.get_ccomplex_cr().real - 151., 11) == 0 + assert round(c.get_ccomplex_cr().imag - 161., 11) == 0 + assert round(c.get_ccomplex_r().real - 151., 11) == 0 + assert round(c.get_ccomplex_r().imag - 161., 11) == 0 + + # complex overloads + cppyy.cppdef(""" + namespace ComplexOverload { + template + struct CO { + CO(std::size_t sz) : m_size(sz), m_cplx(std::complex(7,42)) {} + CO(std::complex cplx) : m_size(42), m_cplx(cplx) {} + + std::size_t m_size; + std::complex m_cplx; + }; + }""") + + COd = cppyy.gbl.ComplexOverload.CO['double'] + COf = cppyy.gbl.ComplexOverload.CO['float'] + scf = cppyy.gbl.std.complex['float'] + + assert COd(2).m_size == 2 + assert COd(2).m_cplx == 7.+42j + assert COd(3.14).m_size == 42 + assert COd(3.14).m_cplx == 3.14+0j + assert COd(9.+7j).m_size == 42 + assert COd(9.+7j).m_cplx == 9.+7j + + assert COf(2).m_size == 2 + assert COf(2).m_cplx == scf(7, 42) + assert COf(3.14).m_size == 42 + assert COf(3.14).m_cplx == scf(3.14, 0) + assert COf(9.+7j).m_size == 42 + assert COf(9.+7j).m_cplx == scf(9., 7.) + # reading of enum types assert c.m_enum == CppyyTestData.kNothing assert c.m_enum == c.kNothing @@ -107,8 +152,8 @@ def test01_instance_data_read_access(self): assert c.get_bool_array2()[i] == bool((i+1)%2) # reading of integer array types - names = ['schar', 'uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong'] - alpha = [ (1, 2), (1, 2), (-1, -2), (3, 4), (-5, -6), (7, 8), (-9, -10), (11, 12)] + names = ['schar', 'uchar', 'int8', 'uint8', 'short', 'ushort', 'int', 'uint', 'long', 'ulong'] + alpha = [ (1, 2), (1, 2), (-1, -2), (3, 4), (-5, -6), (7, 8), (-9, -10), (11, 12), (-13, -14), (15, 16)] if self.has_byte: names.append('byte'); alpha.append((3,4)) for j in range(self.N): @@ -129,6 +174,8 @@ def test01_instance_data_read_access(self): raises(IndexError, c.m_uchar_array.__getitem__, self.N) if self.has_byte: raises(IndexError, c.m_byte_array.__getitem__, self.N) + raises(IndexError, c.m_int8_array.__getitem__, self.N) + raises(IndexError, c.m_uint8_array.__getitem__, self.N) raises(IndexError, c.m_short_array.__getitem__, self.N) raises(IndexError, c.m_ushort_array.__getitem__, self.N) raises(IndexError, c.m_int_array.__getitem__, self.N) @@ -219,6 +266,18 @@ def test02_instance_data_write_access(self): getattr(c, 'set_'+names[i]+'_rv')(4*i) assert eval('c.m_%s' % names[i]) == 4*i + for i in range(len(names)): + setattr(c, 'm_'+names[i], cppyy.default) + assert eval('c.get_%s()' % names[i]) == 0 + + for i in range(len(names)): + getattr(c, 'set_'+names[i])(cppyy.default) + assert eval('c.m_%s' % names[i]) == 0 + + for i in range(len(names)): + getattr(c, 'set_'+names[i]+'_cr')(cppyy.default) + assert eval('c.m_%s' % names[i]) == 0 + # float types through functions c.set_float(0.123); assert round(c.get_float() - 0.123, 5) == 0 c.set_double(0.456); assert round(c.get_double() - 0.456, 8) == 0 @@ -235,13 +294,26 @@ def test02_instance_data_write_access(self): c.set_ldouble(0.098); assert round(c.m_ldouble - 0.098, 8) == 0 c.set_ldouble_cr(0.210); assert round(c.m_ldouble - 0.210, 8) == 0 + names = ['float', 'double', 'ldouble'] + for i in range(len(names)): + setattr(c, 'm_'+names[i], cppyy.default) + assert eval('c.get_%s()' % names[i]) == 0. + + for i in range(len(names)): + getattr(c, 'set_'+names[i])(cppyy.default) + assert eval('c.m_%s' % names[i]) == 0. + + for i in range(len(names)): + getattr(c, 'set_'+names[i]+'_cr')(cppyy.default) + assert eval('c.m_%s' % names[i]) == 0. + # (non-)writing of enum types raises(TypeError, setattr, CppyyTestData, 'kNothing', 42) # arrays; there will be pointer copies, so destroy the current ones c.destroy_arrays() - # integer arrays + # integer arrays (skip int8_t and uint8_t as these are presented as (unsigned) char still) names = ['uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong'] if self.has_byte: names.append('byte') @@ -283,14 +355,16 @@ def test03_array_passing(self): # typed passing ca = c.pass_array(b) - assert type(ca[0]) == type(b[0]) + if t != 'l': assert type(ca[0]) == type(b[0]) + else: assert type(ca[0]) == pylong # 'l' returns PyInt for small values in p2 assert len(b) == self.N for i in range(self.N): assert ca[i] == b[i] # void* passing ca = eval('c.pass_void_array_%s(b)' % t) - assert type(ca[0]) == type(b[0]) + if t != 'l': assert type(ca[0]) == type(b[0]) + else: assert type(ca[0]) == pylong # 'l' returns PyInt for small values in p2 assert len(b) == self.N for i in range(self.N): assert ca[i] == b[i] @@ -316,9 +390,8 @@ def test04_class_read_access(self): # char types assert CppyyTestData.s_char == 'c' assert c.s_char == 'c' - assert c.s_uchar == 'u' assert CppyyTestData.s_uchar == 'u' - assert c.s_wchar == u'U' + assert c.s_uchar == 'u' assert CppyyTestData.s_wchar == u'U' assert c.s_wchar == u'U' assert CppyyTestData.s_char16 == u'\u6c29' @@ -407,10 +480,6 @@ def test05_class_data_write_access(self): assert CppyyTestData.s_byte == 66 CppyyTestData.s_byte = 66 assert c.s_byte == 66 - c.s_short = - 88 - assert CppyyTestData.s_short == - 88 - CppyyTestData.s_short = 88 - assert c.s_short == 88 c.s_short = -102 assert CppyyTestData.s_short == -102 CppyyTestData.s_short = -203 @@ -488,6 +557,22 @@ def test07_type_conversions(self): raises(TypeError, setattr, c.m_double, 'c') raises(TypeError, setattr, c.m_int, -1.) raises(TypeError, setattr, c.m_int, 1.) + raises(TypeError, setattr, c.m_long, 3.14) + raises(TypeError, setattr, c.m_ulong, 3.14) + raises(TypeError, setattr, c.m_llong, 3.14) + raises(TypeError, setattr, c.m_ullong, 3.14) + + raises(TypeError, c.set_int, 3.14) + raises(TypeError, c.set_long, 3.14) + raises(TypeError, c.set_ulong, 3.14) + raises(TypeError, c.set_llong, 3.14) + raises(TypeError, c.set_ullong, 3.14) + + raises(TypeError, c.set_int_cr, 3.14) + raises(TypeError, c.set_long_cr, 3.14) + raises(TypeError, c.set_ulong_cr, 3.14) + raises(TypeError, c.set_llong_cr, 3.14) + raises(TypeError, c.set_ullong_cr, 3.14) c.__destruct__() @@ -650,7 +735,105 @@ def test10_enum(self): assert gbl.EnumSpace.AA == 1 assert gbl.EnumSpace.BB == 2 - def test11_string_passing(self): + def test11_typed_enums(self): + """Determine correct types of enums""" + + import cppyy + + cppyy.cppdef("""\ + namespace TrueEnumTypes { + class Test { + enum nums { ZERO = 0, ONE = 1 }; + enum dir : char { left = 'l', right = 'r' }; + enum rank : long { FIRST = 1, SECOND }; + enum vraioufaux : bool { faux = false, vrai = true }; + }; }""") + + sc = cppyy.gbl.TrueEnumTypes.Test + + assert sc.nums.ZERO == 0 + assert sc.nums.ONE == 1 + assert type(sc.nums.ZERO) == sc.nums + assert isinstance(sc.nums.ZERO, int) + assert 'int' in repr(sc.nums.ZERO) + assert str(sc.nums.ZERO) == '0' + + assert sc.dir.left == 'l' + assert sc.dir.right == 'r' + assert type(sc.dir.left) == sc.dir + assert isinstance(sc.dir.left, str) + assert 'char' in repr(sc.dir.left) + assert str(sc.dir.left) == "'l'" + + assert sc.rank.FIRST == 1 + assert sc.rank.SECOND == 2 + assert type(sc.rank.FIRST) == sc.rank + assert isinstance(sc.rank.FIRST, pylong) + assert 'long' in repr(sc.rank.FIRST) + assert str(sc.rank.FIRST) == '1' or str(sc.rank.FIRST) == '1L' + + assert sc.vraioufaux.faux == False + assert sc.vraioufaux.vrai == True + assert type(sc.vraioufaux.faux) == bool # no bool as base class + assert isinstance(sc.vraioufaux.faux, bool) + + def test12_enum_scopes(self): + """Enum accessibility and scopes""" + + import cppyy + + cppyy.cppdef("""\ + enum { g_one = 1, g_two = 2 }; + enum GEnum1 { g_three = 3, g_four = 4 }; + enum class GEnum2 { g_five = 5, g_six = 6 }; + + namespace EnumScopes { + enum { n_one = 1, n_two = 2 }; + enum NEnum1 { n_three = 3, n_four = 4 }; + enum class NEnum2 { n_five = 5, n_six = 6 }; + } + + class EnumClass { + public: + enum { c_one = 1, c_two = 2 }; + enum CEnum1 { c_three = 3, c_four = 4 }; + enum class CEnum2 { c_five = 5, c_six = 6 }; + }; """) + + gn = cppyy.gbl + assert not hasattr(gn, 'n_one') + assert not hasattr(gn, 'c_one') + assert gn.g_two == 2 + assert gn.g_four == 4 + assert gn.GEnum1.g_three == 3 + assert gn.GEnum1.g_three == gn.g_three + assert type(gn.GEnum1.g_three) == type(gn.g_three) + assert not hasattr(gn, 'g_five') + assert gn.GEnum2.g_six == 6 + + ns = cppyy.gbl.EnumScopes + assert not hasattr(ns, 'g_one') + assert not hasattr(ns, 'c_one') + assert ns.n_two == 2 + assert ns.n_four == 4 + assert ns.NEnum1.n_three == 3 + assert ns.NEnum1.n_three == ns.n_three + assert type(ns.NEnum1.n_three) == type(ns.n_three) + assert not hasattr(ns, 'n_five') + assert ns.NEnum2.n_six == 6 + + cl = cppyy.gbl.EnumClass + assert not hasattr(cl, 'g_one') + assert not hasattr(cl, 'n_one') + assert cl.c_two == 2 + assert cl.c_four == 4 + assert cl.CEnum1.c_three == 3 + assert cl.CEnum1.c_three == cl.c_three + assert type(cl.CEnum1.c_three) == type(cl.c_three) + assert not hasattr(cl, 'c_five') + assert cl.CEnum2.c_six == 6 + + def test13_string_passing(self): """Test passing/returning of a const char*""" import cppyy @@ -669,7 +852,7 @@ def test11_string_passing(self): assert c.get_valid_string32(u'z\u00df\u6c34\U0001f34c') == u'z\u00df\u6c34\U0001f34c' assert c.get_invalid_string32() == u'' - def test12_copy_constructor(self): + def test14_copy_constructor(self): """Test copy constructor""" import cppyy @@ -685,7 +868,7 @@ def test12_copy_constructor(self): for i in range(4): assert t1[i] == t3[i] - def test13_object_returns(self): + def test15_object_returns(self): """Test access to and return of PODs""" import cppyy @@ -712,7 +895,7 @@ def test13_object_returns(self): assert c.get_pod_ptrref().m_int == 666 assert c.get_pod_ptrref().m_double == 3.14 - def test14_object_arguments(self): + def test16_object_arguments(self): """Test setting and returning of a POD through arguments""" import cppyy @@ -780,7 +963,7 @@ def test14_object_arguments(self): assert p.m_int == 888 assert p.m_double == 3.14 - def test15_nullptr_passing(self): + def test17_nullptr_passing(self): """Integer 0 ('NULL') and nullptr allowed to pass through instance*""" import cppyy @@ -795,7 +978,7 @@ def test15_nullptr_passing(self): assert not c.m_ppod assert not c.get_pod_ptr() - def test16_respect_privacy(self): + def test18_respect_privacy(self): """Test that privacy settings are respected""" import cppyy @@ -808,7 +991,7 @@ def test16_respect_privacy(self): c.__destruct__() - def test17_object_and_pointer_comparisons(self): + def test19_object_and_pointer_comparisons(self): """Verify object and pointer comparisons""" import cppyy @@ -845,7 +1028,70 @@ def test17_object_and_pointer_comparisons(self): assert l3 != l5 assert l5 != l3 - def test18_object_validity(self): + def test20_object_comparisons_with_cpp__eq__(self): + """Comparisons with C++ providing __eq__/__ne__""" + + import cppyy + + cppyy.cppdef(""" + namespace MoreComparisons { + struct Comparable1 { + Comparable1(int i) : fInt(i) {} + int fInt; + static bool __eq__(const Comparable1& self, const Comparable1& other){ + return self.fInt == other.fInt; + } + static bool __ne__(const Comparable1& self, const Comparable1& other){ + return self.fInt != other.fInt; + } + }; + + struct Comparable2 { + Comparable2(int i) : fInt(i) {} + int fInt; + bool __eq__(const Comparable2& other){ + return fInt == other.fInt; + } + bool __ne__(const Comparable2& other){ + return fInt != other.fInt; + } + }; }""") + + ns = cppyy.gbl.MoreComparisons + + c1_1 = ns.Comparable1(42) + c1_2 = ns.Comparable1(42) + c1_3 = ns.Comparable1(43) + + assert ns.Comparable1.__dict__['__eq__'](c1_1, c1_2) + assert not ns.Comparable1.__dict__['__eq__'](c1_1, c1_3) + assert not ns.Comparable1.__dict__['__ne__'](c1_1, c1_2) + assert ns.Comparable1.__dict__['__ne__'](c1_1, c1_3) + + # the following works as a side-effect of a workaround for vector calls and + # it is probably preferable to have it working, so leave the discrepancy for + # now: python's aggressive end-of-life schedule will catch up soon enough + if 0x3080000 <= sys.hexversion: + assert c1_1 == c1_2 + assert not c1_1 != c1_2 + else: + with raises(TypeError): + c1_1 == c1_2 + with raises(TypeError): + c1_1 != c1_2 + + c2_1 = ns.Comparable2(27) + c2_2 = ns.Comparable2(27) + c2_3 = ns.Comparable2(28) + + assert c2_1 == c2_1 + assert c2_1 == c2_2 + assert not c2_1 == c2_3 + assert not c2_1 != c2_1 + assert not c2_1 != c2_2 + assert c2_1 != c2_3 + + def test21_object_validity(self): """Test object validity checking""" from cppyy import gbl @@ -859,7 +1105,36 @@ def test18_object_validity(self): assert not d2 - def test19_buffer_reshaping(self): + def test22_buffer_shapes(self): + """Correctness of declared buffer shapes""" + + import cppyy + + cppyy.cppdef("""\ + namespace ShapeTester { + enum Enum{One, Two, Three}; + + template + struct Foo { + T a[5]; + T aa[5][4]; + T aaa[5][4][3]; + }; }""") + + ns = cppyy.gbl.ShapeTester + + for dtype in ["int", "double", "long", "bool", "char", "void*", "ShapeTester::Enum"]: + foo = ns.Foo[dtype]() + if dtype != 'char': + assert foo.a.shape == (5,) + else: + # TODO: verify the following is for historic reasons and should be modified + # once bug #344 (bitbucket) is fixed + assert len(foo.a) == 5 + assert foo.aa.shape == (5, 4) + assert foo.aaa.shape == (5, 4, 3) + + def test23_buffer_reshaping(self): """Test usage of buffer sizing""" import cppyy @@ -871,6 +1146,9 @@ def test19_buffer_reshaping(self): byte_array_names = ['get_byte_array', 'get_byte_array2'] for func in ['get_bool_array', 'get_bool_array2', 'get_uchar_array', 'get_uchar_array2', + 'get_int8_array', 'get_int8_array2', + 'get_uint8_array', 'get_uint8_array2', + 'get_short_array', 'get_short_array2', 'get_ushort_array', 'get_ushort_array2', 'get_int_array', 'get_int_array2', 'get_uint_array', 'get_uint_array2', @@ -881,7 +1159,7 @@ def test19_buffer_reshaping(self): arr.reshape((self.N,)) assert len(arr) == self.N - raises(TypeError, arr.reshape, (1, 2)) + raises(ValueError, arr.reshape, (1, 2)) assert len(arr) == self.N raises(TypeError, arr.reshape, 2*self.N) @@ -891,7 +1169,7 @@ def test19_buffer_reshaping(self): for i in range(self.N): assert arr[i] == l[i] - def test20_voidp(self): + def test24_voidp(self): """Test usage of void* data""" import cppyy @@ -939,7 +1217,27 @@ def null_test(null): c.s_voidp = c2 address_equality_test(c.s_voidp, c2) - def test21_byte_arrays(self): + cppyy.cppdef("""\ + namespace VoidP { + void* vvv[3][5][7]; + struct Init { + Init() { + for (size_t i = 0; i < 3; ++i) { + for (size_t j = 0; j < 5; ++j) { + for (size_t k = 0; k < 7; ++k) + vvv[i][j][k] = (void*)(i+j+k); + } + } + } + } _init; }""") + + ns = cppyy.gbl.VoidP + for i in range(3): + for j in range(5): + for k in range(7): + assert int(ns.vvv[i,j,k]) == i+j+k + + def test25_byte_arrays(self): """Usage of unsigned char* as byte array and std::byte*""" import array, cppyy, ctypes @@ -974,7 +1272,7 @@ def run(self, f, buf, total): if self.has_byte: run(self, cppyy.gbl.sum_byte_data, buf, total) - def test22_function_pointers(self): + def test26_function_pointers(self): """Function pointer passing""" import cppyy @@ -999,8 +1297,9 @@ def test22_function_pointers(self): assert 7 == cppyy.gbl.sum_of_int_ptr(2, 3) cppyy.gbl.sum_of_int_ptr = cppyy.nullptr - with raises(TypeError): # not attribute error! - cppyy.gbl.sum_of_int_ptr + assert not cppyy.gbl.sum_of_int_ptr + with raises(cppyy.gbl.std.bad_function_call): + cppyy.gbl.sum_of_int_ptr(2, 3) with raises(AttributeError): cppyy.gbl.sim_of_int_ptr # incorrect spelling @@ -1025,10 +1324,20 @@ def sum_in_python(i1, i2, i3): with raises(TypeError): cppyy.gbl.call_sum_of_int(3, 2) - def test23_callable_passing(self): + cppyy.cppdef(r"""\ + namespace FuncPtrReturn { + typedef std::string (*func_t)(void); + std::string hello() { return "Hello, World!"; } + func_t foo() { return hello; } + }""") + + ns = cppyy.gbl.FuncPtrReturn + assert ns.foo()() == "Hello, World!" + + def test27_callable_passing(self): """Passing callables through function pointers""" - import cppyy + import cppyy, gc fdd = cppyy.gbl.call_double_double fii = cppyy.gbl.call_int_int @@ -1091,12 +1400,16 @@ def pyd(arg0, arg1): assert c(3, 3) == 9. c.set_callable(lambda x, y: x*y) - raises(TypeError, c, 3, 3) # lambda gone out of scope + assert c(3, 3) == 9. # life line protected - def test24_callable_through_function_passing(self): + c.__dict__.clear() # destroys life lines + gc.collect() + raises(TypeError, c, 3, 3) # lambda gone out of scope + + def test28_callable_through_function_passing(self): """Passing callables through std::function""" - import cppyy + import cppyy, gc fdd = cppyy.gbl.call_double_double_sf fii = cppyy.gbl.call_int_int_sf @@ -1159,9 +1472,50 @@ def pyd(arg0, arg1): assert c(3, 3) == 9. c.set_callable(lambda x, y: x*y) - raises(TypeError, c, 3, 3) # lambda gone out of scope + assert c(3, 3) == 9. # life line protected + + c.__dict__.clear() # destroys life lines + gc.collect() + raises(TypeError, c, 3, 3) # lambda gone out of scope + + def test29_std_function_life_lines(self): + """Life lines to std::function data members""" + + import cppyy, gc + + cppyy.cppdef("""\ + namespace BoundMethod2StdFunction { + class Base { + public: + virtual ~Base() {} - def test25_multi_dim_arrays_of_builtins(test): + std::function execute; + std::string do_execute() { + return execute(); + } + }; } """) + + ns = cppyy.gbl.BoundMethod2StdFunction + + class Derived(ns.Base): + def __init__(self): + super(Derived, self).__init__() + self.execute = self.xyz + + def xyz(self): + return "xyz" + + d = Derived() + assert d.do_execute() == "xyz" + assert d.do_execute() == "xyz" + + gc.collect() + assert d.do_execute() == "xyz" + + d.execute = d.xyz + assert d.do_execute() == "xyz" + + def test30_multi_dim_arrays_of_builtins(test): """Multi-dim arrays of builtins""" import cppyy, ctypes @@ -1195,12 +1549,13 @@ def test25_multi_dim_arrays_of_builtins(test): p = (ctype * len(buf)).from_buffer(buf) assert [p[j] for j in range(width*height)] == [2*j for j in range(width*height)] - def test26_anonymous_union(self): + def test31_anonymous_union(self): """Anonymous unions place there fields in the parent scope""" import cppyy - cppyy.cppdef("""namespace AnonUnion { + cppyy.cppdef("""\ + namespace AnonUnion { struct Event1 { Event1() : num(1) { shrd.a = 5.; } int num; @@ -1286,3 +1641,715 @@ def test26_anonymous_union(self): assert type(p.x) == float assert type(p.data_c[0]) == float assert p.intensity == 5. + + def test32_anonymous_struct(self): + """Anonymous struct creates an unnamed type""" + + import cppyy + + cppyy.cppdef("""\ + namespace AnonStruct { + class Foo1 { + public: + Foo1() { bar.x = 5; } + struct { int x; } bar; + }; + + class Foo2 { + public: + Foo2() { bar.x = 5; baz.x = 7; } + struct { int x; } bar; + struct { int x; } baz; + }; + + typedef struct { + struct { + struct { + struct { + struct { + struct { + const char* (*foo)(const char* s); + } kmmdemo; + } justamouse; + } com; + } root; + } kotlin; + } libuntitled1_ExportedSymbols; + + } """) + + ns = cppyy.gbl.AnonStruct + + foo = ns.Foo1() + assert foo.bar.x == 5 + assert not hasattr(foo.bar, 'bar') + + foo = ns.Foo2() + assert foo.bar.x == 5 + assert foo.baz.x == 7 + + assert 'foo' in dir(ns.libuntitled1_ExportedSymbols().kotlin.root.com.justamouse.kmmdemo) + + def test33_pointer_to_array(self): + """Usability of pointer to array""" + + import cppyy + + AoS = cppyy.gbl.ArrayOfStruct + + bar = AoS.Bar1() + assert bar.fArr[0].fVal == 42 + assert bar.fArr[1].fVal == 13 + + bar = AoS.Bar2(4) + for i in range(4): + assert bar.fArr[i].fVal == 2*i + + cppyy.cppdef(""" + namespace ArrayOfStruct { + union Bar3 { // not supported in dictionary + Foo fArr[]; // clang only + int fBuf; // to allow indexing fArr w/o crashing + }; + }""") + + bar = AoS.Bar3() + assert cppyy.sizeof(AoS.Bar3) >= cppyy.sizeof(AoS.Foo) + arr = bar.fArr + arr.size = 1 + for f in arr: + assert type(f) == AoS.Foo + assert type(bar.fArr[0]) == AoS.Foo + + def test34_object_pointers(self): + """Read/write access to objects through pointers""" + + import cppyy + + c = cppyy.gbl.CppyyTestData() + + assert cppyy.gbl.CppyyTestData.s_strv == "Hello" + assert c.s_strv == "Hello" + assert not cppyy.gbl.CppyyTestData.s_strp + assert not c.s_strp + + c.s_strv = "World" + assert cppyy.gbl.CppyyTestData.s_strv == "World" + + # assign on nullptr is a pointer copy + sn = cppyy.gbl.std.string("aap") + cppyy.gbl.CppyyTestData.s_strp = sn + assert c.s_strp == "aap" + + # assign onto the existing object + cppyy.gbl.CppyyTestData.s_strp.__assign__(cppyy.gbl.std.string("noot")) + assert c.s_strp == "noot" + assert sn == "noot" # set through pointer + + def test35_restrict(self): + """Strip __restrict keyword from use""" + + import cppyy + + cppyy.cppdef("std::string restrict_call(const char*__restrict s) { return s; }") + + assert cppyy.gbl.restrict_call("aap") == "aap" + + def test36_legacy_matrix(self): + """Handling of legacy matrix""" + + import cppyy + + cppyy.cppdef("""\ + namespace Pointer2D { + int** g_matrix; + + int** create_matrix(int n, int m) { + int** mat = (int**)malloc(n*sizeof(int*)); + int* arr = (int*)malloc(n*m*sizeof(int)); + for (int i = 0; i < n; ++i) { + mat[i] = arr + i*m; + for(int j = 0; j < m; ++j) { + mat[i][j] = 13+i*n+j; + } + } + g_matrix = mat; + return mat; + } + + bool destroy_matrix(int** mat, int n, int m) { + for (int i = 0; i < n; ++i) { + for (int j = 0; j < m; ++j) { + if (mat[i][j] != 13+i*n+j) + return false; + } + } + g_matrix = nullptr; + free(mat[0]); + free(mat); + return true; + } }""") + + ns = cppyy.gbl.Pointer2D; + + N, M = 2, 3 + m = ns.create_matrix(N, M) + g = ns.g_matrix; + + for i in range(N): + for j in range(M): + assert m[i][j] == 13+i*N+j + assert g[i][j] == 13+i*N+j + + assert ns.destroy_matrix(m, N, M) + + m = ns.create_matrix(N, M) + assert ns.destroy_matrix(ns.g_matrix, N, M) + + def test37_legacy_matrix_of_structs(self): + """Handling of legacy matrix of structs""" + + import cppyy + + cppyy.cppdef("""\ + namespace StructPointer2D { + typedef struct { + int x,y,z; // 'z' so that sizeof(xy) != sizeof(void*) + } xy; + + xy** g_matrix; + + xy** create_matrix(int n, int m) { + xy** mat = (xy**)malloc(n*sizeof(xy*)); + xy* arr = (xy*)malloc(n*m*sizeof(xy)); + for(int i=0; i fArr; }; + struct SomePOD_E { int* fPtrInt; }; + struct SomePOD_F { std::array* fPtrArr; }; + + namespace LotsOfPODS { + struct SomePOD_A { }; + struct SomePOD_B { int fInt; }; + struct SomePOD_C { int fInt; double fDouble; }; + struct SomePOD_D { std::array fArr; }; + struct SomePOD_E { int* fPtrInt; }; + struct SomePOD_F { std::array* fPtrArr; }; + }""") + + for ns in [cppyy.gbl, cppyy.gbl.LotsOfPODS]: + # no data member POD + assert ns.SomePOD_A() + + # single data member POD + b0 = ns.SomePOD_B() + assert b0.__python_owns__ + assert b0.fInt == 0 + b1 = ns.SomePOD_B(42) + assert b1.__python_owns__ + assert b1.fInt == 42 + b2 = ns.SomePOD_B(fInt = 17) + assert b2.__python_owns__ + assert b2.fInt == 17 + + # dual data member POD + c0 = ns.SomePOD_C() + assert c0.__python_owns__ + assert c0.fInt == 0 + assert c0.fDouble == 0. + c1a = ns.SomePOD_C(42) + assert c1a.__python_owns__ + assert c1a.fInt == 42 + assert c1a.fDouble == 0. + c1b = ns.SomePOD_C(fInt = 17) + assert c1b.__python_owns__ + assert c1b.fInt == 17 + assert c1b.fDouble == 0. + c1c = ns.SomePOD_C(fDouble = 5.) + assert c1c.__python_owns__ + assert c1c.fInt == 0 + assert c1c.fDouble == 5. + c2a = ns.SomePOD_C(88, 10.) + assert c2a.__python_owns__ + assert c2a.fInt == 88 + assert c2a.fDouble == 10. + c2b = ns.SomePOD_C(fDouble=5., fInt=77) + assert c2b.__python_owns__ + assert c2b.fInt == 77 + assert c2b.fDouble ==5. + + # object type data member POD + d0 = ns.SomePOD_D() + assert d0.__python_owns__ + assert len(d0.fArr) == 3 + assert d0.fArr[0] == 0. + d1 = ns.SomePOD_D((1., 2., 3.)) + assert d1.__python_owns__ + assert len(d1.fArr) == 3 + assert list(d1.fArr) == [1., 2., 3] + + # ptr type data member POD + e0 = ns.SomePOD_E() + assert e0.__python_owns__ + + # ptr to object type data member pOD + f0 = ns.SomePOD_F() + assert f0.__python_owns__ + arr = cppyy.gbl.std.array['double', 3]((1., 2., 3.)) + f1 = ns.SomePOD_F(arr) + assert f1.__python_owns__ + assert len(f1.fPtrArr) == 3 + assert list(f1.fPtrArr) == [1., 2., 3] + + def test39_aggregates(self): + """Initializer construction of aggregates""" + + import cppyy + + cppyy.cppdef("""\ + namespace AggregateTest { + class Atom { + public: + using size_type = std::size_t; + + struct AtomicNumber { + size_type a_n = 0; + }; + + std::array coords; + }; }""") + + ns = cppyy.gbl.AggregateTest + + Z = ns.Atom.AtomicNumber(5) + assert Z.a_n == 5 + + a = ns.Aggregate1() + assert a.sInt == 17 + + a = ns.Aggregate2() + assert a.sInt == 27 + assert a.fInt == 42 + + a = ns.Aggregate2(fInt=13) + assert a.fInt == 13 + + cppyy.cppdef("""\ + namespace AggregateTest { + + typedef enum _TYPE { DATA=0, SHAPE } TYPE; + + typedef struct _Buf { + int val; + const char *name; + TYPE buf_type; + } Buf; }""") + + ns = cppyy.gbl.AggregateTest + + b = ns.Buf(val=10, name="aap", buf_type=ns.SHAPE) + + assert b.val == 10 + assert b.name == "aap" + assert b.buf_type == ns.SHAPE + + def test40_more_aggregates(self): + """More aggregate testings (used to fail/report errors)""" + + import cppyy + + cppyy.cppdef("""\ + namespace AggregateTest { + enum E { A=1 }; + + struct R1 { E e; }; + R1 make_R1() { return {A}; } + + struct S { + S(int x): x(x) {} + S(const S&) = delete; + int x; + }; }""") + + ns = cppyy.gbl.AggregateTest + + r1 = ns.make_R1() + assert r1.e == ns.E.A + + if self.has_optional: + cppyy.cppdef("""\ + namespace AggregateTest { + struct R2 { + std::optional s = {}; + }; + + R2 make_R2() { + return {1}; + } }""") + + r2 = ns.make_R2() + assert r2.s.x == 1 + + def test41_complex_numpy_arrays(self): + """Usage of complex numpy arrays""" + + import cppyy + + try: + import numpy as np + except ImportError: + skip('numpy is not installed') + + cppyy.cppdef("""\ + namespace ComplexArrays { + typedef std::complex fcomp; + fcomp fcompdot(const fcomp* arr1, const fcomp* arr2, const int N) { + fcomp res{0.f, 0.f}; + for (int i=0; i dcomp; + dcomp dcompdot(const dcomp* arr1, const dcomp* arr2, const int N) { + dcomp res{0., 0.}; + for (int i=0; i