diff --git a/Include/internal/pycore_genericalias.h b/Include/internal/pycore_genericalias.h new file mode 100644 index 00000000000000..ead45465dca668 --- /dev/null +++ b/Include/internal/pycore_genericalias.h @@ -0,0 +1,19 @@ +#ifndef Py_INTERNAL_GENERICALIAS_H +#define Py_INTERNAL_GENERICALIAS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* runtime lifecycle */ + +extern PyStatus _PyGenericAlias_Init(PyInterpreterState *interp); +extern void _PyGenericAlias_Fini(PyInterpreterState *interp); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_GENERICALIAS_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 86ae3d8dfc1860..0a9ef044336d26 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -29,7 +29,7 @@ extern "C" { #include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_object_state.h" // struct _py_object_state #include "pycore_tuple.h" // struct _Py_tuple_state -#include "pycore_typeobject.h" // struct type_cache +#include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -139,6 +139,9 @@ struct _is { created and then deleted again. */ PySliceObject *slice_cache; + // Dict with existing generic alias objects: + PyObject* genericalias_cache; + struct _Py_tuple_state tuple; struct _Py_list_state list; struct _Py_dict_state dict_state; diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index fb13fa0e243ba7..b773c4b7945924 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -210,6 +210,13 @@ def clear_caches(): else: fractions._hash_algorithm.cache_clear() + try: + _testcapi = sys.modules['_testcapi'] + except KeyError: + pass + else: + _testcapi.genericalias_cache_clear() + def get_build_info(): # Get most important configure and build options as a list of strings. diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 9b59d1e3e0aad2..a393a4c5319253 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -56,6 +56,8 @@ import typing from typing import Unpack +from test import support + from typing import TypeVar T = TypeVar('T') K = TypeVar('K') @@ -94,8 +96,11 @@ class BaseTest(unittest.TestCase): """Test basics.""" - generic_types = [type, tuple, list, dict, set, frozenset, enumerate, - defaultdict, deque, + c_generic_types = [ + tuple, list, dict, set, frozenset, enumerate, + defaultdict, deque, + ] + generic_types = [*c_generic_types, SequenceMatcher, dircmp, FileInput, @@ -173,6 +178,50 @@ def default(): else: self.assertEqual(alias(iter((1, 2, 3))), t((1, 2, 3))) + @support.cpython_only + def test_c_genericaliases_are_cached(self): + for t in self.c_generic_types: + with self.subTest(t=t): + self.assertIs(t[int], t[int]) + self.assertEqual(t[int], t[int]) + self.assertIsNot(t[int], t[str]) + + @support.cpython_only + def test_c_genericaliases_uncachable_still_work(self): + for t in self.c_generic_types: + with self.subTest(t=t): + # Cache does not work for these args, + # but no error is present + self.assertIsNot(t[{}], t[{}]) + self.assertEqual(t[{}], t[{}]) + + @support.cpython_only + def test_generic_alias_unpacks_are_cached(self): + self.assertIs((*tuple[int, str],)[0], (*tuple[int, str],)[0]) + self.assertIsNot((*tuple[str, int],)[0], (*tuple[int, str],)[0]) + self.assertIs((*tuple[T, ...],)[0], (*tuple[T, ...],)[0]) + self.assertIsNot((*tuple[int, str],)[0], tuple[int, str]) + + @support.cpython_only + def test_generic_alias_unpacks_uncachable_still_work(self): + self.assertIsNot((*tuple[{}],)[0], (*tuple[{}],)[0]) + self.assertEqual((*tuple[{}],)[0], (*tuple[{}],)[0]) + + @support.cpython_only + def test_genericalias_constructor_is_no_cached(self): + for t in self.generic_types: + if t is None: + continue + tname = t.__name__ + with self.subTest(f"Testing {tname}"): + self.assertIsNot(GenericAlias(t, [int]), GenericAlias(t, [int])) + + @support.cpython_only + def test_c_union_arg_order(self): + self.assertIsNot(int | str, str | int) + self.assertIsNot(int | str, int | None) + self.assertIsNot(int | str, int | str | None) + def test_unbound_methods(self): t = list[int] a = t() diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index af095632a36fcb..098c38143574c7 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -985,6 +985,11 @@ def __module__(self): def test_or_type_operator_reference_cycle(self): if not hasattr(sys, 'gettotalrefcount'): self.skipTest('Cannot get total reference count.') + try: + import _testcapi + except ImportError: + self.skipTest('Cannot clear types.GenericAlias cache.') + gc.collect() before = sys.gettotalrefcount() for _ in range(30): @@ -993,6 +998,7 @@ def test_or_type_operator_reference_cycle(self): T.blah = U del T del U + _testcapi.genericalias_cache_clear() gc.collect() leeway = 15 self.assertLessEqual(sys.gettotalrefcount() - before, leeway, diff --git a/Makefile.pre.in b/Makefile.pre.in index afd503ef126339..a5f064479f1269 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1688,6 +1688,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_frame.h \ $(srcdir)/Include/internal/pycore_function.h \ $(srcdir)/Include/internal/pycore_genobject.h \ + $(srcdir)/Include/internal/pycore_genericalias.h \ $(srcdir)/Include/internal/pycore_getopt.h \ $(srcdir)/Include/internal/pycore_gil.h \ $(srcdir)/Include/internal/pycore_global_objects.h \ diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index fe1b9f8f5380c1..36d800fc127b24 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/genericalias.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/genericalias.c b/Modules/_testcapi/genericalias.c new file mode 100644 index 00000000000000..4067d5c60a65ba --- /dev/null +++ b/Modules/_testcapi/genericalias.c @@ -0,0 +1,36 @@ +#include "parts.h" + +#define Py_BUILD_CORE +// Needed to include both +// `Include/internal/pycore_gc.h` and +// `Include/cpython/objimpl.h` +#undef _PyGC_FINALIZED +#include "pycore_interp.h" // PyInterpreterState + +static PyObject * +genericalias_cache_clear(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_Get(); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + assert(interp != NULL); + assert(interp->genericalias_cache != NULL); + + PyDict_Clear(interp->genericalias_cache); // needs full PyInterpreterState + + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"genericalias_cache_clear", genericalias_cache_clear, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_GenericAlias(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 60ec81dad2ba9e..17e734910b32a5 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -39,6 +39,7 @@ int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_GenericAlias(PyObject *mod); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 557a6d46ed4632..ea9f449db4d3dc 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3343,7 +3343,7 @@ test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), } state.target = obj; state.found = 0; - + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); Py_DECREF(obj); if (!state.found) { @@ -4189,6 +4189,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyOS(m) < 0) { return NULL; } + if (_PyTestCapi_Init_GenericAlias(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 888cb16edd1b46..7e588e6391340d 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -3,6 +3,8 @@ #include "Python.h" #include "pycore_object.h" #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check +#include "pycore_genericalias.h" // _PyGenericAlias_Init, _PyGenericAlias_Fini +#include "pycore_initconfig.h" // _PyStatus_OK() #include "structmember.h" // PyMemberDef #include @@ -23,6 +25,14 @@ typedef struct { PyObject *obj; /* Set to NULL when iterator is exhausted */ } gaiterobject; +// Forward references: +// ----- + +PyObject * +_Py_GenericAliasStarred(PyObject *origin, PyObject *args, bool starred); + +// ----- + static void ga_dealloc(PyObject *self) { @@ -527,8 +537,8 @@ ga_getitem(PyObject *self, PyObject *item) return NULL; } - PyObject *res = Py_GenericAlias(alias->origin, newargs); - ((gaobject *)res)->starred = alias->starred; + PyObject *res = _Py_GenericAliasStarred(alias->origin, newargs, + alias->starred); Py_DECREF(newargs); return res; @@ -781,21 +791,34 @@ static PyGetSetDef ga_properties[] = { {0} }; -/* A helper function to create GenericAlias' args tuple and set its attributes. - * Returns 1 on success, 0 on failure. - */ -static inline int -setup_ga(gaobject *alias, PyObject *origin, PyObject *args) { - if (!PyTuple_Check(args)) { - args = PyTuple_Pack(1, args); - if (args == NULL) { - return 0; +/* +A helper function to create GenericAlias' args tuple. + +Returns -1 on error, +Returns 0 on no action, +Returns 1 if args were modified. +*/ +static int +normalize_args(PyObject **args) { + PyObject *normalized; + + if (!PyTuple_Check(*args)) { + normalized = PyTuple_Pack(1, *args); + if (normalized == NULL) { + return -1; } + + *args = normalized; + return 1; } - else { - Py_INCREF(args); - } + Py_INCREF(*args); + return 0; +} + +static inline void +setup_ga(gaobject *alias, PyObject *origin, PyObject *args, bool starred) { + // Make sure to call `normalize_args` before calling `setup_ga`. alias->origin = Py_NewRef(origin); alias->args = args; alias->parameters = NULL; @@ -808,7 +831,7 @@ setup_ga(gaobject *alias, PyObject *origin, PyObject *args) { alias->vectorcall = NULL; } - return 1; + alias->starred = starred; } static PyObject * @@ -822,14 +845,16 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); - gaobject *self = (gaobject *)type->tp_alloc(type, 0); - if (self == NULL) { + int arg_res = normalize_args(&arguments); + if (arg_res < 0) { return NULL; } - if (!setup_ga(self, origin, arguments)) { - Py_DECREF(self); + gaobject *self = (gaobject *)type->tp_alloc(type, 0); + if (self == NULL) { + Py_DECREF(arguments); return NULL; } + setup_ga(self, origin, arguments, false); return (PyObject *)self; } @@ -844,11 +869,11 @@ ga_iternext(gaiterobject *gi) { return NULL; } gaobject *alias = (gaobject *)gi->obj; - PyObject *starred_alias = Py_GenericAlias(alias->origin, alias->args); + PyObject *starred_alias = _Py_GenericAliasStarred(alias->origin, + alias->args, true); if (starred_alias == NULL) { return NULL; } - ((gaobject *)starred_alias)->starred = true; Py_SETREF(gi->obj, NULL); return starred_alias; } @@ -923,7 +948,6 @@ ga_iter(PyObject *self) { // TODO: // - argument clinic? -// - cache? PyTypeObject Py_GenericAliasType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "types.GenericAlias", @@ -950,17 +974,120 @@ PyTypeObject Py_GenericAliasType = { .tp_vectorcall_offset = offsetof(gaobject, vectorcall), }; -PyObject * -Py_GenericAlias(PyObject *origin, PyObject *args) -{ +static PyObject * +_Py_GenericAlias_impl_nocache(PyObject *origin, PyObject *args, bool starred) { + // args must be normalized at this point. gaobject *alias = (gaobject*) PyType_GenericAlloc( (PyTypeObject *)&Py_GenericAliasType, 0); if (alias == NULL) { return NULL; } - if (!setup_ga(alias, origin, args)) { - Py_DECREF(alias); + setup_ga(alias, origin, args, starred); + return (PyObject *)alias; +} + +static PyObject * +_Py_GenericAlias_impl(PyObject *origin, PyObject *args, bool starred) { + Py_hash_t hash; + bool unhashable = false; + PyObject *star = NULL; + PyObject *key = NULL; + PyObject *result = NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + + int arg_res = normalize_args(&args); + if (arg_res < 0) { return NULL; } - return (PyObject *)alias; + + star = PyBool_FromLong(starred); + if (star == NULL) { + goto error; + } + key = PyTuple_Pack(3, origin, args, star); + if (key == NULL) { + goto error; + } + + hash = PyObject_Hash(key); + if (hash == -1) { + // `key` contains unhashable objects, stop right here. + // Just return the object itself without setting cache. + PyErr_Clear(); + unhashable = true; + } else { + result = _PyDict_GetItem_KnownHash(interp->genericalias_cache, + key, hash); + if (result) { + Py_INCREF(result); + Py_DECREF(args); + goto finally; + } + if (PyErr_Occurred()) { + goto error; + } + } + + result = _Py_GenericAlias_impl_nocache(origin, args, starred); + if (result == NULL) { + goto error; + } + if (unhashable) { + goto finally; + } + + if (_PyDict_SetItem_KnownHash(interp->genericalias_cache, + key, result, hash) < 0) { + Py_DECREF(origin); + goto error; + } else { + goto finally; + } + +error: + Py_XDECREF(result); + Py_DECREF(args); +finally: + Py_XDECREF(key); + Py_XDECREF(star); + return result; } + +PyObject * +Py_GenericAlias(PyObject *origin, PyObject *args) { + return _Py_GenericAlias_impl(origin, args, false); +} + +// TODO: make this API public? +PyObject * +_Py_GenericAliasStarred(PyObject *origin, PyObject *args, bool starred) { + return _Py_GenericAlias_impl(origin, args, starred); +} + +/* Runtime lifecycle */ + +PyStatus +_PyGenericAlias_Init(PyInterpreterState *interp) { + PyObject *cache; + + if (PyType_Ready(&Py_GenericAliasType) < 0) { + return _PyStatus_ERR("Can't initialize 'types.GenericAlias' type"); + } + + cache = PyDict_New(); + if (cache == NULL) { + return _PyStatus_ERR("Can't initialize 'types.GenericAlias' cache"); + } + + interp->genericalias_cache = cache; + return _PyStatus_OK(); +}; + +void +_PyGenericAlias_Fini(PyInterpreterState *interp) { + if (interp->genericalias_cache != NULL) { + PyObject *cache = interp->genericalias_cache; + Py_DECREF(cache); + interp->genericalias_cache = NULL; + } +}; diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 439cd687fda61d..84c4ac9c6d7f67 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -110,6 +110,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 0e42e4982c21ff..d0889ff3849df8 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -60,6 +60,9 @@ Source Files + + Source Files + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8aafcb786a6064..6e434b551d6064 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -224,6 +224,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 07476f30833372..ed37f49c1904dd 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -573,6 +573,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d6627bc6b7e86b..84bdeeb68b6e23 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -29,6 +29,7 @@ #include "pycore_tuple.h" // _PyTuple_InitTypes() #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() +#include "pycore_genericalias.h" // _PyGenericAlias_Init() #include "opcode.h" extern PyStatus _PyIO_InitTypes(PyInterpreterState *interp); @@ -722,6 +723,11 @@ pycore_init_types(PyInterpreterState *interp) if (_PyStatus_EXCEPTION(status)) { return status; } + + status = _PyGenericAlias_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } return _PyStatus_OK(); } @@ -1665,6 +1671,7 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyGenericAlias_Fini(interp); _PyUnicode_FiniTypes(interp); _PySys_Fini(interp); _PyExc_Fini(interp);