Fail if PyMem_Malloc() is called without holding the GIL

Issue python#26563: Debug hooks on Python memory allocators now raise a fatal error
if functions of the PyMem_Malloc() family are called without holding the GIL.

Author: vstinner

Lib/test/test_capi.py

@@ -602,15 +602,24 @@ def test_api_misuse(self):
         regex = regex.format(ptr=self.PTR_REGEX)
         self.assertRegex(out, regex)
 
-    def test_pyobject_malloc_without_gil(self):
-        # Calling PyObject_Malloc() without holding the GIL must raise an
-        # error in debug mode.
-        code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
+    def check_malloc_without_gil(self, code):
         out = self.check(code)
         expected = ('Fatal Python error: Python memory allocator called '
                     'without holding the GIL')
         self.assertIn(expected, out)
 
+    def test_pymem_malloc_without_gil(self):
+        # Debug hooks must raise an error if PyMem_Malloc() is called
+        # without holding the GIL
+        code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
+        self.check_malloc_without_gil(code)
+
+    def test_pyobject_malloc_without_gil(self):
+        # Debug hooks must raise an error if PyObject_Malloc() is called
+        # without holding the GIL
+        code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
+        self.check_malloc_without_gil(code)
+
 
 class PyMemMallocDebugTests(PyMemDebugTests):
     PYTHONMALLOC = 'malloc_debug'

Misc/NEWS

@@ -10,6 +10,10 @@ Release date: tba
 Core and Builtins
 -----------------
 
+- Issue #26563: Debug hooks on Python memory allocators now raise a fatal
+  error if functions of the :c:func:`PyMem_Malloc` family are called without
+  holding the GIL.
+
 - Issue #26564: On error, the debug hooks on Python memory allocators now use
   the :mod:`tracemalloc` module to get the traceback where a memory block was
   allocated.

Modules/_testcapimodule.c

@@ -3643,11 +3643,29 @@ pymem_api_misuse(PyObject *self, PyObject *args)
     Py_RETURN_NONE;
 }
 
+static PyObject*
+pymem_malloc_without_gil(PyObject *self, PyObject *args)
+{
+    char *buffer;
+
+    /* Deliberate bug to test debug hooks on Python memory allocators:
+       call PyMem_Malloc() without holding the GIL */
+    Py_BEGIN_ALLOW_THREADS
+    buffer = PyMem_Malloc(10);
+    Py_END_ALLOW_THREADS
+
+    PyMem_Free(buffer);
+
+    Py_RETURN_NONE;
+}
+
 static PyObject*
 pyobject_malloc_without_gil(PyObject *self, PyObject *args)
 {
     char *buffer;
 
+    /* Deliberate bug to test debug hooks on Python memory allocators:
+       call PyObject_Malloc() without holding the GIL */
     Py_BEGIN_ALLOW_THREADS
     buffer = PyObject_Malloc(10);
     Py_END_ALLOW_THREADS
@@ -3841,6 +3859,7 @@ static PyMethodDef TestMethods[] = {
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
     {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS},
     {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
+    {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS},
     {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };

Objects/obmalloc.c

@@ -198,7 +198,7 @@ static PyMemAllocatorEx _PyMem_Raw = {
 
 static PyMemAllocatorEx _PyMem = {
 #ifdef Py_DEBUG
-    &_PyMem_Debug.mem, PYRAWDBG_FUNCS
+    &_PyMem_Debug.mem, PYDBG_FUNCS
 #else
     NULL, PYMEM_FUNCS
 #endif
@@ -321,17 +321,17 @@ PyMem_SetupDebugHooks(void)
         PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
     }
 
-    if (_PyMem.malloc != _PyMem_DebugRawMalloc) {
-        alloc.ctx = &_PyMem_Debug.mem;
-        PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);
-        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
-    }
-
     alloc.malloc = _PyMem_DebugMalloc;
     alloc.calloc = _PyMem_DebugCalloc;
     alloc.realloc = _PyMem_DebugRealloc;
     alloc.free = _PyMem_DebugFree;
 
+    if (_PyMem.malloc != _PyMem_DebugMalloc) {
+        alloc.ctx = &_PyMem_Debug.mem;
+        PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
+    }
+
     if (_PyObject.malloc != _PyMem_DebugMalloc) {
         alloc.ctx = &_PyMem_Debug.obj;
         PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc);