merge 3.3-slp (Stackless python#117, fix various reference leaks)

Author: Anselm Kruis

Python/ceval.c

@@ -1216,6 +1216,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
     f->f_execute = slp_eval_frame_noval;
     return slp_eval_frame_value(f, throwflag, retval);
 exit_eval_frame:
+    Py_XDECREF(retval);
     Py_LeaveRecursiveCall();
     f->f_executing = 0;
     tstate->frame = f->f_back;

Stackless/changelog.txt

@@ -20,8 +20,16 @@ What's New in Stackless 3.X.X?
 
 - https://bitbucket.org/stackless-dev/stackless/issues/117
   Fix various reference leaks:
-  - Leak of a reference to Py_None in generator.throw()
-  - Leak of a reference to the thread-id of every thread returned by stackless.threads
+  - Leak of a reference to None in generator.throw()
+  - Leak of a reference to the thread-id of every thread returned by
+    stackless.threads
+  - Leak of a reference to None (usually) in the C-implementations of
+    channel.send_throw() and channel.send_exception()
+  - Leak of a reference to the caught exception (type and value) and traceback,
+    if Stackless invokes an user defined error-handler.
+  - Leak to an object, if a Python profile or trace function fails.
+  - Various leaks, if stackless._wrap.frame.__setstate__() fails, because its
+    state argument is invalid.
 
   Additionally this change brings the handling of caught exceptions more in
   line with C-Python.

Stackless/module/channelobject.c

@@ -664,7 +664,7 @@ channel_send_exception(PyObject *myself, PyObject *args)
         goto err_exit;
     }
     Py_INCREF(Py_None);
-    retval = Py_None;
+    Py_SETREF(retval, Py_None);
 err_exit:
     Py_DECREF(klass);
     Py_XDECREF(args);
@@ -734,7 +734,7 @@ channel_send_throw(PyObject *myself, PyObject *args)
         goto err_exit;
     }
     Py_INCREF(Py_None);
-    retval = Py_None;
+    Py_SETREF(retval, Py_None);
 err_exit:
     return retval;
 }

Stackless/module/stacklessmodule.c

@@ -807,12 +807,20 @@ PyStackless_CallErrorHandler(void)
     if (slp_error_handler != NULL) {
         PyObject *exc, *val, *tb;
         PyObject *result;
-        PyErr_Fetch(&exc, &val, &tb);
-        if (!val)
+        PyErr_Fetch(&exc, &val, &tb); /* we own the refs in exc, val and tb */
+        assert(exc != NULL);
+        if (!val) {
+            Py_INCREF(Py_None);
             val = Py_None;
-        if (!tb)
+        }
+        if (!tb) {
+            Py_INCREF(Py_None);
             tb = Py_None;
+        }
         result = PyObject_CallFunction(slp_error_handler, "OOO", exc, val, tb);
+        Py_DECREF(exc);
+        Py_DECREF(val);
+        Py_DECREF(tb);
         Py_XDECREF(result);
         if (!result)
             return -1; /* an error in the handler! */

Stackless/pickling/prickelpit.c

@@ -873,6 +873,7 @@ frame_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (globals == NULL)
         return NULL;
     f = PyFrame_New(ts, (PyCodeObject *) f_code, globals, globals);
+    assert(f->f_execute == NULL); /* frame is not executable */
     if (f != NULL)
         Py_TYPE(f) = &wrap_PyFrame_Type;
     Py_DECREF(globals);
@@ -883,7 +884,7 @@ frame_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 static PyObject *
 frame_setstate(PyFrameObject *f, PyObject *args)
 {
-    int /*f_restricted, */ f_lasti, f_lineno, i;
+    int f_lasti, f_lineno, i;
     PyObject *f_globals, *f_locals, *blockstack_as_tuple;
     PyObject *localsplus_as_tuple, *exc_as_tuple, *trace, *f_code;
     PyObject *exec_name = NULL;
@@ -893,6 +894,9 @@ frame_setstate(PyFrameObject *f, PyObject *args)
 
     if (is_wrong_type(Py_TYPE(f))) return NULL;
 
+    Py_CLEAR(f->f_globals);
+    Py_CLEAR(f->f_locals);
+
     if (!PyArg_ParseTuple (args, frametuplesetstatefmt,
                            &PyCode_Type, &f_code,
                            &valid,
@@ -901,7 +905,6 @@ frame_setstate(PyFrameObject *f, PyObject *args)
                            &have_locals,
                            &PyDict_Type, &f_locals,
                            &trace,
-/*                             &f_restricted, */
                    &exc_as_tuple,
                    &f_lasti,
                    &f_lineno,
@@ -936,17 +939,21 @@ frame_setstate(PyFrameObject *f, PyObject *args)
             goto err_exit;
         }
         Py_INCREF(trace);
+        assert(f->f_trace == NULL);
         f->f_trace = trace;
     }
 
-/*      f->f_restricted = f_restricted; */
-
     if (exc_as_tuple != Py_None) {
         if (PyTuple_GET_SIZE(exc_as_tuple) != 4) {
             PyErr_SetString(PyExc_ValueError,
                             "bad exception tuple for frame");
             goto err_exit;
         }
+        assert(f->f_exc_type == NULL);
+        assert(f->f_exc_value == NULL);
+        assert(f->f_exc_traceback == NULL);
+        assert(((&f->f_exc_type) + 1 == &f->f_exc_value) &&
+            ((&f->f_exc_type) + 2 == &f->f_exc_traceback));
         slp_from_tuple_with_nulls(&f->f_exc_type, exc_as_tuple);
     }
 
@@ -1022,14 +1029,18 @@ frame_setstate(PyFrameObject *f, PyObject *args)
         }
     }
 
-    /* see if this frame is valid to be run */
+    /* See if this frame is valid to be run. */
     f->f_execute = valid ? good_func : bad_func;
 
     Py_TYPE(f) = &PyFrame_Type;
     Py_INCREF(f);
     return (PyObject *) f;
 err_exit:
-    Py_XDECREF(f);
+    /* Make sure that the frame is not executable. */
+    f->f_execute = NULL;
+    /* Clear members that could leak. */
+    PyFrame_Type.tp_clear((PyObject*)f);
+
     return NULL;
 }
 

Stackless/unittests/test_pickle.py

@@ -532,6 +532,35 @@ def func(current):
             self.assertIsInstance(cell, cell_type)
             self.assertIs(cell.cell_contents, result)
 
+    def testSetstateFailure(self):
+        # incomplete, just one of many failure modes of stackless._wrap.frame.__setstate__
+        foo = "foo"
+
+        def f1(bar="bar"):
+            try:
+                1 / 0
+            except ZeroDivisionError:
+                x = foo
+                locals()
+                return stackless._wrap.frame.__reduce__(sys._getframe())
+            assert False
+
+        def f2():
+            try:
+                1 / 0
+            except ZeroDivisionError:
+                return f1()
+
+        r = f2()
+        self.assertEqual(len(r), 3)
+        wrap_frame = r[0](*r[1])
+        self.assertIsInstance(wrap_frame, stackless._wrap.frame)
+        invalid_state = r[2][:-2] + ((("Not a", "tuple of 3", "integers"),), r[2][-1])
+        self.assertRaisesRegexp(TypeError, "an integer is required", wrap_frame.__setstate__, invalid_state)
+        # must not raise an assertion
+        wrap_frame.__setstate__(r[2])
+        self.assertIs(type(wrap_frame), types.FrameType)
+
 
 class TestDictViewPickling(StacklessPickleTestCase):