more recount testing

Author: hackaugusto

.ycm_extra_conf.py

@@ -21,8 +21,8 @@
     # translate the call to c++ instead o cc and youcompleteme add -x c++
     # instead of -x c
     env = {'CC': 'clang++'}
-
     build = subprocess.Popen(['bear', 'python3', 'setup.py', 'build'], env=env)
+
     build.wait()
     shutil.rmtree(os.path.join(ROOT_DIR, 'build'))
 

Makefile

@@ -0,0 +1,25 @@
+mix.py
+------
+
+mix.py is a coroutine library. Coroutines might also be called Fibers
+(Windows), Green Threads (Java), or Lightweight Threads.
+
+vs. Greenlet
+--------
+
+The main difference between mix.py and greenlet is the stack handling, while
+greenlet relies heavily on the use of copying to exchange stacks mix.py uses a
+fixed size stack.
+
+That said, greenlet has the advantage that it's stacks can grow, while mix.py
+cannot (at least not without spagetthi stacks, and that is not tested for the
+moment), and mix.py sacrifices that benefit for the improved performance of
+_not_ copying stacks to and from the heap.
+
+compiling
+---------
+
+mix.py is coded in C++ with boost, it requires a least a C++11 compatible
+compiler and boost 1.60.0, and obviously Python's headers are required.
+
+Use the package setup.py to compile the package with `python setup.py bdist`

README

@@ -0,0 +1,28 @@
+# PYTHON_VERSION = 3.5m
+PYTHON_VERSION = 2.7
+PYTHON_INCLUDE = /usr/include/python$(PYTHON_VERSION)
+
+BOOST_INC = /usr/include
+BOOST_LIB = /usr/lib
+
+# CXX = clang
+CXX = gcc
+CXXFLAGS = -ggdb3 -Wall -Wextra -Werror -Wno-long-long -Wno-variadic-macros -fexceptions -DNDEBUG -std=c++14 -pipe -fstack-protector-strong --param=ssp-buffer-size=4
+LDFLAGS = -Wl,-O1,--sort-common,--as-needed,-z,relro,--export-dynamic
+INCLUDES = -I . -I ${BOOST_INC} -I $(PYTHON_INCLUDE)
+# LIBS = -L$(BOOST_LIB) -L/usr/lib/python$(PYTHON_VERSION)/config -lpython3 -lboost_python3 -lboost_context
+LIBS = -L$(BOOST_LIB) -L/usr/lib/python$(PYTHON_VERSION)/config -lpython2.7 -lboost_python -lboost_context
+
+TARGET = fiber
+
+$(TARGET).so: $(TARGET).o
+	$(CXX) $(CXXFLAGS) $(INCLUDES) $(LIBS) -shared $(LDFLAGS) $(TARGET).o -o $(TARGET).so
+
+$(TARGET).o: $(TARGET).cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -fPIC -c $(TARGET).cpp
+
+# python setup.py clean leaves most of it behind (dist, the build dir, and libraries from develop command)
+clean:
+	rm *.so *.o || true
+	# rm compile_commands.json || true
+	# rm -r build mix.py.egg-info .tox || true

debug.sh

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# vim: set sts=4 sw=4 ts=4 et:
+
+from mix.fiber import context_switch, Fiber  # noqa

mix/Makefile

@@ -1,7 +1,7 @@
 // vim: set sts=4 sw=4 ts=4 et:
 
 // Python.h includes pyconfig.h that unconditionally defines the
-// _POSIX_C_SOURCE and other constants, since the extension is compile with
+// _POSIX_C_SOURCE and other constants, since the extension is compiled with
 // -Werror the compilation fail if Python.h is included after any system level
 // header
 #include <Python.h>
@@ -16,8 +16,7 @@ namespace ctx = boost::context;
 /**
  * This is a implementation of cooperatively scheduled M:1 fibers for python,
  * the motivation for this project is to have the equivalent semantics as
- * greenlets but with no copying of the stack data and an implementation that
- * does not have assumptions about the stack.
+ * greenlets but with no copying of the stack data.
  *
  * The initial ideia was to use UNIX's makecontext() family of functions, this
  * idea was dropped because the signal mask requires SYSCALLs to update the
@@ -34,8 +33,8 @@ namespace ctx = boost::context;
 /**
  * Some notes:
  *
- * mix does not keep track of a hierarchy of executaiton the same way that
- * greenlet does, this logic should be emulated in python.
+ * mix does not keep track of a hierarchy of execution like the greenlet
+ * library, this logic should be emulated in python.
  *
  * Currently there is no support for spagetthi/segmented stacks.
  *
@@ -129,9 +128,27 @@ struct Fiber
  **/
 void first_call(intptr_t arguments_ptr) {
     callback_t *callback;
+    PyObject* fiber;
 
     callback = (callback_t*)arguments_ptr;
-    PyObject_Call(callback->callback, callback->args, callback->kwds);
+
+    // PyObject_Call call will create a new frame object that will keep the
+    // variables alive, we don't need to increase the refcount
+    fiber = PyObject_Call(callback->callback, callback->args, callback->kwds);
+    // If we get here either the function returned or a exception was raised
+    // and not handled
+
+    // The fiber needs to exit to guarantee proper cleanup, otherwise callback
+    // could have references and keep objects alive.
+    if (fiber) {
+        // TODO: check the type of fiber and switch context
+    }
+
+    // Unlike greenlet we don't keep in the compiled code a chain of parent and
+    // child threads of execution, that means we don't have another stack to
+    // switch to and continue running, also we cannot raise an exception
+    // SystemExit exception because there is no stack to unwind, so just
+    // finalize the running interpreter.
 
 #if PY_MAJOR_VERSION >= 3
     PyObject *pystdout, *pystderr;
@@ -156,18 +173,25 @@ void first_call(intptr_t arguments_ptr) {
     _PyObject_CallMethodId(pystdout, &PyId_flush, "");
 #endif
 
-    printf("mix: Fiber got to the end of stack\n");
-
-    // At this point we cannot raise an SystemExit exception because there is
-    // no stack to unwind, so just finalize the running interpreter.
-    //
-    // XXX: this is probably wrong for python embedded
-    Py_Finalize();
-    std::exit(1);
+    // XXX: calling exit() is probably wrong for python embedded
+    // let the user know why we are exiting
+    // if (result == NULL) {
+    if (PyErr_Occurred() != NULL) {
+        // PyObject *exc, *val, *tb;
+        // PyErr_Fetch(&exc, &val, &tb);
+        // PyErr_NormalizeException(&exc, &val, &c_tb);
+        PyErr_Print();
+
+        // Calling Py_Finalize here will segfault
+        fprintf(stderr, "mix: Fiber got to the end of stack\n");
+        std::exit(2);
+    } else {
+        Py_Finalize();
+        fprintf(stderr, "mix: Fiber got to the end of stack\n");
+        std::exit(1);
+    }
 
-    // Unlike greenlet we don't keep in the compiled code a chain of parent and
-    // child threads of execution, that means we don't have another stack to
-    // continue running and beyond this point we will segfault.
+    // beyond this point the application will segfault
 }
 
 /**
@@ -279,7 +303,7 @@ py::object py_context_switch(Fiber* origin, Fiber *target) {
     return py_context_switch_args_kwds(origin, target, py::tuple(), py::dict());
 }
 
-BOOST_PYTHON_MODULE(mix)
+BOOST_PYTHON_MODULE(fiber)
 {
     using namespace boost::python;
 

mix/__init__.py

@@ -0,0 +1 @@
+tox

context.py renamed to mix/context.py

@@ -0,0 +1,79 @@
+#!/usr/bin/bash
+
+error() {
+    printf "$(tput setaf 1)$@$(tput sgr0)\n" >&2
+    exit
+}
+
+(
+    cd ./mix
+    make clean
+    make
+)
+pip install -e .
+
+[ -z "${PYTHON}" ] && [ ! -z "${PYENV_ROOT}" ] && {
+    PYTHON=$(pyenv which python)
+}
+
+: ${PYTHON:=python}
+
+[ -n "${PYTHON_SOURCE}" ] && {
+    ADD_SOURCE="
+expect -exact \"(gdb)\"
+send \"dir ${PYTHON_SOURCE}\n\"
+"
+}
+
+# if the user didn't ask to run as test just initialize a session
+[ $# -eq 0 ] && {
+    RUN_CMD='
+expect -exact "(gdb)"
+send "run\n"
+
+expect -exact ">>>"
+
+send -- "
+import mix
+
+def noop():
+    pass
+
+f1 = mix.Fiber(noop)
+"
+'
+}
+
+# expect -exact "(gdb)"
+# send "break fiber.cpp:111\n"
+
+# else check that the test file exists and run a session with it
+[ $# -ne 0 -a ! -e "${1}" ] && error "file ${1} does not exists"
+[ $# -ne 0 -a ! -f "${1}" ] && error "${1} is not a file"
+[ $# -ne 0 -a ! -x "${1}" ] && error "${1} is not an executable file"
+[ $# -ne 0 ] && {
+    RUN_CMD="
+expect -exact \"(gdb)\"
+send \"run ${1}\"
+"
+}
+
+read -r -d '' SCRIPT <<EOS
+spawn gdb
+
+expect -exact "(gdb)"
+send "file ${PYTHON}\n"
+
+${ADD_SOURCE}
+
+expect -exact "(gdb)"
+send "dir mix\n"
+
+$RUN_CMD
+
+interact
+
+EOS
+
+/usr/bin/expect -c "${SCRIPT}";
+exit $?;