Enhancements to QThreadExecutor

examples/executor_example.py

@@ -16,7 +16,7 @@ async def master():
 
     await first_50(progress)
     loop = asyncio.get_running_loop()
-    with QThreadExecutor(1) as exec:
+    with QThreadExecutor(1).closing() as exec:
         await loop.run_in_executor(exec, functools.partial(last_50, progress), loop)
 
 

src/qasync/__init__.py

@@ -22,6 +22,7 @@
 import time
 from concurrent.futures import Future
 from queue import Queue
+from threading import Lock
 
 logger = logging.getLogger(__name__)
 
@@ -191,50 +192,90 @@ def __init__(self, max_workers=10, stack_size=None):
         self.__workers = [
             _QThreadWorker(self.__queue, i + 1, stack_size) for i in range(max_workers)
         ]
+        self.__shutdown_lock = Lock()
         self.__been_shutdown = False
 
         for w in self.__workers:
             w.start()
 
     def submit(self, callback, *args, **kwargs):
-        if self.__been_shutdown:
-            raise RuntimeError("QThreadExecutor has been shutdown")
+        with self.__shutdown_lock:
+            if self.__been_shutdown:
+                raise RuntimeError("QThreadExecutor has been shutdown")
 
-        future = Future()
-        self._logger.debug(
-            "Submitting callback %s with args %s and kwargs %s to thread worker queue",
-            callback,
-            args,
-            kwargs,
-        )
-        self.__queue.put((future, callback, args, kwargs))
-        return future
+            future = Future()
+            self._logger.debug(
+                "Submitting callback %s with args %s and kwargs %s to thread worker queue",
+                callback,
+                args,
+                kwargs,
+            )
+            self.__queue.put((future, callback, args, kwargs))
+            return future
 
     def map(self, func, *iterables, timeout=None):
-        raise NotImplementedError("use as_completed on the event loop")
-
-    def shutdown(self, wait=True):
-        if self.__been_shutdown:
-            raise RuntimeError("QThreadExecutor has been shutdown")
+        deadline = time.monotonic() + timeout if timeout is not None else None
+        futures = [self.submit(func, *args) for args in zip(*iterables)]
 
-        self.__been_shutdown = True
-
-        self._logger.debug("Shutting down")
-        for i in range(len(self.__workers)):
-            # Signal workers to stop
-            self.__queue.put(None)
-        if wait:
-            for w in self.__workers:
-                w.wait()
+        # must have generator as a closure so that the submit occurs before first iteration
+        def generator():
+            try:
+                futures.reverse()
+                while futures:
+                    if deadline is not None:
+                        yield _result_or_cancel(
+                            futures.pop(), timeout=deadline - time.monotonic()
+                        )
+                    else:
+                        yield _result_or_cancel(futures.pop())
+            finally:
+                for future in futures:
+                    future.cancel()
+
+        return generator()
+
+    def shutdown(self, wait=True, *, cancel_futures=False):
+        with self.__shutdown_lock:
+            self.__been_shutdown = True
+            self._logger.debug("Shutting down")
+            if cancel_futures:
+                # pop all the futures and cancel them
+                while not self.__queue.empty():
+                    item = self.__queue.get_nowait()
+                    if item is not None:
+                        future, _, _, _ = item
+                        future.cancel()
+            for i in range(len(self.__workers)):
+                # Signal workers to stop
+                self.__queue.put(None)
+            if wait:
+                for w in self.__workers:
+                    w.wait()
 
     def __enter__(self, *args):
-        if self.__been_shutdown:
-            raise RuntimeError("QThreadExecutor has been shutdown")
         return self
 
     def __exit__(self, *args):
         self.shutdown()
 
+    @contextlib.contextmanager
+    def closing(self, *, wait=False, cancel_futures=False):
+        """Explicit context manager to do shutdown, with Wait=False by default"""
+        try:
+            yield self
+        finally:
+            self.shutdown(wait=wait, cancel_futures=cancel_futures)
+
+
+def _result_or_cancel(fut, timeout=None):
+    try:
+        try:
+            return fut.result(timeout)
+        finally:
+            fut.cancel()
+    finally:
+        del fut  # break reference cycle in exceptions
+
 
 def _format_handle(handle: asyncio.Handle):
     cb = getattr(handle, "_callback", None)

tests/test_qeventloop.py

@@ -18,6 +18,18 @@
 import pytest
 
 import qasync
+from qasync import QtModuleName
+
+# flags for skipping certain tests for PySide6
+# This module has known issues causing crashes if certain
+# combinations of tests are invoked, likely caused by some internal problems.
+# current experimentation shows that it is sufficient to skip the socket pair
+# tests and the event-loop-on-qthread tests to avoid crashes later on in the suite.
+# Setting any one of these flags true reduces the frequency of crashes
+# but does not eliminate them.
+skip_process = False
+skip_socket = True
+skip_qthread = True
 
 
 @pytest.fixture
@@ -65,6 +77,8 @@ def excepthook(type, *args):
 )
 def executor(request):
     exc_cls = request.param
+    if exc_cls is ProcessPoolExecutor and QtModuleName == "PySide6" and skip_process:
+        pytest.skip("subprocess is unreliable for  PySide6")
     if exc_cls is None:
         return None
 
@@ -130,6 +144,7 @@ async def blocking_task(self, loop, executor, was_invoked):
         logging.debug("start blocking task()")
 
 
+@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6")
 def test_can_execute_subprocess(loop):
     """Verify that a subprocess can be executed."""
 
@@ -143,6 +158,7 @@ async def mycoro():
     loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=10.0))
 
 
+@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6")
 def test_can_read_subprocess(loop):
     """Verify that a subprocess's data can be read from stdout."""
 
@@ -163,6 +179,7 @@ async def mycoro():
     loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=10.0))
 
 
+@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6")
 def test_can_communicate_subprocess(loop):
     """Verify that a subprocess's data can be passed in/out via stdin/stdout."""
 
@@ -184,6 +201,7 @@ async def mycoro():
     loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=10.0))
 
 
+@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6")
 def test_can_terminate_subprocess(loop):
     """Verify that a subprocess can be terminated."""
 
@@ -309,6 +327,8 @@ def sock_pair(request):
 
     If socket.socketpair isn't available, we emulate it.
     """
+    if QtModuleName == "PySide6" and skip_socket:
+        pytest.skip("SocketPairs are unreliable on PySide6")
 
     def fin():
         if client_sock is not None:
@@ -887,6 +907,7 @@ def test_run_forever_custom_exit_code(loop, application):
             application.exec_ = orig_exec
 
 
+@pytest.mark.skipif(QtModuleName == "PySide6" and skip_qthread, reason="unreliable on PySide6")
 def test_qeventloop_in_qthread():
     class CoroutineExecutorThread(qasync.QtCore.QThread):
         def __init__(self, coro):