From fc6fe131ed36211a68abca8ca35068f3dfc9606a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Tue, 19 Aug 2025 12:57:11 +0000 Subject: [PATCH 1/5] Skip unreliable thread/process/socket tests in PySide6 --- tests/test_qeventloop.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_qeventloop.py b/tests/test_qeventloop.py index c4365ec..a5a0eed 100644 --- a/tests/test_qeventloop.py +++ b/tests/test_qeventloop.py @@ -18,6 +18,18 @@ import pytest import qasync +from qasync import QtModuleName + +# flags for skipping certain tests for PySide6 +# This module has known issues causing crashes if certain +# combinations of tests are invoked, likely caused by some internal problems. +# current experimentation shows that it is sufficient to skip the socket pair +# tests and the event-loop-on-qthread tests to avoid crashes later on in the suite. +# Setting any one of these flags true reduces the frequency of crashes +# but does not eliminate them. +skip_process = False +skip_socket = True +skip_qthread = True @pytest.fixture @@ -65,6 +77,8 @@ def excepthook(type, *args): ) def executor(request): exc_cls = request.param + if exc_cls is ProcessPoolExecutor and QtModuleName == "PySide6" and skip_process: + pytest.skip("subprocess is unreliable for PySide6") if exc_cls is None: return None @@ -130,6 +144,7 @@ async def blocking_task(self, loop, executor, was_invoked): logging.debug("start blocking task()") +@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6") def test_can_execute_subprocess(loop): """Verify that a subprocess can be executed.""" @@ -143,6 +158,7 @@ async def mycoro(): loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=10.0)) +@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6") def test_can_read_subprocess(loop): """Verify that a subprocess's data can be read from stdout.""" @@ -163,6 +179,7 @@ async def mycoro(): loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=10.0)) +@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6") def test_can_communicate_subprocess(loop): """Verify that a subprocess's data can be passed in/out via stdin/stdout.""" @@ -184,6 +201,7 @@ async def mycoro(): loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=10.0)) +@pytest.mark.skipif(QtModuleName == "PySide6" and skip_process, reason="subprocess is unreliable on PySide6") def test_can_terminate_subprocess(loop): """Verify that a subprocess can be terminated.""" @@ -309,6 +327,8 @@ def sock_pair(request): If socket.socketpair isn't available, we emulate it. """ + if QtModuleName == "PySide6" and skip_socket: + pytest.skip("SocketPairs are unreliable on PySide6") def fin(): if client_sock is not None: @@ -887,6 +907,7 @@ def test_run_forever_custom_exit_code(loop, application): application.exec_ = orig_exec +@pytest.mark.skipif(QtModuleName == "PySide6" and skip_qthread, reason="unreliable on PySide6") def test_qeventloop_in_qthread(): class CoroutineExecutorThread(qasync.QtCore.QThread): def __init__(self, coro): From 4fd4cd97ffe1a62993777bb3e1325cb1ceec51a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Mon, 18 Aug 2025 11:10:41 +0000 Subject: [PATCH 2/5] Protect shutdown with lock. Allow shutdown more than once. --- src/qasync/__init__.py | 48 +++++++++++++++++++-------------------- tests/test_qthreadexec.py | 26 ++++++++++++++++----- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/src/qasync/__init__.py b/src/qasync/__init__.py index d7702c9..15b57e1 100644 --- a/src/qasync/__init__.py +++ b/src/qasync/__init__.py @@ -22,6 +22,7 @@ import time from concurrent.futures import Future from queue import Queue +from threading import Lock logger = logging.getLogger(__name__) @@ -191,45 +192,42 @@ def __init__(self, max_workers=10, stack_size=None): self.__workers = [ _QThreadWorker(self.__queue, i + 1, stack_size) for i in range(max_workers) ] + self.__shutdown_lock = Lock() self.__been_shutdown = False for w in self.__workers: w.start() def submit(self, callback, *args, **kwargs): - if self.__been_shutdown: - raise RuntimeError("QThreadExecutor has been shutdown") + with self.__shutdown_lock: + if self.__been_shutdown: + raise RuntimeError("QThreadExecutor has been shutdown") - future = Future() - self._logger.debug( - "Submitting callback %s with args %s and kwargs %s to thread worker queue", - callback, - args, - kwargs, - ) - self.__queue.put((future, callback, args, kwargs)) - return future + future = Future() + self._logger.debug( + "Submitting callback %s with args %s and kwargs %s to thread worker queue", + callback, + args, + kwargs, + ) + self.__queue.put((future, callback, args, kwargs)) + return future def map(self, func, *iterables, timeout=None): raise NotImplementedError("use as_completed on the event loop") def shutdown(self, wait=True): - if self.__been_shutdown: - raise RuntimeError("QThreadExecutor has been shutdown") - - self.__been_shutdown = True - - self._logger.debug("Shutting down") - for i in range(len(self.__workers)): - # Signal workers to stop - self.__queue.put(None) - if wait: - for w in self.__workers: - w.wait() + with self.__shutdown_lock: + self.__been_shutdown = True + self._logger.debug("Shutting down") + for i in range(len(self.__workers)): + # Signal workers to stop + self.__queue.put(None) + if wait: + for w in self.__workers: + w.wait() def __enter__(self, *args): - if self.__been_shutdown: - raise RuntimeError("QThreadExecutor has been shutdown") return self def __exit__(self, *args): diff --git a/tests/test_qthreadexec.py b/tests/test_qthreadexec.py index 67c1833..b311f41 100644 --- a/tests/test_qthreadexec.py +++ b/tests/test_qthreadexec.py @@ -44,15 +44,16 @@ def shutdown_executor(): return exe -def test_shutdown_after_shutdown(shutdown_executor): - with pytest.raises(RuntimeError): - shutdown_executor.shutdown() +@pytest.mark.parametrize("wait", [True, False]) +def test_shutdown_after_shutdown(shutdown_executor, wait): + # it is safe to shutdown twice + shutdown_executor.shutdown(wait=wait) def test_ctx_after_shutdown(shutdown_executor): - with pytest.raises(RuntimeError): - with shutdown_executor: - pass + # it is safe to enter and exit the context after shutdown + with shutdown_executor: + pass def test_submit_after_shutdown(shutdown_executor): @@ -104,3 +105,16 @@ def test_no_stale_reference_as_result(executor, disable_executor_logging): assert collected is True, ( "Stale reference to executor result not collected within timeout." ) + + +def test_context(executor): + """Test that the context manager will shutdown executor""" + with executor: + f = executor.submit(lambda: 42) + assert f.result() == 42 + + # it can be entered again + with executor: + # but will fail when we submit + with pytest.raises(RuntimeError): + executor.submit(lambda: 42) From f67236b536457b9dd23769a59cbe12ccf7da76ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Sat, 16 Aug 2025 16:06:04 +0000 Subject: [PATCH 3/5] Support the `cancel_futures` parameter to executor.shutdown() --- src/qasync/__init__.py | 9 ++++++++- tests/test_qthreadexec.py | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/qasync/__init__.py b/src/qasync/__init__.py index 15b57e1..52cc882 100644 --- a/src/qasync/__init__.py +++ b/src/qasync/__init__.py @@ -216,10 +216,17 @@ def submit(self, callback, *args, **kwargs): def map(self, func, *iterables, timeout=None): raise NotImplementedError("use as_completed on the event loop") - def shutdown(self, wait=True): + def shutdown(self, wait=True, *, cancel_futures=False): with self.__shutdown_lock: self.__been_shutdown = True self._logger.debug("Shutting down") + if cancel_futures: + # pop all the futures and cancel them + while not self.__queue.empty(): + item = self.__queue.get_nowait() + if item is not None: + future, _, _, _ = item + future.cancel() for i in range(len(self.__workers)): # Signal workers to stop self.__queue.put(None) diff --git a/tests/test_qthreadexec.py b/tests/test_qthreadexec.py index b311f41..bffc969 100644 --- a/tests/test_qthreadexec.py +++ b/tests/test_qthreadexec.py @@ -4,7 +4,9 @@ # BSD License import logging import threading +import time import weakref +from concurrent.futures import CancelledError import pytest @@ -118,3 +120,28 @@ def test_context(executor): # but will fail when we submit with pytest.raises(RuntimeError): executor.submit(lambda: 42) + + +@pytest.mark.parametrize("cancel", [True, False]) +def test_shutdown_cancel_futures(executor, cancel): + """Test that shutdown with cancel_futures=True cancels all remaining futures in the queue.""" + + def task(): + time.sleep(0.01) + + # Submit ten tasks to the executor + futures = [executor.submit(task) for _ in range(10)] + # shut it down + executor.shutdown(cancel_futures=cancel) + + cancels = 0 + for future in futures: + try: + future.result(timeout=0.01) + except CancelledError: + cancels += 1 + + if cancel: + assert cancels > 0 + else: + assert cancels == 0 From 787ee1261e52c5cea1664f4bb0d14428ccd03cf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Sat, 16 Aug 2025 17:13:29 +0000 Subject: [PATCH 4/5] add executor.map() functionality --- src/qasync/__init__.py | 30 ++++++++++- tests/test_qthreadexec.py | 103 +++++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/src/qasync/__init__.py b/src/qasync/__init__.py index 52cc882..55f8627 100644 --- a/src/qasync/__init__.py +++ b/src/qasync/__init__.py @@ -214,7 +214,25 @@ def submit(self, callback, *args, **kwargs): return future def map(self, func, *iterables, timeout=None): - raise NotImplementedError("use as_completed on the event loop") + deadline = time.monotonic() + timeout if timeout is not None else None + futures = [self.submit(func, *args) for args in zip(*iterables)] + + # must have generator as a closure so that the submit occurs before first iteration + def generator(): + try: + futures.reverse() + while futures: + if deadline is not None: + yield _result_or_cancel( + futures.pop(), timeout=deadline - time.monotonic() + ) + else: + yield _result_or_cancel(futures.pop()) + finally: + for future in futures: + future.cancel() + + return generator() def shutdown(self, wait=True, *, cancel_futures=False): with self.__shutdown_lock: @@ -241,6 +259,16 @@ def __exit__(self, *args): self.shutdown() +def _result_or_cancel(fut, timeout=None): + try: + try: + return fut.result(timeout) + finally: + fut.cancel() + finally: + del fut # break reference cycle in exceptions + + def _format_handle(handle: asyncio.Handle): cb = getattr(handle, "_callback", None) if isinstance(getattr(cb, "__self__", None), asyncio.tasks.Task): diff --git a/tests/test_qthreadexec.py b/tests/test_qthreadexec.py index bffc969..30eff1e 100644 --- a/tests/test_qthreadexec.py +++ b/tests/test_qthreadexec.py @@ -6,7 +6,8 @@ import threading import time import weakref -from concurrent.futures import CancelledError +from concurrent.futures import CancelledError, TimeoutError +from itertools import islice import pytest @@ -145,3 +146,103 @@ def task(): assert cancels > 0 else: assert cancels == 0 + + +def test_map(executor): + """Basic test of executor map functionality""" + results = list(executor.map(lambda x: x + 1, range(10))) + assert results == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + results = list(executor.map(lambda x, y: x + y, range(10), range(9))) + assert results == [0, 2, 4, 6, 8, 10, 12, 14, 16] + + +def test_map_timeout(executor): + """Test that map with timeout raises TimeoutError and cancels futures""" + results = [] + + def func(x): + nonlocal results + time.sleep(0.05) + results.append(x) + return x + + start = time.monotonic() + with pytest.raises(TimeoutError): + list(executor.map(func, range(10), timeout=0.01)) + duration = time.monotonic() - start + # this test is flaky on some platforms, so we give it a wide bearth. + assert duration < 0.1 + + executor.shutdown(wait=True) + # only about half of the tasks should have completed + # because the max number of workers is 5 and the rest of + # the tasks were not started at the time of the cancel. + assert set(results) != {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + + +def test_map_error(executor): + """Test that map with an exception will raise, and remaining tasks are cancelled""" + results = [] + + def func(x): + nonlocal results + time.sleep(0.05) + if len(results) == 5: + raise ValueError("Test error") + results.append(x) + return x + + with pytest.raises(ValueError): + list(executor.map(func, range(15))) + + executor.shutdown(wait=True, cancel_futures=False) + assert len(results) <= 10, "Final 5 at least should have been cancelled" + + +@pytest.mark.parametrize("cancel", [True, False]) +def test_map_shutdown(executor, cancel): + results = [] + + def func(x): + nonlocal results + time.sleep(0.05) + results.append(x) + return x + + # Get the first few results. + # Keep the iterator alive so that it isn't closed when its reference is dropped. + m = executor.map(func, range(15)) + values = list(islice(m, 5)) + assert values == [0, 1, 2, 3, 4] + + executor.shutdown(wait=True, cancel_futures=cancel) + if cancel: + assert len(results) < 15, "Some tasks should have been cancelled" + else: + assert len(results) == 15, "All tasks should have been completed" + m.close() + + +def test_map_start(executor): + """Test that map starts tasks immediately, before iterating""" + e = threading.Event() + m = executor.map(lambda x: (e.set(), x), range(1)) + e.wait(timeout=0.1) + assert list(m) == [(None, 0)] + + +def test_map_close(executor): + """Test that closing a running map cancels all remaining tasks.""" + results = [] + def func(x): + nonlocal results + time.sleep(0.05) + results.append(x) + return x + m = executor.map(func, range(10)) + # must start the generator so that close() has any effect + assert next(m) == 0 + m.close() + executor.shutdown(wait=True, cancel_futures=False) + assert len(results) < 10, "Some tasks should have been cancelled" From c6807c52da14815fae60547925ccefaedcf1f011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Sat, 16 Aug 2025 17:14:03 +0000 Subject: [PATCH 5/5] add `closing()` context manager, which by default does a `shutdown(wait=False)` for better performance in async --- examples/executor_example.py | 2 +- src/qasync/__init__.py | 8 ++++++++ tests/test_qthreadexec.py | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/examples/executor_example.py b/examples/executor_example.py index 2dee234..b1823cb 100644 --- a/examples/executor_example.py +++ b/examples/executor_example.py @@ -16,7 +16,7 @@ async def master(): await first_50(progress) loop = asyncio.get_running_loop() - with QThreadExecutor(1) as exec: + with QThreadExecutor(1).closing() as exec: await loop.run_in_executor(exec, functools.partial(last_50, progress), loop) diff --git a/src/qasync/__init__.py b/src/qasync/__init__.py index 55f8627..640e940 100644 --- a/src/qasync/__init__.py +++ b/src/qasync/__init__.py @@ -258,6 +258,14 @@ def __enter__(self, *args): def __exit__(self, *args): self.shutdown() + @contextlib.contextmanager + def closing(self, *, wait=False, cancel_futures=False): + """Explicit context manager to do shutdown, with Wait=False by default""" + try: + yield self + finally: + self.shutdown(wait=wait, cancel_futures=cancel_futures) + def _result_or_cancel(fut, timeout=None): try: diff --git a/tests/test_qthreadexec.py b/tests/test_qthreadexec.py index 30eff1e..abb3217 100644 --- a/tests/test_qthreadexec.py +++ b/tests/test_qthreadexec.py @@ -8,6 +8,7 @@ import weakref from concurrent.futures import CancelledError, TimeoutError from itertools import islice +from unittest import mock import pytest @@ -246,3 +247,21 @@ def func(x): m.close() executor.shutdown(wait=True, cancel_futures=False) assert len(results) < 10, "Some tasks should have been cancelled" + + +def test_closing(executor): + """Test that closing context manager works as expected""" + # mock the shutdown method of the executor + with mock.patch.object(executor, "shutdown") as mock_shutdown: + with executor.closing(): + pass + + # ensure that shutdown was called with (False, cancel_futures=False) + mock_shutdown.assert_called_once_with(wait=False, cancel_futures=False) + + with mock.patch.object(executor, "shutdown") as mock_shutdown: + with executor.closing(wait=True, cancel_futures=True): + pass + + # ensure that shutdown was called with (False, cancel_futures=False) + mock_shutdown.assert_called_once_with(wait=True, cancel_futures=True)