Handle failure during thread creation (#2471)

In Python 3.12 when you try to start a thread during shutdown a RuntimeError is raised. Handle this case with grace.

---------

Co-authored-by: Ivana Kellyerova <[email protected]>

Author: antonpirker

sentry_sdk/metrics.py

@@ -348,7 +348,7 @@ def _ensure_thread(self):
             try:
                 self._flusher.start()
             except RuntimeError:
-                # Unfortunately at this point the interpreter is in a start that no
+                # Unfortunately at this point the interpreter is in a state that no
                 # longer allows us to spawn a thread and we have to bail.
                 self._running = False
                 return False

sentry_sdk/monitor.py

@@ -37,6 +37,13 @@ def __init__(self, transport, interval=10):
 
     def _ensure_running(self):
         # type: () -> None
+        """
+        Check that the monitor has an active thread to run in, or create one if not.
+
+        Note that this might fail (e.g. in Python 3.12 it's not possible to
+        spawn new threads at interpreter shutdown). In that case self._running
+        will be False after running this function.
+        """
         if self._thread_for_pid == os.getpid() and self._thread is not None:
             return None
 
@@ -53,7 +60,14 @@ def _thread():
 
             thread = Thread(name=self.name, target=_thread)
             thread.daemon = True
-            thread.start()
+            try:
+                thread.start()
+            except RuntimeError:
+                # Unfortunately at this point the interpreter is in a state that no
+                # longer allows us to spawn a thread and we have to bail.
+                self._running = False
+                return None
+
             self._thread = thread
             self._thread_for_pid = os.getpid()
 

sentry_sdk/profiler.py

@@ -898,6 +898,14 @@ def teardown(self):
 
     def ensure_running(self):
         # type: () -> None
+        """
+        Check that the profiler has an active thread to run in, and start one if
+        that's not the case.
+
+        Note that this might fail (e.g. in Python 3.12 it's not possible to
+        spawn new threads at interpreter shutdown). In that case self.running
+        will be False after running this function.
+        """
         pid = os.getpid()
 
         # is running on the right process
@@ -918,7 +926,14 @@ def ensure_running(self):
             # can keep the application running after other threads
             # have exited
             self.thread = threading.Thread(name=self.name, target=self.run, daemon=True)
-            self.thread.start()
+            try:
+                self.thread.start()
+            except RuntimeError:
+                # Unfortunately at this point the interpreter is in a state that no
+                # longer allows us to spawn a thread and we have to bail.
+                self.running = False
+                self.thread = None
+                return
 
     def run(self):
         # type: () -> None
@@ -1004,7 +1019,14 @@ def ensure_running(self):
             self.running = True
 
             self.thread = ThreadPool(1)
-            self.thread.spawn(self.run)
+            try:
+                self.thread.spawn(self.run)
+            except RuntimeError:
+                # Unfortunately at this point the interpreter is in a state that no
+                # longer allows us to spawn a thread and we have to bail.
+                self.running = False
+                self.thread = None
+                return
 
     def run(self):
         # type: () -> None

sentry_sdk/sessions.py

@@ -105,6 +105,13 @@ def flush(self):
 
     def _ensure_running(self):
         # type: (...) -> None
+        """
+        Check that we have an active thread to run in, or create one if not.
+
+        Note that this might fail (e.g. in Python 3.12 it's not possible to
+        spawn new threads at interpreter shutdown). In that case self._running
+        will be False after running this function.
+        """
         if self._thread_for_pid == os.getpid() and self._thread is not None:
             return None
         with self._thread_lock:
@@ -120,9 +127,17 @@ def _thread():
 
             thread = Thread(target=_thread)
             thread.daemon = True
-            thread.start()
+            try:
+                thread.start()
+            except RuntimeError:
+                # Unfortunately at this point the interpreter is in a state that no
+                # longer allows us to spawn a thread and we have to bail.
+                self._running = False
+                return None
+
             self._thread = thread
             self._thread_for_pid = os.getpid()
+
         return None
 
     def add_aggregate_session(

tests/test_monitor.py

@@ -3,6 +3,11 @@
 from sentry_sdk import Hub, start_transaction
 from sentry_sdk.transport import Transport
 
+try:
+    from unittest import mock  # python 3.3 and above
+except ImportError:
+    import mock  # python < 3.3
+
 
 class HealthyTestTransport(Transport):
     def _send_event(self, event):
@@ -82,3 +87,18 @@ def test_transaction_uses_downsampled_rate(
         assert transaction.sample_rate == 0.5
 
     assert reports == [("backpressure", "transaction")]
+
+
+def test_monitor_no_thread_on_shutdown_no_errors(sentry_init):
+    sentry_init(transport=HealthyTestTransport())
+
+    # make it seem like the interpreter is shutting down
+    with mock.patch(
+        "threading.Thread.start",
+        side_effect=RuntimeError("can't create new thread at interpreter shutdown"),
+    ):
+        monitor = Hub.current.client.monitor
+        assert monitor is not None
+        assert monitor._thread is None
+        monitor.run()
+        assert monitor._thread is None

tests/test_profiler.py

@@ -661,6 +661,51 @@ def test_thread_scheduler_single_background_thread(scheduler_class):
     assert len(get_scheduler_threads(scheduler)) == 0
 
 
+@requires_python_version(3, 3)
+@pytest.mark.parametrize(
+    ("scheduler_class",),
+    [
+        pytest.param(ThreadScheduler, id="thread scheduler"),
+        pytest.param(
+            GeventScheduler,
+            marks=[
+                requires_gevent,
+                pytest.mark.skip(
+                    reason="cannot find this thread via threading.enumerate()"
+                ),
+            ],
+            id="gevent scheduler",
+        ),
+    ],
+)
+def test_thread_scheduler_no_thread_on_shutdown(scheduler_class):
+    scheduler = scheduler_class(frequency=1000)
+
+    # not yet setup, no scheduler threads yet
+    assert len(get_scheduler_threads(scheduler)) == 0
+
+    scheduler.setup()
+
+    # setup but no profiles started so still no threads
+    assert len(get_scheduler_threads(scheduler)) == 0
+
+    # mock RuntimeError as if the 3.12 intepreter was shutting down
+    with mock.patch(
+        "threading.Thread.start",
+        side_effect=RuntimeError("can't create new thread at interpreter shutdown"),
+    ):
+        scheduler.ensure_running()
+
+    assert scheduler.running is False
+
+    # still no thread
+    assert len(get_scheduler_threads(scheduler)) == 0
+
+    scheduler.teardown()
+
+    assert len(get_scheduler_threads(scheduler)) == 0
+
+
 @requires_python_version(3, 3)
 @pytest.mark.parametrize(
     ("scheduler_class",),

tests/test_sessions.py

@@ -3,6 +3,11 @@
 from sentry_sdk import Hub
 from sentry_sdk.sessions import auto_session_tracking
 
+try:
+    from unittest import mock  # python 3.3 and above
+except ImportError:
+    import mock  # python < 3.3
+
 
 def sorted_aggregates(item):
     aggregates = item["aggregates"]
@@ -119,3 +124,32 @@ def test_aggregates_explicitly_disabled_session_tracking_request_mode(
     assert len(aggregates) == 1
     assert aggregates[0]["exited"] == 1
     assert "errored" not in aggregates[0]
+
+
+def test_no_thread_on_shutdown_no_errors(sentry_init):
+    sentry_init(
+        release="fun-release",
+        environment="not-fun-env",
+    )
+
+    hub = Hub.current
+
+    # make it seem like the interpreter is shutting down
+    with mock.patch(
+        "threading.Thread.start",
+        side_effect=RuntimeError("can't create new thread at interpreter shutdown"),
+    ):
+        with auto_session_tracking(session_mode="request"):
+            with sentry_sdk.push_scope():
+                try:
+                    raise Exception("all is wrong")
+                except Exception:
+                    sentry_sdk.capture_exception()
+
+        with auto_session_tracking(session_mode="request"):
+            pass
+
+        hub.start_session(session_mode="request")
+        hub.end_session()
+
+        sentry_sdk.flush()

tests/test_transport.py

@@ -18,6 +18,10 @@
 from sentry_sdk.envelope import Envelope, parse_json
 from sentry_sdk.integrations.logging import LoggingIntegration
 
+try:
+    from unittest import mock  # python 3.3 and above
+except ImportError:
+    import mock  # python < 3.3
 
 CapturedData = namedtuple("CapturedData", ["path", "event", "envelope", "compressed"])
 
@@ -165,6 +169,21 @@ def test_transport_infinite_loop(capturing_server, request, make_client):
     assert len(capturing_server.captured) == 1
 
 
+def test_transport_no_thread_on_shutdown_no_errors(capturing_server, make_client):
+    client = make_client()
+
+    # make it seem like the interpreter is shutting down
+    with mock.patch(
+        "threading.Thread.start",
+        side_effect=RuntimeError("can't create new thread at interpreter shutdown"),
+    ):
+        with Hub(client):
+            capture_message("hi")
+
+    # nothing exploded but also no events can be sent anymore
+    assert len(capturing_server.captured) == 0
+
+
 NOW = datetime(2014, 6, 2)