Set thread names to help debugging

.github/workflows/proof-alarm.yml

@@ -15,7 +15,9 @@ jobs:
 
     - name: Check
       run: |
-        git diff --quiet 754ba168f source/linux/epoll_event_loop.c
+        TMPFILE=$(mktemp)
+        echo "495506dac0244fd96f66dc1d11740db8  source/linux/epoll_event_loop.c" > $TMPFILE
+        md5sum --check $TMPFILE
 
     # No further steps if successful
 

source/bsd/kqueue_event_loop.c

@@ -41,7 +41,7 @@ static int s_unsubscribe_from_io_events(struct aws_event_loop *event_loop, struc
 static void s_free_io_event_resources(void *user_data);
 static bool s_is_event_thread(struct aws_event_loop *event_loop);
 
-static void s_event_thread_main(void *user_data);
+static void aws_event_loop_thread(void *user_data);
 
 int aws_open_nonblocking_posix_pipe(int pipe_fds[2]);
 
@@ -368,7 +368,7 @@ static int s_run(struct aws_event_loop *event_loop) {
 
     aws_thread_increment_unjoined_count();
     int err =
-        aws_thread_launch(&impl->thread_created_on, s_event_thread_main, (void *)event_loop, &impl->thread_options);
+        aws_thread_launch(&impl->thread_created_on, aws_event_loop_thread, (void *)event_loop, &impl->thread_options);
 
     if (err) {
         aws_thread_decrement_unjoined_count();
@@ -808,7 +808,20 @@ static int s_aws_event_flags_from_kevent(struct kevent *kevent) {
     return event_flags;
 }
 
-static void s_event_thread_main(void *user_data) {
+/**
+ * This just calls kevent()
+ *
+ * We broke this out into its own function so that the stacktrace clearly shows
+ * what this thread is doing. We've had a lot of cases where users think this
+ * thread is deadlocked because it's stuck here. We want it to be clear
+ * that it's doing nothing on purpose. It's waiting for events to happen...
+ */
+AWS_NO_INLINE
+static int aws_event_loop_listen_for_io_events(int kq_fd, struct kevent kevents[MAX_EVENTS], struct timespec *timeout) {
+    return kevent(kq_fd, NULL /*changelist*/, 0 /*nchanges*/, kevents /*eventlist*/, MAX_EVENTS /*nevents*/, timeout);
+}
+
+static void aws_event_loop_thread(void *user_data) {
     struct aws_event_loop *event_loop = user_data;
     AWS_LOGF_INFO(AWS_LS_IO_EVENT_LOOP, "id=%p: main loop started", (void *)event_loop);
     struct kqueue_loop *impl = event_loop->impl_data;
@@ -851,8 +864,7 @@ static void s_event_thread_main(void *user_data) {
             (unsigned long long)timeout.tv_nsec);
 
         /* Process kqueue events */
-        int num_kevents = kevent(
-            impl->kq_fd, NULL /*changelist*/, 0 /*nchanges*/, kevents /*eventlist*/, MAX_EVENTS /*nevents*/, &timeout);
+        int num_kevents = aws_event_loop_listen_for_io_events(impl->kq_fd, kevents, &timeout);
 
         aws_event_loop_register_tick_start(event_loop);
         AWS_LOGF_TRACE(

source/event_loop.c

@@ -65,10 +65,9 @@ static void s_aws_event_loop_group_shutdown_async(struct aws_event_loop_group *e
 
     aws_thread_init(&cleanup_thread, el_group->allocator);
 
-    struct aws_thread_options thread_options;
-    AWS_ZERO_STRUCT(thread_options);
-    thread_options.cpu_id = -1;
+    struct aws_thread_options thread_options = *aws_default_thread_options();
     thread_options.join_strategy = AWS_TJS_MANAGED;
+    thread_options.name = aws_byte_cursor_from_c_str("EvntLoopCleanup"); /* 15 characters is max for Linux */
 
     aws_thread_launch(&cleanup_thread, s_event_loop_destroy_async_thread_fn, el_group, &thread_options);
 }
@@ -124,13 +123,21 @@ static struct aws_event_loop_group *s_event_loop_group_new(
 
             struct aws_event_loop_options options = {
                 .clock = clock,
+                .thread_options = &thread_options,
             };
 
             if (pin_threads) {
                 thread_options.cpu_id = usable_cpus[i].cpu_id;
-                options.thread_options = &thread_options;
             }
 
+            /* Thread name should be <= 15 characters */
+            char thread_name[32] = {0};
+            int thread_name_len = snprintf(thread_name, sizeof(thread_name), "AwsEventLoop %d", (int)i + 1);
+            if (thread_name_len > AWS_THREAD_NAME_RECOMMENDED_STRLEN) {
+                snprintf(thread_name, sizeof(thread_name), "AwsEventLoop");
+            }
+            thread_options.name = aws_byte_cursor_from_c_str(thread_name);
+
             struct aws_event_loop *loop = new_loop_fn(alloc, &options, new_loop_user_data);
 
             if (!loop) {

source/host_resolver.c

@@ -961,7 +961,7 @@ static bool s_is_host_entry_pinned_by_listener(struct aws_linked_list *listener_
     return false;
 }
 
-static void resolver_thread_fn(void *arg) {
+static void aws_host_resolver_thread(void *arg) {
     struct host_entry *host_entry = arg;
 
     size_t unsolicited_resolve_max = host_entry->resolution_config.max_ttl;
@@ -1359,8 +1359,9 @@ static inline int create_and_init_host_entry(
 
     struct aws_thread_options thread_options = *aws_default_thread_options();
     thread_options.join_strategy = AWS_TJS_MANAGED;
+    thread_options.name = aws_byte_cursor_from_c_str("AwsHostResolver"); /* 15 characters is max for Linux */
 
-    aws_thread_launch(&new_host_entry->resolver_thread, resolver_thread_fn, new_host_entry, &thread_options);
+    aws_thread_launch(&new_host_entry->resolver_thread, aws_host_resolver_thread, new_host_entry, &thread_options);
     ++default_host_resolver->pending_host_entry_shutdown_completion_callbacks;
 
     return AWS_OP_SUCCESS;

source/linux/epoll_event_loop.c

@@ -60,7 +60,7 @@ static int s_unsubscribe_from_io_events(struct aws_event_loop *event_loop, struc
 static void s_free_io_event_resources(void *user_data);
 static bool s_is_on_callers_thread(struct aws_event_loop *event_loop);
 
-static void s_main_loop(void *args);
+static void aws_event_loop_thread(void *args);
 
 static struct aws_event_loop_vtable s_vtable = {
     .destroy = s_destroy,
@@ -272,7 +272,9 @@ static int s_run(struct aws_event_loop *event_loop) {
 
     epoll_loop->should_continue = true;
     aws_thread_increment_unjoined_count();
-    if (aws_thread_launch(&epoll_loop->thread_created_on, &s_main_loop, event_loop, &epoll_loop->thread_options)) {
+    if (aws_thread_launch(
+            &epoll_loop->thread_created_on, &aws_event_loop_thread, event_loop, &epoll_loop->thread_options)) {
+
         aws_thread_decrement_unjoined_count();
         AWS_LOGF_FATAL(AWS_LS_IO_EVENT_LOOP, "id=%p: thread creation failed.", (void *)event_loop);
         epoll_loop->should_continue = false;
@@ -547,7 +549,20 @@ static void s_process_task_pre_queue(struct aws_event_loop *event_loop) {
     }
 }
 
-static void s_main_loop(void *args) {
+/**
+ * This just calls epoll_wait()
+ *
+ * We broke this out into its own function so that the stacktrace clearly shows
+ * what this thread is doing. We've had a lot of cases where users think this
+ * thread is deadlocked because it's stuck here. We want it to be clear
+ * that it's doing nothing on purpose. It's waiting for events to happen...
+ */
+AWS_NO_INLINE
+static int aws_event_loop_listen_for_io_events(int epoll_fd, struct epoll_event events[MAX_EVENTS], int timeout) {
+    return epoll_wait(epoll_fd, events, MAX_EVENTS, timeout);
+}
+
+static void aws_event_loop_thread(void *args) {
     struct aws_event_loop *event_loop = args;
     AWS_LOGF_INFO(AWS_LS_IO_EVENT_LOOP, "id=%p: main loop started", (void *)event_loop);
     struct epoll_loop *epoll_loop = event_loop->impl_data;
@@ -586,7 +601,7 @@ static void s_main_loop(void *args) {
     while (epoll_loop->should_continue) {
 
         AWS_LOGF_TRACE(AWS_LS_IO_EVENT_LOOP, "id=%p: waiting for a maximum of %d ms", (void *)event_loop, timeout);
-        int event_count = epoll_wait(epoll_loop->epoll_fd, events, MAX_EVENTS, timeout);
+        int event_count = aws_event_loop_listen_for_io_events(epoll_loop->epoll_fd, events, timeout);
         aws_event_loop_register_tick_start(event_loop);
 
         AWS_LOGF_TRACE(

source/windows/iocp/iocp_event_loop.c

@@ -106,7 +106,7 @@ static int s_connect_to_io_completion_port(struct aws_event_loop *event_loop, st
 static bool s_is_event_thread(struct aws_event_loop *event_loop);
 static int s_unsubscribe_from_io_events(struct aws_event_loop *event_loop, struct aws_io_handle *handle);
 static void s_free_io_event_resources(void *user_data);
-static void s_event_thread_main(void *user_data);
+static void aws_event_loop_thread(void *user_data);
 
 void aws_overlapped_init(
     struct aws_overlapped *overlapped,
@@ -357,7 +357,7 @@ static int s_run(struct aws_event_loop *event_loop) {
 
     AWS_LOGF_INFO(AWS_LS_IO_EVENT_LOOP, "id=%p: Starting event-loop thread.", (void *)event_loop);
     aws_thread_increment_unjoined_count();
-    int err = aws_thread_launch(&impl->thread_created_on, s_event_thread_main, event_loop, &impl->thread_options);
+    int err = aws_thread_launch(&impl->thread_created_on, aws_event_loop_thread, event_loop, &impl->thread_options);
     if (err) {
         aws_thread_decrement_unjoined_count();
         AWS_LOGF_FATAL(AWS_LS_IO_EVENT_LOOP, "id=%p: thread creation failed.", (void *)event_loop);
@@ -643,8 +643,32 @@ static void s_free_io_event_resources(void *user_data) {
     (void)user_data;
 }
 
+/**
+ * This just calls GetQueuedCompletionStatusEx()
+ *
+ * We broke this out into its own function so that the stacktrace clearly shows
+ * what this thread is doing. We've had a lot of cases where users think this
+ * thread is deadlocked because it's stuck here. We want it to be clear
+ * that it's doing nothing on purpose. It's waiting for events to happen...
+ */
+AWS_NO_INLINE
+static bool aws_event_loop_listen_for_io_events(
+    HANDLE iocp_handle,
+    OVERLAPPED_ENTRY completion_packets[MAX_COMPLETION_PACKETS_PER_LOOP],
+    ULONG *num_entries,
+    DWORD timeout_ms) {
+
+    return GetQueuedCompletionStatusEx(
+        iocp_handle,                     /* Completion port */
+        completion_packets,              /* Out: completion port entries */
+        MAX_COMPLETION_PACKETS_PER_LOOP, /* max number of entries to remove */
+        num_entries,                     /* Out: number of entries removed */
+        timeout_ms,                      /* Timeout in ms. If timeout reached then FALSE is returned. */
+        false);                          /* Alertable */
+}
+
 /* Called from event-thread */
-static void s_event_thread_main(void *user_data) {
+static void aws_event_loop_thread(void *user_data) {
     struct aws_event_loop *event_loop = user_data;
     AWS_LOGF_INFO(AWS_LS_IO_EVENT_LOOP, "id=%p: main loop started", (void *)event_loop);
 
@@ -667,13 +691,11 @@ static void s_event_thread_main(void *user_data) {
         ULONG num_entries = 0;
         bool should_process_synced_data = false;
         AWS_LOGF_TRACE(AWS_LS_IO_EVENT_LOOP, "id=%p: waiting for a maximum of %d ms", (void *)event_loop, timeout_ms);
-        bool has_completion_entries = GetQueuedCompletionStatusEx(
-            impl->iocp_handle,               /* Completion port */
-            completion_packets,              /* Out: completion port entries */
-            MAX_COMPLETION_PACKETS_PER_LOOP, /* max number of entries to remove */
-            &num_entries,                    /* Out: number of entries removed */
-            timeout_ms,                      /* Timeout in ms. If timeout reached then FALSE is returned. */
-            false);                          /* fAlertable */
+        bool has_completion_entries = aws_event_loop_listen_for_io_events(
+            impl->iocp_handle,  /* Completion port */
+            completion_packets, /* Out: completion port entries */
+            &num_entries,       /* Out: number of entries removed */
+            timeout_ms);        /* Timeout in ms. If timeout reached then FALSE is returned. */
 
         aws_event_loop_register_tick_start(event_loop);