diff --git a/CMakeLists.txt b/CMakeLists.txt index ac6e56857..51f1f1b71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,10 @@ if(POLICY CMP0157 AND CMAKE_Swift_COMPILER_USE_OLD_DRIVER) cmake_policy(SET CMP0157 OLD) endif() +if(POLICY CMP0181) + cmake_policy(SET CMP0181 NEW) +endif() + project(dispatch VERSION 1.3 LANGUAGES C CXX) @@ -118,6 +122,8 @@ include(DispatchSanitization) include(DispatchCompilerWarnings) include(DTrace) +include(EnableFramePointers) + # NOTE(abdulras) this is the CMake supported way to control whether we generate # shared or static libraries. This impacts the behaviour of `add_library` in # what type of library it generates. @@ -191,6 +197,7 @@ check_function_exists(pthread_attr_setcpupercent_np HAVE_PTHREAD_ATTR_SETCPUPERC check_function_exists(pthread_yield_np HAVE_PTHREAD_YIELD_NP) check_function_exists(pthread_main_np HAVE_PTHREAD_MAIN_NP) check_function_exists(pthread_workqueue_setdispatch_np HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP) +check_function_exists(pthread_setname_np HAVE_PTHREAD_SETNAME_NP) check_function_exists(strlcpy HAVE_STRLCPY) check_function_exists(sysconf HAVE_SYSCONF) check_function_exists(arc4random HAVE_ARC4RANDOM) @@ -266,6 +273,7 @@ check_symbol_exists(__printflike "bsd/sys/cdefs.h" HAVE_PRINTFLIKE) if(ANDROID) set(ENABLE_DTRACE_DEFAULT OFF) + add_link_options("LINKER:-z,max-page-size=16384") endif() if(BSD) @@ -305,15 +313,10 @@ add_compile_definitions($<$:HAVE_CONFIG_H>) if(ENABLE_SWIFT) - if(NOT SWIFT_SYSTEM_NAME) - if(APPLE) - set(SWIFT_SYSTEM_NAME macosx) - else() - set(SWIFT_SYSTEM_NAME "$") - endif() - endif() + include(SwiftSupport) + option(dispatch_INSTALL_ARCH_SUBDIR "Install libraries under an architecture subdirectory" NO) - set(INSTALL_TARGET_DIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${SWIFT_SYSTEM_NAME}" CACHE PATH "Path where the libraries will be installed") + set(INSTALL_TARGET_DIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${dispatch_PLATFORM}$<$:/${dispatch_ARCH}>" CACHE PATH "Path where the libraries will be installed") set(INSTALL_DISPATCH_HEADERS_DIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/dispatch" CACHE PATH "Path where the headers will be installed for libdispatch") set(INSTALL_BLOCK_HEADERS_DIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/Block" CACHE PATH "Path where the headers will be installed for the blocks runtime") set(INSTALL_OS_HEADERS_DIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/os" CACHE PATH "Path where the os/ headers will be installed") @@ -329,6 +332,9 @@ add_subdirectory(dispatch) add_subdirectory(man) add_subdirectory(os) add_subdirectory(private) +if(NOT APPLE) + add_subdirectory(src/BlocksRuntime) +endif() add_subdirectory(src) if(BUILD_TESTING) add_subdirectory(tests) diff --git a/cmake/config.h.in b/cmake/config.h.in index 27737c991..fc630ea67 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -208,6 +208,9 @@ /* Define to 1 if you have the `_pthread_workqueue_init' function. */ #cmakedefine HAVE__PTHREAD_WORKQUEUE_INIT +/* Define to 1 if you have the `pthread_setname_np' function. */ +#cmakedefine01 HAVE_PTHREAD_SETNAME_NP + /* Define to use non-portable pthread TSD optimizations for Mac OS X) */ #cmakedefine USE_APPLE_TSD_OPTIMIZATIONS diff --git a/cmake/modules/EnableFramePointers.cmake b/cmake/modules/EnableFramePointers.cmake new file mode 100644 index 000000000..8573d8dc4 --- /dev/null +++ b/cmake/modules/EnableFramePointers.cmake @@ -0,0 +1,13 @@ +# +# Including this file enables frame pointers, if we know how. +# + +include(CheckCompilerFlag) + +# Check if the compiler supports -fno-omit-frame-pointer +check_compiler_flag(C -fno-omit-frame-pointer SUPPORTS_NO_OMIT_FP) + +# If it does, use it +if (SUPPORTS_NO_OMIT_FP) + add_compile_options($<$:-fno-omit-frame-pointer>) +endif() diff --git a/cmake/modules/SwiftSupport.cmake b/cmake/modules/SwiftSupport.cmake index 3da519ec5..697d98516 100644 --- a/cmake/modules/SwiftSupport.cmake +++ b/cmake/modules/SwiftSupport.cmake @@ -1,30 +1,67 @@ include_guard() -if(NOT dispatch_MODULE_TRIPLE) +if(NOT dispatch_MODULE_TRIPLE OR NOT dispatch_ARCH OR NOT dispatch_PLATFORM) + # Get the target information from the Swift compiler. set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) if(CMAKE_Swift_COMPILER_TARGET) list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) endif() execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) +endif() +if(NOT dispatch_MODULE_TRIPLE) string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") set(dispatch_MODULE_TRIPLE "${module_triple}" CACHE STRING "Triple used to install swiftmodule files") mark_as_advanced(dispatch_MODULE_TRIPLE) - message(CONFIGURE_LOG "Swift module triple: ${module_triple}") endif() +if(NOT dispatch_ARCH) + if(CMAKE_Swift_COMPILER_VERSION VERSION_EQUAL 0.0.0 OR CMAKE_Swift_COMPILER_VERSION VERSION_GREATER_EQUAL 6.2) + # For newer compilers, we can use the -print-target-info command to get the architecture. + string(JSON module_arch GET "${target_info_json}" "target" "arch") + else() + # For older compilers, extract the value from `dispatch_MODULE_TRIPLE`. + string(REGEX MATCH "^[^-]+" module_arch "${dispatch_MODULE_TRIPLE}") + endif() + + set(dispatch_ARCH "${module_arch}" CACHE STRING "Arch folder name used to install libraries") + mark_as_advanced(dispatch_ARCH) + message(CONFIGURE_LOG "Swift arch: ${dispatch_ARCH}") +endif() + +if(NOT dispatch_PLATFORM) + if(CMAKE_Swift_COMPILER_VERSION VERSION_EQUAL 0.0.0 OR CMAKE_Swift_COMPILER_VERSION VERSION_GREATER_EQUAL 6.2) + # For newer compilers, we can use the -print-target-info command to get the platform. + string(JSON swift_platform GET "${target_info_json}" "target" "platform") + else() + # For older compilers, compile the value from `CMAKE_SYSTEM_NAME`. + if(APPLE) + set(swift_platform macosx) + else() + set(swift_platform "$") + endif() + endif() + + set(dispatch_PLATFORM "${swift_platform}" CACHE STRING "Platform folder name used to install libraries") + mark_as_advanced(dispatch_PLATFORM) + message(CONFIGURE_LOG "Swift platform: ${dispatch_PLATFORM}") +endif() + function(install_swift_module target) get_target_property(module ${target} Swift_MODULE_NAME) if(NOT module) set(module ${target}) endif() + + set(INSTALL_SWIFT_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${dispatch_PLATFORM}" CACHE PATH "Path where the swift modules will be installed") + install( FILES $/${module}.swiftdoc - DESTINATION ${INSTALL_TARGET_DIR}/${module}.swiftmodule + DESTINATION ${INSTALL_SWIFT_MODULE_DIR}/${module}.swiftmodule RENAME ${dispatch_MODULE_TRIPLE}.swiftdoc) install( FILES $/${module}.swiftmodule - DESTINATION ${INSTALL_TARGET_DIR}/${module}.swiftmodule + DESTINATION ${INSTALL_SWIFT_MODULE_DIR}/${module}.swiftmodule RENAME ${dispatch_MODULE_TRIPLE}.swiftmodule) endfunction() diff --git a/dispatch/base.h b/dispatch/base.h index 8e7728525..00c723b36 100644 --- a/dispatch/base.h +++ b/dispatch/base.h @@ -194,7 +194,7 @@ # endif #else # if defined(_WIN32) -# if defined(dispatch_EXPORT) || defined(__DISPATCH_BUILDING_DISPATCH__) +# if defined(dispatch_EXPORTS) # define DISPATCH_EXPORT DISPATCH_EXTERN __declspec(dllexport) # else # define DISPATCH_EXPORT DISPATCH_EXTERN __declspec(dllimport) diff --git a/dispatch/queue.h b/dispatch/queue.h index e2f7e05e9..ff68d2308 100644 --- a/dispatch/queue.h +++ b/dispatch/queue.h @@ -173,12 +173,12 @@ DISPATCH_DECL_SUBCLASS(dispatch_queue_main, dispatch_queue_serial); * * @discussion * Dispatch concurrent queues are lightweight objects to which regular and - * barrier workitems may be submited. Barrier workitems are invoked in + * barrier workitems may be submitted. Barrier workitems are invoked in * exclusion of any other kind of workitem in FIFO order. * * Regular workitems can be invoked concurrently for the same concurrent queue, * in any order. However, regular workitems will not be invoked before any - * barrier workitem submited ahead of them has been invoked. + * barrier workitem submitted ahead of them has been invoked. * * In other words, if a serial queue is equivalent to a mutex in the Dispatch * world, a concurrent queue is equivalent to a reader-writer lock, where diff --git a/src/BlocksRuntime/CMakeLists.txt b/src/BlocksRuntime/CMakeLists.txt index fc3d60252..3732b0aec 100644 --- a/src/BlocksRuntime/CMakeLists.txt +++ b/src/BlocksRuntime/CMakeLists.txt @@ -7,8 +7,11 @@ if(WIN32) BlocksRuntime.def) if(NOT BUILD_SHARED_LIBS) - target_compile_definitions(BlocksRuntime PRIVATE + target_compile_definitions(BlocksRuntime PUBLIC BlocksRuntime_STATIC) + target_compile_options(BlocksRuntime PUBLIC + "$<$:SHELL:$<$:-Xclang >-static-libclosure>" + $<$:SHELL:-Xcc -static-libclosure>) endif() endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a1f8a0c58..c10583054 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,8 +1,4 @@ -if(NOT APPLE) - add_subdirectory(BlocksRuntime) -endif() - add_library(dispatch allocator.c apply.c @@ -96,6 +92,12 @@ target_include_directories(dispatch PUBLIC target_include_directories(dispatch PRIVATE ${PROJECT_SOURCE_DIR}/private) +if(NOT BUILD_SHARED_LIBS) + target_compile_definitions(dispatch PUBLIC + dispatch_STATIC) + target_compile_options(dispatch PUBLIC + "$<$:SHELL:-Xcc -Ddispatch_STATIC>") +endif() if(WIN32) target_compile_definitions(dispatch PRIVATE _CRT_NONSTDC_NO_WARNINGS @@ -109,7 +111,7 @@ if(DISPATCH_ENABLE_ASSERTS) DISPATCH_DEBUG=1) endif() -if("${CMAKE_C_SIMULATE_ID}" STREQUAL "MSVC") +if("${CMAKE_C_COMPILER_FRONTEND_VARIANT}" STREQUAL "MSVC") target_compile_options(dispatch PRIVATE /EHs-c-) target_compile_options(dispatch PRIVATE /W3) else() diff --git a/src/event/event_config.h b/src/event/event_config.h index 4f4b6e5a3..fac801256 100644 --- a/src/event/event_config.h +++ b/src/event/event_config.h @@ -118,7 +118,11 @@ // FreeBSD's kevent does not support those # ifndef NOTE_ABSOLUTE -# define NOTE_ABSOLUTE 0 +# ifdef NOTE_ABSTIME +# define NOTE_ABSOLUTE NOTE_ABSTIME +# else +# define NOTE_ABSOLUTE 0 +# endif # endif # ifndef NOTE_EXITSTATUS # define NOTE_EXITSTATUS 0 diff --git a/src/event/event_epoll.c b/src/event/event_epoll.c index f31d13ee0..b7fe8379b 100644 --- a/src/event/event_epoll.c +++ b/src/event/event_epoll.c @@ -89,7 +89,9 @@ DISPATCH_ALWAYS_INLINE static inline uint32_t _dispatch_muxnote_armed_events(dispatch_muxnote_t dmn) { - return dmn->dmn_events & ~dmn->dmn_disarmed_events; + uint32_t events = dmn->dmn_events; + uint16_t disarmed_events = dmn->dmn_disarmed_events; + return events & ~(uint32_t)disarmed_events; } DISPATCH_ALWAYS_INLINE diff --git a/src/event/event_kevent.c b/src/event/event_kevent.c index 0d8db09f1..eb80082c8 100644 --- a/src/event/event_kevent.c +++ b/src/event/event_kevent.c @@ -50,6 +50,10 @@ DISPATCH_STATIC_GLOBAL(struct dispatch_muxnote_bucket_s _dispatch_sources[DSL_HA #define DISPATCH_NOTE_CLOCK_WALL NOTE_NSECONDS | NOTE_MACH_CONTINUOUS_TIME #define DISPATCH_NOTE_CLOCK_MONOTONIC NOTE_MACHTIME | NOTE_MACH_CONTINUOUS_TIME #define DISPATCH_NOTE_CLOCK_UPTIME NOTE_MACHTIME +#elif __FreeBSD__ +#define DISPATCH_NOTE_CLOCK_WALL NOTE_NSECONDS +#define DISPATCH_NOTE_CLOCK_MONOTONIC NOTE_NSECONDS +#define DISPATCH_NOTE_CLOCK_UPTIME NOTE_NSECONDS #else #define DISPATCH_NOTE_CLOCK_WALL 0 #define DISPATCH_NOTE_CLOCK_MONOTONIC 0 diff --git a/src/event/event_windows.c b/src/event/event_windows.c index 31613ca00..af67dfd4c 100644 --- a/src/event/event_windows.c +++ b/src/event/event_windows.c @@ -260,8 +260,16 @@ _dispatch_muxnote_retain(dispatch_muxnote_t dmn) static void _dispatch_muxnote_release(dispatch_muxnote_t dmn) { - uintptr_t refcount = os_atomic_dec(&dmn->dmn_refcount, relaxed); + // We perform a minor optimization here - perform the decrement with + // release semantics. In the case that we are going to dispose of the + // value, we perform the acquire fence. This reduces the cost on the + // normal path by avoiding the acquire fence. This should be more + // beneficial on ARM64, as X64 being TSO'ed doesn't gain much. However, + // `mfence` being isolated should hopefully be a bit more efficient than + // the repeated `lock` if there is contention. + uintptr_t refcount = os_atomic_dec(&dmn->dmn_refcount, release); if (refcount == 0) { + os_atomic_thread_fence(acquire); _dispatch_muxnote_dispose(dmn); } else if (refcount == UINTPTR_MAX) { DISPATCH_INTERNAL_CRASH(0, "muxnote refcount underflow"); @@ -277,9 +285,6 @@ _dispatch_pipe_monitor_thread(void *context) char cBuffer[1]; DWORD dwNumberOfBytesTransferred; OVERLAPPED ov = {0}; - // Block on a 0-byte read; this will only resume when data is - // available in the pipe. The pipe must be PIPE_WAIT or this thread - // will spin. BOOL bSuccess = ReadFile(hPipe, cBuffer, /* nNumberOfBytesToRead */ 0, &dwNumberOfBytesTransferred, &ov); DWORD dwBytesAvailable; diff --git a/src/init.c b/src/init.c index 2f2efb0ae..b7364fea3 100644 --- a/src/init.c +++ b/src/init.c @@ -383,9 +383,9 @@ dispatch_get_global_queue(intptr_t priority, uintptr_t flags) } dispatch_qos_t qos = _dispatch_qos_from_queue_priority(priority); #if !HAVE_PTHREAD_WORKQUEUE_QOS - if (qos == QOS_CLASS_MAINTENANCE) { + if (qos == DISPATCH_QOS_MAINTENANCE) { qos = DISPATCH_QOS_BACKGROUND; - } else if (qos == QOS_CLASS_USER_INTERACTIVE) { + } else if (qos == DISPATCH_QOS_USER_INTERACTIVE) { qos = DISPATCH_QOS_USER_INITIATED; } #endif diff --git a/src/io.c b/src/io.c index 95c9d2d60..7ae1e0efe 100644 --- a/src/io.c +++ b/src/io.c @@ -1462,20 +1462,20 @@ _dispatch_fd_entry_create_with_fd(dispatch_fd_t fd, uintptr_t hash) int result = ioctlsocket((SOCKET)fd, (long)FIONBIO, &value); (void)dispatch_assume_zero(result); } else { - // The _dispatch_pipe_monitor_thread expects pipes to be - // PIPE_WAIT and exploits this assumption by using a blocking - // 0-byte read as a synchronization mechanism. + // Try to make writing nonblocking, although pipes not coming + // from Foundation.Pipe may not have FILE_WRITE_ATTRIBUTES. DWORD dwPipeMode = 0; if (GetNamedPipeHandleState((HANDLE)fd, &dwPipeMode, NULL, - NULL, NULL, NULL, 0) && !(dwPipeMode & PIPE_WAIT)) { - dwPipeMode |= PIPE_WAIT; + NULL, NULL, NULL, 0) && !(dwPipeMode & PIPE_NOWAIT)) { + dwPipeMode |= PIPE_NOWAIT; if (!SetNamedPipeHandleState((HANDLE)fd, &dwPipeMode, NULL, NULL)) { - // If setting the pipe to PIPE_WAIT fails, the - // monitoring thread will spin constantly, saturating - // a core, which is undesirable but non-fatal. - // The semantics will still be correct in this case. - _dispatch_fd_entry_debug("failed to set PIPE_WAIT", + // We may end up blocking on subsequent writes, but we + // don't have a good alternative. + // The WriteQuotaAvailable from NtQueryInformationFile + // erroneously returns 0 when there is a blocking read + // on the other end of the pipe. + _dispatch_fd_entry_debug("failed to set PIPE_NOWAIT", fd_entry); } } @@ -2550,40 +2550,13 @@ _dispatch_operation_perform(dispatch_operation_t op) NTSTATUS status = _dispatch_NtQueryInformationFile(hFile, &iosb, &fpli, sizeof(fpli), FilePipeLocalInformation); if (NT_SUCCESS(status)) { - // WriteQuotaAvailable is the free space in the output buffer - // that has not already been reserved for reading. In other words, - // WriteQuotaAvailable = - // OutboundQuota - WriteQuotaUsed - QueuedReadSize. - // It is not documented that QueuedReadSize is part of this - // calculation, but this behavior has been observed experimentally. - // Unfortunately, this means that it is not possible to distinguish - // between a full output buffer and a reader blocked waiting for a - // full buffer's worth of data. This is a problem because if the - // output buffer is full and no reader is waiting for data, then - // attempting to write to the buffer of a PIPE_WAIT, non- - // overlapped I/O pipe will block the dispatch queue thread. - // - // In order to work around this idiosyncrasy, we bound the size of - // the write to be OutboundQuota - 1. This affords us a sentinel value - // in WriteQuotaAvailable that can be used to detect if a reader is - // making progress or not. - // WriteQuotaAvailable = 0 => a reader is blocked waiting for data. - // WriteQuotaAvailable = 1 => the pipe has been written to, but no - // reader is making progress. - // When we detect that WriteQuotaAvailable == 1, we write 0 bytes to - // avoid blocking the dispatch queue thread. - if (fpli.WriteQuotaAvailable == 0) { - // This condition can only occur when we have a reader blocked - // waiting for data on the pipe. In this case, write a full - // buffer's worth of data (less one byte to preserve this - // sentinel value of WriteQuotaAvailable == 0). - len = MIN(len, fpli.OutboundQuota - 1); - } else { - // Subtract 1 from WriteQuotaAvailable to ensure we do not fill - // the pipe and preserve the sentinel value of - // WriteQuotaAvailable == 1. - len = MIN(len, fpli.WriteQuotaAvailable - 1); + // WriteQuotaAvailable is unreliable in the presence + // of a blocking reader, when it can return zero, so only + // account for it otherwise + if (fpli.WriteQuotaAvailable > 0) { + len = MIN(len, fpli.WriteQuotaAvailable); } + len = MIN(len, fpli.OutboundQuota); } OVERLAPPED ovlOverlapped = {}; diff --git a/src/queue.c b/src/queue.c index 1d948e947..e369e5b9f 100644 --- a/src/queue.c +++ b/src/queue.c @@ -23,6 +23,33 @@ #include "protocol.h" // _dispatch_send_wakeup_runloop_thread #endif +#if defined(__linux__) +#include +#include +#endif + +#if defined(_WIN32) +// Wrapper around SetThreadDescription for UTF-8 strings +void _dispatch_win32_set_thread_description(HANDLE hThread, const char *description) { + int wcsize = MultiByteToWideChar(CP_UTF8, 0, description, -1, NULL, 0); + if (wcsize == 0) { + return; + } + + wchar_t* wcstr = (wchar_t*)malloc(wcsize * sizeof(wchar_t)); + if (wcstr == NULL) { + return; + } + + int result = MultiByteToWideChar(CP_UTF8, 0, description, -1, wcstr, wcsize); + if (result != 0) { + SetThreadDescription(hThread, wcstr); + } + + free(wcstr); +} +#endif + static inline void _dispatch_root_queues_init(void); static void _dispatch_lane_barrier_complete(dispatch_lane_class_t dqu, dispatch_qos_t qos, dispatch_wakeup_flags_t flags); @@ -6216,10 +6243,61 @@ _dispatch_worker_thread(void *context) _dispatch_sigmask(); #endif _dispatch_introspection_thread_add(); + dispatch_priority_t pri = dq->dq_priority; + pthread_priority_t pp = _dispatch_get_priority(); + +#if defined(__linux__) + // The Linux kernel does not have a direct analogue to the QoS-based + // thread policy engine found in XNU. + // + // We cannot use 'pthread_setschedprio', because all threads with default + // scheduling policy (SCHED_OTHER) have the same pthread 'priority'. + // For both CFS, which was introduced in Linux 2.6.23, and its successor + // EEVDF (since 6.6) 'sched_get_priority_max' and 'sched_get_priority_min' + // will just return 0. + // + // However, as outlined in "man 2 setpriority", the nice value is a + // per‐thread attribute: different threads in the same process can have + // different nice values. We can thus setup the thread's initial priority + // by converting the QoS class and relative priority to a 'nice' value. + pp = _dispatch_priority_to_pp_strip_flags(pri); + int nice = _dispatch_pp_to_nice(pp); + + #if HAVE_PTHREAD_SETNAME_NP + // pthread thread names are restricted to just 16 characters + // including NUL. It does not make sense to pass the queue's + // label as a name. + pthread_setname_np(pthread_self(), "DispatchWorker"); + #endif + + errno = 0; + int rc = setpriority(PRIO_PROCESS, 0, nice); + if (rc != -1 || errno == 0) { + _dispatch_thread_setspecific(dispatch_priority_key, (void *)(uintptr_t)pp); + } else { + _dispatch_log("Failed to set thread priority for worker thread: pqc=%p errno=%d\n", pqc, errno); + } +#elif defined(_WIN32) + pp = _dispatch_priority_to_pp_strip_flags(pri); + int win_priority = _dispatch_pp_to_win32_priority(pp); + + HANDLE current = GetCurrentThread(); + + // Set thread description to the label of the root queue + if (dq->dq_label) { + _dispatch_win32_set_thread_description(current, dq->dq_label); + } + + int rc = SetThreadPriority(current, win_priority); + if (rc) { + _dispatch_thread_setspecific(dispatch_priority_key, (void *)(uintptr_t)pp); + } else { + DWORD dwError = GetLastError(); + _dispatch_log("Failed to set thread priority for worker thread: pqc=%p win_priority=%d dwError=%lu\n", pqc, win_priority, dwError); + } +#endif const int64_t timeout = 5ull * NSEC_PER_SEC; - pthread_priority_t pp = _dispatch_get_priority(); - dispatch_priority_t pri = dq->dq_priority; // If the queue is neither // - the manager @@ -6258,6 +6336,14 @@ _dispatch_worker_thread(void *context) (void)os_atomic_inc2o(dq, dgq_thread_pool_size, release); _dispatch_root_queue_poke(dq, 1, 0); _dispatch_release(dq); // retained in _dispatch_root_queue_poke_slow + +#if defined(_WIN32) + // Make sure to properly end the background processing mode + if (win_priority == THREAD_MODE_BACKGROUND_BEGIN) { + SetThreadPriority(current, THREAD_MODE_BACKGROUND_END); + } +#endif + return NULL; } #if defined(_WIN32) diff --git a/src/shims.h b/src/shims.h index e14697a9f..a65052dd0 100644 --- a/src/shims.h +++ b/src/shims.h @@ -58,9 +58,6 @@ #define DISPATCH_WORKQ_MAX_PTHREAD_COUNT 255 #endif -#include "shims/hw_config.h" -#include "shims/priority.h" - #if HAVE_PTHREAD_NP_H #include #endif @@ -69,6 +66,9 @@ #include #endif +#include "shims/hw_config.h" +#include "shims/priority.h" + #if !HAVE_DECL_FD_COPY #define FD_COPY(f, t) (void)(*(t) = *(f)) #endif diff --git a/src/shims/hw_config.h b/src/shims/hw_config.h index 06b8921d9..4e6f7c3c9 100644 --- a/src/shims/hw_config.h +++ b/src/shims/hw_config.h @@ -102,14 +102,14 @@ static inline uint32_t _dispatch_hw_get_config(_dispatch_hw_config_t c) { uint32_t val = 1; -#if defined(__linux__) && HAVE_SYSCONF +#if defined(__FreeBSD__) || (defined(__linux__) && HAVE_SYSCONF) switch (c) { case _dispatch_hw_config_logical_cpus: case _dispatch_hw_config_physical_cpus: return (uint32_t)sysconf(_SC_NPROCESSORS_CONF); case _dispatch_hw_config_active_cpus: { -#ifdef __USE_GNU +#if defined(__FreeBSD__) || __USE_GNU // Prefer pthread_getaffinity_np because it considers // scheduler cpu affinity. This matters if the program // is restricted to a subset of the online cpus (eg via numactl). diff --git a/src/shims/lock.c b/src/shims/lock.c index 6d0ab8764..85e44544c 100644 --- a/src/shims/lock.c +++ b/src/shims/lock.c @@ -231,10 +231,10 @@ _dispatch_sema4_timedwait(_dispatch_sema4_t *sema, dispatch_time_t timeout) struct timespec _timeout; int ret; + uint64_t nsec = _dispatch_time_nanoseconds_since_epoch(timeout); + _timeout.tv_sec = (__typeof__(_timeout.tv_sec))(nsec / NSEC_PER_SEC); + _timeout.tv_nsec = (__typeof__(_timeout.tv_nsec))(nsec % NSEC_PER_SEC); do { - uint64_t nsec = _dispatch_time_nanoseconds_since_epoch(timeout); - _timeout.tv_sec = (__typeof__(_timeout.tv_sec))(nsec / NSEC_PER_SEC); - _timeout.tv_nsec = (__typeof__(_timeout.tv_nsec))(nsec % NSEC_PER_SEC); ret = sem_timedwait(sema, &_timeout); } while (unlikely(ret == -1 && errno == EINTR)); diff --git a/src/shims/priority.h b/src/shims/priority.h index 3a79c5efb..92646c3db 100644 --- a/src/shims/priority.h +++ b/src/shims/priority.h @@ -210,6 +210,71 @@ _dispatch_qos_to_pp(dispatch_qos_t qos) return pp | _PTHREAD_PRIORITY_PRIORITY_MASK; } + +#if defined(__linux__) +// These presets roughly match the `android.os.Process' constants +// used for `setThreadPriority()'. +// +// Be aware that with the Completely Fair Scheduler (CFS) the weight is computed +// as 1024 / (1.25) ^ (nice) where nice is in the range -20 to 19. +// This means that nice is not a linear scale. +#define DISPATCH_NICE_BACKGROUND 10 +#define DISPATCH_NICE_UTILITY 2 +#define DISPATCH_NICE_DEFAULT 0 +// Note that you might not have permission to increase the priority +// of a thread beyond the default priority. +#define DISPATCH_NICE_USER_INITIATED -2 +#define DISPATCH_NICE_USER_INTERACTIVE -4 + +DISPATCH_ALWAYS_INLINE +static inline int _dispatch_pp_to_nice(pthread_priority_t pp) +{ + // FIXME: What about relative priorities? + uint32_t qos = _dispatch_qos_from_pp(pp); + + switch (qos) { + case DISPATCH_QOS_BACKGROUND: + return DISPATCH_NICE_BACKGROUND; + case DISPATCH_QOS_UTILITY: + return DISPATCH_NICE_UTILITY; + case DISPATCH_QOS_DEFAULT: + return DISPATCH_NICE_DEFAULT; + case DISPATCH_QOS_USER_INITIATED: + return DISPATCH_NICE_USER_INITIATED; + case DISPATCH_QOS_USER_INTERACTIVE: + return DISPATCH_NICE_USER_INTERACTIVE; + } + + return DISPATCH_NICE_DEFAULT; +} +#endif // defined(__linux__) + +#if defined(_WIN32) +DISPATCH_ALWAYS_INLINE +static inline int _dispatch_pp_to_win32_priority(pthread_priority_t pp) { + uint32_t qos = _dispatch_qos_from_pp(pp); + + switch (qos) { + case DISPATCH_QOS_BACKGROUND: + // Make sure to end background mode before exiting the thread! + return THREAD_MODE_BACKGROUND_BEGIN; + case DISPATCH_QOS_UTILITY: + return THREAD_PRIORITY_BELOW_NORMAL; + case DISPATCH_QOS_DEFAULT: + return THREAD_PRIORITY_NORMAL; + // User input threads should be THREAD_PRIORITY_NORMAL, to + // avoid unintentionally starving the system + case DISPATCH_QOS_USER_INITIATED: + return THREAD_PRIORITY_NORMAL; + case DISPATCH_QOS_USER_INTERACTIVE: + return THREAD_PRIORITY_NORMAL; + } + + return THREAD_PRIORITY_NORMAL; +} +#endif // defined(_WIN32) + + // including maintenance DISPATCH_ALWAYS_INLINE static inline bool diff --git a/src/swift/CMakeLists.txt b/src/swift/CMakeLists.txt index d255f2cd7..a0082fb1e 100644 --- a/src/swift/CMakeLists.txt +++ b/src/swift/CMakeLists.txt @@ -1,5 +1,3 @@ -include(SwiftSupport) - if(HAVE_OBJC) add_library(DispatchStubs STATIC DispatchStubs.m) diff --git a/src/swift/Source.swift b/src/swift/Source.swift index 92201f8e0..0c3abc7f9 100644 --- a/src/swift/Source.swift +++ b/src/swift/Source.swift @@ -116,7 +116,7 @@ extension DispatchSource { } #endif -#if !os(Linux) && !os(Android) && !os(Windows) && !os(OpenBSD) +#if !os(Linux) && !os(Android) && !os(Windows) public struct ProcessEvent : OptionSet, RawRepresentable { public let rawValue: UInt public init(rawValue: UInt) { self.rawValue = rawValue } @@ -124,16 +124,19 @@ extension DispatchSource { public static let exit = ProcessEvent(rawValue: 0x80000000) public static let fork = ProcessEvent(rawValue: 0x40000000) public static let exec = ProcessEvent(rawValue: 0x20000000) -#if os(FreeBSD) - public static let track = ProcessEvent(rawValue: 0x00000001) -#else + +#if canImport(Darwin) public static let signal = ProcessEvent(rawValue: 0x08000000) #endif +#if os(FreeBSD) || os(OpenBSD) + public static let track = ProcessEvent(rawValue: 0x00000001) +#endif -#if os(FreeBSD) - public static let all: ProcessEvent = [.exit, .fork, .exec, .track] -#else +#if canImport(Darwin) public static let all: ProcessEvent = [.exit, .fork, .exec, .signal] +#endif +#if os(FreeBSD) || os(OpenBSD) + public static let all: ProcessEvent = [.exit, .fork, .exec, .track] #endif } #endif @@ -183,7 +186,7 @@ extension DispatchSource { } #endif -#if !os(Linux) && !os(Android) && !os(Windows) && !os(OpenBSD) +#if !os(Linux) && !os(Android) && !os(Windows) public class func makeProcessSource(identifier: pid_t, eventMask: ProcessEvent, queue: DispatchQueue? = nil) -> DispatchSourceProcess { let source = dispatch_source_create(_swift_dispatch_source_type_PROC(), UInt(identifier), eventMask.rawValue, queue?.__wrapped) return DispatchSource(source: source) as DispatchSourceProcess @@ -299,7 +302,7 @@ extension DispatchSourceMemoryPressure { } #endif -#if !os(Linux) && !os(Android) && !os(Windows) && !os(OpenBSD) +#if !os(Linux) && !os(Android) && !os(Windows) extension DispatchSourceProcess { public var handle: pid_t { return pid_t(CDispatch.dispatch_source_get_handle((self as! DispatchSource).__wrapped)) diff --git a/src/swift/Wrapper.swift b/src/swift/Wrapper.swift index 1118bb6e4..a260697f2 100644 --- a/src/swift/Wrapper.swift +++ b/src/swift/Wrapper.swift @@ -182,7 +182,7 @@ extension DispatchSource : DispatchSourceMachSend, } #endif -#if !os(Linux) && !os(Android) && !os(Windows) && !os(OpenBSD) +#if !os(Linux) && !os(Android) && !os(Windows) extension DispatchSource : DispatchSourceProcess { } #endif @@ -277,7 +277,7 @@ public protocol DispatchSourceMemoryPressure : DispatchSourceProtocol { } #endif -#if !os(Linux) && !os(Android) && !os(Windows) && !os(OpenBSD) +#if !os(Linux) && !os(Android) && !os(Windows) public protocol DispatchSourceProcess : DispatchSourceProtocol { var handle: pid_t { get } diff --git a/src/swift/shims/DispatchOverlayShims.h b/src/swift/shims/DispatchOverlayShims.h index 5ed8dd228..fbcf0116e 100644 --- a/src/swift/shims/DispatchOverlayShims.h +++ b/src/swift/shims/DispatchOverlayShims.h @@ -88,7 +88,7 @@ SWIFT_DISPATCH_SOURCE_TYPE(PROC) SWIFT_DISPATCH_SOURCE_TYPE(VNODE) #endif -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__OpenBSD__) SWIFT_DISPATCH_SOURCE_TYPE(PROC) SWIFT_DISPATCH_SOURCE_TYPE(VNODE) #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f06162874..ed5d2f444 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,6 +23,8 @@ add_library(bsdtests STATIC bsdtests.c dispatch_test.c) +target_link_libraries(bsdtests PUBLIC + dispatch) target_include_directories(bsdtests PRIVATE ${CMAKE_CURRENT_BINARY_DIR} @@ -81,7 +83,7 @@ function(add_unit_test name) target_include_directories(${name} SYSTEM BEFORE PRIVATE "${BlocksRuntime_INCLUDE_DIR}") - if("${CMAKE_C_SIMULATE_ID}" STREQUAL "MSVC") + if("${CMAKE_C_COMPILER_FRONTEND_VARIANT}" STREQUAL "MSVC") target_compile_options(${name} PRIVATE -Xclang -fblocks) target_compile_options(${name} PRIVATE /W3 -Wno-deprecated-declarations) else() diff --git a/tests/bsdtests.c b/tests/bsdtests.c index 6307180fd..3ea91a381 100644 --- a/tests/bsdtests.c +++ b/tests/bsdtests.c @@ -164,6 +164,20 @@ test_uint32_format(uint32_t actual, uint32_t expected, const char *format, ...) _test_uint32(NULL, 0, desc, actual, expected); } +void +_test_uint32_not(const char* file, long line, const char* desc, uint32_t actual, uint32_t unexpected) +{ + _test_print(file, line, desc, + (actual != unexpected), "%u", actual, "!%u", unexpected); +} + +void +test_uint32_not_format(uint32_t actual, uint32_t unexpected, const char *format, ...) +{ + GENERATE_DESC + _test_uint32_not(NULL, 0, desc, actual, unexpected); +} + void _test_int32(const char* file, long line, const char* desc, int32_t actual, int32_t expected) { diff --git a/tests/bsdtests.h b/tests/bsdtests.h index e8e292e6e..3437a51ed 100644 --- a/tests/bsdtests.h +++ b/tests/bsdtests.h @@ -111,6 +111,10 @@ void _test_uint32(const char* file, long line, const char* desc, uint32_t actual #define test_uint32(a,b,c) _test_uint32(__SOURCE_FILE__, __LINE__, a, b, c) void test_uint32_format(uint32_t actual, uint32_t expected, const char *format, ...) __printflike(3,4); +void _test_uint32_not(const char* file, long line, const char* desc, uint32_t actual, uint32_t unexpected); +#define test_uint32_not(a,b,c) _test_uint32_not(__SOURCE_FILE__, __LINE__, a, b, c) +void test_uint32_not_format(uint32_t actual, uint32_t unexpected, const char *format, ...) __printflike(3,4); + void _test_int32(const char* file, long line, const char* desc, int32_t actual, int32_t expected); #define test_int32(a,b,c) _test_int32(__SOURCE_FILE__, __LINE__, a, b, c) void test_int32_format(int32_t actual, int32_t expected, const char* format, ...) __printflike(3,4); diff --git a/tests/dispatch_io_pipe.c b/tests/dispatch_io_pipe.c index eb32b6b87..90c212236 100644 --- a/tests/dispatch_io_pipe.c +++ b/tests/dispatch_io_pipe.c @@ -408,12 +408,7 @@ test_dispatch_write(int kind, int delay) dispatch_group_t g = dispatch_group_create(); dispatch_group_enter(g); - // The libdispatch implementation writes at most bufsize-1 bytes - // before requiring a reader to start making progress. Because - // these tests operate serially, the reader will not make progress - // until the write finishes, and a write of >= bufsize will not - // finish until the reader starts draining the pipe. - const size_t bufsize = test_get_pipe_buffer_size(kind) - 1; + const size_t bufsize = test_get_pipe_buffer_size(kind); char *buf = calloc(bufsize, 1); assert(buf); diff --git a/tests/dispatch_workqueue.c b/tests/dispatch_workqueue.c index c12598b69..cce8071d3 100644 --- a/tests/dispatch_workqueue.c +++ b/tests/dispatch_workqueue.c @@ -4,6 +4,10 @@ #if defined(__linux__) // For pthread_getaffinity_np() #include +#elif defined(__FreeBSD__) +// for pthread_getaffinity_np / cpu_set_t +#include +#include #endif struct test_context { @@ -37,11 +41,11 @@ spin(void *context) static uint32_t activecpu(void) { - uint32_t activecpu; -#if defined(__linux__) || defined(__OpenBSD__) + uint32_t activecpu = 0xa1a1a1; +#if defined(__linux__) || defined(__OpenBSD__) || defined(__FreeBSD__) activecpu = (uint32_t)sysconf(_SC_NPROCESSORS_ONLN); -#if defined(__linux__) && __USE_GNU +#if defined(__FreeBSD__) || (defined(__linux__) && __USE_GNU) cpu_set_t cpuset; if (pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), @@ -55,7 +59,8 @@ activecpu(void) activecpu = si.dwNumberOfProcessors; #else size_t s = sizeof(activecpu); - sysctlbyname("hw.activecpu", &activecpu, &s, NULL, 0); + if (sysctlbyname("hw.activecpu", &activecpu, &s, NULL, 0) != 0) + return 0; #endif return activecpu; } @@ -66,6 +71,7 @@ int main(void) { uint32_t ncpu = activecpu(); + test_uint32_not("Failed to get CPU count", ncpu, 0); dispatch_test_start("Dispatch workqueue");