From b41fc553cecc9ba5c81cda41bb85e2ee1937586e Mon Sep 17 00:00:00 2001 From: Zebediah Figura Date: Mon, 2 Nov 2020 20:24:07 -0600 Subject: [PATCH] ntdll: Reimplement Win32 futexes on top of thread-ID alerts. Signed-off-by: Zebediah Figura --- dlls/ntdll/ntdll_misc.h | 2 + dlls/ntdll/sync.c | 185 ++++++++++++++++++++++++++++++++++++++- dlls/ntdll/thread.c | 2 + dlls/ntdll/unix/loader.c | 3 - dlls/ntdll/unix/sync.c | 162 ---------------------------------- dlls/ntdll/unixlib.h | 6 +- 6 files changed, 187 insertions(+), 173 deletions(-) diff --git a/dlls/ntdll/ntdll_misc.h b/dlls/ntdll/ntdll_misc.h index b8f9dc28e63..eab3b5e5248 100644 --- a/dlls/ntdll/ntdll_misc.h +++ b/dlls/ntdll/ntdll_misc.h @@ -85,6 +85,8 @@ extern const struct unix_funcs *unix_funcs DECLSPEC_HIDDEN; extern struct _KUSER_SHARED_DATA *user_shared_data DECLSPEC_HIDDEN; +extern void addr_wait_free_entry(void) DECLSPEC_HIDDEN; + extern int CDECL NTDLL__vsnprintf( char *str, SIZE_T len, const char *format, __ms_va_list args ) DECLSPEC_HIDDEN; extern int CDECL NTDLL__vsnwprintf( WCHAR *str, SIZE_T len, const WCHAR *format, __ms_va_list args ) DECLSPEC_HIDDEN; diff --git a/dlls/ntdll/sync.c b/dlls/ntdll/sync.c index f1263ae33fd..d652f55b630 100644 --- a/dlls/ntdll/sync.c +++ b/dlls/ntdll/sync.c @@ -36,6 +36,13 @@ #include "wine/debug.h" #include "ntdll_misc.h" +WINE_DEFAULT_DEBUG_CHANNEL(sync); + +static const char *debugstr_timeout( const LARGE_INTEGER *timeout ) +{ + if (!timeout) return "(infinite)"; + return wine_dbgstr_longlong( timeout->QuadPart ); +} /****************************************************************** * RtlRunOnceInitialize (NTDLL.@) @@ -530,13 +537,143 @@ NTSTATUS WINAPI RtlSleepConditionVariableSRW( RTL_CONDITION_VARIABLE *variable, return status; } +/* The following functions define a lock-free array mapping thread IDs to + * values, which can be grown but not shrunk. We do this by allocating one slice + * of the array at a time, and storing a pointer to the next slice at the end. + * + * This is both for efficiency (we want this function to be as fast as possible) + * and because locking the TEB list is hard otherwise—we need to safely access + * the TEB list, but cannot do so using any of these synchronization primitives, + * and we may need to access the TEB list before being inserted into it (e.g. + * from heap locks, or the TEB list lock itself.) + */ + +struct addr_wait_entry +{ + void *addr; + HANDLE tid; +}; + +struct addr_wait_array +{ + struct addr_wait_entry entries[(0x1000 - sizeof(struct addr_wait_entry *)) / sizeof(struct addr_wait_entry)]; + struct addr_wait_array *next; +}; + +static struct addr_wait_array first_addr_wait_array; + +static struct addr_wait_entry *addr_wait_allocate_entry( HANDLE tid ) +{ + struct addr_wait_array *array = &first_addr_wait_array; + + for (;;) + { + struct addr_wait_array *new_array = NULL; + SIZE_T size = sizeof(*new_array); + unsigned int i; + + for (;;) + { + for (i = 0; i < ARRAY_SIZE(array->entries); ++i) + { + if (!array->entries[i].tid && !InterlockedCompareExchangePointer( &array->entries[i].tid, tid, NULL )) + return &array->entries[i]; + } + + if (!array->next) break; + array = array->next; + } + + if (NtAllocateVirtualMemory( NtCurrentProcess(), (void **)&new_array, 0, &size, MEM_COMMIT, PAGE_READWRITE )) + return NULL; + + if (InterlockedCompareExchangePointer( (void **)&array->next, new_array, NULL )) + { + /* another thread beat us to it */ + NtFreeVirtualMemory( NtCurrentProcess(), (void **)&new_array, &size, MEM_RELEASE ); + } + /* start searching again from the new array */ + array = array->next; + } +} + +void addr_wait_free_entry(void) +{ + struct addr_wait_entry *entry = NtCurrentTeb()->ReservedForPerf; + if (entry) + InterlockedExchangePointer( &entry->tid, NULL ); +} + +static BOOL compare_addr( const void *addr, const void *cmp, SIZE_T size ) +{ + switch (size) + { + case 1: + return (*(const UCHAR *)addr == *(const UCHAR *)cmp); + case 2: + return (*(const USHORT *)addr == *(const USHORT *)cmp); + case 4: + return (*(const ULONG *)addr == *(const ULONG *)cmp); + case 8: + return (*(const ULONG64 *)addr == *(const ULONG64 *)cmp); + } + + return FALSE; +} + /*********************************************************************** * RtlWaitOnAddress (NTDLL.@) */ NTSTATUS WINAPI RtlWaitOnAddress( const void *addr, const void *cmp, SIZE_T size, const LARGE_INTEGER *timeout ) { - return unix_funcs->RtlWaitOnAddress( addr, cmp, size, timeout ); + struct addr_wait_entry *entry = NtCurrentTeb()->ReservedForPerf; + NTSTATUS ret; + + TRACE("addr %p cmp %p size %#Ix timeout %s\n", addr, cmp, size, debugstr_timeout( timeout )); + + if (size != 1 && size != 2 && size != 4 && size != 8) + return STATUS_INVALID_PARAMETER; + + if (!entry) + { + if (!(entry = addr_wait_allocate_entry( NtCurrentTeb()->ClientId.UniqueThread ))) + return STATUS_NO_MEMORY; + NtCurrentTeb()->ReservedForPerf = entry; + } + + InterlockedExchangePointer( &entry->addr, (void *)addr ); + + /* Ensure that the compare-and-swap above is ordered before the comparison + * below. This barrier is paired with another in RtlWakeByAddress*(). + * + * In more detail, given the following sequence: + * + * Thread A Thread B + * ----------------------------------------------------------------- + * RtlWaitOnAddress( &val ); val = 1; + * queue thread RtlWakeByAddress( &val ); + * MemoryBarrier(); <---- paired with ----> MemoryBarrier(); + * compare_addr( &val ); if (thread is queued) + * + * We must ensure that the thread is queued [through the above + * InterlockedExchangePointer()] before reading "val", and that writes to + * "val" by the application happen before we check for queued threads. + * Otherwise, thread A can deadlock: "val" may appear unchanged, while + * thread B observed that thread A was not queued. + */ + MemoryBarrier(); + + if (!compare_addr( addr, cmp, size )) + { + InterlockedExchangePointer( &entry->addr, NULL ); + return STATUS_SUCCESS; + } + + ret = NtWaitForAlertByThreadId( NULL, timeout ); + InterlockedExchangePointer( &entry->addr, NULL ); + if (ret == STATUS_ALERTED) ret = STATUS_SUCCESS; + return ret; } /*********************************************************************** @@ -544,7 +681,26 @@ NTSTATUS WINAPI RtlWaitOnAddress( const void *addr, const void *cmp, SIZE_T size */ void WINAPI RtlWakeAddressAll( const void *addr ) { - return unix_funcs->RtlWakeAddressAll( addr ); + struct addr_wait_array *array; + unsigned int i; + + TRACE("%p\n", addr); + + if (!addr) return; + + /* Ensure that memory stores to "addr" are ordered before reading the + * array below. Paired with another barrier in RtlWaitOnAddress() [q.v.]. + */ + MemoryBarrier(); + + for (array = &first_addr_wait_array; array != NULL; array = array->next) + { + for (i = 0; i < ARRAY_SIZE(array->entries); ++i) + { + if (array->entries[i].addr == addr) + NtAlertThreadByThreadId( array->entries[i].tid ); + } + } } /*********************************************************************** @@ -552,5 +708,28 @@ void WINAPI RtlWakeAddressAll( const void *addr ) */ void WINAPI RtlWakeAddressSingle( const void *addr ) { - return unix_funcs->RtlWakeAddressSingle( addr ); + struct addr_wait_array *array; + unsigned int i; + + TRACE("%p\n", addr); + + if (!addr) return; + + /* Ensure that memory stores to "addr" are ordered before reading the + * array below. Paired with another barrier in RtlWaitOnAddress() [q.v.]. + */ + MemoryBarrier(); + + for (array = &first_addr_wait_array; array != NULL; array = array->next) + { + for (i = 0; i < ARRAY_SIZE(array->entries); ++i) + { + if (array->entries[i].addr == addr + && InterlockedCompareExchangePointer( &array->entries[i].addr, NULL, (void *)addr ) == addr) + { + NtAlertThreadByThreadId( array->entries[i].tid ); + return; + } + } + } } diff --git a/dlls/ntdll/thread.c b/dlls/ntdll/thread.c index 25496609f08..4f395336428 100644 --- a/dlls/ntdll/thread.c +++ b/dlls/ntdll/thread.c @@ -84,6 +84,8 @@ void WINAPI RtlExitUserThread( ULONG status ) NtQueryInformationThread( GetCurrentThread(), ThreadAmILastThread, &last, sizeof(last), NULL ); if (last) RtlExitUserProcess( status ); LdrShutdownThread(); + /* must be done last, in particular after any heap allocations */ + addr_wait_free_entry(); for (;;) NtTerminateThread( GetCurrentThread(), status ); } diff --git a/dlls/ntdll/unix/loader.c b/dlls/ntdll/unix/loader.c index 4c76865019b..d7fc60cad2f 100644 --- a/dlls/ntdll/unix/loader.c +++ b/dlls/ntdll/unix/loader.c @@ -1753,9 +1753,6 @@ static struct unix_funcs unix_funcs = #endif DbgUiIssueRemoteBreakin, RtlGetSystemTimePrecise, - RtlWaitOnAddress, - RtlWakeAddressAll, - RtlWakeAddressSingle, fast_RtlpWaitForCriticalSection, fast_RtlpUnWaitCriticalSection, fast_RtlDeleteCriticalSection, diff --git a/dlls/ntdll/unix/sync.c b/dlls/ntdll/unix/sync.c index 7d6423083e1..41005425a90 100644 --- a/dlls/ntdll/unix/sync.c +++ b/dlls/ntdll/unix/sync.c @@ -77,10 +77,6 @@ WINE_DEFAULT_DEBUG_CHANNEL(sync); HANDLE keyed_event = 0; -static const LARGE_INTEGER zero_timeout; - -static pthread_mutex_t addr_mutex = PTHREAD_MUTEX_INITIALIZER; - static const char *debugstr_timeout( const LARGE_INTEGER *timeout ) { if (!timeout) return "(infinite)"; @@ -190,24 +186,6 @@ static void timespec_from_timeout( struct timespec *timespec, const LARGE_INTEGE #endif -static BOOL compare_addr( const void *addr, const void *cmp, SIZE_T size ) -{ - switch (size) - { - case 1: - return (*(const UCHAR *)addr == *(const UCHAR *)cmp); - case 2: - return (*(const USHORT *)addr == *(const USHORT *)cmp); - case 4: - return (*(const ULONG *)addr == *(const ULONG *)cmp); - case 8: - return (*(const ULONG64 *)addr == *(const ULONG64 *)cmp); - } - - return FALSE; -} - - /* create a struct security_descriptor and contained information in one contiguous piece of memory */ NTSTATUS alloc_object_attributes( const OBJECT_ATTRIBUTES *attr, struct object_attributes **ret, data_size_t *ret_len ) @@ -2836,71 +2814,6 @@ NTSTATUS CDECL fast_RtlWakeConditionVariable( RTL_CONDITION_VARIABLE *variable, return STATUS_SUCCESS; } - -/* We can't map addresses to futex directly, because an application can wait on - * 8 bytes, and we can't pass all 8 as the compare value to futex(). Instead we - * map all addresses to a small fixed table of futexes. This may result in - * spurious wakes, but the application is already expected to handle those. */ - -static int addr_futex_table[256]; - -static inline int *hash_addr( const void *addr ) -{ - ULONG_PTR val = (ULONG_PTR)addr; - - return &addr_futex_table[(val >> 2) & 255]; -} - -static inline NTSTATUS fast_wait_addr( const void *addr, const void *cmp, SIZE_T size, - const LARGE_INTEGER *timeout ) -{ - int *futex; - int val; - struct timespec timespec; - int ret; - - if (!use_futexes()) - return STATUS_NOT_IMPLEMENTED; - - futex = hash_addr( addr ); - - /* We must read the previous value of the futex before checking the value - * of the address being waited on. That way, if we receive a wake between - * now and waiting on the futex, we know that val will have changed. - * Use an atomic load so that memory accesses are ordered between this read - * and the increment below. */ - val = InterlockedCompareExchange( futex, 0, 0 ); - if (!compare_addr( addr, cmp, size )) - return STATUS_SUCCESS; - - if (timeout) - { - timespec_from_timeout( ×pec, timeout ); - ret = futex_wait( futex, val, ×pec ); - } - else - ret = futex_wait( futex, val, NULL ); - - if (ret == -1 && errno == ETIMEDOUT) - return STATUS_TIMEOUT; - return STATUS_SUCCESS; -} - -static inline NTSTATUS fast_wake_addr( const void *addr ) -{ - int *futex; - - if (!use_futexes()) - return STATUS_NOT_IMPLEMENTED; - - futex = hash_addr( addr ); - - InterlockedIncrement( futex ); - - futex_wake( futex, INT_MAX ); - return STATUS_SUCCESS; -} - #else NTSTATUS CDECL fast_RtlTryAcquireSRWLockExclusive( RTL_SRWLOCK *lock ) @@ -2943,79 +2856,4 @@ NTSTATUS CDECL fast_wait_cv( RTL_CONDITION_VARIABLE *variable, const void *value return STATUS_NOT_IMPLEMENTED; } -static inline NTSTATUS fast_wait_addr( const void *addr, const void *cmp, SIZE_T size, - const LARGE_INTEGER *timeout ) -{ - return STATUS_NOT_IMPLEMENTED; -} - -static inline NTSTATUS fast_wake_addr( const void *addr ) -{ - return STATUS_NOT_IMPLEMENTED; -} - #endif - - -/*********************************************************************** - * RtlWaitOnAddress (NTDLL.@) - */ -NTSTATUS WINAPI RtlWaitOnAddress( const void *addr, const void *cmp, SIZE_T size, - const LARGE_INTEGER *timeout ) -{ - select_op_t select_op; - NTSTATUS ret; - timeout_t abs_timeout = timeout ? timeout->QuadPart : TIMEOUT_INFINITE; - - if (size != 1 && size != 2 && size != 4 && size != 8) - return STATUS_INVALID_PARAMETER; - - if ((ret = fast_wait_addr( addr, cmp, size, timeout )) != STATUS_NOT_IMPLEMENTED) - return ret; - - mutex_lock( &addr_mutex ); - if (!compare_addr( addr, cmp, size )) - { - mutex_unlock( &addr_mutex ); - return STATUS_SUCCESS; - } - - if (abs_timeout < 0) - { - LARGE_INTEGER now; - - NtQueryPerformanceCounter( &now, NULL ); - abs_timeout -= now.QuadPart; - } - - select_op.keyed_event.op = SELECT_KEYED_EVENT_WAIT; - select_op.keyed_event.handle = wine_server_obj_handle( keyed_event ); - select_op.keyed_event.key = wine_server_client_ptr( addr ); - - return server_select( &select_op, sizeof(select_op.keyed_event), SELECT_INTERRUPTIBLE, - abs_timeout, NULL, &addr_mutex, NULL ); -} - -/*********************************************************************** - * RtlWakeAddressAll (NTDLL.@) - */ -void WINAPI RtlWakeAddressAll( const void *addr ) -{ - if (fast_wake_addr( addr ) != STATUS_NOT_IMPLEMENTED) return; - - mutex_lock( &addr_mutex ); - while (NtReleaseKeyedEvent( 0, addr, 0, &zero_timeout ) == STATUS_SUCCESS) {} - mutex_unlock( &addr_mutex ); -} - -/*********************************************************************** - * RtlWakeAddressSingle (NTDLL.@) - */ -void WINAPI RtlWakeAddressSingle( const void *addr ) -{ - if (fast_wake_addr( addr ) != STATUS_NOT_IMPLEMENTED) return; - - mutex_lock( &addr_mutex ); - NtReleaseKeyedEvent( 0, addr, 0, &zero_timeout ); - mutex_unlock( &addr_mutex ); -} diff --git a/dlls/ntdll/unixlib.h b/dlls/ntdll/unixlib.h index 04ae8230b1a..21e34842778 100644 --- a/dlls/ntdll/unixlib.h +++ b/dlls/ntdll/unixlib.h @@ -26,7 +26,7 @@ struct _DISPATCHER_CONTEXT; /* increment this when you change the function table */ -#define NTDLL_UNIXLIB_VERSION 120 +#define NTDLL_UNIXLIB_VERSION 121 struct unix_funcs { @@ -38,10 +38,6 @@ struct unix_funcs /* other Win32 API functions */ NTSTATUS (WINAPI *DbgUiIssueRemoteBreakin)( HANDLE process ); LONGLONG (WINAPI *RtlGetSystemTimePrecise)(void); - NTSTATUS (WINAPI *RtlWaitOnAddress)( const void *addr, const void *cmp, SIZE_T size, - const LARGE_INTEGER *timeout ); - void (WINAPI *RtlWakeAddressAll)( const void *addr ); - void (WINAPI *RtlWakeAddressSingle)( const void *addr ); /* fast locks */ NTSTATUS (CDECL *fast_RtlpWaitForCriticalSection)( RTL_CRITICAL_SECTION *crit, int timeout ); -- 2.30.2