diff options
Diffstat (limited to 'nptl')
45 files changed, 941 insertions, 259 deletions
diff --git a/nptl/ChangeLog b/nptl/ChangeLog index 2e850439de..f46f53af3b 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,3 +1,195 @@ +2007-08-13 Jakub Jelinek <jakub@redhat.com> + + * sysdeps/unix/sysv/linux/i386/lowlevellock.h (__lll_private_flag): + Fix a pasto. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S + (__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or + FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. + Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S + (__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or + FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Include + kernel-features.h. + (__pthread_cond_wait, __condvar_w_cleanup): Pass LLL_PRIVATE to + lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is + process private. Switch DW_CFA_advance_loc1 and some + DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S + (__pthread_cond_timedwait, __condvar_tw_cleanup): Pass LLL_PRIVATE to + lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is + process private. Switch DW_CFA_advance_loc{1,2} and some + DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Use + #ifdef __ASSUME_PRIVATE_FUTEX instead of #if __ASSUME_PRIVATE_FUTEX. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: + Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: + Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S + (__pthread_cond_broadcast): Compare %r8 instead of + dep_mutex-cond_*(%rdi) with $-1. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S + (__pthread_cond_signal): Xor FUTEX_WAKE_OP with FUTEX_WAKE instead + of oring. + +2007-08-13 Ulrich Drepper <drepper@redhat.com> + + * sysdeps/unix/sysv/linux/i386/i786/Implies: New file. + +2007-08-13 Jakub Jelinek <jakub@redhat.com> + + * allocatestack.c: Include kernel-features.h. + * pthread_create.c: Likewise. + * pthread_mutex_init.c: Likewise. + * init.c: Likewise. + * pthread_cond_timedwait.c: Likewise. + * sysdeps/unix/sysv/linux/alpha/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/ia64/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: + Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: + Likewise. + * sysdeps/unix/sysv/linux/s390/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/sparc/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S: Likewise. + +2007-08-12 Jakub Jelinek <jakub@redhat.com> + + * sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h + [__WORDSIZE=32] (pthread_rwlock_t): Split __flags element into four + byte elements. One of them is the new __shared element. + [__WORDSIZE=64] (pthread_rwlock_t): Renamed __pad1 element to __shared, + adjust names of other padding elements. + * sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h + [__WORDSIZE=32] (pthread_rwlock_t): Split __flags element into four + byte elements. One of them is the new __shared element. + [__WORDSIZE=64] (pthread_rwlock_t): Renamed __pad1 element to __shared, + adjust names of other padding elements. + * sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h (pthread_rwlock_t): + Renamed __pad1 element to __shared, adjust names of other padding + elements. + * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h + (pthread_rwlock_t): Likewise. + * sysdeps/unix/sysv/linux/ia64/lowlevellock.h (__lll_lock): Fix a + typo. + +2007-08-09 Anton Blanchard <anton@samba.org> + + * sysdeps/unix/sysv/linux/powerpc/pthread_spin_unlock.c: New file. + +2007-08-12 Ulrich Drepper <drepper@redhat.com> + + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Include + <kernel-features.h>. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise. + +2007-08-11 Ulrich Drepper <drepper@redhat.com> + + * pthreadP.h (PTHREAD_ROBUST_MUTEX_PSHARED): Define. + * pthread_mutex_lock.c: Use it instead of PTHREAD_MUTEX_PSHARED when + dealing with robust mutexes. + * pthread_mutex_timedlock.c: Likewise. + * pthread_mutex_trylock.c: Likewise. + * pthread_mutex_unlock.c: Likewise. + * sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise. + +2007-08-06 Jakub Jelinek <jakub@redhat.com> + + * pthreadP.h (PTHREAD_MUTEX_PSHARED_BIT): Define. + (PTHREAD_MUTEX_TYPE): Mask __kind with 127. + (PTHREAD_MUTEX_PSHARED): Define. + * pthread_mutex_init.c (__pthread_mutex_init): Set + PTHREAD_MUTEX_PSHARED_BIT for pshared or robust + mutexes. + * pthread_mutex_lock.c (LLL_MUTEX_LOCK): Take mutex as argument + instead of its __data.__lock field, pass PTHREAD_MUTEX_PSHARED + as second argument to lll_lock. + (LLL_MUTEX_TRYLOCK): Take mutex as argument + instead of its __data.__lock field. + (LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its + __data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument + to lll_robust_lock. + (__pthread_mutex_lock): Update LLL_MUTEX_LOCK, LLL_MUTEX_TRYLOCK, + LLL_ROBUST_MUTEX_LOCK users, use PTHREAD_MUTEX_TYPE (mutex) + instead of mutex->__data.__kind directly, pass + PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock and lll_futex_wait. + * pthread_mutex_trylock.c (__pthread_mutex_trylock): Use + PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind + directly, pass PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock. + (pthread_mutex_timedlock): Pass PTHREAD_MUTEX_PSHARED (mutex) + to lll_timedlock, lll_robust_timedlock, lll_unlock and + lll_futex_timed_wait. Use PTHREAD_MUTEX_TYPE (mutex) instead + of mutex->__data.__kind directly. + * pthread_mutex_timedlock.c (pthread_mutex_timedlock): Pass + PTHREAD_MUTEX_PSHARED (mutex) to lll_timedlock, + lll_robust_timedlock, lll_unlock and lll_futex_timed_wait. Use + PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind directly. + * pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): Pass + PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock, lll_robust_unlock + and lll_futex_wake. + * pthread_mutex_setprioceiling.c (pthread_mutex_setprioceiling): Pass + PTHREAD_MUTEX_PSHARED (mutex) to lll_futex_wait and lll_futex_wake. + Use PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind + directly. + * sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c (LLL_MUTEX_LOCK): + Take mutex as argument instead of its __data.__lock field, pass + PTHREAD_MUTEX_PSHARED as second argument to lll_cond_lock. + (LLL_MUTEX_TRYLOCK): Take mutex as argument instead of its + __data.__lock field. + (LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its + __data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument + to lll_robust_cond_lock. + * pthread_cond_broadcast.c (__pthread_cond_broadcast): Add pshared + variable, pass it to lll_lock, lll_unlock, lll_futex_requeue and + lll_futex_wake. Don't use lll_futex_requeue if dependent mutex + has PTHREAD_MUTEX_PSHARED_BIT bit set in its __data.__kind. + * pthread_cond_destroy.c (__pthread_cond_destroy): Add pshared + variable, pass it to lll_lock, lll_unlock, lll_futex_wake and + lll_futex_wait. + * pthread_cond_signal.c (__pthread_cond_signal): Add pshared + variable, pass it to lll_lock, lll_unlock, lll_futex_wake_unlock and + lll_futex_wake. + * pthread_cond_timedwait.c (__pthread_cond_wait): Add + pshared variable, pass it to lll_lock, lll_unlock, + lll_futex_timedwait and lll_futex_wake. + * pthread_cond_wait.c (__condvar_cleanup, __pthread_cond_wait): Add + pshared variable, pass it to lll_lock, lll_unlock, lll_futex_wait + and lll_futex_wake. + * sysdeps/unix/sysv/linux/alpha/lowlevellock.h (lll_futex_requeue, + lll_futex_wake_unlock): Add private argument, use __lll_private_flag + macro. + * sysdeps/unix/sysv/linux/ia64/lowlevellock.h (lll_futex_requeue, + lll_futex_wake_unlock): Likewise. + * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_futex_requeue): + Likewise. + * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_futex_requeue, + lll_futex_wake_unlock): Likewise. + * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (lll_futex_requeue): + Likewise. + * sysdeps/unix/sysv/linux/s390/lowlevellock.h (lll_futex_requeue, + lll_futex_wake_unlock): Likewise. + (lll_futex_wake): Fix a typo. + * sysdeps/unix/sysv/linux/pthread-pi-defines.sym (PS_BIT): Add. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S + (__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or + FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. + Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S + (__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or + FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S + (__pthread_cond_timedwait): Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: + (__condvar_cleanup, __pthread_cond_wait): Likewise. + 2007-08-05 Jakub Jelinek <jakub@redhat.com> * sysdeps/unix/sysv/linux/powerpc/powerpc32/sysdep-cancel.h (PSEUDO): diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index f30c88f301..02a84f4d9b 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -28,6 +28,7 @@ #include <dl-sysdep.h> #include <tls.h> #include <lowlevellock.h> +#include <kernel-features.h> #ifndef NEED_SEPARATE_REGISTER_STACK diff --git a/nptl/init.c b/nptl/init.c index 95b9c7be5d..ffb50b4cca 100644 --- a/nptl/init.c +++ b/nptl/init.c @@ -33,6 +33,7 @@ #include <shlib-compat.h> #include <smp.h> #include <lowlevellock.h> +#include <kernel-features.h> /* Size and alignment of static TLS block. */ diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index 82c0f1ecf6..17b6492ad8 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -96,9 +96,22 @@ enum PTHREAD_MUTEX_PP_ADAPTIVE_NP = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ADAPTIVE_NP }; +#define PTHREAD_MUTEX_PSHARED_BIT 128 #define PTHREAD_MUTEX_TYPE(m) \ - ((m)->__data.__kind) + ((m)->__data.__kind & 127) + +#if LLL_PRIVATE == 0 && LLL_SHARED == 128 +# define PTHREAD_MUTEX_PSHARED(m) \ + ((m)->__data.__kind & 128) +#else +# define PTHREAD_MUTEX_PSHARED(m) \ + (((m)->__data.__kind & 128) ? LLL_SHARED : LLL_PRIVATE) +#endif + +/* The kernel when waking robust mutexes on exit never uses + FUTEX_PRIVATE_FLAG FUTEX_WAKE. */ +#define PTHREAD_ROBUST_MUTEX_PSHARED(m) LLL_SHARED /* Ceiling in __data.__lock. __data.__lock is signed, so don't use the MSB bit in there, but in the mask also include that bit, diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c index 5c0d76effc..22523c2973 100644 --- a/nptl/pthread_cond_broadcast.c +++ b/nptl/pthread_cond_broadcast.c @@ -32,8 +32,10 @@ int __pthread_cond_broadcast (cond) pthread_cond_t *cond; { + int pshared = (cond->__data.__mutex == (void *) ~0l) + ? LLL_SHARED : LLL_PRIVATE; /* Make sure we are alone. */ - lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); /* Are there any waiters to be woken? */ if (cond->__data.__total_seq > cond->__data.__wakeup_seq) @@ -47,7 +49,7 @@ __pthread_cond_broadcast (cond) ++cond->__data.__broadcast_seq; /* We are done. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); /* Do not use requeue for pshared condvars. */ if (cond->__data.__mutex == (void *) ~0l) @@ -57,21 +59,22 @@ __pthread_cond_broadcast (cond) pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; /* XXX: Kernel so far doesn't support requeue to PI futex. */ - if (__builtin_expect (mut->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP, - 0)) + /* XXX: Kernel so far can only requeue to the same type of futex, + in this case private (we don't requeue for pshared condvars). */ + if (__builtin_expect (mut->__data.__kind + & (PTHREAD_MUTEX_PRIO_INHERIT_NP + | PTHREAD_MUTEX_PSHARED_BIT), 0)) goto wake_all; /* lll_futex_requeue returns 0 for success and non-zero for errors. */ if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1, INT_MAX, &mut->__data.__lock, - futex_val), 0)) + futex_val, LLL_PRIVATE), 0)) { /* The requeue functionality is not available. */ wake_all: - lll_futex_wake (&cond->__data.__futex, INT_MAX, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared); } /* That's all. */ @@ -79,7 +82,7 @@ __pthread_cond_broadcast (cond) } /* We are done. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); return 0; } diff --git a/nptl/pthread_cond_destroy.c b/nptl/pthread_cond_destroy.c index 53b5cd272f..35135a68bc 100644 --- a/nptl/pthread_cond_destroy.c +++ b/nptl/pthread_cond_destroy.c @@ -26,14 +26,17 @@ int __pthread_cond_destroy (cond) pthread_cond_t *cond; { + int pshared = (cond->__data.__mutex == (void *) ~0l) + ? LLL_SHARED : LLL_PRIVATE; + /* Make sure we are alone. */ - lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); if (cond->__data.__total_seq > cond->__data.__wakeup_seq) { /* If there are still some waiters which have not been woken up, this is an application bug. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); return EBUSY; } @@ -60,19 +63,16 @@ __pthread_cond_destroy (cond) { pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; lll_futex_wake (&mut->__data.__lock, INT_MAX, - // XYZ check mutex flag - LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mut)); } do { - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); - lll_futex_wait (&cond->__data.__nwaiters, nwaiters, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wait (&cond->__data.__nwaiters, nwaiters, pshared); - lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); nwaiters = cond->__data.__nwaiters; } diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c index f2de58fa1d..023bbb5e9b 100644 --- a/nptl/pthread_cond_signal.c +++ b/nptl/pthread_cond_signal.c @@ -32,8 +32,11 @@ int __pthread_cond_signal (cond) pthread_cond_t *cond; { + int pshared = (cond->__data.__mutex == (void *) ~0l) + ? LLL_SHARED : LLL_PRIVATE; + /* Make sure we are alone. */ - lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); /* Are there any waiters to be woken? */ if (cond->__data.__total_seq > cond->__data.__wakeup_seq) @@ -45,18 +48,14 @@ __pthread_cond_signal (cond) /* Wake one. */ if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1, 1, &cond->__data.__lock, - // XYZ check mutex flag - LLL_SHARED), - 0)) + pshared), 0)) return 0; - lll_futex_wake (&cond->__data.__futex, 1, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wake (&cond->__data.__futex, 1, pshared); } /* We are done. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); return 0; } diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c index a8d95dc224..9d268e911e 100644 --- a/nptl/pthread_cond_timedwait.c +++ b/nptl/pthread_cond_timedwait.c @@ -23,6 +23,7 @@ #include <lowlevellock.h> #include <pthread.h> #include <pthreadP.h> +#include <kernel-features.h> #include <shlib-compat.h> @@ -53,14 +54,17 @@ __pthread_cond_timedwait (cond, mutex, abstime) if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) return EINVAL; + int pshared = (cond->__data.__mutex == (void *) ~0l) + ? LLL_SHARED : LLL_PRIVATE; + /* Make sure we are along. */ - lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); /* Now we can release the mutex. */ int err = __pthread_mutex_unlock_usercnt (mutex, 0); if (err) { - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); return err; } @@ -146,22 +150,20 @@ __pthread_cond_timedwait (cond, mutex, abstime) unsigned int futex_val = cond->__data.__futex; /* Prepare to wait. Release the condvar futex. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); /* Enable asynchronous cancellation. Required by the standard. */ cbuffer.oldtype = __pthread_enable_asynccancel (); /* Wait until woken by signal or broadcast. */ err = lll_futex_timed_wait (&cond->__data.__futex, - futex_val, &rt, - // XYZ check mutex flag - LLL_SHARED); + futex_val, &rt, pshared); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (cbuffer.oldtype); /* We are going to look at shared data again, so get the lock. */ - lll_lock(cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); /* If a broadcast happened, we are done. */ if (cbuffer.bc_seq != cond->__data.__broadcast_seq) @@ -198,12 +200,10 @@ __pthread_cond_timedwait (cond, mutex, abstime) and it can be successfully destroyed. */ if (cond->__data.__total_seq == -1ULL && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) - lll_futex_wake (&cond->__data.__nwaiters, 1, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wake (&cond->__data.__nwaiters, 1, pshared); /* We are done with the condvar. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); /* The cancellation handling is back to normal, remove the handler. */ __pthread_cleanup_pop (&buffer, 0); diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c index 679655f8fd..670fba5736 100644 --- a/nptl/pthread_cond_wait.c +++ b/nptl/pthread_cond_wait.c @@ -43,9 +43,11 @@ __condvar_cleanup (void *arg) struct _condvar_cleanup_buffer *cbuffer = (struct _condvar_cleanup_buffer *) arg; unsigned int destroying; + int pshared = (cbuffer->cond->__data.__mutex == (void *) ~0l) + ? LLL_SHARED : LLL_PRIVATE; /* We are going to modify shared data. */ - lll_lock (cbuffer->cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cbuffer->cond->__data.__lock, pshared); if (cbuffer->bc_seq == cbuffer->cond->__data.__broadcast_seq) { @@ -71,20 +73,16 @@ __condvar_cleanup (void *arg) if (cbuffer->cond->__data.__total_seq == -1ULL && cbuffer->cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) { - lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1, pshared); destroying = 1; } /* We are done. */ - lll_unlock (cbuffer->cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cbuffer->cond->__data.__lock, pshared); /* Wake everybody to make sure no condvar signal gets lost. */ if (! destroying) - lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, pshared); /* Get the mutex before returning unless asynchronous cancellation is in effect. */ @@ -100,15 +98,17 @@ __pthread_cond_wait (cond, mutex) struct _pthread_cleanup_buffer buffer; struct _condvar_cleanup_buffer cbuffer; int err; + int pshared = (cond->__data.__mutex == (void *) ~0l) + ? LLL_SHARED : LLL_PRIVATE; /* Make sure we are along. */ - lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); /* Now we can release the mutex. */ err = __pthread_mutex_unlock_usercnt (mutex, 0); if (__builtin_expect (err, 0)) { - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); return err; } @@ -144,21 +144,19 @@ __pthread_cond_wait (cond, mutex) unsigned int futex_val = cond->__data.__futex; /* Prepare to wait. Release the condvar futex. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); /* Enable asynchronous cancellation. Required by the standard. */ cbuffer.oldtype = __pthread_enable_asynccancel (); /* Wait until woken by signal or broadcast. */ - lll_futex_wait (&cond->__data.__futex, futex_val, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wait (&cond->__data.__futex, futex_val, pshared); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (cbuffer.oldtype); /* We are going to look at shared data again, so get the lock. */ - lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_lock (cond->__data.__lock, pshared); /* If a broadcast happened, we are done. */ if (cbuffer.bc_seq != cond->__data.__broadcast_seq) @@ -181,12 +179,10 @@ __pthread_cond_wait (cond, mutex) and it can be successfully destroyed. */ if (cond->__data.__total_seq == -1ULL && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) - lll_futex_wake (&cond->__data.__nwaiters, 1, - // XYZ check mutex flag - LLL_SHARED); + lll_futex_wake (&cond->__data.__nwaiters, 1, pshared); /* We are done with the condvar. */ - lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (cond->__data.__lock, pshared); /* The cancellation handling is back to normal, remove the handler. */ __pthread_cleanup_pop (&buffer, 0); diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c index 3ab2fa498d..e08b76a9b8 100644 --- a/nptl/pthread_create.c +++ b/nptl/pthread_create.c @@ -27,6 +27,7 @@ #include <atomic.h> #include <libc-internal.h> #include <resolv.h> +#include <kernel-features.h> #include <shlib-compat.h> diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c index 96f1fb00f8..d9b1ef0b98 100644 --- a/nptl/pthread_mutex_init.c +++ b/nptl/pthread_mutex_init.c @@ -1,4 +1,5 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 + Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -20,6 +21,7 @@ #include <assert.h> #include <errno.h> #include <string.h> +#include <kernel-features.h> #include "pthreadP.h" static const struct pthread_mutexattr default_attr = @@ -120,6 +122,12 @@ __pthread_mutex_init (mutex, mutexattr) break; } + /* The kernel when waking robust mutexes on exit never uses + FUTEX_PRIVATE_FLAG FUTEX_WAKE. */ + if ((imutexattr->mutexkind & (PTHREAD_MUTEXATTR_FLAG_PSHARED + | PTHREAD_MUTEXATTR_FLAG_ROBUST)) != 0) + mutex->__data.__kind |= PTHREAD_MUTEX_PSHARED_BIT; + /* Default values: mutex not used yet. */ // mutex->__count = 0; already done by memset // mutex->__owner = 0; already done by memset diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c index a82922e99a..ed98dfc6c3 100644 --- a/nptl/pthread_mutex_lock.c +++ b/nptl/pthread_mutex_lock.c @@ -27,9 +27,13 @@ #ifndef LLL_MUTEX_LOCK -# define LLL_MUTEX_LOCK(mutex) lll_lock (mutex, /* XYZ */ LLL_SHARED) -# define LLL_MUTEX_TRYLOCK(mutex) lll_trylock (mutex) -# define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_lock (mutex, id, /* XYZ */ LLL_SHARED) +# define LLL_MUTEX_LOCK(mutex) \ + lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)) +# define LLL_MUTEX_TRYLOCK(mutex) \ + lll_trylock ((mutex)->__data.__lock) +# define LLL_ROBUST_MUTEX_LOCK(mutex, id) \ + lll_robust_lock ((mutex)->__data.__lock, id, \ + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)) #endif @@ -62,7 +66,7 @@ __pthread_mutex_lock (mutex) } /* We have to get the mutex. */ - LLL_MUTEX_LOCK (mutex->__data.__lock); + LLL_MUTEX_LOCK (mutex); assert (mutex->__data.__owner == 0); mutex->__data.__count = 1; @@ -79,7 +83,7 @@ __pthread_mutex_lock (mutex) case PTHREAD_MUTEX_TIMED_NP: simple: /* Normal mutex. */ - LLL_MUTEX_LOCK (mutex->__data.__lock); + LLL_MUTEX_LOCK (mutex); assert (mutex->__data.__owner == 0); break; @@ -87,7 +91,7 @@ __pthread_mutex_lock (mutex) if (! __is_smp) goto simple; - if (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0) + if (LLL_MUTEX_TRYLOCK (mutex) != 0) { int cnt = 0; int max_cnt = MIN (MAX_ADAPTIVE_COUNT, @@ -96,7 +100,7 @@ __pthread_mutex_lock (mutex) { if (cnt++ >= max_cnt) { - LLL_MUTEX_LOCK (mutex->__data.__lock); + LLL_MUTEX_LOCK (mutex); break; } @@ -104,7 +108,7 @@ __pthread_mutex_lock (mutex) BUSY_WAIT_NOP; #endif } - while (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0); + while (LLL_MUTEX_TRYLOCK (mutex) != 0); mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8; } @@ -166,16 +170,15 @@ __pthread_mutex_lock (mutex) /* Check whether we already hold the mutex. */ if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0)) { - if (mutex->__data.__kind - == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) + int kind = PTHREAD_MUTEX_TYPE (mutex); + if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) { THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); return EDEADLK; } - if (mutex->__data.__kind - == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) + if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) { THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); @@ -191,14 +194,15 @@ __pthread_mutex_lock (mutex) } } - oldval = LLL_ROBUST_MUTEX_LOCK (mutex->__data.__lock, id); + oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id); if (__builtin_expect (mutex->__data.__owner == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) { /* This mutex is now not recoverable. */ mutex->__data.__count = 0; - lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (mutex->__data.__lock, + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); return ENOTRECOVERABLE; } @@ -410,8 +414,7 @@ __pthread_mutex_lock (mutex) if (oldval != ceilval) lll_futex_wait (&mutex->__data.__lock, ceilval | 2, - // XYZ check mutex flag - LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); } while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, ceilval | 2, ceilval) diff --git a/nptl/pthread_mutex_setprioceiling.c b/nptl/pthread_mutex_setprioceiling.c index 301fb63d21..836c9a3e93 100644 --- a/nptl/pthread_mutex_setprioceiling.c +++ b/nptl/pthread_mutex_setprioceiling.c @@ -47,12 +47,13 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling) /* Check whether we already hold the mutex. */ bool locked = false; + int kind = PTHREAD_MUTEX_TYPE (mutex); if (mutex->__data.__owner == THREAD_GETMEM (THREAD_SELF, tid)) { - if (mutex->__data.__kind == PTHREAD_MUTEX_PP_ERRORCHECK_NP) + if (kind == PTHREAD_MUTEX_PP_ERRORCHECK_NP) return EDEADLK; - if (mutex->__data.__kind == PTHREAD_MUTEX_PP_RECURSIVE_NP) + if (kind == PTHREAD_MUTEX_PP_RECURSIVE_NP) locked = true; } @@ -81,8 +82,7 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling) if (oldval != ceilval) lll_futex_wait (&mutex->__data.__lock, ceilval | 2, - // XYZ check mutex flag - LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); } while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, ceilval | 2, ceilval) @@ -113,8 +113,7 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling) atomic_full_barrier (); lll_futex_wake (&mutex->__data.__lock, INT_MAX, - // XYZ check mutex flag - LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); return 0; } diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c index 7a0ed57eaa..4bf0efea34 100644 --- a/nptl/pthread_mutex_timedlock.c +++ b/nptl/pthread_mutex_timedlock.c @@ -57,7 +57,7 @@ pthread_mutex_timedlock (mutex, abstime) /* We have to get the mutex. */ result = lll_timedlock (mutex->__data.__lock, abstime, - /* XYZ */ LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); if (result != 0) goto out; @@ -78,7 +78,7 @@ pthread_mutex_timedlock (mutex, abstime) simple: /* Normal mutex. */ result = lll_timedlock (mutex->__data.__lock, abstime, - /* XYZ */ LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); break; case PTHREAD_MUTEX_ADAPTIVE_NP: @@ -95,7 +95,7 @@ pthread_mutex_timedlock (mutex, abstime) if (cnt++ >= max_cnt) { result = lll_timedlock (mutex->__data.__lock, abstime, - /* XYZ */ LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); break; } @@ -152,16 +152,15 @@ pthread_mutex_timedlock (mutex, abstime) /* Check whether we already hold the mutex. */ if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0)) { - if (mutex->__data.__kind - == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) + int kind = PTHREAD_MUTEX_TYPE (mutex); + if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) { THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); return EDEADLK; } - if (mutex->__data.__kind - == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) + if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) { THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); @@ -178,14 +177,15 @@ pthread_mutex_timedlock (mutex, abstime) } result = lll_robust_timedlock (mutex->__data.__lock, abstime, id, - /* XYZ */ LLL_SHARED); + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); if (__builtin_expect (mutex->__data.__owner == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) { /* This mutex is now not recoverable. */ mutex->__data.__count = 0; - lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (mutex->__data.__lock, + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); return ENOTRECOVERABLE; } @@ -446,8 +446,7 @@ pthread_mutex_timedlock (mutex, abstime) lll_futex_timed_wait (&mutex->__data.__lock, ceilval | 2, &rt, - // XYZ check mutex flag - LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); } } while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c index 9478d382c2..f6e24d4138 100644 --- a/nptl/pthread_mutex_trylock.c +++ b/nptl/pthread_mutex_trylock.c @@ -115,16 +115,15 @@ __pthread_mutex_trylock (mutex) /* Check whether we already hold the mutex. */ if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0)) { - if (mutex->__data.__kind - == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) + int kind = PTHREAD_MUTEX_TYPE (mutex); + if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) { THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); return EDEADLK; } - if (mutex->__data.__kind - == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) + if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) { THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); @@ -154,7 +153,8 @@ __pthread_mutex_trylock (mutex) /* This mutex is now not recoverable. */ mutex->__data.__count = 0; if (oldval == id) - lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (mutex->__data.__lock, + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); return ENOTRECOVERABLE; } diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c index 6226089ebe..d33d0593d8 100644 --- a/nptl/pthread_mutex_unlock.c +++ b/nptl/pthread_mutex_unlock.c @@ -61,7 +61,7 @@ __pthread_mutex_unlock_usercnt (mutex, decr) --mutex->__data.__nusers; /* Unlock. */ - lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED); + lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)); break; case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP: @@ -115,7 +115,8 @@ __pthread_mutex_unlock_usercnt (mutex, decr) --mutex->__data.__nusers; /* Unlock. */ - lll_robust_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED); + lll_robust_unlock (mutex->__data.__lock, + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); break; @@ -242,8 +243,7 @@ __pthread_mutex_unlock_usercnt (mutex, decr) if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1) lll_futex_wake (&mutex->__data.__lock, 1, - // XYZ check mutex flag - LLL_SHARED); + PTHREAD_MUTEX_PSHARED (mutex)); int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT; return __pthread_tpp_change_priority (oldprio, -1); diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h index 41a54d4b0d..41c0be1978 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h @@ -1,5 +1,5 @@ /* Machine-specific pthread type layouts. Alpha version. - Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -126,9 +126,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h index f3f291979a..4f6796449a 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h @@ -24,6 +24,7 @@ #include <bits/pthreadtypes.h> #include <atomic.h> #include <sysdep.h> +#include <kernel-features.h> #define __NR_futex 394 @@ -103,24 +104,24 @@ while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ - (nr_move), (mutex), (val)); \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ + (nr_wake), (nr_move), (mutex), (val)); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \ +#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_WAKE_OP, (nr_wake), \ - (nr_wake2), (futexp2), \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_WAKE_OP, private), \ + (nr_wake), (nr_wake2), (futexp2), \ FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 122d83afee..776c47f6cc 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -83,11 +83,18 @@ __pthread_cond_broadcast: je 9f /* XXX: The kernel so far doesn't support requeue to PI futex. */ - testl $PI_BIT, MUTEX_KIND(%edi) + /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same + type of futex (private resp. shared). */ + testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi) jne 9f /* Wake up all threads. */ - movl $FUTEX_CMP_REQUEUE, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx +#else + movl %gs:PRIVATE_FUTEX, %ecx + orl $FUTEX_CMP_REQUEUE, %ecx +#endif movl $SYS_futex, %eax movl $0x7fffffff, %esi movl $1, %edx @@ -132,28 +139,63 @@ __pthread_cond_broadcast: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b /* Unlock in loop requires waekup. */ 5: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b /* Unlock in loop requires waekup. */ 7: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 8b 9: /* The futex requeue functionality is not available. */ movl $0x7fffffff, %edx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax ENTER_KERNEL jmp 10b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index e3510c8ab1..36a18036c5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -70,7 +70,18 @@ __pthread_cond_signal: /* Wake up one thread. */ pushl %esi pushl %ebp - movl $FUTEX_WAKE_OP, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE_OP, %ecx movl $SYS_futex, %eax movl $1, %edx movl $1, %esi @@ -92,7 +103,9 @@ __pthread_cond_signal: popl %ebx ret -7: movl $FUTEX_WAKE, %ecx +7: /* %ecx should be either FUTEX_WAKE_OP or + FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx movl $SYS_futex, %eax /* %edx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ @@ -106,8 +119,16 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: movl %edi, %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b @@ -118,8 +139,16 @@ __pthread_cond_signal: #else leal cond_lock(%edi), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%edi) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index 79a7497e8c..83f8db25bb 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -22,6 +22,7 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -157,7 +158,20 @@ __pthread_cond_timedwait: movl %eax, (%esp) leal 4(%esp), %esi - xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -231,7 +245,18 @@ __pthread_cond_timedwait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -279,8 +304,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -292,8 +325,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -304,8 +345,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -316,8 +365,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -338,8 +395,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -400,8 +465,16 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -440,7 +513,18 @@ __condvar_tw_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -459,15 +543,34 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -587,12 +690,12 @@ __condvar_tw_cleanup: .uleb128 20 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 .byte 0x40+.Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -614,7 +717,8 @@ __condvar_tw_cleanup: .byte 0x40+.LSbl1-.Lpop_edi # DW_CFA_advance_loc+N .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE .byte 0x85 # DW_CFA_offset %ebp @@ -625,14 +729,15 @@ __condvar_tw_cleanup: .uleb128 4 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 + .byte 4 # DW_CFA_advance_loc4 #if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .4byte .LSbl4-.LSbl3 #else - .byte 4 # DW_CFA_advance_loc4 - .long .LSbl5-.LSbl3 + .4byte .LSbl5-.LSbl3 #endif .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 68741bffe8..5b301979b5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -22,6 +22,7 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <tcb-offsets.h> +#include <kernel-features.h> .text @@ -100,7 +101,20 @@ __pthread_cond_wait: 4: call __pthread_enable_asynccancel movl %eax, (%esp) - movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -161,7 +175,18 @@ __pthread_cond_wait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -197,8 +222,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -210,8 +243,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -222,8 +263,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -234,8 +283,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -256,8 +313,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -292,8 +357,16 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -332,7 +405,18 @@ __condvar_w_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -351,15 +435,34 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -473,12 +576,12 @@ __condvar_w_cleanup: .uleb128 16 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 2 # DW_CFA_advance_loc1 - .byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 .byte 0x40+ .Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -502,13 +605,16 @@ __condvar_w_cleanup: .uleb128 3 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl4-.LSbl3 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE .align 4 diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S index d8a62ed2b7..d8f1bd54a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -68,7 +69,7 @@ __pthread_rwlock_rdlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S index 71b97c60f5..0d96e03252 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -99,7 +100,7 @@ pthread_rwlock_timedrdlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S index c002472085..e78fdf6dda 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -97,7 +98,7 @@ pthread_rwlock_timedwrlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S index fdad432e30..a23e1b50a8 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S @@ -20,6 +20,7 @@ #include <sysdep.h> #include <lowlevellock.h> #include <lowlevelrwlock.h> +#include <kernel-features.h> .text @@ -73,7 +74,7 @@ __pthread_rwlock_unlock: jne 7f 8: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%edi), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S index 3f55c82930..65b99fe7d3 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -66,7 +67,7 @@ __pthread_rwlock_wrlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i786/Implies b/nptl/sysdeps/unix/sysv/linux/i386/i786/Implies new file mode 100644 index 0000000000..7cb7d9a678 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i786/Implies @@ -0,0 +1,2 @@ +# The PPro and PII cores are mostly the same. +unix/sysv/linux/i386/i686 diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 2f663aa68b..8f67616af7 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -83,7 +83,7 @@ ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) \ : (fl)) \ : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \ - asm ("andl %%fs:%P1, %0" : "+r" (__fl) \ + asm ("andl %%gs:%P1, %0" : "+r" (__fl) \ : "i" (offsetof (struct pthread, header.private_futex))); \ __fl | (fl); })) # endif diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h index 892769dca4..330717f079 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -126,9 +126,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h index 3c28a397ea..ada79851e2 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h @@ -25,6 +25,7 @@ #include <bits/pthreadtypes.h> #include <ia64intrin.h> #include <atomic.h> +#include <kernel-features.h> #define __NR_futex 1230 #define FUTEX_WAIT 0 @@ -103,18 +104,20 @@ do \ while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \ ({ \ - DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_CMP_REQUEUE, \ + DO_INLINE_SYSCALL(futex, 6, (long) (ftx), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private), \ (int) (nr_wake), (int) (nr_move), (long) (mutex), \ (int) val); \ _r10 == -1; \ }) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2) \ +#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2, private) \ ({ \ - DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_WAKE_OP, \ + DO_INLINE_SYSCALL(futex, 6, (long) (ftx), \ + __lll_private_flag (FUTEX_WAKE_OP, private), \ (int) (nr_wake), (int) (nr_wake2), (long) (ftx2), \ FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ _r10 == -1; \ @@ -152,6 +155,7 @@ extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden; __lll_lock_wait_private (__futex); \ else \ __lll_lock_wait (__futex, private); \ + } \ })) #define lll_lock(futex, private) __lll_lock (&(futex), private) diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h index 41804d1372..80b9a4369e 100644 --- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h @@ -24,6 +24,7 @@ #include <sys/param.h> #include <bits/pthreadtypes.h> #include <atomic.h> +#include <kernel-features.h> #ifndef __NR_futex # define __NR_futex 221 @@ -107,14 +108,14 @@ while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ - (nr_move), (mutex), (val)); \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ + (nr_wake), (nr_move), (mutex), (val)); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/pthread_spin_unlock.c b/nptl/sysdeps/unix/sysv/linux/powerpc/pthread_spin_unlock.c new file mode 100644 index 0000000000..90f2dc67c0 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/pthread_spin_unlock.c @@ -0,0 +1,29 @@ +/* pthread_spin_unlock -- unlock a spin lock. PowerPC version. + Copyright (C) 2007 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include "pthreadP.h" +#include <lowlevellock.h> + +int +pthread_spin_unlock (pthread_spinlock_t *lock) +{ + __asm __volatile (__lll_rel_instr ::: "memory"); + *lock = 0; + return 0; +} diff --git a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym index a1b6794260..d985c6a79b 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym +++ b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym @@ -4,3 +4,4 @@ MUTEX_KIND offsetof (pthread_mutex_t, __data.__kind) PI_BIT PTHREAD_MUTEX_PRIO_INHERIT_NP +PS_BIT PTHREAD_MUTEX_PSHARED_BIT diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c index 81ecd6556b..93841c5b3e 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c +++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -1,8 +1,12 @@ #include <pthreadP.h> -#define LLL_MUTEX_LOCK(mutex) lll_cond_lock (mutex, /* XYZ */ LLL_SHARED) -#define LLL_MUTEX_TRYLOCK(mutex) lll_cond_trylock (mutex) -#define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_cond_lock (mutex, id, /* XYZ */ LLL_SHARED) +#define LLL_MUTEX_LOCK(mutex) \ + lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)) +#define LLL_MUTEX_TRYLOCK(mutex) \ + lll_cond_trylock ((mutex)->__data.__lock) +#define LLL_ROBUST_MUTEX_LOCK(mutex, id) \ + lll_robust_cond_lock ((mutex)->__data.__lock, id, \ + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)) #define __pthread_mutex_lock __pthread_mutex_cond_lock #define NO_INCR diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h index c77031d7bb..c7345cd9a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -159,9 +159,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; @@ -175,9 +175,12 @@ typedef union unsigned int __writer_wakeup; unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; + unsigned char __pad1; + unsigned char __pad2; + unsigned char __shared; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ - unsigned int __flags; + unsigned char __flags; int __writer; } __data; # endif diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h index ad4d27300f..7fee435f12 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h @@ -24,6 +24,7 @@ #include <sys/param.h> #include <bits/pthreadtypes.h> #include <atomic.h> +#include <kernel-features.h> #define SYS_futex 238 #define FUTEX_WAIT 0 @@ -93,7 +94,7 @@ ({ \ register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ register unsigned long int __r3 asm ("3") \ - __lll_private_flag (FUTEX_WAKE, private); \ + = __lll_private_flag (FUTEX_WAKE, private); \ register unsigned long int __r4 asm ("4") = (unsigned long int) (nr); \ register unsigned long int __result asm ("2"); \ \ @@ -117,10 +118,11 @@ /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \ ({ \ register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ - register unsigned long int __r3 asm ("3") = FUTEX_CMP_REQUEUE; \ + register unsigned long int __r3 asm ("3") \ + = __lll_private_flag (FUTEX_CMP_REQUEUE, private); \ register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \ register unsigned long int __r5 asm ("5") = (long int) (nr_move); \ register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex); \ @@ -137,10 +139,11 @@ /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2) \ +#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \ ({ \ register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ - register unsigned long int __r3 asm ("3") = FUTEX_WAKE_OP; \ + register unsigned long int __r3 asm ("3") \ + = __lll_private_flag (FUTEX_WAKE_OP, private); \ register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \ register unsigned long int __r5 asm ("5") = (long int) (nr_wake2); \ register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \ diff --git a/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S index 17c1e6f567..fde4f57b2a 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S @@ -22,6 +22,7 @@ #include <lowlevelcond.h> #include <pthread-errnos.h> #include "lowlevel-atomic.h" +#include <kernel-features.h> .text diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h index e734c1205e..faf0584868 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h @@ -1,5 +1,5 @@ /* Machine-specific pthread type layouts. SPARC version. - Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -160,9 +160,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; @@ -176,9 +176,12 @@ typedef union unsigned int __writer_wakeup; unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; + unsigned char __pad1; + unsigned char __pad2; + unsigned char __shared; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ - unsigned int __flags; + unsigned char __flags; int __writer; } __data; # endif diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h index 38692bbd2d..f4512b2622 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h @@ -24,6 +24,7 @@ #include <sys/param.h> #include <bits/pthreadtypes.h> #include <atomic.h> +#include <kernel-features.h> #define FUTEX_WAIT 0 @@ -96,14 +97,14 @@ }) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ - (nr_move), (mutex), (val)); \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ + (nr_wake), (nr_move), (mutex), (val)); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) @@ -121,14 +122,14 @@ /* Avoid FUTEX_WAKE_OP if supporting pre-v9 CPUs. */ # define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) 1 #else -# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \ +# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_WAKE_OP, (nr_wake), \ - (nr_wake2), (futexp2), \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_WAKE_OP, private), \ + (nr_wake), (nr_wake2), (futexp2), \ FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 192d203926..2cd69a14ce 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -533,7 +533,7 @@ LLL_STUB_UNWIND_INFO_END while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \ ({ int __res; \ register int __nr_move __asm ("r10") = nr_move; \ register void *__mutex __asm ("r8") = mutex; \ @@ -541,7 +541,8 @@ LLL_STUB_UNWIND_INFO_END __asm __volatile ("syscall" \ : "=a" (__res) \ : "0" (__NR_futex), "D" ((void *) ftx), \ - "S" (FUTEX_CMP_REQUEUE), "d" (nr_wake), \ + "S" (__lll_private_flag (FUTEX_CMP_REQUEUE, \ + private)), "d" (nr_wake), \ "r" (__nr_move), "r" (__mutex), "r" (__val) \ : "cx", "r11", "cc", "memory"); \ __res < 0; }) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 0c619bf271..6155255eb0 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -71,11 +71,18 @@ __pthread_cond_broadcast: je 9f /* XXX: The kernel so far doesn't support requeue to PI futex. */ - testl $PI_BIT, MUTEX_KIND(%r8) + /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same + type of futex (private resp. shared). */ + testl $(PI_BIT | PS_BIT), MUTEX_KIND(%r8) jne 9f /* Wake up all threads. */ - movl $FUTEX_CMP_REQUEUE, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi +#else + movl %fs:PRIVATE_FUTEX, %esi + orl $FUTEX_CMP_REQUEUE, %esi +#endif movl $SYS_futex, %eax movl $1, %edx movl $0x7fffffff, %r10d @@ -104,8 +111,10 @@ __pthread_cond_broadcast: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -114,22 +123,36 @@ __pthread_cond_broadcast: /* Unlock in loop requires wakeup. */ 5: addq $cond_lock-cond_futex, %rdi - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 6b /* Unlock in loop requires wakeup. */ 7: addq $cond_lock-cond_futex, %rdi - /* XYZ */ + cmpq $-1, %r8 + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake subq $cond_lock-cond_futex, %rdi jmp 8b 9: /* The futex requeue functionality is not available. */ + cmpq $-1, %r8 movl $0x7fffffff, %edx - movl $FUTEX_WAKE, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif movl $SYS_futex, %eax syscall jmp 10b diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 2fc9d1fad7..8f65f2cd69 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -55,10 +55,20 @@ __pthread_cond_signal: addl $1, (%rdi) /* Wake up one thread. */ - movl $FUTEX_WAKE_OP, %esi - movl $SYS_futex, %eax + cmpq $-1, dep_mutex(%r8) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE_OP, %eax + movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE_OP, %esi +#endif movl $1, %r10d + movl $SYS_futex, %eax #if cond_lock != 0 addq $cond_lock, %r8 #endif @@ -75,7 +85,9 @@ __pthread_cond_signal: xorl %eax, %eax retq -7: movl $FUTEX_WAKE, %esi +7: /* %esi should be either FUTEX_WAKE_OP or + FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi movl $SYS_futex, %eax /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ @@ -98,8 +110,10 @@ __pthread_cond_signal: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -109,8 +123,13 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: movq %r8, %rdi - /* XYZ */ +#if cond_lock != 0 + addq $cond_lock, %rdi +#endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 6b .size __pthread_cond_signal, .-__pthread_cond_signal diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 003069fb6b..a5de670866 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -23,6 +23,8 @@ #include <lowlevelcond.h> #include <pthread-errnos.h> +#include <kernel-features.h> + /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 @@ -186,12 +188,20 @@ __pthread_cond_timedwait: movl %eax, (%rsp) leaq 24(%rsp), %r10 -#if FUTEX_WAIT == 0 - xorl %esi, %esi + cmpq $-1, dep_mutex(%rdi) + movq %r12, %rdx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT, %eax + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi #else - movl $FUTEX_WAIT, %esi + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi +# if FUTEX_WAIT != 0 + orl $FUTEX_WAIT, %esi +# endif #endif - movq %r12, %rdx addq $cond_futex, %rdi movl $SYS_futex, %eax syscall @@ -251,9 +261,19 @@ __pthread_cond_timedwait: jne 25f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi @@ -292,8 +312,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait jmp 2b @@ -302,8 +324,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 4b @@ -312,8 +336,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -325,8 +351,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 11b @@ -344,8 +372,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake 17: movq (%rsp), %rax diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 34ef2c7b77..2c17dc03a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -23,6 +23,8 @@ #include <lowlevelcond.h> #include <tcb-offsets.h> +#include <kernel-features.h> + .text @@ -49,8 +51,10 @@ __condvar_cleanup: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -81,9 +85,19 @@ __condvar_cleanup: jne 4f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi movl $1, %r12d @@ -98,16 +112,28 @@ __condvar_cleanup: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testq %r12, %r12 jnz 5f addq $cond_futex, %rdi - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_futex(%rdi) movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif movl $SYS_futex, %eax syscall @@ -216,12 +242,20 @@ __pthread_cond_wait: xorq %r10, %r10 movq %r12, %rdx addq $cond_futex-cond_lock, %rdi - movl $SYS_futex, %eax -#if FUTEX_WAIT == 0 - xorl %esi, %esi + cmpq $-1, dep_mutex-cond_futex(%rdi) +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT, %eax + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi #else - movl $FUTEX_WAIT, %esi + movl $FUTEX_WAIT, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi +# if FUTEX_WAIT != 0 + orl $FUTEX_WAIT, %esi +# endif #endif + movl $SYS_futex, %eax syscall movl (%rsp), %edi @@ -267,9 +301,19 @@ __pthread_cond_wait: jne 17f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi @@ -302,8 +346,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait jmp 2b @@ -312,8 +358,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 4b @@ -322,8 +370,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -335,8 +385,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 11b @@ -354,8 +406,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake 13: movq %r10, %rax |