From 203c77a45eb7b63f3a4339e1a74174cce2ff7b25 Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Mon, 29 Aug 2005 19:34:43 +0000 Subject: * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. (lll_futex_wake_unlock): Define. * sysdeps/unix/sysv/linux/alpha/lowlevellock.h (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. (lll_futex_wake_unlock): Define. * sysdeps/unix/sysv/linux/ia64/lowlevellock.h (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. (lll_futex_wake_unlock): Define. * sysdeps/unix/sysv/linux/s390/lowlevellock.h (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. (lll_futex_wake_unlock): Define. * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. (lll_futex_wake_unlock): Define. * sysdeps/pthread/pthread_cond_signal.c (__pthread_cond_signal): Use lll_futex_wake_unlock. * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. (__pthread_cond_signal): Use FUTEX_WAKE_OP. * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. (__pthread_cond_signal): Use FUTEX_WAKE_OP. --- nptl/ChangeLog | 26 ++++++++++++ nptl/sysdeps/pthread/pthread_cond_signal.c | 5 +++ nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h | 16 ++++++++ .../sysv/linux/i386/i486/pthread_cond_signal.S | 47 ++++++++++++++++------ nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h | 11 +++++ .../sysdeps/unix/sysv/linux/powerpc/lowlevellock.h | 15 +++++++ nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h | 23 +++++++++++ nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h | 15 +++++++ .../unix/sysv/linux/x86_64/pthread_cond_signal.S | 27 ++++++++++++- 9 files changed, 171 insertions(+), 14 deletions(-) diff --git a/nptl/ChangeLog b/nptl/ChangeLog index b6d66d87bb..d728d92316 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,3 +1,29 @@ +2005-08-29 Jakub Jelinek + + * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (FUTEX_WAKE_OP, + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. + (lll_futex_wake_unlock): Define. + * sysdeps/unix/sysv/linux/alpha/lowlevellock.h (FUTEX_WAKE_OP, + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. + (lll_futex_wake_unlock): Define. + * sysdeps/unix/sysv/linux/ia64/lowlevellock.h (FUTEX_WAKE_OP, + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. + (lll_futex_wake_unlock): Define. + * sysdeps/unix/sysv/linux/s390/lowlevellock.h (FUTEX_WAKE_OP, + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. + (lll_futex_wake_unlock): Define. + * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (FUTEX_WAKE_OP, + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. + (lll_futex_wake_unlock): Define. + * sysdeps/pthread/pthread_cond_signal.c (__pthread_cond_signal): Use + lll_futex_wake_unlock. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S + (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. + (__pthread_cond_signal): Use FUTEX_WAKE_OP. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S + (FUTEX_WAKE_OP, FUTEX_OP_CLEAR_WAKE_IF_GT_ONE): Define. + (__pthread_cond_signal): Use FUTEX_WAKE_OP. + 2005-08-23 Ulrich Drepper * sysdeps/unix/sysv/linux/timer_routines.c (timer_helper_thread): diff --git a/nptl/sysdeps/pthread/pthread_cond_signal.c b/nptl/sysdeps/pthread/pthread_cond_signal.c index f5623480f8..5a9bbcad91 100644 --- a/nptl/sysdeps/pthread/pthread_cond_signal.c +++ b/nptl/sysdeps/pthread/pthread_cond_signal.c @@ -43,6 +43,11 @@ __pthread_cond_signal (cond) ++cond->__data.__futex; /* Wake one. */ + if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1, + 1, &cond->__data.__lock), + 0)) + return 0; + lll_futex_wake (&cond->__data.__futex, 1); } diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h index fd4a7ca4bb..ab325d2b06 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h @@ -31,6 +31,8 @@ #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 #define FUTEX_CMP_REQUEUE 4 +#define FUTEX_WAKE_OP 5 +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -73,6 +75,20 @@ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \ + ({ \ + INTERNAL_SYSCALL_DECL (__err); \ + long int __ret; \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, \ + (futexp), FUTEX_WAKE_OP, (nr_wake), \ + (nr_wake2), (futexp2), \ + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ + INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ + }) + + + static inline int __attribute__((always_inline)) __lll_mutex_trylock(int *futex) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index 3c5a1db59c..d0f931ff15 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -31,7 +31,9 @@ #define SYS_futex 240 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 -#define FUTEX_REQUEUE 3 +#define FUTEX_WAKE_OP 5 + +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) #define EINVAL 22 @@ -79,21 +81,46 @@ __pthread_cond_signal: addl $1, (%ebx) /* Wake up one thread. */ - movl $FUTEX_WAKE, %ecx + pushl %esi + pushl %ebp + movl $FUTEX_WAKE_OP, %ecx movl $SYS_futex, %eax movl $1, %edx + movl $1, %esi + movl $FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %ebp + /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for + sysenter. + ENTER_KERNEL */ + int $0x80 + popl %ebp + popl %esi + + /* For any kind of error, we try again with WAKE. + The general test also covers running on old kernels. */ + cmpl $-4095, %eax + jae 7f + +6: xorl %eax, %eax + popl %edi + popl %ebx + ret + +7: movl $FUTEX_WAKE, %ecx + movl $SYS_futex, %eax + /* %edx should be 1 already from $FUTEX_WAKE_OP syscall. + movl $1, %edx */ ENTER_KERNEL /* Unlock. Note that at this point %edi always points to cond_lock. */ 4: LOCK subl $1, (%edi) - jne 5f + je 6b -6: xorl %eax, %eax - popl %edi - popl %ebx - ret + /* Unlock in loop requires wakeup. */ +5: movl %edi, %eax + call __lll_mutex_unlock_wake + jmp 6b /* Initial locking failed. */ 1: @@ -105,10 +132,6 @@ __pthread_cond_signal: call __lll_mutex_lock_wait jmp 2b - /* Unlock in loop requires wakeup. */ -5: movl %edi, %eax - call __lll_mutex_unlock_wake - jmp 6b .size __pthread_cond_signal, .-__pthread_cond_signal versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h index e13358ffef..4219fe2716 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h @@ -31,6 +31,8 @@ #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 #define FUTEX_CMP_REQUEUE 4 +#define FUTEX_WAKE_OP 5 +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) /* Delay in spinlock loop. */ #define BUSY_WAIT_NOP asm ("hint @pause") @@ -62,6 +64,15 @@ _r10 == -1; \ }) +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2) \ +({ \ + DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_WAKE_OP, \ + (int) (nr_wake), (int) (nr_wake2), (long) (ftx2), \ + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ + _r10 == -1; \ +}) + #define __lll_mutex_trylock(futex) \ (atomic_compare_and_exchange_val_acq (futex, 1, 0) != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h index 1f2f481d64..f9eaa11e98 100644 --- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h @@ -33,6 +33,8 @@ #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 #define FUTEX_CMP_REQUEUE 4 +#define FUTEX_WAKE_OP 5 +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -79,6 +81,19 @@ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \ + ({ \ + INTERNAL_SYSCALL_DECL (__err); \ + long int __ret; \ + \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, \ + (futexp), FUTEX_WAKE_OP, (nr_wake), \ + (nr_wake2), (futexp2), \ + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ + INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ + }) + #ifdef UP # define __lll_acq_instr "" # define __lll_rel_instr "" diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h index f4ed98a503..5f20537943 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h @@ -30,6 +30,8 @@ #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 #define FUTEX_CMP_REQUEUE 4 +#define FUTEX_WAKE_OP 5 +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -103,6 +105,27 @@ }) +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2) \ + ({ \ + register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ + register unsigned long int __r3 asm ("3") = FUTEX_WAKE_OP; \ + register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \ + register unsigned long int __r5 asm ("5") = (long int) (nr_wake2); \ + register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \ + register unsigned long int __r7 asm ("7") \ + = (int) FUTEX_OP_CLEAR_WAKE_IF_GT_ONE; \ + register unsigned long __result asm ("2"); \ + \ + __asm __volatile ("svc %b1" \ + : "=d" (__result) \ + : "i" (SYS_futex), "0" (__r2), "d" (__r3), \ + "d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7) \ + : "cc", "memory" ); \ + __result > -4096UL; \ + }) + + #define lll_compare_and_swap(futex, oldval, newval, operation) \ do { \ __typeof (futex) __futex = (futex); \ diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h index 4626aec524..8d12db3a16 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h @@ -30,6 +30,8 @@ #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 #define FUTEX_CMP_REQUEUE 4 +#define FUTEX_WAKE_OP 5 +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -76,6 +78,19 @@ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \ + ({ \ + INTERNAL_SYSCALL_DECL (__err); \ + long int __ret; \ + \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, \ + (futexp), FUTEX_WAKE_OP, (nr_wake), \ + (nr_wake2), (futexp2), \ + FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ + INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ + }) + #ifdef __sparc32_atomic_do_lock #error SPARC < v9 does not support compare and swap which is essential for futex based locking #endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index de9e109f45..3dbb9e81e3 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -31,7 +31,9 @@ #define SYS_futex 202 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 -#define FUTEX_REQUEUE 3 +#define FUTEX_WAKE_OP 5 + +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) #define EINVAL 22 @@ -66,9 +68,30 @@ __pthread_cond_signal: addl $1, (%rdi) /* Wake up one thread. */ - movl $FUTEX_WAKE, %esi + movl $FUTEX_WAKE_OP, %esi movl $SYS_futex, %eax movl $1, %edx + movl $1, %r10d +#if cond_lock != 0 + addq $cond_lock, %r8 +#endif + movl $FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %r9d + syscall +#if cond_lock != 0 + subq $cond_lock, %r8 +#endif + /* For any kind of error, we try again with WAKE. + The general test also covers running on old kernels. */ + cmpq $-4095, %rax + jae 7f + + xorl %eax, %eax + retq + +7: movl $FUTEX_WAKE, %esi + movl $SYS_futex, %eax + /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. + movl $1, %edx */ syscall /* Unlock. */ -- cgit v1.2.3