diff options
Diffstat (limited to 'nptl/sysdeps')
29 files changed, 626 insertions, 159 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h index 41a54d4b0d..41c0be1978 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h @@ -1,5 +1,5 @@ /* Machine-specific pthread type layouts. Alpha version. - Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -126,9 +126,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h index f3f291979a..4f6796449a 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h @@ -24,6 +24,7 @@ #include <bits/pthreadtypes.h> #include <atomic.h> #include <sysdep.h> +#include <kernel-features.h> #define __NR_futex 394 @@ -103,24 +104,24 @@ while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ - (nr_move), (mutex), (val)); \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ + (nr_wake), (nr_move), (mutex), (val)); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \ +#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_WAKE_OP, (nr_wake), \ - (nr_wake2), (futexp2), \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_WAKE_OP, private), \ + (nr_wake), (nr_wake2), (futexp2), \ FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 122d83afee..776c47f6cc 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -83,11 +83,18 @@ __pthread_cond_broadcast: je 9f /* XXX: The kernel so far doesn't support requeue to PI futex. */ - testl $PI_BIT, MUTEX_KIND(%edi) + /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same + type of futex (private resp. shared). */ + testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi) jne 9f /* Wake up all threads. */ - movl $FUTEX_CMP_REQUEUE, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx +#else + movl %gs:PRIVATE_FUTEX, %ecx + orl $FUTEX_CMP_REQUEUE, %ecx +#endif movl $SYS_futex, %eax movl $0x7fffffff, %esi movl $1, %edx @@ -132,28 +139,63 @@ __pthread_cond_broadcast: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b /* Unlock in loop requires waekup. */ 5: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b /* Unlock in loop requires waekup. */ 7: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 8b 9: /* The futex requeue functionality is not available. */ movl $0x7fffffff, %edx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax ENTER_KERNEL jmp 10b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index e3510c8ab1..36a18036c5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -70,7 +70,18 @@ __pthread_cond_signal: /* Wake up one thread. */ pushl %esi pushl %ebp - movl $FUTEX_WAKE_OP, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE_OP, %ecx movl $SYS_futex, %eax movl $1, %edx movl $1, %esi @@ -92,7 +103,9 @@ __pthread_cond_signal: popl %ebx ret -7: movl $FUTEX_WAKE, %ecx +7: /* %ecx should be either FUTEX_WAKE_OP or + FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx movl $SYS_futex, %eax /* %edx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ @@ -106,8 +119,16 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: movl %edi, %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b @@ -118,8 +139,16 @@ __pthread_cond_signal: #else leal cond_lock(%edi), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%edi) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index 79a7497e8c..83f8db25bb 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -22,6 +22,7 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -157,7 +158,20 @@ __pthread_cond_timedwait: movl %eax, (%esp) leal 4(%esp), %esi - xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -231,7 +245,18 @@ __pthread_cond_timedwait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -279,8 +304,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -292,8 +325,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -304,8 +345,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -316,8 +365,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -338,8 +395,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -400,8 +465,16 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -440,7 +513,18 @@ __condvar_tw_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -459,15 +543,34 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -587,12 +690,12 @@ __condvar_tw_cleanup: .uleb128 20 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 .byte 0x40+.Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -614,7 +717,8 @@ __condvar_tw_cleanup: .byte 0x40+.LSbl1-.Lpop_edi # DW_CFA_advance_loc+N .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE .byte 0x85 # DW_CFA_offset %ebp @@ -625,14 +729,15 @@ __condvar_tw_cleanup: .uleb128 4 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 + .byte 4 # DW_CFA_advance_loc4 #if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .4byte .LSbl4-.LSbl3 #else - .byte 4 # DW_CFA_advance_loc4 - .long .LSbl5-.LSbl3 + .4byte .LSbl5-.LSbl3 #endif .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 68741bffe8..5b301979b5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -22,6 +22,7 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <tcb-offsets.h> +#include <kernel-features.h> .text @@ -100,7 +101,20 @@ __pthread_cond_wait: 4: call __pthread_enable_asynccancel movl %eax, (%esp) - movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -161,7 +175,18 @@ __pthread_cond_wait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -197,8 +222,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -210,8 +243,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -222,8 +263,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -234,8 +283,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -256,8 +313,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -292,8 +357,16 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -332,7 +405,18 @@ __condvar_w_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -351,15 +435,34 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -473,12 +576,12 @@ __condvar_w_cleanup: .uleb128 16 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 2 # DW_CFA_advance_loc1 - .byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 .byte 0x40+ .Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -502,13 +605,16 @@ __condvar_w_cleanup: .uleb128 3 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl4-.LSbl3 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE .align 4 diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S index d8a62ed2b7..d8f1bd54a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -68,7 +69,7 @@ __pthread_rwlock_rdlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S index 71b97c60f5..0d96e03252 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -99,7 +100,7 @@ pthread_rwlock_timedrdlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S index c002472085..e78fdf6dda 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -97,7 +98,7 @@ pthread_rwlock_timedwrlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S index fdad432e30..a23e1b50a8 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S @@ -20,6 +20,7 @@ #include <sysdep.h> #include <lowlevellock.h> #include <lowlevelrwlock.h> +#include <kernel-features.h> .text @@ -73,7 +74,7 @@ __pthread_rwlock_unlock: jne 7f 8: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%edi), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S index 3f55c82930..65b99fe7d3 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <lowlevelrwlock.h> #include <pthread-errnos.h> +#include <kernel-features.h> .text @@ -66,7 +67,7 @@ __pthread_rwlock_wrlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i786/Implies b/nptl/sysdeps/unix/sysv/linux/i386/i786/Implies new file mode 100644 index 0000000000..7cb7d9a678 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i786/Implies @@ -0,0 +1,2 @@ +# The PPro and PII cores are mostly the same. +unix/sysv/linux/i386/i686 diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 2f663aa68b..8f67616af7 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -83,7 +83,7 @@ ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) \ : (fl)) \ : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \ - asm ("andl %%fs:%P1, %0" : "+r" (__fl) \ + asm ("andl %%gs:%P1, %0" : "+r" (__fl) \ : "i" (offsetof (struct pthread, header.private_futex))); \ __fl | (fl); })) # endif diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h index 892769dca4..330717f079 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -126,9 +126,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h index 3c28a397ea..ada79851e2 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h @@ -25,6 +25,7 @@ #include <bits/pthreadtypes.h> #include <ia64intrin.h> #include <atomic.h> +#include <kernel-features.h> #define __NR_futex 1230 #define FUTEX_WAIT 0 @@ -103,18 +104,20 @@ do \ while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \ ({ \ - DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_CMP_REQUEUE, \ + DO_INLINE_SYSCALL(futex, 6, (long) (ftx), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private), \ (int) (nr_wake), (int) (nr_move), (long) (mutex), \ (int) val); \ _r10 == -1; \ }) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2) \ +#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2, private) \ ({ \ - DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_WAKE_OP, \ + DO_INLINE_SYSCALL(futex, 6, (long) (ftx), \ + __lll_private_flag (FUTEX_WAKE_OP, private), \ (int) (nr_wake), (int) (nr_wake2), (long) (ftx2), \ FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ _r10 == -1; \ @@ -152,6 +155,7 @@ extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden; __lll_lock_wait_private (__futex); \ else \ __lll_lock_wait (__futex, private); \ + } \ })) #define lll_lock(futex, private) __lll_lock (&(futex), private) diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h index 41804d1372..80b9a4369e 100644 --- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h @@ -24,6 +24,7 @@ #include <sys/param.h> #include <bits/pthreadtypes.h> #include <atomic.h> +#include <kernel-features.h> #ifndef __NR_futex # define __NR_futex 221 @@ -107,14 +108,14 @@ while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ - (nr_move), (mutex), (val)); \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ + (nr_wake), (nr_move), (mutex), (val)); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/pthread_spin_unlock.c b/nptl/sysdeps/unix/sysv/linux/powerpc/pthread_spin_unlock.c new file mode 100644 index 0000000000..90f2dc67c0 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/pthread_spin_unlock.c @@ -0,0 +1,29 @@ +/* pthread_spin_unlock -- unlock a spin lock. PowerPC version. + Copyright (C) 2007 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include "pthreadP.h" +#include <lowlevellock.h> + +int +pthread_spin_unlock (pthread_spinlock_t *lock) +{ + __asm __volatile (__lll_rel_instr ::: "memory"); + *lock = 0; + return 0; +} diff --git a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym index a1b6794260..d985c6a79b 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym +++ b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym @@ -4,3 +4,4 @@ MUTEX_KIND offsetof (pthread_mutex_t, __data.__kind) PI_BIT PTHREAD_MUTEX_PRIO_INHERIT_NP +PS_BIT PTHREAD_MUTEX_PSHARED_BIT diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c index 81ecd6556b..93841c5b3e 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c +++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -1,8 +1,12 @@ #include <pthreadP.h> -#define LLL_MUTEX_LOCK(mutex) lll_cond_lock (mutex, /* XYZ */ LLL_SHARED) -#define LLL_MUTEX_TRYLOCK(mutex) lll_cond_trylock (mutex) -#define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_cond_lock (mutex, id, /* XYZ */ LLL_SHARED) +#define LLL_MUTEX_LOCK(mutex) \ + lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)) +#define LLL_MUTEX_TRYLOCK(mutex) \ + lll_cond_trylock ((mutex)->__data.__lock) +#define LLL_ROBUST_MUTEX_LOCK(mutex, id) \ + lll_robust_cond_lock ((mutex)->__data.__lock, id, \ + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)) #define __pthread_mutex_lock __pthread_mutex_cond_lock #define NO_INCR diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h index c77031d7bb..c7345cd9a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -159,9 +159,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; @@ -175,9 +175,12 @@ typedef union unsigned int __writer_wakeup; unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; + unsigned char __pad1; + unsigned char __pad2; + unsigned char __shared; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ - unsigned int __flags; + unsigned char __flags; int __writer; } __data; # endif diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h index ad4d27300f..7fee435f12 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h @@ -24,6 +24,7 @@ #include <sys/param.h> #include <bits/pthreadtypes.h> #include <atomic.h> +#include <kernel-features.h> #define SYS_futex 238 #define FUTEX_WAIT 0 @@ -93,7 +94,7 @@ ({ \ register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ register unsigned long int __r3 asm ("3") \ - __lll_private_flag (FUTEX_WAKE, private); \ + = __lll_private_flag (FUTEX_WAKE, private); \ register unsigned long int __r4 asm ("4") = (unsigned long int) (nr); \ register unsigned long int __result asm ("2"); \ \ @@ -117,10 +118,11 @@ /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \ ({ \ register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ - register unsigned long int __r3 asm ("3") = FUTEX_CMP_REQUEUE; \ + register unsigned long int __r3 asm ("3") \ + = __lll_private_flag (FUTEX_CMP_REQUEUE, private); \ register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \ register unsigned long int __r5 asm ("5") = (long int) (nr_move); \ register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex); \ @@ -137,10 +139,11 @@ /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2) \ +#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \ ({ \ register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ - register unsigned long int __r3 asm ("3") = FUTEX_WAKE_OP; \ + register unsigned long int __r3 asm ("3") \ + = __lll_private_flag (FUTEX_WAKE_OP, private); \ register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \ register unsigned long int __r5 asm ("5") = (long int) (nr_wake2); \ register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \ diff --git a/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S index 17c1e6f567..fde4f57b2a 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/sh/pthread_cond_timedwait.S @@ -22,6 +22,7 @@ #include <lowlevelcond.h> #include <pthread-errnos.h> #include "lowlevel-atomic.h" +#include <kernel-features.h> .text diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h index e734c1205e..faf0584868 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h @@ -1,5 +1,5 @@ /* Machine-specific pthread type layouts. SPARC version. - Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -160,9 +160,9 @@ typedef union unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; int __writer; - int __pad1; + int __shared; + unsigned long int __pad1; unsigned long int __pad2; - unsigned long int __pad3; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; @@ -176,9 +176,12 @@ typedef union unsigned int __writer_wakeup; unsigned int __nr_readers_queued; unsigned int __nr_writers_queued; + unsigned char __pad1; + unsigned char __pad2; + unsigned char __shared; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ - unsigned int __flags; + unsigned char __flags; int __writer; } __data; # endif diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h index 38692bbd2d..f4512b2622 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h @@ -24,6 +24,7 @@ #include <sys/param.h> #include <bits/pthreadtypes.h> #include <atomic.h> +#include <kernel-features.h> #define FUTEX_WAIT 0 @@ -96,14 +97,14 @@ }) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ - (nr_move), (mutex), (val)); \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ + (nr_wake), (nr_move), (mutex), (val)); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) @@ -121,14 +122,14 @@ /* Avoid FUTEX_WAKE_OP if supporting pre-v9 CPUs. */ # define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) 1 #else -# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \ +# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 6, \ - (futexp), FUTEX_WAKE_OP, (nr_wake), \ - (nr_wake2), (futexp2), \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ + __lll_private_flag (FUTEX_WAKE_OP, private), \ + (nr_wake), (nr_wake2), (futexp2), \ FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 192d203926..2cd69a14ce 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -533,7 +533,7 @@ LLL_STUB_UNWIND_INFO_END while (0) /* Returns non-zero if error happened, zero if success. */ -#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \ +#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \ ({ int __res; \ register int __nr_move __asm ("r10") = nr_move; \ register void *__mutex __asm ("r8") = mutex; \ @@ -541,7 +541,8 @@ LLL_STUB_UNWIND_INFO_END __asm __volatile ("syscall" \ : "=a" (__res) \ : "0" (__NR_futex), "D" ((void *) ftx), \ - "S" (FUTEX_CMP_REQUEUE), "d" (nr_wake), \ + "S" (__lll_private_flag (FUTEX_CMP_REQUEUE, \ + private)), "d" (nr_wake), \ "r" (__nr_move), "r" (__mutex), "r" (__val) \ : "cx", "r11", "cc", "memory"); \ __res < 0; }) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 0c619bf271..6155255eb0 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -71,11 +71,18 @@ __pthread_cond_broadcast: je 9f /* XXX: The kernel so far doesn't support requeue to PI futex. */ - testl $PI_BIT, MUTEX_KIND(%r8) + /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same + type of futex (private resp. shared). */ + testl $(PI_BIT | PS_BIT), MUTEX_KIND(%r8) jne 9f /* Wake up all threads. */ - movl $FUTEX_CMP_REQUEUE, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi +#else + movl %fs:PRIVATE_FUTEX, %esi + orl $FUTEX_CMP_REQUEUE, %esi +#endif movl $SYS_futex, %eax movl $1, %edx movl $0x7fffffff, %r10d @@ -104,8 +111,10 @@ __pthread_cond_broadcast: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -114,22 +123,36 @@ __pthread_cond_broadcast: /* Unlock in loop requires wakeup. */ 5: addq $cond_lock-cond_futex, %rdi - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 6b /* Unlock in loop requires wakeup. */ 7: addq $cond_lock-cond_futex, %rdi - /* XYZ */ + cmpq $-1, %r8 + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake subq $cond_lock-cond_futex, %rdi jmp 8b 9: /* The futex requeue functionality is not available. */ + cmpq $-1, %r8 movl $0x7fffffff, %edx - movl $FUTEX_WAKE, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif movl $SYS_futex, %eax syscall jmp 10b diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 2fc9d1fad7..8f65f2cd69 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -55,10 +55,20 @@ __pthread_cond_signal: addl $1, (%rdi) /* Wake up one thread. */ - movl $FUTEX_WAKE_OP, %esi - movl $SYS_futex, %eax + cmpq $-1, dep_mutex(%r8) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE_OP, %eax + movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE_OP, %esi +#endif movl $1, %r10d + movl $SYS_futex, %eax #if cond_lock != 0 addq $cond_lock, %r8 #endif @@ -75,7 +85,9 @@ __pthread_cond_signal: xorl %eax, %eax retq -7: movl $FUTEX_WAKE, %esi +7: /* %esi should be either FUTEX_WAKE_OP or + FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi movl $SYS_futex, %eax /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ @@ -98,8 +110,10 @@ __pthread_cond_signal: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -109,8 +123,13 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: movq %r8, %rdi - /* XYZ */ +#if cond_lock != 0 + addq $cond_lock, %rdi +#endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 6b .size __pthread_cond_signal, .-__pthread_cond_signal diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 003069fb6b..a5de670866 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -23,6 +23,8 @@ #include <lowlevelcond.h> #include <pthread-errnos.h> +#include <kernel-features.h> + /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 @@ -186,12 +188,20 @@ __pthread_cond_timedwait: movl %eax, (%rsp) leaq 24(%rsp), %r10 -#if FUTEX_WAIT == 0 - xorl %esi, %esi + cmpq $-1, dep_mutex(%rdi) + movq %r12, %rdx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT, %eax + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi #else - movl $FUTEX_WAIT, %esi + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi +# if FUTEX_WAIT != 0 + orl $FUTEX_WAIT, %esi +# endif #endif - movq %r12, %rdx addq $cond_futex, %rdi movl $SYS_futex, %eax syscall @@ -251,9 +261,19 @@ __pthread_cond_timedwait: jne 25f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi @@ -292,8 +312,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait jmp 2b @@ -302,8 +324,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 4b @@ -312,8 +336,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -325,8 +351,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 11b @@ -344,8 +372,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake 17: movq (%rsp), %rax diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 34ef2c7b77..2c17dc03a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -23,6 +23,8 @@ #include <lowlevelcond.h> #include <tcb-offsets.h> +#include <kernel-features.h> + .text @@ -49,8 +51,10 @@ __condvar_cleanup: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -81,9 +85,19 @@ __condvar_cleanup: jne 4f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi movl $1, %r12d @@ -98,16 +112,28 @@ __condvar_cleanup: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testq %r12, %r12 jnz 5f addq $cond_futex, %rdi - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_futex(%rdi) movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif movl $SYS_futex, %eax syscall @@ -216,12 +242,20 @@ __pthread_cond_wait: xorq %r10, %r10 movq %r12, %rdx addq $cond_futex-cond_lock, %rdi - movl $SYS_futex, %eax -#if FUTEX_WAIT == 0 - xorl %esi, %esi + cmpq $-1, dep_mutex-cond_futex(%rdi) +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT, %eax + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi #else - movl $FUTEX_WAIT, %esi + movl $FUTEX_WAIT, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi +# if FUTEX_WAIT != 0 + orl $FUTEX_WAIT, %esi +# endif #endif + movl $SYS_futex, %eax syscall movl (%rsp), %edi @@ -267,9 +301,19 @@ __pthread_cond_wait: jne 17f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi @@ -302,8 +346,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait jmp 2b @@ -312,8 +358,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 4b @@ -322,8 +370,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi @@ -335,8 +385,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake jmp 11b @@ -354,8 +406,10 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - /* XYZ */ + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi + cmovne %eax, %esi callq __lll_unlock_wake 13: movq %r10, %rax |