* sysdeps/unix/sysv/linux/kernel-features.h (__ASSUME_PSELECT,cvs/fedora-glibc-20070814T0725cvs/fedora-glibc-20070814T0643

	__ASSUME_PPOLL, __ASSUME_ATFCTS, __ASSUME_SET_ROBUST_LIST,
	__ASSUME_UTIMENSAT, __ASSUME_FALLOCATE): Update per-arch conditions
	when each feature was introduced.

	* sysdeps/unix/sysv/linux/dl-vdso.c: Don't include dl-hash.h.
	* sysdeps/unix/sysv/linux/dl-vdso.h: Don't include dl-hash.h if NDEBUG.
	(CHECK_HASH): New macro.
	(PREPARE_VERSION): Use it.

	* sysdeps/unix/sysv/linux/pselect.c (__generic_pselect): Only provide
	prototype if not __ASSUME_PSELECT.
	* sysdeps/unix/sysv/linux/ppoll.c (__generic_ppoll): Only provide
	prototype if not __ASSUME_PPOLL.

	* sysdeps/unix/sysv/linux/dl-osinfo.h (ROUND): #undef after use.

	* sysdeps/unix/clock_settime.c (freq, __pthread_clock_settime,
	hp_timing_settime): Don't define or declare if HANDLED_CPUTIME
	is defined.

13 files changed, 394 insertions, 76 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 345829cf49..e56ece02bd 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,40 @@
+2007-08-13  Jakub Jelinek  <jakub@redhat.com>
+
+	* sysdeps/unix/sysv/linux/i386/lowlevellock.h (__lll_private_flag):
+	Fix a pasto.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+	(__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or
+	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
+	Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+	(__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or
+	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Include
+	kernel-features.h.
+	(__pthread_cond_wait, __condvar_w_cleanup): Pass LLL_PRIVATE to
+	lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is
+	process private.  Switch DW_CFA_advance_loc1 and some
+	DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+	(__pthread_cond_timedwait, __condvar_tw_cleanup): Pass LLL_PRIVATE to
+	lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is
+	process private.  Switch DW_CFA_advance_loc{1,2} and some
+	DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Use
+	#ifdef __ASSUME_PRIVATE_FUTEX instead of #if __ASSUME_PRIVATE_FUTEX.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+	(__pthread_cond_broadcast): Compare %r8 instead of
+	dep_mutex-cond_*(%rdi) with $-1.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+	(__pthread_cond_signal): Xor FUTEX_WAKE_OP with FUTEX_WAKE instead
+	of oring.
+
 2007-08-13  Ulrich Drepper  <drepper@redhat.com>
 
 	* sysdeps/unix/sysv/linux/i386/i786/Implies: New file.
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 122d83afee..776c47f6cc 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -83,11 +83,18 @@ __pthread_cond_broadcast:
 	je	9f
 
 	/* XXX: The kernel so far doesn't support requeue to PI futex.  */
-	testl	$PI_BIT, MUTEX_KIND(%edi)
+	/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
+	   type of futex (private resp. shared).  */
+	testl	$(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
 	jne	9f
 
 	/* Wake up all threads.  */
-	movl	$FUTEX_CMP_REQUEUE, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
+#else
+	movl	%gs:PRIVATE_FUTEX, %ecx
+	orl	$FUTEX_CMP_REQUEUE, %ecx
+#endif
 	movl	$SYS_futex, %eax
 	movl	$0x7fffffff, %esi
 	movl	$1, %edx
@@ -132,28 +139,63 @@ __pthread_cond_broadcast:
 #else
 	leal	cond_lock(%ebx), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 	jmp	2b
 
 	/* Unlock in loop requires waekup.  */
 5:	leal	cond_lock-cond_futex(%ebx), %eax
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_futex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 	jmp	6b
 
 	/* Unlock in loop requires waekup.  */
 7:	leal	cond_lock-cond_futex(%ebx), %eax
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_futex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 	jmp	8b
 
 9:	/* The futex requeue functionality is not available.  */
 	movl	$0x7fffffff, %edx
-	movl	$FUTEX_WAKE, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_futex(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE, %ecx
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 	jmp	10b
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index e3510c8ab1..36a18036c5 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -70,7 +70,18 @@ __pthread_cond_signal:
 	/* Wake up one thread.  */
 	pushl	%esi
 	pushl	%ebp
-	movl	$FUTEX_WAKE_OP, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_futex(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE_OP, %ecx
 	movl	$SYS_futex, %eax
 	movl	$1, %edx
 	movl	$1, %esi
@@ -92,7 +103,9 @@ __pthread_cond_signal:
 	popl	%ebx
 	ret
 
-7:	movl	$FUTEX_WAKE, %ecx
+7:	/* %ecx should be either FUTEX_WAKE_OP or
+	   FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall.  */
+	xorl	$(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
 	movl	$SYS_futex, %eax
 	/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
 	movl	$1, %edx  */
@@ -106,8 +119,16 @@ __pthread_cond_signal:
 
 	/* Unlock in loop requires wakeup.  */
 5:	movl	%edi, %eax
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_futex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 	jmp	6b
 
@@ -118,8 +139,16 @@ __pthread_cond_signal:
 #else
 	leal	cond_lock(%edi), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%edi)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 	jmp	2b
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index ff56c7ae4a..83f8db25bb 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -158,7 +158,20 @@ __pthread_cond_timedwait:
 	movl	%eax, (%esp)
 
 	leal	4(%esp), %esi
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+#if FUTEX_WAIT != 0
+	addl	$FUTEX_WAIT, %ecx
+#endif
 	movl	%edi, %edx
 	addl	$cond_futex, %ebx
 .Ladd_cond_futex:
@@ -232,7 +245,18 @@ __pthread_cond_timedwait:
 
 	addl	$cond_nwaiters, %ebx
 	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_nwaiters(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE, %ecx
 	movl	$1, %edx
 	ENTER_KERNEL
 	subl	$cond_nwaiters, %ebx
@@ -280,8 +304,16 @@ __pthread_cond_timedwait:
 #else
 	leal	cond_lock(%ebx), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 	jmp	2b
 
@@ -293,8 +325,16 @@ __pthread_cond_timedwait:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 	jmp	4b
 
@@ -305,8 +345,16 @@ __pthread_cond_timedwait:
 #else
 	leal	cond_lock(%ebx), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 	jmp	6b
 
@@ -317,8 +365,16 @@ __pthread_cond_timedwait:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 	jmp	11b
 
@@ -339,8 +395,16 @@ __pthread_cond_timedwait:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 
 	movl	%esi, %eax
@@ -401,8 +465,16 @@ __condvar_tw_cleanup:
 #else
 	leal	cond_lock(%ebx), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 
 1:	movl	broadcast_seq(%ebx), %eax
@@ -441,7 +513,18 @@ __condvar_tw_cleanup:
 
 	addl	$cond_nwaiters, %ebx
 	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_nwaiters(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE, %ecx
 	movl	$1, %edx
 	ENTER_KERNEL
 	subl	$cond_nwaiters, %ebx
@@ -460,15 +543,34 @@ __condvar_tw_cleanup:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
 2:	testl	%edi, %edi
 	jnz	5f
 	addl	$cond_futex, %ebx
-	movl	$FUTEX_WAKE, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_futex(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE, %ecx
 	movl	$SYS_futex, %eax
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
@@ -588,12 +690,12 @@ __condvar_tw_cleanup:
 	.uleb128 20
 	.byte	0x83				# DW_CFA_offset %ebx
 	.uleb128 5
-	.byte	2				# DW_CFA_advance_loc1
-	.byte	.Lsubl-.Lpush_ebx
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.Lsubl-.Lpush_ebx
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 20+FRAME_SIZE
-	.byte	3				# DW_CFA_advance_loc2
-	.2byte	.Laddl-.Lsubl
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.Laddl-.Lsubl
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 20
 	.byte	0x40+.Lpop_ebx-.Laddl		# DW_CFA_advance_loc+N
@@ -615,7 +717,8 @@ __condvar_tw_cleanup:
 	.byte	0x40+.LSbl1-.Lpop_edi		# DW_CFA_advance_loc+N
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 20
-	.byte	0x40+.LSbl2-.LSbl1		# DW_CFA_advance_loc+N
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.LSbl2-.LSbl1
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 20+FRAME_SIZE
 	.byte	0x85				# DW_CFA_offset %ebp
@@ -626,14 +729,15 @@ __condvar_tw_cleanup:
 	.uleb128 4
 	.byte	0x83				# DW_CFA_offset %ebx
 	.uleb128 5
-	.byte	0x40+.LSbl3-.LSbl2		# DW_CFA_advance_loc+N
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.LSbl3-.LSbl2
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 20
+	.byte	4				# DW_CFA_advance_loc4
 #if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS
-	.byte	0x40+.LSbl4-.LSbl3		# DW_CFA_advance_loc+N
+	.4byte	.LSbl4-.LSbl3
 #else
-	.byte	4				# DW_CFA_advance_loc4
-	.long	.LSbl5-.LSbl3
+	.4byte	.LSbl5-.LSbl3
 #endif
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 20+FRAME_SIZE
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 68741bffe8..5b301979b5 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -22,6 +22,7 @@
 #include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <tcb-offsets.h>
+#include <kernel-features.h>
 
 
 	.text
@@ -100,7 +101,20 @@ __pthread_cond_wait:
 4:	call	__pthread_enable_asynccancel
 	movl	%eax, (%esp)
 
-	movl	%esi, %ecx	/* movl $FUTEX_WAIT, %ecx */
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+#if FUTEX_WAIT != 0
+	addl	$FUTEX_WAIT, %ecx
+#endif
 	movl	%edi, %edx
 	addl	$cond_futex, %ebx
 .Ladd_cond_futex:
@@ -161,7 +175,18 @@ __pthread_cond_wait:
 
 	addl	$cond_nwaiters, %ebx
 	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_nwaiters(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE, %ecx
 	movl	$1, %edx
 	ENTER_KERNEL
 	subl	$cond_nwaiters, %ebx
@@ -197,8 +222,16 @@ __pthread_cond_wait:
 #else
 	leal	cond_lock(%ebx), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 	jmp	2b
 
@@ -210,8 +243,16 @@ __pthread_cond_wait:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 	jmp	4b
 
@@ -222,8 +263,16 @@ __pthread_cond_wait:
 #else
 	leal	cond_lock(%ebx), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 	jmp	6b
 
@@ -234,8 +283,16 @@ __pthread_cond_wait:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 	jmp	11b
 
@@ -256,8 +313,16 @@ __pthread_cond_wait:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 
 	movl	%esi, %eax
@@ -292,8 +357,16 @@ __condvar_w_cleanup:
 #else
 	leal	cond_lock(%ebx), %edx
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_lock_wait
 
 1:	movl	broadcast_seq(%ebx), %eax
@@ -332,7 +405,18 @@ __condvar_w_cleanup:
 
 	addl	$cond_nwaiters, %ebx
 	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_nwaiters(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE, %ecx
 	movl	$1, %edx
 	ENTER_KERNEL
 	subl	$cond_nwaiters, %ebx
@@ -351,15 +435,34 @@ __condvar_w_cleanup:
 #else
 	leal	cond_lock(%ebx), %eax
 #endif
-	/* XYZ */
-	movl	$LLL_SHARED, %ecx
+#if (LLL_SHARED-LLL_PRIVATE) > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex(%ebx)
+	setne	%cl
+	subl	$1, %ecx
+	andl	$(LLL_SHARED-LLL_PRIVATE), %ecx
+#if LLL_PRIVATE != 0
+	addl	$LLL_PRIVATE, %ecx
+#endif
 	call	__lll_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
 2:	testl	%edi, %edi
 	jnz	5f
 	addl	$cond_futex, %ebx
-	movl	$FUTEX_WAKE, %ecx
+#if FUTEX_PRIVATE_FLAG > 255
+	xorl	%ecx, %ecx
+#endif
+	cmpl	$-1, dep_mutex-cond_futex(%ebx)
+	sete	%cl
+	subl	$1, %ecx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$FUTEX_WAKE, %ecx
 	movl	$SYS_futex, %eax
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
@@ -473,12 +576,12 @@ __condvar_w_cleanup:
 	.uleb128 16
 	.byte	0x83				# DW_CFA_offset %ebx
 	.uleb128 4
-	.byte	2				# DW_CFA_advance_loc1
-	.byte	.Lsubl-.Lpush_ebx
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.Lsubl-.Lpush_ebx
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 16+FRAME_SIZE
-	.byte	2				# DW_CFA_advance_loc1
-	.byte	.Laddl-.Lsubl
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.Laddl-.Lsubl
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 16
 	.byte	0x40+ .Lpop_ebx-.Laddl		# DW_CFA_advance_loc+N
@@ -502,13 +605,16 @@ __condvar_w_cleanup:
 	.uleb128 3
 	.byte	0x83				# DW_CFA_offset %ebx
 	.uleb128 4
-	.byte	0x40+.LSbl2-.LSbl1		# DW_CFA_advance_loc+N
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.LSbl2-.LSbl1
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 16+FRAME_SIZE
-	.byte	0x40+.LSbl3-.LSbl2		# DW_CFA_advance_loc+N
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.LSbl3-.LSbl2
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 16
-	.byte	0x40+.LSbl4-.LSbl3		# DW_CFA_advance_loc+N
+	.byte	4				# DW_CFA_advance_loc4
+	.4byte	.LSbl4-.LSbl3
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 16+FRAME_SIZE
 	.align	4
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
index d30bc459bb..d8f1bd54a2 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
@@ -69,7 +69,7 @@ __pthread_rwlock_rdlock:
 	jne	10f
 
 11:
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
 	movzbl	PSHARED(%ebx), %ecx
 	xorl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
 #else
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
index 86c9873ec7..0d96e03252 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
@@ -100,7 +100,7 @@ pthread_rwlock_timedrdlock:
 	movl	%edx, 4(%esp)
 
 	movl	%esi, %edx
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
 	movzbl	PSHARED(%ebp), %ecx
 	xorl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
 #else
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
index 5f6c80f8e5..e78fdf6dda 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
@@ -98,7 +98,7 @@ pthread_rwlock_timedwrlock:
 	movl	%edx, 4(%esp)
 
 	movl	%esi, %edx
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
 	movzbl	PSHARED(%ebp), %ecx
 	xorl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
 #else
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
index 3f19cd1802..a23e1b50a8 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
@@ -74,7 +74,7 @@ __pthread_rwlock_unlock:
 	jne	7f
 
 8:
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
 	movzbl	PSHARED(%edi), %ecx
 	xorl	$FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %ecx
 #else
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
index 70d0fcfde8..65b99fe7d3 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
@@ -67,7 +67,7 @@ __pthread_rwlock_wrlock:
 	jne	10f
 
 11:
-#if __ASSUME_PRIVATE_FUTEX
+#ifdef __ASSUME_PRIVATE_FUTEX
 	movzbl	PSHARED(%ebx), %ecx
 	xorl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
 #else
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index 2f663aa68b..8f67616af7 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -83,7 +83,7 @@
       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))	      \
       : (fl))								      \
    : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);		      \
-	asm ("andl %%fs:%P1, %0" : "+r" (__fl)				      \
+	asm ("andl %%gs:%P1, %0" : "+r" (__fl)				      \
 	     : "i" (offsetof (struct pthread, header.private_futex)));	      \
 	__fl | (fl); }))
 # endif	      
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index 6b8a29e768..6155255eb0 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -132,7 +132,7 @@ __pthread_cond_broadcast:
 
 	/* Unlock in loop requires wakeup.  */
 7:	addq	$cond_lock-cond_futex, %rdi
-	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	cmpq	$-1, %r8
 	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
 	cmovne	%eax, %esi
@@ -141,7 +141,7 @@ __pthread_cond_broadcast:
 	jmp	8b
 
 9:	/* The futex requeue functionality is not available.  */
-	cmpq	$-1, dep_mutex-cond_futex(%rdi)
+	cmpq	$-1, %r8
 	movl	$0x7fffffff, %edx
 #ifdef __ASSUME_PRIVATE_FUTEX
 	movl	$FUTEX_WAKE, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 8be6d4a21b..8f65f2cd69 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -87,7 +87,7 @@ __pthread_cond_signal:
 
 7:	/* %esi should be either FUTEX_WAKE_OP or
 	   FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall.  */
-	xorl	$(FUTEX_WAKE | FUTEX_WAKE_OP), %esi
+	xorl	$(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi
 	movl	$SYS_futex, %eax
 	/* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
 	movl	$1, %edx  */