* pthreadP.h (PTHREAD_ROBUST_MUTEX_PSHARED): Define.

	* pthread_mutex_lock.c: Use it instead of PTHREAD_MUTEX_PSHARED when
	dealing with robust mutexes.
	* pthread_mutex_timedlock.c: Likewise.
	* pthread_mutex_trylock.c: Likewise.
	* pthread_mutex_unlock.c: Likewise.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.

2007-08-06  Jakub Jelinek  <jakub@redhat.com>

	* pthreadP.h (PTHREAD_MUTEX_PSHARED_BIT): Define.
	(PTHREAD_MUTEX_TYPE): Mask __kind with 127.
	(PTHREAD_MUTEX_PSHARED): Define.
	* pthread_mutex_init.c (__pthread_mutex_init): Set
	PTHREAD_MUTEX_PSHARED_BIT for pshared or robust
	mutexes.
	* pthread_mutex_lock.c (LLL_MUTEX_LOCK): Take mutex as argument
	instead of its __data.__lock field, pass PTHREAD_MUTEX_PSHARED
	as second argument to lll_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument
	instead of its __data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_lock.
	(__pthread_mutex_lock): Update LLL_MUTEX_LOCK, LLL_MUTEX_TRYLOCK,
	LLL_ROBUST_MUTEX_LOCK users, use PTHREAD_MUTEX_TYPE (mutex)
	instead of mutex->__data.__kind directly, pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock and lll_futex_wait.
	* pthread_mutex_trylock.c (__pthread_mutex_trylock): Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly, pass PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock.
	(pthread_mutex_timedlock): Pass PTHREAD_MUTEX_PSHARED (mutex)
	to lll_timedlock, lll_robust_timedlock, lll_unlock and
	lll_futex_timed_wait.  Use PTHREAD_MUTEX_TYPE (mutex) instead
	of mutex->__data.__kind directly.
	* pthread_mutex_timedlock.c (pthread_mutex_timedlock): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_timedlock,
	lll_robust_timedlock, lll_unlock and lll_futex_timed_wait.  Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind directly.
	* pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock, lll_robust_unlock
	and lll_futex_wake.
	* pthread_mutex_setprioceiling.c (pthread_mutex_setprioceiling): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_futex_wait and lll_futex_wake.
	Use PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c (LLL_MUTEX_LOCK):
	Take mutex as argument instead of its __data.__lock field, pass
	PTHREAD_MUTEX_PSHARED as second argument to lll_cond_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument instead of its
	__data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_cond_lock.
	* pthread_cond_broadcast.c (__pthread_cond_broadcast): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_requeue and
	lll_futex_wake.  Don't use lll_futex_requeue if dependent mutex
	has PTHREAD_MUTEX_PSHARED_BIT bit set in its __data.__kind.
	* pthread_cond_destroy.c (__pthread_cond_destroy): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake and
	lll_futex_wait.
	* pthread_cond_signal.c (__pthread_cond_signal): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake_unlock and
	lll_futex_wake.
	* pthread_cond_timedwait.c (__pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock,
	lll_futex_timedwait and lll_futex_wake.
	* pthread_cond_wait.c (__condvar_cleanup, __pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock, lll_futex_wait
	and lll_futex_wake.
	* sysdeps/unix/sysv/linux/alpha/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Add private argument, use __lll_private_flag
	macro.
	* sysdeps/unix/sysv/linux/ia64/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/s390/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	(lll_futex_wake): Fix a typo.
	* sysdeps/unix/sysv/linux/pthread-pi-defines.sym (PS_BIT): Add.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
	(__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
	(__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
	(__pthread_cond_timedwait): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:
	(__condvar_cleanup, __pthread_cond_wait): Likewise.

5 files changed, 163 insertions, 40 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index 192d203926..2cd69a14ce 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -533,7 +533,7 @@ LLL_STUB_UNWIND_INFO_END
   while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \
   ({ int __res;								      \
      register int __nr_move __asm ("r10") = nr_move;			      \
      register void *__mutex __asm ("r8") = mutex;			      \
@@ -541,7 +541,8 @@ LLL_STUB_UNWIND_INFO_END
      __asm __volatile ("syscall"					      \
 		       : "=a" (__res)					      \
 		       : "0" (__NR_futex), "D" ((void *) ftx),		      \
-			 "S" (FUTEX_CMP_REQUEUE), "d" (nr_wake),	      \
+			 "S" (__lll_private_flag (FUTEX_CMP_REQUEUE,	      \
+						  private)), "d" (nr_wake),   \
 			 "r" (__nr_move), "r" (__mutex), "r" (__val)	      \
 		       : "cx", "r11", "cc", "memory");			      \
      __res < 0; })
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index 0c619bf271..6b8a29e768 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -71,11 +71,18 @@ __pthread_cond_broadcast:
 	je	9f
 
 	/* XXX: The kernel so far doesn't support requeue to PI futex.  */
-	testl	$PI_BIT, MUTEX_KIND(%r8)
+	/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
+	   type of futex (private resp. shared).  */
+	testl	$(PI_BIT | PS_BIT), MUTEX_KIND(%r8)
 	jne	9f
 
 	/* Wake up all threads.  */
-	movl	$FUTEX_CMP_REQUEUE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi
+#else
+	movl	%fs:PRIVATE_FUTEX, %esi
+	orl	$FUTEX_CMP_REQUEUE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	movl	$1, %edx
 	movl	$0x7fffffff, %r10d
@@ -104,8 +111,10 @@ __pthread_cond_broadcast:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -114,22 +123,36 @@ __pthread_cond_broadcast:
 
 	/* Unlock in loop requires wakeup.  */
 5:	addq	$cond_lock-cond_futex, %rdi
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	6b
 
 	/* Unlock in loop requires wakeup.  */
 7:	addq	$cond_lock-cond_futex, %rdi
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	subq	$cond_lock-cond_futex, %rdi
 	jmp	8b
 
 9:	/* The futex requeue functionality is not available.  */
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
 	movl	$0x7fffffff, %edx
-	movl	$FUTEX_WAKE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 	jmp	10b
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 2fc9d1fad7..8be6d4a21b 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -55,10 +55,20 @@ __pthread_cond_signal:
 	addl	$1, (%rdi)
 
 	/* Wake up one thread.  */
-	movl	$FUTEX_WAKE_OP, %esi
-	movl	$SYS_futex, %eax
+	cmpq	$-1, dep_mutex(%r8)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE_OP, %eax
+	movl	$(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE_OP, %esi
+#endif
 	movl	$1, %r10d
+	movl	$SYS_futex, %eax
 #if cond_lock != 0
 	addq	$cond_lock, %r8
 #endif
@@ -75,7 +85,9 @@ __pthread_cond_signal:
 	xorl	%eax, %eax
 	retq
 
-7:	movl	$FUTEX_WAKE, %esi
+7:	/* %esi should be either FUTEX_WAKE_OP or
+	   FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall.  */
+	xorl	$(FUTEX_WAKE | FUTEX_WAKE_OP), %esi
 	movl	$SYS_futex, %eax
 	/* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
 	movl	$1, %edx  */
@@ -98,8 +110,10 @@ __pthread_cond_signal:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -109,8 +123,13 @@ __pthread_cond_signal:
 	/* Unlock in loop requires wakeup.  */
 5:
 	movq	%r8, %rdi
-	/* XYZ */
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
+#endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	6b
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 003069fb6b..415f06f467 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -186,12 +186,20 @@ __pthread_cond_timedwait:
 	movl	%eax, (%rsp)
 
 	leaq	24(%rsp), %r10
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	cmpq	$-1, dep_mutex(%rdi)
+	movq	%r12, %rdx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	movl	$FUTEX_WAIT, %esi
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+# if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+# endif
 #endif
-	movq	%r12, %rdx
 	addq	$cond_futex, %rdi
 	movl	$SYS_futex, %eax
 	syscall
@@ -251,9 +259,19 @@ __pthread_cond_timedwait:
 	jne	25f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -292,8 +310,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 	jmp	2b
 
@@ -302,8 +322,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	4b
 
@@ -312,8 +334,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -325,8 +349,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	11b
 
@@ -344,8 +370,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 17:	movq	(%rsp), %rax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 34ef2c7b77..db2683b87b 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -49,8 +49,10 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -81,9 +83,19 @@ __condvar_cleanup:
 	jne	4f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 	movl	$1, %r12d
@@ -98,16 +110,28 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
 2:	testq	%r12, %r12
 	jnz	5f
 	addq	$cond_futex, %rdi
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -216,12 +240,20 @@ __pthread_cond_wait:
 	xorq	%r10, %r10
 	movq	%r12, %rdx
 	addq	$cond_futex-cond_lock, %rdi
-	movl	$SYS_futex, %eax
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	movl	$FUTEX_WAIT, %esi
+	movl	$FUTEX_WAIT, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+# if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+# endif
 #endif
+	movl	$SYS_futex, %eax
 	syscall
 
 	movl	(%rsp), %edi
@@ -267,9 +299,19 @@ __pthread_cond_wait:
 	jne	17f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -302,8 +344,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 	jmp	2b
 
@@ -312,8 +356,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	4b
 
@@ -322,8 +368,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -335,8 +383,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	11b
 
@@ -354,8 +404,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 13:	movq	%r10, %rax