Update.

2004-06-03  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/i386/i486/bits/atomic.h: Optimize a bit.

4 files changed, 56 insertions, 40 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index 9da84c66d6..8d7858a071 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -76,11 +76,12 @@ typedef union
   struct
   {
     int __lock;
-    int __clock;
+    unsigned int __futex;
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
+    int __clock;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 456f3dbfb1..5471c1c927 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -32,6 +32,7 @@
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
 #define FUTEX_REQUEUE		3
+#define FUTEX_CMP_REQUEUE	4
 
 #define EINVAL			22
 
@@ -47,8 +48,9 @@ __pthread_cond_broadcast:
 	pushl	%ebx
 	pushl	%esi
 	pushl	%edi
+	pushl	%ebp
 
-	movl	16(%esp), %ebx
+	movl	20(%esp), %ebx
 
 	/* Get internal lock.  */
 	movl	$1, %edx
@@ -61,29 +63,31 @@ __pthread_cond_broadcast:
 #endif
 	jnz	1f
 
-2:	addl	$wakeup_seq, %ebx
-	movl	total_seq+4-wakeup_seq(%ebx), %eax
-	movl	total_seq-wakeup_seq(%ebx), %ecx
-	cmpl	4(%ebx), %eax
+2:	addl	$cond_futex, %ebx
+	movl	total_seq+4-cond_futex(%ebx), %eax
+	movl	total_seq-cond_futex(%ebx), %ebp
+	cmpl	wakeup_seq+4-cond_futex(%ebx), %eax
 	ja	3f
 	jb	4f
-	cmpl	(%ebx), %ecx
+	cmpl	wakeup_seq-cond_futex(%ebx), %ebp
 	jna	4f
 
 	/* Cause all currently waiting threads to recognize they are
 	   woken up.  */
-3:	movl	%ecx, (%ebx)
-	movl	%eax, 4(%ebx)
-	movl	%ecx, woken_seq-wakeup_seq(%ebx)
-	movl	%eax, woken_seq-wakeup_seq+4(%ebx)
-	addl	$1, broadcast_seq-wakeup_seq(%ebx)
+3:	movl	%ebp, wakeup_seq-cond_futex(%ebx)
+	movl	%eax, wakeup_seq-cond_futex+4(%ebx)
+	movl	%ebp, woken_seq-cond_futex(%ebx)
+	movl	%eax, woken_seq-cond_futex+4(%ebx)
+	addl	%ebp, %ebp
+	addl	$1, broadcast_seq-cond_futex(%ebx)
+	movl	%ebp, (%ebx)
 
 	/* Get the address of the mutex used.  */
-	movl	dep_mutex-wakeup_seq(%ebx), %edi
+	movl	dep_mutex-cond_futex(%ebx), %edi
 
 	/* Unlock.  */
 	LOCK
-	subl	$1, cond_lock-wakeup_seq(%ebx)
+	subl	$1, cond_lock-cond_futex(%ebx)
 	jne	7f
 
 	/* Don't use requeue for pshared condvars.  */
@@ -91,7 +95,7 @@ __pthread_cond_broadcast:
 	je	9f
 
 	/* Wake up all threads.  */
-	movl	$FUTEX_REQUEUE, %ecx
+	movl	$FUTEX_CMP_REQUEUE, %ecx
 	movl	$SYS_futex, %eax
 	movl	$0x7fffffff, %esi
 	movl	$1, %edx
@@ -99,14 +103,18 @@ __pthread_cond_broadcast:
 # if MUTEX_FUTEX != 0
 	addl	$MUTEX_FUTEX, %edi
 # endif
-	ENTER_KERNEL
+/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for sysenter.
+	ENTER_KERNEL  */
+	int	$0x80
 
-#ifndef __ASSUME_FUTEX_REQUEUE
-	cmpl	$-EINVAL, %eax
-	je	9f
-#endif
+	/* For any kind of error, which mainly is EAGAIN, we try again
+	   with WAKE.  The general test also covers running on old
+	   kernels.  */
+	cmpl	$0xfffff001, %eax
+	jae	9f
 
 10:	xorl	%eax, %eax
+	popl	%ebp
 	popl	%edi
 	popl	%esi
 	popl	%ebx
@@ -115,10 +123,11 @@ __pthread_cond_broadcast:
 	.align	16
 	/* Unlock.  */
 4:	LOCK
-	subl	$1, cond_lock-wakeup_seq(%ebx)
+	subl	$1, cond_lock-cond_futex(%ebx)
 	jne	5f
 
 6:	xorl	%eax, %eax
+	popl	%ebp
 	popl	%edi
 	popl	%esi
 	popl	%ebx
@@ -135,12 +144,12 @@ __pthread_cond_broadcast:
 	jmp	2b
 
 	/* Unlock in loop requires waekup.  */
-5:	leal	cond_lock-wakeup_seq(%ebx), %eax
+5:	leal	cond_lock-cond_futex(%ebx), %eax
 	call	__lll_mutex_unlock_wake
 	jmp	6b
 
 	/* Unlock in loop requires waekup.  */
-7:	leal	cond_lock-wakeup_seq(%ebx), %eax
+7:	leal	cond_lock-cond_futex(%ebx), %eax
 	call	__lll_mutex_unlock_wake
 	jmp	8b
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index 4722b4c0e0..3c5a1db59c 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -60,22 +60,23 @@ __pthread_cond_signal:
 #endif
 	jnz	1f
 
-2:	leal	wakeup_seq(%edi), %ebx
+2:	leal	cond_futex(%edi), %ebx
 	movl	total_seq+4(%edi), %eax
 	movl	total_seq(%edi), %ecx
-	cmpl	4(%ebx), %eax
+	cmpl	wakeup_seq+4(%edi), %eax
 #if cond_lock != 0
 	/* Must use leal to preserve the flags.  */
 	leal	cond_lock(%edi), %edi
 #endif
 	ja	3f
 	jb	4f
-	cmpl	(%ebx), %ecx
+	cmpl	wakeup_seq-cond_futex(%ebx), %ecx
 	jbe	4f
 
 	/* Bump the wakeup number.  */
-3:	addl	$1, (%ebx)
-	adcl	$0, 4(%ebx)
+3:	addl	$1, wakeup_seq-cond_futex(%ebx)
+	adcl	$0, wakeup_seq-cond_futex+4(%ebx)
+	addl	$1, (%ebx)
 
 	/* Wake up one thread.  */
 	movl	$FUTEX_WAKE, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 74e3172ab0..3fe7f8c17a 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -79,6 +79,7 @@ __pthread_cond_wait:
 
 	addl	$1, total_seq(%ebx)
 	adcl	$0, total_seq+4(%ebx)
+	addl	$1, cond_futex(%ebx)
 
 #define FRAME_SIZE 16
 	subl	$FRAME_SIZE, %esp
@@ -92,8 +93,10 @@ __pthread_cond_wait:
 	movl	%edx, 8(%esp)
 	movl	%eax, 12(%esp)
 
+8:	movl	cond_futex(%ebx), %edi
+
 	/* Unlock.  */
-8:	LOCK
+	LOCK
 #if cond_lock == 0
 	subl	$1, (%ebx)
 #else
@@ -107,12 +110,12 @@ __pthread_cond_wait:
 
 	movl	%esi, %ecx	/* movl $FUTEX_WAIT, %ecx */
 	movl	%edi, %edx
-	addl	$wakeup_seq, %ebx
-.Ladd_wakeup:
+	addl	$cond_futex, %ebx
+.Ladd_cond_futex:
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
-	subl	$wakeup_seq, %ebx
-.Lsub_wakeup:
+	subl	$cond_futex, %ebx
+.Lsub_cond_futex:
 
 	movl	(%esp), %eax
 	call	__pthread_disable_asynccancel
@@ -246,7 +249,7 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
 
 	.type	__condvar_w_cleanup2, @function
 __condvar_w_cleanup2:
-	subl	$wakeup_seq, %ebx
+	subl	$cond_futex, %ebx
 	.size	__condvar_w_cleanup2, .-__condvar_w_cleanup2
 .LSbl4:
 	.type	__condvar_w_cleanup, @function
@@ -278,6 +281,8 @@ __condvar_w_cleanup:
 	addl	$1, wakeup_seq(%ebx)
 	adcl	$0, wakeup_seq+4(%ebx)
 
+	addl	$1, cond_futex(%ebx)
+
 	addl	$1, woken_seq(%ebx)
 	adcl	$0, woken_seq+4(%ebx)
 
@@ -297,7 +302,7 @@ __condvar_w_cleanup:
 	call	__lll_mutex_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
-2:	addl	$wakeup_seq, %ebx
+2:	addl	$cond_futex, %ebx
 	movl	$FUTEX_WAKE, %ecx
 	movl	$SYS_futex, %eax
 	movl	$0x7fffffff, %edx
@@ -323,15 +328,15 @@ __condvar_w_cleanup:
 	.uleb128 .Lcstend-.Lcstbegin
 .Lcstbegin:
 	.long	.LcleanupSTART-.LSTARTCODE
-	.long	.Ladd_wakeup-.LcleanupSTART
+	.long	.Ladd_cond_futex-.LcleanupSTART
 	.long	__condvar_w_cleanup-.LSTARTCODE
 	.uleb128  0
-	.long	.Ladd_wakeup-.LSTARTCODE
-	.long	.Lsub_wakeup-.Ladd_wakeup
+	.long	.Ladd_cond_futex-.LSTARTCODE
+	.long	.Lsub_cond_futex-.Ladd_cond_futex
 	.long	__condvar_w_cleanup2-.LSTARTCODE
 	.uleb128  0
-	.long	.Lsub_wakeup-.LSTARTCODE
-	.long	.LcleanupEND-.Lsub_wakeup
+	.long	.Lsub_cond_futex-.LSTARTCODE
+	.long	.LcleanupEND-.Lsub_cond_futex
 	.long	__condvar_w_cleanup-.LSTARTCODE
 	.uleb128  0
 	.long	.LcallUR-.LSTARTCODE