Update.

2004-06-03  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/i386/i486/bits/atomic.h: Optimize a bit.

5 files changed, 53 insertions, 43 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
index e29c77529e..661e476aa7 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
@@ -76,11 +76,12 @@ typedef union
   struct
   {
     int __lock;
-    int __clock;
+    unsigned int __futex;
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
+    int __clock;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index 6d29d761fa..e8d7bd9bb6 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -32,6 +32,7 @@
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
 #define FUTEX_REQUEUE		3
+#define FUTEX_CMP_REQUEUE	4
 
 #define EINVAL			22
 
@@ -55,39 +56,42 @@ __pthread_cond_broadcast:
 #endif
 	jnz	1f
 
-2:	addq	$wakeup_seq, %rdi
-	movq	total_seq-wakeup_seq(%rdi), %rcx
-	cmpq	(%rdi), %rcx
+2:	addq	$cond_futex, %rdi
+	movq	total_seq-cond_futex(%rdi), %r9
+	cmpq	wakeup_seq-cond_futex(%rdi), %r9
 	jna	4f
 
 	/* Cause all currently waiting threads to recognize they are
 	   woken up.  */
-	movq	%rcx, (%rdi)
-	movq	%rcx, woken_seq-wakeup_seq(%rdi)
-	incl	broadcast_seq-wakeup_seq(%rdi)
+	movq	%r9, wakeup_seq-cond_futex(%rdi)
+	movq	%r9, woken_seq-cond_futex(%rdi)
+	addq	%r9, %r9
+	movl	%r9d, (%rdi)
+	incl	broadcast_seq-cond_futex(%rdi)
 
 	/* Get the address of the mutex used.  */
-	movq	dep_mutex-wakeup_seq(%rdi), %r8
+	movq	dep_mutex-cond_futex(%rdi), %r8
 
 	/* Unlock.  */
 	LOCK
-	decl	cond_lock-wakeup_seq(%rdi)
+	decl	cond_lock-cond_futex(%rdi)
 	jne	7f
 
 8:	cmpq	$-1, %r8
 	je	9f
 
 	/* Wake up all threads.  */
-	movq	$FUTEX_REQUEUE, %rsi
+	movq	$FUTEX_CMP_REQUEUE, %rsi
 	movq	$SYS_futex, %rax
 	movl	$1, %edx
 	movq	$0x7fffffff, %r10
 	syscall
 
-#ifndef __ASSUME_FUTEX_REQUEUE
-	cmpq	$-EINVAL, %rax
-	je	9f
-#endif
+	/* For any kind of error, which mainly is EAGAIN, we try again
+	   with WAKE.  The general test also covers running on old
+	   kernels.  */
+	cmpq	$-4095, %rax
+	jae	9f
 
 10:	xorl	%eax, %eax
 	retq
@@ -95,7 +99,7 @@ __pthread_cond_broadcast:
 	.align	16
 	/* Unlock.  */
 4:	LOCK
-	decl	cond_lock-wakeup_seq(%rdi)
+	decl	cond_lock-cond_futex(%rdi)
 	jne	5f
 
 6:	xorl	%eax, %eax
@@ -113,14 +117,14 @@ __pthread_cond_broadcast:
 	jmp	2b
 
 	/* Unlock in loop requires wakeup.  */
-5:	addq	$cond_lock-wakeup_seq, %rdi
+5:	addq	$cond_lock-cond_futex, %rdi
 	callq	__lll_mutex_unlock_wake
 	jmp	6b
 
 	/* Unlock in loop requires wakeup.  */
-7:	addq	$cond_lock-wakeup_seq, %rdi
+7:	addq	$cond_lock-cond_futex, %rdi
 	callq	__lll_mutex_unlock_wake
-	subq	$cond_lock-wakeup_seq, %rdi
+	subq	$cond_lock-cond_futex, %rdi
 	jmp	8b
 
 9:	/* The futex requeue functionality is not available.  */
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 92eadfc433..62bb74cc1a 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -56,13 +56,14 @@ __pthread_cond_signal:
 #endif
 	jnz	1f
 
-2:	addq	$wakeup_seq, %rdi
+2:	addq	$cond_futex, %rdi
 	movq	total_seq(%r8), %rcx
-	cmpq	(%rdi), %rcx
+	cmpq	wakeup_seq(%r8), %rcx
 	jbe	4f
 
 	/* Bump the wakeup number.  */
-	addq	$1, (%rdi)
+	addq	$1, wakeup_seq(%r8)
+	addl	$1, (%rdi)
 
 	/* Wake up one thread.  */
 	movq	$FUTEX_WAKE, %rsi
@@ -95,11 +96,7 @@ __pthread_cond_signal:
 
 	/* Unlock in loop requires wakeup.  */
 5:
-#if cond_lock != 0
-	addq	$cond_lock-wakeup_seq, %rdi
-#else
 	movq	%r8, %rdi
-#endif
 	callq	__lll_mutex_unlock_wake
 	jmp	6b
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 80cbf7e430..e75f05e07f 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -105,6 +105,7 @@ __pthread_cond_timedwait:
 
 	movq	8(%rsp), %rdi
 	incq	total_seq(%rdi)
+	incl	cond_futex(%rdi)
 
 	/* Install cancellation handler.  */
 #ifdef PIC
@@ -118,9 +119,9 @@ __pthread_cond_timedwait:
 
 	/* Get and store current wakeup_seq value.  */
 	movq	8(%rsp), %rdi
-	movq	wakeup_seq(%rdi), %r12
+	movq	wakeup_seq(%rdi), %r9
 	movl	broadcast_seq(%rdi), %edx
-	movq	%r12, 40(%rsp)
+	movq	%r9, 40(%rsp)
 	movl	%edx, 4(%rsp)
 
 	/* Get the current time.  */
@@ -172,6 +173,8 @@ __pthread_cond_timedwait:
 21:	movq	%rcx, 24(%rsp)
 	movq	%rdx, 32(%rsp)
 
+	movl	cond_futex(%rdi), %r12d
+
 	/* Unlock.  */
 	LOCK
 #if cond_lock == 0
@@ -187,7 +190,7 @@ __pthread_cond_timedwait:
 	leaq	24(%rsp), %r10
 	xorq	%rsi, %rsi	/* movq $FUTEX_WAIT, %rsi */
 	movq	%r12, %rdx
-	addq	$wakeup_seq-cond_lock, %rdi
+	addq	$cond_futex, %rdi
 	movq	$SYS_futex, %rax
 	syscall
 	movq	%rax, %r14
@@ -211,21 +214,22 @@ __pthread_cond_timedwait:
 
 	movq	woken_seq(%rdi), %rax
 
-	movq	wakeup_seq(%rdi), %r12
+	movq	wakeup_seq(%rdi), %r9
 
 	cmpl	4(%rsp), %edx
 	jne	23f
 
-	cmpq	40(%rsp), %r12
+	cmpq	40(%rsp), %r9
 	jbe	15f
 
-	cmpq	%rax, %r12
+	cmpq	%rax, %r9
 	ja	9f
 
 15:	cmpq	$-ETIMEDOUT, %r14
 	jne	8b
 
 13:	incq	wakeup_seq(%rdi)
+	incl	cond_futex(%rdi)
 	movq	$ETIMEDOUT, %r14
 	jmp	14f
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 065eb11813..9e7da301d3 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -69,6 +69,8 @@ __condvar_cleanup:
 
 	incq	woken_seq(%rdi)
 
+	incl	cond_futex(%rdi)
+
 3:	LOCK
 #if cond_lock == 0
 	decl	(%rdi)
@@ -82,7 +84,7 @@ __condvar_cleanup:
 	callq	__lll_mutex_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
-2:	addq	$wakeup_seq, %rdi
+2:	addq	$cond_futex, %rdi
 	movq	$FUTEX_WAKE, %rsi
 	movl	$0x7fffffff, %edx
 	movq	$SYS_futex, %rax
@@ -154,6 +156,7 @@ __pthread_cond_wait:
 
 	movq	8(%rsp), %rdi
 	incq	total_seq(%rdi)
+	incl	cond_futex(%rdi)
 
 	/* Install cancellation handler.  */
 #ifdef PIC
@@ -167,13 +170,14 @@ __pthread_cond_wait:
 
 	/* Get and store current wakeup_seq value.  */
 	movq	8(%rsp), %rdi
-	movq	wakeup_seq(%rdi), %r12
+	movq	wakeup_seq(%rdi), %r9
 	movl	broadcast_seq(%rdi), %edx
-	movq	%r12, 24(%rsp)
+	movq	%r9, 24(%rsp)
 	movl	%edx, 4(%rsp)
 
 	/* Unlock.  */
-8:	LOCK
+8:	movl	cond_futex(%rdi), %r12d
+	LOCK
 #if cond_lock == 0
 	decl	(%rdi)
 #else
@@ -187,7 +191,7 @@ __pthread_cond_wait:
 	movq	8(%rsp), %rdi
 	xorq	%r10, %r10
 	movq	%r12, %rdx
-	addq	$wakeup_seq-cond_lock, %rdi
+	addq	$cond_futex-cond_lock, %rdi
 	movq	$SYS_futex, %rax
 	movq	%r10, %rsi	/* movq $FUTEX_WAIT, %rsi */
 	syscall
@@ -211,15 +215,15 @@ __pthread_cond_wait:
 
 	movq	woken_seq(%rdi), %rax
 
-	movq	wakeup_seq(%rdi), %r12
+	movq	wakeup_seq(%rdi), %r9
 
 	cmpl	4(%rsp), %edx
 	jne	16f
 
-	cmpq	24(%rsp), %r12
+	cmpq	24(%rsp), %r9
 	jbe	8b
 
-	cmpq	%rax, %r12
+	cmpq	%rax, %r9
 	jna	8b
 
 	incq	woken_seq(%rdi)
@@ -359,8 +363,8 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
 	.byte	0x40+.Lsubq-.Lpush_r12		# DW_CFA_advance_loc+N
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 16+FRAME_SIZE
-	.byte	2				# DW_CFA_advance_loc1
-	.byte	.Laddq-.Lsubq
+	.byte	3				# DW_CFA_advance_loc2
+	.2byte	.Laddq-.Lsubq
 	.byte	14				# DW_CFA_def_cfa_offset
 	.uleb128 16
 	.byte	0x40+.Lpop_r12-.Laddq		# DW_CFA_advance_loc+N