summaryrefslogtreecommitdiff
path: root/nptl/sysdeps/unix/sysv/linux
diff options
context:
space:
mode:
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux')
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S30
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S43
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S67
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S71
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S18
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h2
-rw-r--r--nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c21
7 files changed, 197 insertions, 55 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 40fb04b31b..a7ca78f78d 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -91,12 +91,17 @@ __pthread_cond_broadcast:
8: cmpl $-1, %edi
je 9f
- /* XXX: The kernel so far doesn't support requeue to PI futex. */
- /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
- type of futex (private resp. shared). */
- testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
+ /* Do not use requeue for pshared condvars. */
+ testl $PS_BIT, MUTEX_KIND(%edi)
jne 9f
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ je 81f
+
/* Wake up all threads. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
@@ -138,6 +143,23 @@ __pthread_cond_broadcast:
cfi_restore_state
+81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $0x7fffffff, %esi
+ movl $1, %edx
+ /* Get the address of the futex involved. */
+# if MUTEX_FUTEX != 0
+ addl $MUTEX_FUTEX, %edi
+# endif
+ int $0x80
+
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpl $0xfffff001, %eax
+ jb 6b
+ jmp 9f
+
/* Initial locking failed. */
1:
#if cond_lock == 0
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index 013fcc303f..9fc2cbfeaf 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <kernel-features.h>
+#include <pthread-pi-defines.h>
#include <pthread-errnos.h>
@@ -86,7 +87,17 @@ __pthread_cond_signal:
#endif
cmpl $-1, dep_mutex-cond_futex(%ebx)
sete %cl
- subl $1, %ecx
+ je 8f
+
+ movl dep_mutex-cond_futex(%ebx), %edx
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edx), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ je 9f
+
+8: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -124,8 +135,34 @@ __pthread_cond_signal:
cfi_restore_state
-7: /* %ecx should be either FUTEX_WAKE_OP or
- FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
+9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $1, %edx
+ xorl %esi, %esi
+ movl dep_mutex-cond_futex(%ebx), %edi
+ movl (%ebx), %ebp
+ /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+ sysenter.
+ ENTER_KERNEL */
+ int $0x80
+ popl %ebp
+ popl %esi
+
+ leal -cond_futex(%ebx), %edi
+
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ cmpl $-4095, %eax
+ jb 4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+ andl $FUTEX_PRIVATE_FLAG, %ecx
+#else
+ andl %gs:PRIVATE_FUTEX, %ecx
+#endif
+ orl $FUTEX_WAKE, %ecx
+
xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
movl $SYS_futex, %eax
/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index 8f5088a460..218762a8ad 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -95,7 +96,7 @@ __pthread_cond_timedwait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -107,8 +108,10 @@ __pthread_cond_timedwait:
movl %edx, 16(%esp)
movl %eax, 20(%esp)
+ /* Reset the pi-requeued flag. */
+8: movl $0, 24(%esp)
/* Get the current time. */
-8: movl %ebx, %edx
+ movl %ebx, %edx
#ifdef __NR_clock_gettime
/* Get the clock number. */
movl cond_nwaiters(%ebx), %ebx
@@ -158,6 +161,7 @@ __pthread_cond_timedwait:
movl %edx, 8(%esp)
movl cond_futex(%ebx), %edi
+ movl %edi, 28(%esp)
/* Unlock. */
LOCK
@@ -172,13 +176,50 @@ __pthread_cond_timedwait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
- leal 4(%esp), %esi
#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 40f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ jne 40f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ /* The following only works like this because we only support
+ two clocks, represented using a single bit. */
+ testl $1, cond_nwaiters(%ebx)
+ /* XXX Need to implement using sete instead of a jump. */
+ jne 42f
+ orl $FUTEX_CLOCK_REALTIME, %ecx
+
+ /* Requeue-PI uses absolute timeout */
+42: leal (%ebp), %esi
+ movl 28(%esp), %edx
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ movl %eax, %esi
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on ETIMEDOUT or any other error. */
+ cmpl $0, %eax
+ sete 24(%esp)
+ je 41f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 41f
+ xorl %ecx, %ecx
+
+40: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -187,7 +228,8 @@ __pthread_cond_timedwait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ leal 4(%esp), %esi
+ movl 28(%esp), %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -196,7 +238,7 @@ __pthread_cond_timedwait:
.Lsub_cond_futex:
movl %eax, %esi
- movl (%esp), %eax
+41: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -284,10 +326,16 @@ __pthread_cond_timedwait:
#endif
jne 10f
+11: xorl %eax, %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+ movl 24(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 26f
+
/* Remove cancellation handler. */
-11: movl 24+FRAME_SIZE(%esp), %eax
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+26: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
/* We return the result of the mutex_lock operation if it failed. */
@@ -317,6 +365,9 @@ __pthread_cond_timedwait:
cfi_restore_state
+27: call __pthread_mutex_cond_lock_adjust
+ jmp 26b
+
/* Initial locking failed. */
1:
#if cond_lock == 0
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 776b95e40c..b3b8bc4be5 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -22,6 +22,8 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -43,6 +45,9 @@ __pthread_cond_wait:
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
#endif
+ pushl %ebp
+ cfi_adjust_cfa_offset(4)
+ cfi_rel_offset(%ebp, 0)
pushl %edi
cfi_adjust_cfa_offset(4)
cfi_rel_offset(%edi, 0)
@@ -55,7 +60,7 @@ __pthread_cond_wait:
cfi_remember_state
xorl %esi, %esi
- movl 16(%esp), %ebx
+ movl 20(%esp), %ebx
/* Get internal lock. */
movl $1, %edx
@@ -71,7 +76,7 @@ __pthread_cond_wait:
/* Store the reference to the mutex. If there is already a
different value in there this is a bad user bug. */
2: cmpl $-1, dep_mutex(%ebx)
- movl 20(%esp), %eax
+ movl 24(%esp), %eax
je 15f
movl %eax, dep_mutex(%ebx)
@@ -87,7 +92,7 @@ __pthread_cond_wait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -99,7 +104,9 @@ __pthread_cond_wait:
movl %edx, 8(%esp)
movl %eax, 12(%esp)
-8: movl cond_futex(%ebx), %edi
+ /* Reset the pi-requeued flag. */
+8: movl $0, 16(%esp)
+ movl cond_futex(%ebx), %ebp
/* Unlock. */
LOCK
@@ -114,12 +121,39 @@ __pthread_cond_wait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
-#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
-#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a non-robust PI mutex if the PI bit is set and
+ the robust bit is not set. */
+ movl MUTEX_KIND(%edi), %eax
+ andl $(ROBUST_BIT|PI_BIT), %eax
+ cmpl $PI_BIT, %eax
+ jne 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl %ebp, %edx
+ xorl %esi, %esi
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on error. */
+ cmpl $0, %eax
+ sete 16(%esp)
+ je 19f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -128,7 +162,7 @@ __pthread_cond_wait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ movl %ebp, %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -136,7 +170,7 @@ __pthread_cond_wait:
subl $cond_futex, %ebx
.Lsub_cond_futex:
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -212,9 +246,15 @@ __pthread_cond_wait:
#endif
jne 10f
-11: movl 20+FRAME_SIZE(%esp), %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+11: xorl %eax, %eax
+ movl 16(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 20f
+
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+20: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
14: popl %ebx
@@ -226,12 +266,19 @@ __pthread_cond_wait:
popl %edi
cfi_adjust_cfa_offset(-4)
cfi_restore(%edi)
+ popl %ebp
+ cfi_adjust_cfa_offset(-4)
+ cfi_restore(%ebp)
/* We return the result of the mutex_lock operation. */
ret
cfi_restore_state
+21: call __pthread_mutex_cond_lock_adjust
+ xorl %eax, %eax
+ jmp 20b
+
/* Initial locking failed. */
1:
#if cond_lock == 0
@@ -484,7 +531,7 @@ __condvar_w_cleanup:
movl $0x7fffffff, %edx
ENTER_KERNEL
-5: movl 20+FRAME_SIZE(%esp), %eax
+5: movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
movl %esi, (%esp)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S
index 13a36d64cc..18013a8df4 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -131,10 +131,10 @@ sem_timedwait:
xorl %eax, %eax
-10: LOCK
+ LOCK
decl NWAITERS(%ebx)
- addl $12, %esp
+10: addl $12, %esp
.Ladd_esp:
popl %ebx
.Lpop_ebx:
@@ -268,19 +268,19 @@ sem_wait_cleanup:
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 8
.byte 0x86 # DW_CFA_offset %esi
- .uleb128 2
+ .uleb128 2
.byte 4 # DW_CFA_advance_loc4
.long .Lpush_edi-.Lpush_esi
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 12
.byte 0x87 # DW_CFA_offset %edi
- .uleb128 3
+ .uleb128 3
.byte 4 # DW_CFA_advance_loc4
.long .Lpush_ebx-.Lpush_edi
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 16
.byte 0x83 # DW_CFA_offset %ebx
- .uleb128 4
+ .uleb128 4
.byte 4 # DW_CFA_advance_loc4
.long .Lsub_esp-.Lpush_ebx
.byte 14 # DW_CFA_def_cfa_offset
@@ -309,11 +309,11 @@ sem_wait_cleanup:
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 28
.byte 0x86 # DW_CFA_offset %esi
- .uleb128 2
+ .uleb128 2
.byte 0x87 # DW_CFA_offset %edi
- .uleb128 3
+ .uleb128 3
.byte 0x83 # DW_CFA_offset %ebx
- .uleb128 4
+ .uleb128 4
.align 4
.LENDFDE:
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index 66e0e628fd..4bb585af3b 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -54,6 +54,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
diff --git a/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c b/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c
index 5391d5cc8b..82c97c352a 100644
--- a/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c
+++ b/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c
@@ -1,5 +1,5 @@
-/* Special .init and .fini section support for SH. NPTL version.
- Copyright (C) 2003 Free Software Foundation, Inc.
+/* Special .init and .fini section support for SH. NPTL version.
+ Copyright (C) 2003, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it
@@ -62,19 +62,12 @@ _init:\n\
mov.l .L24,r1\n\
add r0,r1\n\
jsr @r1\n\
- nop\n\
- mova .L23,r0\n\
- mov.l .L23,r1\n\
- add r0,r1\n\
- jsr @r1\n\
mov r15,r14\n\
bra 1f\n\
nop\n\
.align 2\n\
.L22:\n\
.long _GLOBAL_OFFSET_TABLE_\n\
-.L23:\n\
- .long __gmon_start__@PLT\n\
.L24:\n\
.long __pthread_initialize_minimal_internal@PLT\n\
1:\n\
@@ -91,16 +84,6 @@ _init:\n\
rts \n\
mov.l @r15+,r12\n\
END_INIT\n\
- .section .text\n\
- .align 5\n\
- .weak __gmon_start__\n\
- .type __gmon_start__,@function\n\
-__gmon_start__:\n\
- mov.l r14,@-r15\n\
- mov r15,r14\n\
- mov r14,r15\n\
- rts \n\
- mov.l @r15+,r14\n\
\n\
/*@_init_EPILOG_ENDS*/\n\
\n\