diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux')
7 files changed, 197 insertions, 55 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 40fb04b31b..a7ca78f78d 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -91,12 +91,17 @@ __pthread_cond_broadcast: 8: cmpl $-1, %edi je 9f - /* XXX: The kernel so far doesn't support requeue to PI futex. */ - /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same - type of futex (private resp. shared). */ - testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi) + /* Do not use requeue for pshared condvars. */ + testl $PS_BIT, MUTEX_KIND(%edi) jne 9f + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%edi), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + je 81f + /* Wake up all threads. */ #ifdef __ASSUME_PRIVATE_FUTEX movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx @@ -138,6 +143,23 @@ __pthread_cond_broadcast: cfi_restore_state +81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx + movl $SYS_futex, %eax + movl $0x7fffffff, %esi + movl $1, %edx + /* Get the address of the futex involved. */ +# if MUTEX_FUTEX != 0 + addl $MUTEX_FUTEX, %edi +# endif + int $0x80 + + /* For any kind of error, which mainly is EAGAIN, we try again + with WAKE. The general test also covers running on old + kernels. */ + cmpl $0xfffff001, %eax + jb 6b + jmp 9f + /* Initial locking failed. */ 1: #if cond_lock == 0 diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index 013fcc303f..9fc2cbfeaf 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -22,6 +22,7 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <kernel-features.h> +#include <pthread-pi-defines.h> #include <pthread-errnos.h> @@ -86,7 +87,17 @@ __pthread_cond_signal: #endif cmpl $-1, dep_mutex-cond_futex(%ebx) sete %cl - subl $1, %ecx + je 8f + + movl dep_mutex-cond_futex(%ebx), %edx + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%edx), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + je 9f + +8: subl $1, %ecx #ifdef __ASSUME_PRIVATE_FUTEX andl $FUTEX_PRIVATE_FLAG, %ecx #else @@ -124,8 +135,34 @@ __pthread_cond_signal: cfi_restore_state -7: /* %ecx should be either FUTEX_WAKE_OP or - FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ +9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx + movl $SYS_futex, %eax + movl $1, %edx + xorl %esi, %esi + movl dep_mutex-cond_futex(%ebx), %edi + movl (%ebx), %ebp + /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for + sysenter. + ENTER_KERNEL */ + int $0x80 + popl %ebp + popl %esi + + leal -cond_futex(%ebx), %edi + + /* For any kind of error, we try again with WAKE. + The general test also covers running on old kernels. */ + cmpl $-4095, %eax + jb 4f + +7: +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + orl $FUTEX_WAKE, %ecx + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx movl $SYS_futex, %eax /* %edx should be 1 already from $FUTEX_WAKE_OP syscall. diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index 8f5088a460..218762a8ad 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -22,6 +22,7 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <pthread-errnos.h> +#include <pthread-pi-defines.h> #include <kernel-features.h> @@ -95,7 +96,7 @@ __pthread_cond_timedwait: addl $1, cond_futex(%ebx) addl $(1 << nwaiters_shift), cond_nwaiters(%ebx) -#define FRAME_SIZE 24 +#define FRAME_SIZE 32 subl $FRAME_SIZE, %esp cfi_adjust_cfa_offset(FRAME_SIZE) @@ -107,8 +108,10 @@ __pthread_cond_timedwait: movl %edx, 16(%esp) movl %eax, 20(%esp) + /* Reset the pi-requeued flag. */ +8: movl $0, 24(%esp) /* Get the current time. */ -8: movl %ebx, %edx + movl %ebx, %edx #ifdef __NR_clock_gettime /* Get the clock number. */ movl cond_nwaiters(%ebx), %ebx @@ -158,6 +161,7 @@ __pthread_cond_timedwait: movl %edx, 8(%esp) movl cond_futex(%ebx), %edi + movl %edi, 28(%esp) /* Unlock. */ LOCK @@ -172,13 +176,50 @@ __pthread_cond_timedwait: 4: call __pthread_enable_asynccancel movl %eax, (%esp) - leal 4(%esp), %esi #if FUTEX_PRIVATE_FLAG > 255 xorl %ecx, %ecx #endif cmpl $-1, dep_mutex(%ebx) sete %cl - subl $1, %ecx + je 40f + + movl dep_mutex(%ebx), %edi + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%edi), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 40f + + movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx + /* The following only works like this because we only support + two clocks, represented using a single bit. */ + testl $1, cond_nwaiters(%ebx) + /* XXX Need to implement using sete instead of a jump. */ + jne 42f + orl $FUTEX_CLOCK_REALTIME, %ecx + + /* Requeue-PI uses absolute timeout */ +42: leal (%ebp), %esi + movl 28(%esp), %edx + addl $cond_futex, %ebx + movl $SYS_futex, %eax + ENTER_KERNEL + subl $cond_futex, %ebx + movl %eax, %esi + /* Set the pi-requeued flag only if the kernel has returned 0. The + kernel does not hold the mutex on ETIMEDOUT or any other error. */ + cmpl $0, %eax + sete 24(%esp) + je 41f + + /* Normal and PI futexes dont mix. Use normal futex functions only + if the kernel does not support the PI futex functions. */ + cmpl $-ENOSYS, %eax + jne 41f + xorl %ecx, %ecx + +40: subl $1, %ecx #ifdef __ASSUME_PRIVATE_FUTEX andl $FUTEX_PRIVATE_FLAG, %ecx #else @@ -187,7 +228,8 @@ __pthread_cond_timedwait: #if FUTEX_WAIT != 0 addl $FUTEX_WAIT, %ecx #endif - movl %edi, %edx + leal 4(%esp), %esi + movl 28(%esp), %edx addl $cond_futex, %ebx .Ladd_cond_futex: movl $SYS_futex, %eax @@ -196,7 +238,7 @@ __pthread_cond_timedwait: .Lsub_cond_futex: movl %eax, %esi - movl (%esp), %eax +41: movl (%esp), %eax call __pthread_disable_asynccancel .LcleanupEND: @@ -284,10 +326,16 @@ __pthread_cond_timedwait: #endif jne 10f +11: xorl %eax, %eax + /* With requeue_pi, the mutex lock is held in the kernel. */ + movl 24(%esp), %ecx + testl %ecx, %ecx + jnz 26f + /* Remove cancellation handler. */ -11: movl 24+FRAME_SIZE(%esp), %eax + movl 24+FRAME_SIZE(%esp), %eax call __pthread_mutex_cond_lock - addl $FRAME_SIZE, %esp +26: addl $FRAME_SIZE, %esp cfi_adjust_cfa_offset(-FRAME_SIZE); /* We return the result of the mutex_lock operation if it failed. */ @@ -317,6 +365,9 @@ __pthread_cond_timedwait: cfi_restore_state +27: call __pthread_mutex_cond_lock_adjust + jmp 26b + /* Initial locking failed. */ 1: #if cond_lock == 0 diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 776b95e40c..b3b8bc4be5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -22,6 +22,8 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <tcb-offsets.h> +#include <pthread-errnos.h> +#include <pthread-pi-defines.h> #include <kernel-features.h> @@ -43,6 +45,9 @@ __pthread_cond_wait: cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) #endif + pushl %ebp + cfi_adjust_cfa_offset(4) + cfi_rel_offset(%ebp, 0) pushl %edi cfi_adjust_cfa_offset(4) cfi_rel_offset(%edi, 0) @@ -55,7 +60,7 @@ __pthread_cond_wait: cfi_remember_state xorl %esi, %esi - movl 16(%esp), %ebx + movl 20(%esp), %ebx /* Get internal lock. */ movl $1, %edx @@ -71,7 +76,7 @@ __pthread_cond_wait: /* Store the reference to the mutex. If there is already a different value in there this is a bad user bug. */ 2: cmpl $-1, dep_mutex(%ebx) - movl 20(%esp), %eax + movl 24(%esp), %eax je 15f movl %eax, dep_mutex(%ebx) @@ -87,7 +92,7 @@ __pthread_cond_wait: addl $1, cond_futex(%ebx) addl $(1 << nwaiters_shift), cond_nwaiters(%ebx) -#define FRAME_SIZE 16 +#define FRAME_SIZE 20 subl $FRAME_SIZE, %esp cfi_adjust_cfa_offset(FRAME_SIZE) @@ -99,7 +104,9 @@ __pthread_cond_wait: movl %edx, 8(%esp) movl %eax, 12(%esp) -8: movl cond_futex(%ebx), %edi + /* Reset the pi-requeued flag. */ +8: movl $0, 16(%esp) + movl cond_futex(%ebx), %ebp /* Unlock. */ LOCK @@ -114,12 +121,39 @@ __pthread_cond_wait: 4: call __pthread_enable_asynccancel movl %eax, (%esp) -#if FUTEX_PRIVATE_FLAG > 255 xorl %ecx, %ecx -#endif cmpl $-1, dep_mutex(%ebx) sete %cl - subl $1, %ecx + je 18f + + movl dep_mutex(%ebx), %edi + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%edi), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 18f + + movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx + movl %ebp, %edx + xorl %esi, %esi + addl $cond_futex, %ebx + movl $SYS_futex, %eax + ENTER_KERNEL + subl $cond_futex, %ebx + /* Set the pi-requeued flag only if the kernel has returned 0. The + kernel does not hold the mutex on error. */ + cmpl $0, %eax + sete 16(%esp) + je 19f + + /* Normal and PI futexes dont mix. Use normal futex functions only + if the kernel does not support the PI futex functions. */ + cmpl $-ENOSYS, %eax + jne 19f + xorl %ecx, %ecx + +18: subl $1, %ecx #ifdef __ASSUME_PRIVATE_FUTEX andl $FUTEX_PRIVATE_FLAG, %ecx #else @@ -128,7 +162,7 @@ __pthread_cond_wait: #if FUTEX_WAIT != 0 addl $FUTEX_WAIT, %ecx #endif - movl %edi, %edx + movl %ebp, %edx addl $cond_futex, %ebx .Ladd_cond_futex: movl $SYS_futex, %eax @@ -136,7 +170,7 @@ __pthread_cond_wait: subl $cond_futex, %ebx .Lsub_cond_futex: - movl (%esp), %eax +19: movl (%esp), %eax call __pthread_disable_asynccancel .LcleanupEND: @@ -212,9 +246,15 @@ __pthread_cond_wait: #endif jne 10f -11: movl 20+FRAME_SIZE(%esp), %eax + /* With requeue_pi, the mutex lock is held in the kernel. */ +11: xorl %eax, %eax + movl 16(%esp), %ecx + testl %ecx, %ecx + jnz 20f + + movl 24+FRAME_SIZE(%esp), %eax call __pthread_mutex_cond_lock - addl $FRAME_SIZE, %esp +20: addl $FRAME_SIZE, %esp cfi_adjust_cfa_offset(-FRAME_SIZE); 14: popl %ebx @@ -226,12 +266,19 @@ __pthread_cond_wait: popl %edi cfi_adjust_cfa_offset(-4) cfi_restore(%edi) + popl %ebp + cfi_adjust_cfa_offset(-4) + cfi_restore(%ebp) /* We return the result of the mutex_lock operation. */ ret cfi_restore_state +21: call __pthread_mutex_cond_lock_adjust + xorl %eax, %eax + jmp 20b + /* Initial locking failed. */ 1: #if cond_lock == 0 @@ -484,7 +531,7 @@ __condvar_w_cleanup: movl $0x7fffffff, %edx ENTER_KERNEL -5: movl 20+FRAME_SIZE(%esp), %eax +5: movl 24+FRAME_SIZE(%esp), %eax call __pthread_mutex_cond_lock movl %esi, (%esp) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S index 13a36d64cc..18013a8df4 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -131,10 +131,10 @@ sem_timedwait: xorl %eax, %eax -10: LOCK + LOCK decl NWAITERS(%ebx) - addl $12, %esp +10: addl $12, %esp .Ladd_esp: popl %ebx .Lpop_ebx: @@ -268,19 +268,19 @@ sem_wait_cleanup: .byte 14 # DW_CFA_def_cfa_offset .uleb128 8 .byte 0x86 # DW_CFA_offset %esi - .uleb128 2 + .uleb128 2 .byte 4 # DW_CFA_advance_loc4 .long .Lpush_edi-.Lpush_esi .byte 14 # DW_CFA_def_cfa_offset .uleb128 12 .byte 0x87 # DW_CFA_offset %edi - .uleb128 3 + .uleb128 3 .byte 4 # DW_CFA_advance_loc4 .long .Lpush_ebx-.Lpush_edi .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 .byte 0x83 # DW_CFA_offset %ebx - .uleb128 4 + .uleb128 4 .byte 4 # DW_CFA_advance_loc4 .long .Lsub_esp-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset @@ -309,11 +309,11 @@ sem_wait_cleanup: .byte 14 # DW_CFA_def_cfa_offset .uleb128 28 .byte 0x86 # DW_CFA_offset %esi - .uleb128 2 + .uleb128 2 .byte 0x87 # DW_CFA_offset %edi - .uleb128 3 + .uleb128 3 .byte 0x83 # DW_CFA_offset %ebx - .uleb128 4 + .uleb128 4 .align 4 .LENDFDE: diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 66e0e628fd..4bb585af3b 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -54,6 +54,8 @@ #define FUTEX_TRYLOCK_PI 8 #define FUTEX_WAIT_BITSET 9 #define FUTEX_WAKE_BITSET 10 +#define FUTEX_WAIT_REQUEUE_PI 11 +#define FUTEX_CMP_REQUEUE_PI 12 #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 diff --git a/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c b/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c index 5391d5cc8b..82c97c352a 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c +++ b/nptl/sysdeps/unix/sysv/linux/sh/pt-initfini.c @@ -1,5 +1,5 @@ -/* Special .init and .fini section support for SH. NPTL version. - Copyright (C) 2003 Free Software Foundation, Inc. +/* Special .init and .fini section support for SH. NPTL version. + Copyright (C) 2003, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it @@ -62,19 +62,12 @@ _init:\n\ mov.l .L24,r1\n\ add r0,r1\n\ jsr @r1\n\ - nop\n\ - mova .L23,r0\n\ - mov.l .L23,r1\n\ - add r0,r1\n\ - jsr @r1\n\ mov r15,r14\n\ bra 1f\n\ nop\n\ .align 2\n\ .L22:\n\ .long _GLOBAL_OFFSET_TABLE_\n\ -.L23:\n\ - .long __gmon_start__@PLT\n\ .L24:\n\ .long __pthread_initialize_minimal_internal@PLT\n\ 1:\n\ @@ -91,16 +84,6 @@ _init:\n\ rts \n\ mov.l @r15+,r12\n\ END_INIT\n\ - .section .text\n\ - .align 5\n\ - .weak __gmon_start__\n\ - .type __gmon_start__,@function\n\ -__gmon_start__:\n\ - mov.l r14,@-r15\n\ - mov r15,r14\n\ - mov r14,r15\n\ - rts \n\ - mov.l @r15+,r14\n\ \n\ /*@_init_EPILOG_ENDS*/\n\ \n\ |