diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux')
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/fork.h | 3 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S | 52 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/kernel-posix-timers.h | 10 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/lowlevellock.c | 35 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/register-atfork.c | 15 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.S | 54 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S | 4 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/timer_create.c | 7 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/timer_delete.c | 23 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/timer_routines.c | 45 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/unregister-atfork.c | 13 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 57 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S | 2 |
13 files changed, 178 insertions, 142 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/fork.h b/nptl/sysdeps/unix/sysv/linux/fork.h index 032b68f083..a00cfabe26 100644 --- a/nptl/sysdeps/unix/sysv/linux/fork.h +++ b/nptl/sysdeps/unix/sysv/linux/fork.h @@ -55,3 +55,6 @@ extern int __register_atfork (void (*__prepare) (void), void (*__child) (void), void *dso_handle); libc_hidden_proto (__register_atfork) + +/* Add a new element to the fork list. */ +extern void __linkin_atfork (struct fork_handler *newp) attribute_hidden; diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S index 745ab91239..9c8a68f5b2 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S @@ -184,6 +184,12 @@ __lll_timedlock_wait: movl %ecx, %ebp movl %edx, %edi + movl $2, %edx + xchgl %edx, (%ebp) + + test %edx, %edx + je 6f + 1: /* Get current time. */ movl %esp, %ebx @@ -203,36 +209,30 @@ __lll_timedlock_wait: addl $1000000000, %edx subl $1, %ecx 4: testl %ecx, %ecx - js 5f /* Time is already up. */ + js 2f /* Time is already up. */ /* Store relative timeout. */ movl %ecx, (%esp) movl %edx, 4(%esp) + /* Futex call. */ movl %ebp, %ebx - - movl $1, %eax movl $2, %edx - LOCK - cmpxchgl %edx, (%ebx) - - testl %eax, %eax - je 8f - - /* Futex call. */ movl %esp, %esi movl 16(%esp), %ecx LOAD_FUTEX_WAIT (%ecx) movl $SYS_futex, %eax ENTER_KERNEL - movl %eax, %ecx -8: /* NB: %edx == 2 */ - xorl %eax, %eax - LOCK - cmpxchgl %edx, (%ebx) + /* NB: %edx == 2 */ + xchgl %edx, (%ebp) + + testl %edx, %edx + je 6f - jnz 7f + cmpl $-ETIMEDOUT, %eax + jne 1b +2: movl $ETIMEDOUT, %edx 6: addl $8, %esp cfi_adjust_cfa_offset(-8) @@ -248,29 +248,11 @@ __lll_timedlock_wait: popl %edi cfi_adjust_cfa_offset(-4) cfi_restore(%edi) + movl %edx, %eax ret 3: movl $EINVAL, %eax ret - - cfi_adjust_cfa_offset(24) - cfi_offset(%edi, -8) - cfi_offset(%esi, -12) - cfi_offset(%ebx, -16) - cfi_offset(%ebp, -20) - /* Check whether the time expired. */ -7: cmpl $-ETIMEDOUT, %ecx - je 5f - - /* Make sure the current holder knows we are going to sleep. */ - movl %edx, %eax - xchgl %eax, (%ebx) - testl %eax, %eax - jz 6b - jmp 1b - -5: movl $ETIMEDOUT, %eax - jmp 6b cfi_endproc .size __lll_timedlock_wait,.-__lll_timedlock_wait #endif diff --git a/nptl/sysdeps/unix/sysv/linux/kernel-posix-timers.h b/nptl/sysdeps/unix/sysv/linux/kernel-posix-timers.h index 3ff4cda3f4..5f419c2144 100644 --- a/nptl/sysdeps/unix/sysv/linux/kernel-posix-timers.h +++ b/nptl/sysdeps/unix/sysv/linux/kernel-posix-timers.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2003 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2003. @@ -35,6 +35,11 @@ extern pthread_once_t __helper_once attribute_hidden; /* TID of the helper thread. */ extern pid_t __helper_tid attribute_hidden; +/* List of active SIGEV_THREAD timers. */ +extern struct timer *__active_timer_sigev_thread attribute_hidden; +/* Lock for the __active_timer_sigev_thread. */ +extern pthread_mutex_t __active_timer_sigev_thread_lock attribute_hidden; + /* Type of timers in the kernel. */ typedef int kernel_timer_t; @@ -57,4 +62,7 @@ struct timer void (*thrfunc) (sigval_t); sigval_t sival; pthread_attr_t attr; + + /* Next element in list of active SIGEV_THREAD timers. */ + struct timer *next; }; diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c index f0e42957c2..01c4f4861a 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c +++ b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c @@ -27,13 +27,11 @@ void __lll_lock_wait_private (int *futex) { - do - { - int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1); - if (oldval != 0) - lll_futex_wait (futex, 2, LLL_PRIVATE); - } - while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0); + if (*futex == 2) + lll_futex_wait (futex, 2, LLL_PRIVATE); + + while (atomic_exchange_acq (futex, 2) != 0) + lll_futex_wait (futex, 2, LLL_PRIVATE); } @@ -42,13 +40,11 @@ __lll_lock_wait_private (int *futex) void __lll_lock_wait (int *futex, int private) { - do - { - int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1); - if (oldval != 0) - lll_futex_wait (futex, 2, private); - } - while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0); + if (*futex == 2) + lll_futex_wait (futex, 2, private); + + while (atomic_exchange_acq (futex, 2) != 0) + lll_futex_wait (futex, 2, private); } @@ -59,15 +55,16 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime, int private) if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) return EINVAL; - do + /* Try locking. */ + while (atomic_exchange_acq (futex, 2) != 0) { struct timeval tv; - struct timespec rt; /* Get the current time. */ (void) __gettimeofday (&tv, NULL); /* Compute relative timeout. */ + struct timespec rt; rt.tv_sec = abstime->tv_sec - tv.tv_sec; rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000; if (rt.tv_nsec < 0) @@ -76,16 +73,12 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime, int private) --rt.tv_sec; } - /* Already timed out? */ if (rt.tv_sec < 0) return ETIMEDOUT; /* Wait. */ - int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1); - if (oldval != 0) - lll_futex_timed_wait (futex, 2, &rt, private); + lll_futex_timed_wait (futex, 2, &rt, private); } - while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0); return 0; } diff --git a/nptl/sysdeps/unix/sysv/linux/register-atfork.c b/nptl/sysdeps/unix/sysv/linux/register-atfork.c index 231fc9b091..bd103295fb 100644 --- a/nptl/sysdeps/unix/sysv/linux/register-atfork.c +++ b/nptl/sysdeps/unix/sysv/linux/register-atfork.c @@ -21,6 +21,7 @@ #include <stdlib.h> #include <string.h> #include <fork.h> +#include <atomic.h> /* Lock to protect allocation and deallocation of fork handlers. */ @@ -97,8 +98,7 @@ __register_atfork (prepare, parent, child, dso_handle) newp->child_handler = child; newp->dso_handle = dso_handle; - newp->next = __fork_handlers; - __fork_handlers = newp; + __linkin_atfork (newp); } /* Release the lock. */ @@ -109,6 +109,17 @@ __register_atfork (prepare, parent, child, dso_handle) libc_hidden_def (__register_atfork) +void +attribute_hidden +__linkin_atfork (struct fork_handler *newp) +{ + do + newp->next = __fork_handlers; + while (catomic_compare_and_exchange_bool_acq (&__fork_handlers, + newp, newp->next) != 0); +} + + libc_freeres_fn (free_mem) { /* Get the lock to not conflict with running forks. */ diff --git a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.S index e929e28499..afcf0be299 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.S @@ -76,7 +76,7 @@ add tmp2, tmp ; \ mov.l @tmp, tmp2 ; \ bra 98f ; \ - mov #FUTEX_PRIVATE_FLAG, tmp + mov #FUTEX_PRIVATE_FLAG, tmp ; \ 99: .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \ 98: extu.b tmp, tmp ; \ xor tmp, reg ; \ @@ -88,7 +88,7 @@ add tmp2, tmp ; \ mov.l @tmp, tmp2 ; \ bra 98f ; \ - mov #FUTEX_PRIVATE_FLAG, tmp + mov #FUTEX_PRIVATE_FLAG, tmp ; \ 99: .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \ 98: extu.b tmp, tmp ; \ xor tmp, reg ; \ @@ -96,13 +96,13 @@ mov #FUTEX_WAIT, tmp ; \ or tmp, reg # endif -# define LOAD_FUTEX_WAKE(reg,tmp) \ +# define LOAD_FUTEX_WAKE(reg,tmp,tmp2) \ stc gbr, tmp ; \ mov.w 99f, tmp2 ; \ add tmp2, tmp ; \ mov.l @tmp, tmp2 ; \ bra 98f ; \ - mov #FUTEX_PRIVATE_FLAG, tmp + mov #FUTEX_PRIVATE_FLAG, tmp ; \ 99: .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \ 98: extu.b tmp, tmp ; \ xor tmp, reg ; \ @@ -225,6 +225,12 @@ __lll_timedlock_wait: add #-8, r15 cfi_adjust_cfa_offset(8) + mov #2, r2 + XCHG (r2, @r8, r3) + + tst r3, r3 + bt 6f + 1: /* Get current time. */ mov r15, r4 @@ -250,17 +256,11 @@ __lll_timedlock_wait: add #-1, r2 4: cmp/pz r2 - bf 5f /* Time is already up. */ + bf 2f /* Time is already up. */ mov.l r2, @r15 /* Store relative timeout. */ mov.l r3, @(4,r15) - mov #1, r3 - mov #2, r4 - CMPXCHG (r3, @r8, r4, r2) - tst r2, r2 - bt 8f - mov r8, r4 mov r11, r5 LOAD_FUTEX_WAIT (r5, r0, r1) @@ -272,39 +272,29 @@ __lll_timedlock_wait: SYSCALL_INST_PAD mov r0, r5 -8: - mov #0, r3 - mov #2, r4 - CMPXCHG (r3, @r8, r4, r2) - bf/s 7f - mov #0, r0 + mov #2, r2 + XCHG (r2, @r8, r3) + + tst r3, r3 + bt/s 6f + mov #-ETIMEDOUT, r1 + cmp/eq r5, r1 + bf 1b + +2: mov #ETIMEDOUT, r3 6: + mov r3, r0 add #8, r15 mov.l @r15+, r8 mov.l @r15+, r9 mov.l @r15+, r10 rts mov.l @r15+, r11 -7: - /* Check whether the time expired. */ - mov #-ETIMEDOUT, r1 - cmp/eq r5, r1 - bt 5f - /* Make sure the current holder knows we are going to sleep. */ - XCHG (r2, @r8, r3) - tst r3, r3 - bt/s 6b - mov #0, r0 - bra 1b - nop 3: rts mov #EINVAL, r0 -5: - bra 6b - mov #ETIMEDOUT, r0 cfi_endproc .L1k: diff --git a/nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S index 0ebfbfe8c9..13093422a3 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S +++ b/nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S @@ -42,7 +42,7 @@ add tmp2, tmp ; \ mov.l @tmp, tmp2 ; \ bra 98f ; \ - mov #FUTEX_PRIVATE_FLAG, tmp + mov #FUTEX_PRIVATE_FLAG, tmp ; \ 99: .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \ 98: extu.b tmp, tmp ; \ xor tmp, reg ; \ @@ -54,7 +54,7 @@ add tmp2, tmp ; \ mov.l @tmp, tmp2 ; \ bra 98f ; \ - mov #FUTEX_PRIVATE_FLAG, tmp + mov #FUTEX_PRIVATE_FLAG, tmp ; \ 99: .word PRIVATE_FUTEX - TLS_PRE_TCB_SIZE ; \ 98: extu.b tmp, tmp ; \ xor tmp, reg ; \ diff --git a/nptl/sysdeps/unix/sysv/linux/timer_create.c b/nptl/sysdeps/unix/sysv/linux/timer_create.c index 497068b554..a07234d7d1 100644 --- a/nptl/sysdeps/unix/sysv/linux/timer_create.c +++ b/nptl/sysdeps/unix/sysv/linux/timer_create.c @@ -206,6 +206,13 @@ timer_create (clock_id, evp, timerid) syscall_clockid, &sev, &newp->ktimerid); if (! INTERNAL_SYSCALL_ERROR_P (res, err)) { + /* Add to the queue of active timers with thread + delivery. */ + pthread_mutex_lock (&__active_timer_sigev_thread_lock); + newp->next = __active_timer_sigev_thread; + __active_timer_sigev_thread = newp; + pthread_mutex_unlock (&__active_timer_sigev_thread_lock); + *timerid = (timer_t) newp; return 0; } diff --git a/nptl/sysdeps/unix/sysv/linux/timer_delete.c b/nptl/sysdeps/unix/sysv/linux/timer_delete.c index 35055212a3..510541fc2a 100644 --- a/nptl/sysdeps/unix/sysv/linux/timer_delete.c +++ b/nptl/sysdeps/unix/sysv/linux/timer_delete.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2003. @@ -54,6 +54,27 @@ timer_delete (timerid) if (res == 0) { + if (kt->sigev_notify == SIGEV_THREAD) + { + /* Remove the timer from the list. */ + pthread_mutex_lock (&__active_timer_sigev_thread_lock); + if (__active_timer_sigev_thread == kt) + __active_timer_sigev_thread = kt->next; + else + { + struct timer *prevp = __active_timer_sigev_thread; + while (prevp->next != NULL) + if (prevp->next == kt) + { + prevp->next = kt->next; + break; + } + else + prevp = prevp->next; + } + pthread_mutex_unlock (&__active_timer_sigev_thread_lock); + } + # ifndef __ASSUME_POSIX_TIMERS /* We know the syscall support is available. */ __no_posix_timers = 1; diff --git a/nptl/sysdeps/unix/sysv/linux/timer_routines.c b/nptl/sysdeps/unix/sysv/linux/timer_routines.c index a5eb442251..b159316fb2 100644 --- a/nptl/sysdeps/unix/sysv/linux/timer_routines.c +++ b/nptl/sysdeps/unix/sysv/linux/timer_routines.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2003. @@ -27,6 +27,12 @@ #include "kernel-posix-timers.h" +/* List of active SIGEV_THREAD timers. */ +struct timer *__active_timer_sigev_thread; +/* Lock for the __active_timer_sigev_thread. */ +pthread_mutex_t __active_timer_sigev_thread_lock = PTHREAD_MUTEX_INITIALIZER; + + struct thread_start_data { void (*thrfunc) (sigval_t); @@ -95,19 +101,36 @@ timer_helper_thread (void *arg) if (si.si_code == SI_TIMER) { struct timer *tk = (struct timer *) si.si_ptr; - struct thread_start_data *td = malloc (sizeof (*td)); - /* There is not much we can do if the allocation fails. */ - if (td != NULL) - { - /* That is the signal we are waiting for. */ - td->thrfunc = tk->thrfunc; - td->sival = tk->sival; + /* Check the timer is still used and will not go away + while we are reading the values here. */ + pthread_mutex_lock (&__active_timer_sigev_thread_lock); - pthread_t th; - (void) pthread_create (&th, &tk->attr, timer_sigev_thread, - td); + struct timer *runp = __active_timer_sigev_thread; + while (runp != NULL) + if (runp == tk) + break; + else + runp = runp->next; + + if (runp != NULL) + { + struct thread_start_data *td = malloc (sizeof (*td)); + + /* There is not much we can do if the allocation fails. */ + if (td != NULL) + { + /* This is the signal we are waiting for. */ + td->thrfunc = tk->thrfunc; + td->sival = tk->sival; + + pthread_t th; + (void) pthread_create (&th, &tk->attr, + timer_sigev_thread, td); + } } + + pthread_mutex_unlock (&__active_timer_sigev_thread_lock); } else if (si.si_code == SI_TKILL) /* The thread is canceled. */ diff --git a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c index 56a4f149e1..c738acd0c3 100644 --- a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c +++ b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c @@ -67,10 +67,21 @@ __unregister_atfork (dso_handle) It's a single linked list so readers are. */ do { + again: if (runp->dso_handle == dso_handle) { if (lastp == NULL) - __fork_handlers = runp->next; + { + /* We have to use an atomic operation here because + __linkin_atfork also uses one. */ + if (catomic_compare_and_exchange_bool_acq (&__fork_handlers, + runp->next, runp) + != 0) + { + runp = __fork_handlers; + goto again; + } + } else lastp->next = runp->next; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S index 7065cfac32..4505e2cec6 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S @@ -178,6 +178,12 @@ __lll_timedlock_wait: movq %rdi, %r12 movq %rdx, %r13 + movl $2, %edx + xchgl %edx, (%r12) + + testl %edx, %edx + je 6f + 1: /* Get current time. */ movq %rsp, %rdi @@ -199,33 +205,31 @@ __lll_timedlock_wait: addq $1000000000, %rsi decq %rdi 4: testq %rdi, %rdi - js 5f /* Time is already up. */ + js 2f /* Time is already up. */ - /* Futex call. */ - movq %rdi, (%rsp) /* Store relative timeout. */ + /* Store relative timeout. */ + movq %rdi, (%rsp) movq %rsi, 8(%rsp) - movl $1, %eax + /* Futex call. */ movl $2, %edx - LOCK - cmpxchgl %edx, (%r12) - - testl %eax, %eax - je 8f - + movl $1, %eax movq %rsp, %r10 movl 24(%rsp), %esi LOAD_FUTEX_WAIT (%esi) movq %r12, %rdi movl $SYS_futex, %eax syscall - movq %rax, %rcx -8: /* NB: %edx == 2 */ - xorl %eax, %eax - LOCK - cmpxchgl %edx, (%r12) - jnz 7f + /* NB: %edx == 2 */ + xchgl %edx, (%r12) + + testl %edx, %edx + je 6f + + cmpl $-ETIMEDOUT, %eax + jne 1b +2: movl $ETIMEDOUT, %edx 6: addq $32, %rsp cfi_adjust_cfa_offset(-32) @@ -244,30 +248,11 @@ __lll_timedlock_wait: popq %r8 cfi_adjust_cfa_offset(-8) cfi_restore(%r8) + movl %edx, %eax retq 3: movl $EINVAL, %eax retq - - cfi_adjust_cfa_offset(72) - cfi_offset(%r8, -16) - cfi_offset(%r9, -24) - cfi_offset(%r12, -32) - cfi_offset(%r13, -40) - cfi_offset(%r14, -48) - /* Check whether the time expired. */ -7: cmpq $-ETIMEDOUT, %rcx - je 5f - - /* Make sure the current holder knows we are going to sleep. */ - movl %edx, %eax - xchgl %eax, (%rdi) - testl %eax, %eax - jz 6b - jmp 1b - -5: movl $ETIMEDOUT, %eax - jmp 6b cfi_endproc .size __lll_timedlock_wait,.-__lll_timedlock_wait #endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index d730a7e12d..20bc59db9c 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -134,12 +134,14 @@ __pthread_cond_timedwait: /* Only clocks 0 and 1 are allowed so far. Both are handled in the kernel. */ leaq 24(%rsp), %rsi +# ifdef SHARED movq __vdso_clock_gettime@GOTPCREL(%rip), %rax movq (%rax), %rax PTR_DEMANGLE (%rax) jz 26f call *%rax jmp 27f +# endif 26: movl $__NR_clock_gettime, %eax syscall 27: |