diff options
author | Andreas Schwab <schwab@redhat.com> | 2009-07-30 14:18:37 +0200 |
---|---|---|
committer | Andreas Schwab <schwab@redhat.com> | 2009-07-30 14:18:37 +0200 |
commit | b870de510d54108c7c839abc17ea1559085e55a3 (patch) | |
tree | d59aca63c9713ac51b929e388187f6ec0bb1273e /nptl/sysdeps | |
parent | ca2a37b64e0347b400e58da9ca238c9320a55edb (diff) | |
parent | 78c4ef475d47a2289635f74b726f52defedb4651 (diff) |
Merge commit 'origin/master' into fedora/master
Diffstat (limited to 'nptl/sysdeps')
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym | 1 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S | 6 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S | 6 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S | 24 | ||||
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | 23 | ||||
-rw-r--r-- | nptl/sysdeps/x86_64/tcb-offsets.sym | 1 | ||||
-rw-r--r-- | nptl/sysdeps/x86_64/tls.h | 80 |
7 files changed, 102 insertions, 39 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym index d985c6a79b..46fbd0de74 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym +++ b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym @@ -3,5 +3,6 @@ -- These PI macros are used by assembly code. MUTEX_KIND offsetof (pthread_mutex_t, __data.__kind) +ROBUST_BIT PTHREAD_MUTEX_ROBUST_NORMAL_NP PI_BIT PTHREAD_MUTEX_PRIO_INHERIT_NP PS_BIT PTHREAD_MUTEX_PSHARED_BIT diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 0f10ec910c..224a56088e 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -75,8 +75,10 @@ __pthread_cond_broadcast: jne 9f /* Requeue to a PI mutex if the PI bit is set. */ - testl $PI_BIT, MUTEX_KIND(%r8) - jne 81f + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + je 81f /* Wake up all threads. */ #ifdef __ASSUME_PRIVATE_FUTEX diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index f1050fea7c..4d001eec7f 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -64,8 +64,10 @@ __pthread_cond_signal: /* Get the address of the mutex used. */ movq dep_mutex(%r8), %rcx - testl $PI_BIT, MUTEX_KIND(%rcx) - jne 9f + movl MUTEX_KIND(%rcx), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + je 9f #ifdef __ASSUME_PRIVATE_FUTEX movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 7486825d5f..4913beb8af 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -165,9 +165,12 @@ __pthread_cond_timedwait: je 60f movq dep_mutex(%rdi), %r8 - /* Requeue to a PI mutex if the PI bit is set. */ - testl $PI_BIT, MUTEX_KIND(%r8) - je 61f + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 61f movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi xorl %eax, %eax @@ -289,11 +292,10 @@ __pthread_cond_timedwait: /* If requeue_pi is used the kernel performs the locking of the mutex. */ -41: xorl %eax, %eax +41: movq 16(%rsp), %rdi testl %r15d, %r15d - jnz 63f + jnz 64f - movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock 63: testq %rax, %rax @@ -316,12 +318,18 @@ __pthread_cond_timedwait: retq - /* Initial locking failed. */ -31: cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE) + cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE) cfi_rel_offset(%r12, FRAME_SIZE + 24) cfi_rel_offset(%r13, FRAME_SIZE + 16) cfi_rel_offset(%r14, FRAME_SIZE + 8) cfi_rel_offset(%r15, FRAME_SIZE) + +64: callq __pthread_mutex_cond_lock_adjust + movq %r14, %rax + jmp 48b + + /* Initial locking failed. */ +31: #if cond_lock != 0 addq $cond_lock, %rdi #endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 2fab38e277..a66523eab6 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -134,9 +134,12 @@ __pthread_cond_wait: je 60f movq dep_mutex-cond_futex(%rdi), %r8 - /* Requeue to a PI mutex if the PI bit is set. */ - testl $PI_BIT, MUTEX_KIND(%r8) - je 61f + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 61f movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi movl $SYS_futex, %eax @@ -234,11 +237,10 @@ __pthread_cond_wait: /* If requeue_pi is used the kernel performs the locking of the mutex. */ -11: xorl %eax, %eax +11: movq 16(%rsp), %rdi testl %r13d, %r13d - jnz 14f + jnz 18f - movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock 14: addq $FRAME_SIZE, %rsp @@ -254,11 +256,16 @@ __pthread_cond_wait: /* We return the result of the mutex_lock operation. */ retq - /* Initial locking failed. */ -1: cfi_adjust_cfa_offset(16 + FRAME_SIZE) cfi_rel_offset(%r12, FRAME_SIZE + 8) cfi_rel_offset(%r13, FRAME_SIZE) + +18: callq __pthread_mutex_cond_lock_adjust + xorl %eax, %eax + jmp 14b + + /* Initial locking failed. */ +1: #if cond_lock != 0 addq $cond_lock, %rdi #endif diff --git a/nptl/sysdeps/x86_64/tcb-offsets.sym b/nptl/sysdeps/x86_64/tcb-offsets.sym index 1c70c6bde7..51f35c61cf 100644 --- a/nptl/sysdeps/x86_64/tcb-offsets.sym +++ b/nptl/sysdeps/x86_64/tcb-offsets.sym @@ -15,3 +15,4 @@ VGETCPU_CACHE_OFFSET offsetof (tcbhead_t, vgetcpu_cache) #ifndef __ASSUME_PRIVATE_FUTEX PRIVATE_FUTEX offsetof (tcbhead_t, private_futex) #endif +RTLD_SAVESPACE_SSE offsetof (tcbhead_t, rtld_savespace_sse) diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h index ea89f3b1a2..4212038ab5 100644 --- a/nptl/sysdeps/x86_64/tls.h +++ b/nptl/sysdeps/x86_64/tls.h @@ -29,6 +29,7 @@ # include <sysdep.h> # include <kernel-features.h> # include <bits/wordsize.h> +# include <xmmintrin.h> /* Type for the dtv. */ @@ -55,16 +56,23 @@ typedef struct uintptr_t stack_guard; uintptr_t pointer_guard; unsigned long int vgetcpu_cache[2]; -#ifndef __ASSUME_PRIVATE_FUTEX +# ifndef __ASSUME_PRIVATE_FUTEX int private_futex; -#else +# else int __unused1; -#endif -#if __WORDSIZE == 64 - int __pad1; -#endif +# endif +# if __WORDSIZE == 64 + int rtld_must_xmm_save; +# endif /* Reservation of some values for the TM ABI. */ void *__private_tm[5]; +# if __WORDSIZE == 64 + long int __unused2; + /* Have space for the post-AVX register size. */ + __m128 rtld_savespace_sse[8][4]; + + void *__padding[8]; +# endif } tcbhead_t; #else /* __ASSEMBLER__ */ @@ -109,7 +117,12 @@ typedef struct # define TLS_TCB_SIZE sizeof (struct pthread) /* Alignment requirements for the TCB. */ -# define TLS_TCB_ALIGN __alignof__ (struct pthread) +//# define TLS_TCB_ALIGN __alignof__ (struct pthread) +// Normally the above would be correct But we have to store post-AVX +// vector registers in the TCB and we want the storage to be aligned. +// unfortunately there isn't yet a type for these values and hence no +// 32-byte alignment requirement. Make this explicit, for now. +# define TLS_TCB_ALIGN 32 /* The TCB can have any size and the memory following the address the thread pointer points to is unspecified. Allocate the TCB there. */ @@ -298,7 +311,7 @@ typedef struct /* Atomic compare and exchange on TLS, returning old value. */ -#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \ +# define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \ ({ __typeof (descr->member) __ret; \ __typeof (oldval) __old = (oldval); \ if (sizeof (descr->member) == 4) \ @@ -313,7 +326,7 @@ typedef struct /* Atomic logical and. */ -#define THREAD_ATOMIC_AND(descr, member, val) \ +# define THREAD_ATOMIC_AND(descr, member, val) \ (void) ({ if (sizeof ((descr)->member) == 4) \ asm volatile (LOCK_PREFIX "andl %1, %%fs:%P0" \ :: "i" (offsetof (struct pthread, member)), \ @@ -324,7 +337,7 @@ typedef struct /* Atomic set bit. */ -#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \ +# define THREAD_ATOMIC_BIT_SET(descr, member, bit) \ (void) ({ if (sizeof ((descr)->member) == 4) \ asm volatile (LOCK_PREFIX "orl %1, %%fs:%P0" \ :: "i" (offsetof (struct pthread, member)), \ @@ -334,7 +347,7 @@ typedef struct abort (); }) -#define CALL_THREAD_FCT(descr) \ +# define CALL_THREAD_FCT(descr) \ ({ void *__res; \ asm volatile ("movq %%fs:%P2, %%rdi\n\t" \ "callq *%%fs:%P1" \ @@ -355,18 +368,18 @@ typedef struct /* Set the pointer guard field in the TCB head. */ -#define THREAD_SET_POINTER_GUARD(value) \ +# define THREAD_SET_POINTER_GUARD(value) \ THREAD_SETMEM (THREAD_SELF, header.pointer_guard, value) -#define THREAD_COPY_POINTER_GUARD(descr) \ +# define THREAD_COPY_POINTER_GUARD(descr) \ ((descr)->header.pointer_guard \ = THREAD_GETMEM (THREAD_SELF, header.pointer_guard)) /* Get and set the global scope generation counter in the TCB head. */ -#define THREAD_GSCOPE_FLAG_UNUSED 0 -#define THREAD_GSCOPE_FLAG_USED 1 -#define THREAD_GSCOPE_FLAG_WAIT 2 -#define THREAD_GSCOPE_RESET_FLAG() \ +# define THREAD_GSCOPE_FLAG_UNUSED 0 +# define THREAD_GSCOPE_FLAG_USED 1 +# define THREAD_GSCOPE_FLAG_WAIT 2 +# define THREAD_GSCOPE_RESET_FLAG() \ do \ { int __res; \ asm volatile ("xchgl %0, %%fs:%P1" \ @@ -377,11 +390,40 @@ typedef struct lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1, LLL_PRIVATE); \ } \ while (0) -#define THREAD_GSCOPE_SET_FLAG() \ +# define THREAD_GSCOPE_SET_FLAG() \ THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED) -#define THREAD_GSCOPE_WAIT() \ +# define THREAD_GSCOPE_WAIT() \ GL(dl_wait_lookup_done) () + +# ifdef SHARED +/* Defined in dl-trampoline.S. */ +extern void _dl_x86_64_save_sse (void); +extern void _dl_x86_64_restore_sse (void); + +# define RTLD_CHECK_FOREIGN_CALL \ + (THREAD_GETMEM (THREAD_SELF, header.rtld_must_xmm_save) != 0) + +# define RTLD_ENABLE_FOREIGN_CALL \ + THREAD_SETMEM (THREAD_SELF, header.rtld_must_xmm_save, 1) + +# define RTLD_PREPARE_FOREIGN_CALL \ + do if (THREAD_GETMEM (THREAD_SELF, header.rtld_must_xmm_save)) \ + { \ + _dl_x86_64_save_sse (); \ + THREAD_SETMEM (THREAD_SELF, header.rtld_must_xmm_save, 0); \ + } \ + while (0) + +# define RTLD_FINALIZE_FOREIGN_CALL \ + do { \ + if (THREAD_GETMEM (THREAD_SELF, header.rtld_must_xmm_save) == 0) \ + _dl_x86_64_restore_sse (); \ + THREAD_SETMEM (THREAD_SELF, header.rtld_must_xmm_save, 0); \ + } while (0) +# endif + + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ |