diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S')
-rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | 432 |
1 files changed, 228 insertions, 204 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index e5e802d531..e6323ea3e2 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc. +/* Copyright (C) 2002-2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -22,146 +22,42 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <tcb-offsets.h> +#include <pthread-pi-defines.h> #include <kernel-features.h> .text - .align 16 - .type __condvar_cleanup, @function - .globl __condvar_cleanup - .hidden __condvar_cleanup -__condvar_cleanup: - pushq %r12 - - /* Get internal lock. */ - movq %rdi, %r8 - movq 8(%rdi), %rdi - movl $1, %esi - xorl %eax, %eax - LOCK -#if cond_lock == 0 - cmpxchgl %esi, (%rdi) -#else - cmpxchgl %esi, cond_lock(%rdi) -#endif - jz 1f - -#if cond_lock != 0 - addq $cond_lock, %rdi -#endif - cmpq $-1, dep_mutex-cond_lock(%rdi) - movl $LLL_PRIVATE, %eax - movl $LLL_SHARED, %esi - cmovne %eax, %esi - callq __lll_lock_wait -#if cond_lock != 0 - subq $cond_lock, %rdi -#endif - -1: movl broadcast_seq(%rdi), %edx - cmpl 4(%r8), %edx - jne 3f - - /* We increment the wakeup_seq counter only if it is lower than - total_seq. If this is not the case the thread was woken and - then canceled. In this case we ignore the signal. */ - movq total_seq(%rdi), %rax - cmpq wakeup_seq(%rdi), %rax - jbe 6f - incq wakeup_seq(%rdi) - incl cond_futex(%rdi) -6: incq woken_seq(%rdi) - -3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) - - /* Wake up a thread which wants to destroy the condvar object. */ - xorq %r12, %r12 - cmpq $0xffffffffffffffff, total_seq(%rdi) - jne 4f - movl cond_nwaiters(%rdi), %eax - andl $~((1 << nwaiters_shift) - 1), %eax - jne 4f - - addq $cond_nwaiters, %rdi - cmpq $-1, dep_mutex-cond_nwaiters(%rdi) - movl $1, %edx -#ifdef __ASSUME_PRIVATE_FUTEX - movl $FUTEX_WAKE, %eax - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi - cmove %eax, %esi -#else - movl $0, %eax - movl %fs:PRIVATE_FUTEX, %esi - cmove %eax, %esi - orl $FUTEX_WAKE, %esi -#endif - movl $SYS_futex, %eax - syscall - subq $cond_nwaiters, %rdi - movl $1, %r12d - -4: LOCK -#if cond_lock == 0 - decl (%rdi) -#else - decl cond_lock(%rdi) -#endif - je 2f -#if cond_lock != 0 - addq $cond_lock, %rdi -#endif - cmpq $-1, dep_mutex-cond_lock(%rdi) - movl $LLL_PRIVATE, %eax - movl $LLL_SHARED, %esi - cmovne %eax, %esi - callq __lll_unlock_wake - - /* Wake up all waiters to make sure no signal gets lost. */ -2: testq %r12, %r12 - jnz 5f - addq $cond_futex, %rdi - cmpq $-1, dep_mutex-cond_futex(%rdi) - movl $0x7fffffff, %edx -#ifdef __ASSUME_PRIVATE_FUTEX - movl $FUTEX_WAKE, %eax - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi - cmove %eax, %esi -#else - movl $0, %eax - movl %fs:PRIVATE_FUTEX, %esi - cmove %eax, %esi - orl $FUTEX_WAKE, %esi -#endif - movl $SYS_futex, %eax - syscall - -5: movq 16(%r8), %rdi - callq __pthread_mutex_cond_lock - - popq %r12 - - retq - .size __condvar_cleanup, .-__condvar_cleanup - - /* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */ .globl __pthread_cond_wait .type __pthread_cond_wait, @function .align 16 __pthread_cond_wait: .LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif + pushq %r12 -.Lpush_r12: -#define FRAME_SIZE 64 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) + pushq %r13 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) +#define FRAME_SIZE 32 subq $FRAME_SIZE, %rsp -.Lsubq: + cfi_adjust_cfa_offset(FRAME_SIZE) + /* Stack frame: - rsp + 64 - +--------------------------+ - rsp + 32 | cleanup buffer | + rsp + 32 +--------------------------+ rsp + 24 | old wake_seq value | +--------------------------+ @@ -208,16 +104,6 @@ __pthread_cond_wait: incl cond_futex(%rdi) addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) - /* Install cancellation handler. */ -#ifdef PIC - leaq __condvar_cleanup(%rip), %rsi -#else - leaq __condvar_cleanup, %rsi -#endif - leaq 32(%rsp), %rdi - movq %rsp, %rdx - callq __pthread_cleanup_push - /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi movq wakeup_seq(%rdi), %r9 @@ -235,31 +121,57 @@ __pthread_cond_wait: #endif jne 3f +.LcleanupSTART: 4: callq __pthread_enable_asynccancel movl %eax, (%rsp) movq 8(%rsp), %rdi xorq %r10, %r10 movq %r12, %rdx - addq $cond_futex-cond_lock, %rdi + // XXX reverse + lea + addq $cond_futex, %rdi cmpq $-1, dep_mutex-cond_futex(%rdi) #ifdef __ASSUME_PRIVATE_FUTEX movl $FUTEX_WAIT, %eax movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi cmove %eax, %esi #else - movl $FUTEX_WAIT, %eax + movl $0, %eax movl %fs:PRIVATE_FUTEX, %esi cmove %eax, %esi # if FUTEX_WAIT != 0 +# error "cc destroyed by following orl" orl $FUTEX_WAIT, %esi # endif #endif + je 60f + + movq dep_mutex-cond_futex(%rdi), %r8 + /* Requeue to a PI mutex if the PI bit is set. */ + testl $PI_BIT, MUTEX_KIND(%r8) + je 60f + + movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + movl $SYS_futex, %eax + syscall + + movl $1, %r13d +#ifdef __ASSUME_REQUEUE_PI + jmp 62f +#else + cmpq $-4095, %rax + jnae 62f + + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi +#endif + +60: xorl %r13d, %r13d movl $SYS_futex, %eax syscall - movl (%rsp), %edi +62: movl (%rsp), %edi callq __pthread_disable_asynccancel +.LcleanupEND: /* Lock. */ movq 8(%rsp), %rdi @@ -325,24 +237,33 @@ __pthread_cond_wait: #endif jne 10f - /* Remove cancellation handler. */ -11: movq 32+CLEANUP_PREV(%rsp), %rdx - movq %rdx, %fs:CLEANUP + /* If requeue_pi is used the kernel performs the locking of the + mutex. */ +11: xorl %eax, %eax + testl %r13d, %r13d + jnz 14f movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock + 14: addq $FRAME_SIZE, %rsp -.Laddq: + cfi_adjust_cfa_offset(-FRAME_SIZE) + popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 -.Lpop_r12: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) /* We return the result of the mutex_lock operation. */ retq /* Initial locking failed. */ 1: -.LSbl1: + cfi_adjust_cfa_offset(16 + FRAME_SIZE) + cfi_rel_offset(%r12, FRAME_SIZE + 8) + cfi_rel_offset(%r13, FRAME_SIZE) #if cond_lock != 0 addq $cond_lock, %rdi #endif @@ -414,75 +335,178 @@ __pthread_cond_wait: 13: movq %r10, %rax jmp 14b -.LENDCODE: .size __pthread_cond_wait, .-__pthread_cond_wait versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, GLIBC_2_3_2) - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long L(ENDCIE)-L(STARTCIE) # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zR" # NUL-terminated augmentation - # string. + .align 16 + .type __condvar_cleanup1, @function + .globl __condvar_cleanup1 + .hidden __condvar_cleanup1 +__condvar_cleanup1: + /* Stack frame: + + rsp + 48 + +--------------------------+ + rsp + 40 | %r12 | + +--------------------------+ + rsp + 32 | %r13 | + +--------------------------+ + rsp + 24 | unused | + +--------------------------+ + rsp + 16 | mutex pointer | + +--------------------------+ + rsp + 8 | condvar pointer | + +--------------------------+ + rsp + 4 | old broadcast_seq value | + +--------------------------+ + rsp + 0 | old cancellation mode | + +--------------------------+ + */ + + movq %rax, 24(%rsp) + + /* Get internal lock. */ + movq 8(%rsp), %rdi + movl $1, %esi + xorl %eax, %eax + LOCK +#if cond_lock == 0 + cmpxchgl %esi, (%rdi) #else - .ascii "\0" # NUL-terminated augmentation - # string. + cmpxchgl %esi, cond_lock(%rdi) #endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 1 # Augmentation value length. - .byte 0x1b # Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. + jz 1f + +#if cond_lock != 0 + addq $cond_lock, %rdi #endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x8 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait +#if cond_lock != 0 + subq $cond_lock, %rdi +#endif + +1: movl broadcast_seq(%rdi), %edx + cmpl 4(%rsp), %edx + jne 3f + + /* We increment the wakeup_seq counter only if it is lower than + total_seq. If this is not the case the thread was woken and + then canceled. In this case we ignore the signal. */ + movq total_seq(%rdi), %rax + cmpq wakeup_seq(%rdi), %rax + jbe 6f + incq wakeup_seq(%rdi) + incl cond_futex(%rdi) +6: incq woken_seq(%rdi) + +3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorq %r12, %r12 + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 4f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << nwaiters_shift) - 1), %eax + jne 4f + + cmpq $-1, dep_mutex(%rdi) + leaq cond_nwaiters(%rdi), %rdi + movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax + syscall + subq $cond_nwaiters, %rdi + movl $1, %r12d + +4: LOCK +#if cond_lock == 0 + decl (%rdi) #else - .long .LSTARTCODE # Start address of the code. + decl cond_lock(%rdi) #endif - .long .LENDCODE-.LSTARTCODE # Length of the code. -#ifdef SHARED - .uleb128 0 # No augmentation data. + je 2f +#if cond_lock != 0 + addq $cond_lock, %rdi #endif - .byte 0x40+.Lpush_r12-.LSTARTCODE # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 0x40+.Lsubq-.Lpush_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddq-.Lsubq - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x40+.Lpop_r12-.Laddq # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 0xcc # DW_CFA_restore %r12 - .byte 0x40+.LSbl1-.Lpop_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 80 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + + /* Wake up all waiters to make sure no signal gets lost. */ +2: testq %r12, %r12 + jnz 5f + addq $cond_futex, %rdi + cmpq $-1, dep_mutex-cond_futex(%rdi) + movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax + syscall + +5: movq 16(%rsp), %rdi + callq __pthread_mutex_cond_lock + + movq 24(%rsp), %rdi + movq 40(%rsp), %r12 + movq 32(%rsp), %r13 +.LcallUR: + call _Unwind_Resume@PLT + hlt +.LENDCODE: + cfi_endproc + .size __condvar_cleanup1, .-__condvar_cleanup1 + + + .section .gcc_except_table,"a",@progbits +.LexceptSTART: + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format + .uleb128 .Lcstend-.Lcstbegin +.Lcstbegin: + .uleb128 .LcleanupSTART-.LSTARTCODE + .uleb128 .LcleanupEND-.LcleanupSTART + .uleb128 __condvar_cleanup1-.LSTARTCODE + .uleb128 0 + .uleb128 .LcallUR-.LSTARTCODE + .uleb128 .LENDCODE-.LcallUR + .uleb128 0 + .uleb128 0 +.Lcstend: + + +#ifdef SHARED + .hidden DW.ref.__gcc_personality_v0 + .weak DW.ref.__gcc_personality_v0 + .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits .align 8 -.LENDFDE: + .type DW.ref.__gcc_personality_v0, @object + .size DW.ref.__gcc_personality_v0, 8 +DW.ref.__gcc_personality_v0: + .quad __gcc_personality_v0 +#endif |