summaryrefslogtreecommitdiff
path: root/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
diff options
context:
space:
mode:
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S')
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S161
1 files changed, 121 insertions, 40 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
index 502f1d442f..7065cfac32 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
@@ -19,33 +19,46 @@
#include <sysdep.h>
#include <pthread-errnos.h>
+#include <kernel-features.h>
+#include <lowlevellock.h>
.text
-#ifndef LOCK
-# ifdef UP
-# define LOCK
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
+ movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
+# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
+ movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
+# define LOAD_FUTEX_WAIT(reg) \
+ xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
+# define LOAD_FUTEX_WAKE(reg) \
+ xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
+#else
+# if FUTEX_WAIT == 0
+# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
+ movl %fs:PRIVATE_FUTEX, reg
# else
-# define LOCK lock
+# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
+ movl %fs:PRIVATE_FUTEX, reg ; \
+ orl $FUTEX_WAIT, reg
# endif
-#endif
-
-#define SYS_futex 202
-#ifndef FUTEX_WAIT
-# define FUTEX_WAIT 0
-# define FUTEX_WAKE 1
-#endif
-
-#ifndef LOAD_FUTEX_WAIT
+# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
+ movl %fs:PRIVATE_FUTEX, reg ; \
+ orl $FUTEX_WAKE, reg
# if FUTEX_WAIT == 0
# define LOAD_FUTEX_WAIT(reg) \
- xorl reg, reg
+ xorl $FUTEX_PRIVATE_FLAG, reg ; \
+ andl %fs:PRIVATE_FUTEX, reg
# else
# define LOAD_FUTEX_WAIT(reg) \
- movl $FUTEX_WAIT, reg
+ xorl $FUTEX_PRIVATE_FLAG, reg ; \
+ andl %fs:PRIVATE_FUTEX, reg ; \
+ orl $FUTEX_WAIT, reg
# endif
# define LOAD_FUTEX_WAKE(reg) \
- movl $FUTEX_WAKE, reg
+ xorl $FUTEX_PRIVATE_FLAG, reg ; \
+ andl %fs:PRIVATE_FUTEX, reg ; \
+ orl $FUTEX_WAKE, reg
#endif
@@ -53,11 +66,11 @@
#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000
- .globl __lll_mutex_lock_wait
- .type __lll_mutex_lock_wait,@function
- .hidden __lll_mutex_lock_wait
+ .globl __lll_lock_wait_private
+ .type __lll_lock_wait_private,@function
+ .hidden __lll_lock_wait_private
.align 16
-__lll_mutex_lock_wait:
+__lll_lock_wait_private:
cfi_startproc
pushq %r10
cfi_adjust_cfa_offset(8)
@@ -67,7 +80,7 @@ __lll_mutex_lock_wait:
cfi_offset(%rdx, -24)
xorq %r10, %r10 /* No timeout. */
movl $2, %edx
- LOAD_FUTEX_WAIT (%esi)
+ LOAD_PRIVATE_FUTEX_WAIT (%esi)
cmpl %edx, %eax /* NB: %edx == 2 */
jne 2f
@@ -89,15 +102,52 @@ __lll_mutex_lock_wait:
cfi_restore(%r10)
retq
cfi_endproc
- .size __lll_mutex_lock_wait,.-__lll_mutex_lock_wait
-
+ .size __lll_lock_wait_private,.-__lll_lock_wait_private
#ifdef NOT_IN_libc
- .globl __lll_mutex_timedlock_wait
- .type __lll_mutex_timedlock_wait,@function
- .hidden __lll_mutex_timedlock_wait
+ .globl __lll_lock_wait
+ .type __lll_lock_wait,@function
+ .hidden __lll_lock_wait
.align 16
-__lll_mutex_timedlock_wait:
+__lll_lock_wait:
+ cfi_startproc
+ pushq %r10
+ cfi_adjust_cfa_offset(8)
+ pushq %rdx
+ cfi_adjust_cfa_offset(8)
+ cfi_offset(%r10, -16)
+ cfi_offset(%rdx, -24)
+ xorq %r10, %r10 /* No timeout. */
+ movl $2, %edx
+ LOAD_FUTEX_WAIT (%esi)
+
+ cmpl %edx, %eax /* NB: %edx == 2 */
+ jne 2f
+
+1: movl $SYS_futex, %eax
+ syscall
+
+2: movl %edx, %eax
+ xchgl %eax, (%rdi) /* NB: lock is implied */
+
+ testl %eax, %eax
+ jnz 1b
+
+ popq %rdx
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore(%rdx)
+ popq %r10
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore(%r10)
+ retq
+ cfi_endproc
+ .size __lll_lock_wait,.-__lll_lock_wait
+
+ .globl __lll_timedlock_wait
+ .type __lll_timedlock_wait,@function
+ .hidden __lll_timedlock_wait
+ .align 16
+__lll_timedlock_wait:
cfi_startproc
/* Check for a valid timeout value. */
cmpq $1000000000, 8(%rdx)
@@ -118,10 +168,12 @@ __lll_mutex_timedlock_wait:
cfi_offset(%r12, -32)
cfi_offset(%r13, -40)
cfi_offset(%r14, -48)
+ pushq %rsi
+ cfi_adjust_cfa_offset(8)
/* Stack frame for the timespec and timeval structs. */
- subq $16, %rsp
- cfi_adjust_cfa_offset(16)
+ subq $24, %rsp
+ cfi_adjust_cfa_offset(24)
movq %rdi, %r12
movq %rdx, %r13
@@ -162,6 +214,7 @@ __lll_mutex_timedlock_wait:
je 8f
movq %rsp, %r10
+ movl 24(%rsp), %esi
LOAD_FUTEX_WAIT (%esi)
movq %r12, %rdi
movl $SYS_futex, %eax
@@ -174,8 +227,8 @@ __lll_mutex_timedlock_wait:
cmpxchgl %edx, (%r12)
jnz 7f
-6: addq $16, %rsp
- cfi_adjust_cfa_offset(-16)
+6: addq $32, %rsp
+ cfi_adjust_cfa_offset(-32)
popq %r14
cfi_adjust_cfa_offset(-8)
cfi_restore(%r14)
@@ -196,7 +249,7 @@ __lll_mutex_timedlock_wait:
3: movl $EINVAL, %eax
retq
- cfi_adjust_cfa_offset(56)
+ cfi_adjust_cfa_offset(72)
cfi_offset(%r8, -16)
cfi_offset(%r9, -24)
cfi_offset(%r12, -32)
@@ -216,15 +269,15 @@ __lll_mutex_timedlock_wait:
5: movl $ETIMEDOUT, %eax
jmp 6b
cfi_endproc
- .size __lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait
+ .size __lll_timedlock_wait,.-__lll_timedlock_wait
#endif
- .globl __lll_mutex_unlock_wake
- .type __lll_mutex_unlock_wake,@function
- .hidden __lll_mutex_unlock_wake
+ .globl __lll_unlock_wake_private
+ .type __lll_unlock_wake_private,@function
+ .hidden __lll_unlock_wake_private
.align 16
-__lll_mutex_unlock_wake:
+__lll_unlock_wake_private:
cfi_startproc
pushq %rsi
cfi_adjust_cfa_offset(8)
@@ -234,7 +287,7 @@ __lll_mutex_unlock_wake:
cfi_offset(%rdx, -24)
movl $0, (%rdi)
- LOAD_FUTEX_WAKE (%esi)
+ LOAD_PRIVATE_FUTEX_WAKE (%esi)
movl $1, %edx /* Wake one thread. */
movl $SYS_futex, %eax
syscall
@@ -247,10 +300,38 @@ __lll_mutex_unlock_wake:
cfi_restore(%rsi)
retq
cfi_endproc
- .size __lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake
-
+ .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
#ifdef NOT_IN_libc
+ .globl __lll_unlock_wake
+ .type __lll_unlock_wake,@function
+ .hidden __lll_unlock_wake
+ .align 16
+__lll_unlock_wake:
+ cfi_startproc
+ pushq %rsi
+ cfi_adjust_cfa_offset(8)
+ pushq %rdx
+ cfi_adjust_cfa_offset(8)
+ cfi_offset(%rsi, -16)
+ cfi_offset(%rdx, -24)
+
+ movl $0, (%rdi)
+ LOAD_FUTEX_WAKE (%esi)
+ movl $1, %edx /* Wake one thread. */
+ movl $SYS_futex, %eax
+ syscall
+
+ popq %rdx
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore(%rdx)
+ popq %rsi
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore(%rsi)
+ retq
+ cfi_endproc
+ .size __lll_unlock_wake,.-__lll_unlock_wake
+
.globl __lll_timedwait_tid
.type __lll_timedwait_tid,@function
.hidden __lll_timedwait_tid
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-27 23:43:02 +0000