diff options
23 files changed, 227 insertions, 5 deletions
@@ -1,5 +1,53 @@ 2014-06-13 Andi Kleen <ak@linux.intel.com> + * nptl/pthread_rwlock_rdlock.c: Include elide.h. + (pthread_rwlock_rdlock): Add elision. + * nptl/pthread_rwlock_wrlock.c: Include elide.h. + (pthread_rwlock_wrlock): Add elision. + * nptl/pthread_rwlock_trywrlock.c: Include elide.h. + (pthread_rwlock_trywrlock): Add elision. + * nptl/pthread_rwlock_tryrdlock.c: Include elide.h. + (pthread_rwlock_tryrdlock): Add elision. + * nptl/pthread_rwlock_unlock.c: Include elide.h. + (pthread_rwlock_tryrdlock): Add elision unlock. + * nptl/sysdeps/pthread/pthread.h: + (__PTHREAD_RWLOCK_ELISION_EXTRA): Handle new define + (PTHREAD_RWLOCK_INITIALIZER, + PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP): + Handle new elision field. + * sysdeps/x86/nptl/elide.h: New file. Add generic elision macros. + * sysdeps/arm/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/sh/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/tile/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/a/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/aarch64/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/alpha/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/hppa/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/ia64/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/m68k/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/microblaze/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/mips/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/powerpc/nptl/bits/pthreadtypes.h + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + * sysdeps/unix/sysv/linux/x86/elision-conf.c: + (elision_init): Set try_xbegin to zero when no RTM. + * sysdeps/x86/nptl/bits/pthreadtypes.h + (pthread_rwlock_t): Change __pad1 to __rwelision. + (__PTHREAD_RWLOCK_ELISION_EXTRA): Add. + +2014-06-13 Andi Kleen <ak@linux.intel.com> + * nptl/pthread_rwlock_rdlock (__pthread_rwlock_rdlock): Split into __do_pthread_rwlock_rdlock and __pthread_rwlock_rdlock. * nptl/pthread_rwlock_wrlock (__pthread_rwlock_wrlock): diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c index 1df0327a22..6eb9e091c4 100644 --- a/nptl/pthread_rwlock_rdlock.c +++ b/nptl/pthread_rwlock_rdlock.c @@ -22,6 +22,7 @@ #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> +#include <elide.h> /* Acquire read lock for RWLOCK. Slow path. */ @@ -102,6 +103,12 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) LIBC_PROBE (rdlock_entry, 1, rwlock); + if (ELIDE_LOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__writer == 0 + && rwlock->__data.__nr_readers == 0)) + return 0; + /* Make sure we are alone. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c index f7b1e6b8ac..cf8d68eb3f 100644 --- a/nptl/pthread_rwlock_tryrdlock.c +++ b/nptl/pthread_rwlock_tryrdlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include "pthreadP.h" #include <lowlevellock.h> +#include <elide.h> int @@ -26,6 +27,12 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) { int result = EBUSY; + if (ELIDE_TRYLOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__nr_readers == 0 + && rwlock->__data.__writer, 0)) + return 0; + lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); if (rwlock->__data.__writer == 0 diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c index 106f157c1d..0291fc9e3c 100644 --- a/nptl/pthread_rwlock_trywrlock.c +++ b/nptl/pthread_rwlock_trywrlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include "pthreadP.h" #include <lowlevellock.h> +#include <elide.h> int @@ -26,6 +27,12 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock) { int result = EBUSY; + if (ELIDE_TRYLOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__nr_readers == 0 + && rwlock->__data.__writer, 1)) + return 0; + lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0) diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c index d4923838e7..3ebddeb246 100644 --- a/nptl/pthread_rwlock_unlock.c +++ b/nptl/pthread_rwlock_unlock.c @@ -22,6 +22,8 @@ #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> +#include <elide.h> + /* Unlock RWLOCK. */ int @@ -29,6 +31,10 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock) { LIBC_PROBE (rwlock_unlock, 1, rwlock); + if (ELIDE_UNLOCK (rwlock->__data.__writer == 0 + && rwlock->__data.__nr_readers == 0)) + return 0; + lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); if (rwlock->__data.__writer) rwlock->__data.__writer = 0; diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c index de54e51432..91ad82a1be 100644 --- a/nptl/pthread_rwlock_wrlock.c +++ b/nptl/pthread_rwlock_wrlock.c @@ -22,6 +22,7 @@ #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> +#include <elide.h> /* Acquire write lock for RWLOCK. */ @@ -91,6 +92,12 @@ __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock) { LIBC_PROBE (wrlock_entry, 1, rwlock); + if (ELIDE_LOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__writer == 0 + && rwlock->__data.__nr_readers == 0)) + return 0; + /* Make sure we are alone. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h index d70f8b35b1..c9f1c833a8 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h @@ -214,6 +214,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/arm/nptl/bits/pthreadtypes.h b/sysdeps/arm/nptl/bits/pthreadtypes.h index 64b9e0963e..3ece5b9a1a 100644 --- a/sysdeps/arm/nptl/bits/pthreadtypes.h +++ b/sysdeps/arm/nptl/bits/pthreadtypes.h @@ -155,6 +155,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/nptl/pthread.h b/sysdeps/nptl/pthread.h index 40a3e215cc..682a1ae597 100644 --- a/sysdeps/nptl/pthread.h +++ b/sysdeps/nptl/pthread.h @@ -132,17 +132,17 @@ enum /* Read-write lock initializers. */ # define PTHREAD_RWLOCK_INITIALIZER \ - { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } } + { { 0, 0, 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, 0 } } # ifdef __USE_GNU # ifdef __PTHREAD_RWLOCK_INT_FLAGS_SHARED # define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \ - { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + { { 0, 0, 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, \ PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP } } # else # if __BYTE_ORDER == __LITTLE_ENDIAN # define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \ { { 0, 0, 0, 0, 0, 0, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP, \ - 0, 0, 0, 0 } } + 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, 0 } } # else # define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \ { { 0, 0, 0, 0, 0, 0, 0, 0, 0, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP,\ diff --git a/sysdeps/sh/nptl/bits/pthreadtypes.h b/sysdeps/sh/nptl/bits/pthreadtypes.h index a177f28347..363a94848c 100644 --- a/sysdeps/sh/nptl/bits/pthreadtypes.h +++ b/sysdeps/sh/nptl/bits/pthreadtypes.h @@ -155,6 +155,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/sparc/nptl/bits/pthreadtypes.h b/sysdeps/sparc/nptl/bits/pthreadtypes.h index 8c0340a963..0d225a3c65 100644 --- a/sysdeps/sparc/nptl/bits/pthreadtypes.h +++ b/sysdeps/sparc/nptl/bits/pthreadtypes.h @@ -194,6 +194,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/tile/nptl/bits/pthreadtypes.h b/sysdeps/tile/nptl/bits/pthreadtypes.h index 5ca3391ce5..f808bb2533 100644 --- a/sysdeps/tile/nptl/bits/pthreadtypes.h +++ b/sysdeps/tile/nptl/bits/pthreadtypes.h @@ -194,6 +194,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/aarch64/nptl/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/aarch64/nptl/bits/pthreadtypes.h index 1a44bb6101..5ad0e700be 100644 --- a/sysdeps/unix/sysv/linux/aarch64/nptl/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/aarch64/nptl/bits/pthreadtypes.h @@ -141,6 +141,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h index 3dbe6127dd..55f1067c70 100644 --- a/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h @@ -142,6 +142,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/hppa/nptl/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/hppa/nptl/bits/pthreadtypes.h index a79c195d9a..d6fdc2c44d 100644 --- a/sysdeps/unix/sysv/linux/hppa/nptl/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/hppa/nptl/bits/pthreadtypes.h @@ -196,6 +196,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/ia64/nptl/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/ia64/nptl/bits/pthreadtypes.h index 9468329000..71e4785e64 100644 --- a/sysdeps/unix/sysv/linux/ia64/nptl/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/ia64/nptl/bits/pthreadtypes.h @@ -143,6 +143,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/m68k/nptl/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/m68k/nptl/bits/pthreadtypes.h index 1e1fed82dc..fd46c781b3 100644 --- a/sysdeps/unix/sysv/linux/m68k/nptl/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/m68k/nptl/bits/pthreadtypes.h @@ -146,6 +146,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/microblaze/nptl/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/microblaze/nptl/bits/pthreadtypes.h index 9c7e620e2e..943445384b 100644 --- a/sysdeps/unix/sysv/linux/microblaze/nptl/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/microblaze/nptl/bits/pthreadtypes.h @@ -150,6 +150,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/mips/nptl/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/mips/nptl/bits/pthreadtypes.h index cefd2b6316..843031e854 100644 --- a/sysdeps/unix/sysv/linux/mips/nptl/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/mips/nptl/bits/pthreadtypes.h @@ -203,6 +203,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h index eda3d1a3e4..4e9c5184aa 100644 --- a/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h +++ b/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h @@ -194,6 +194,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; diff --git a/sysdeps/unix/sysv/linux/x86/elision-conf.c b/sysdeps/unix/sysv/linux/x86/elision-conf.c index e6f5d6d1ab..28e48d9e92 100644 --- a/sysdeps/unix/sysv/linux/x86/elision-conf.c +++ b/sysdeps/unix/sysv/linux/x86/elision-conf.c @@ -66,6 +66,8 @@ elision_init (int argc __attribute__ ((unused)), #ifdef ENABLE_LOCK_ELISION __pthread_force_elision = __libc_enable_secure ? 0 : __elision_available; #endif + if (!HAS_RTM) + __elision_aconf.retry_try_xbegin = 0; /* Disable elision on rwlocks */ } #ifdef SHARED diff --git a/sysdeps/x86/nptl/bits/pthreadtypes.h b/sysdeps/x86/nptl/bits/pthreadtypes.h index b4329f652b..b04c32b11f 100644 --- a/sysdeps/x86/nptl/bits/pthreadtypes.h +++ b/sysdeps/x86/nptl/bits/pthreadtypes.h @@ -184,11 +184,13 @@ typedef union unsigned int __nr_writers_queued; int __writer; int __shared; - unsigned long int __pad1; + signed char __rwelision; + unsigned char __pad1[7]; unsigned long int __pad2; /* FLAGS must stay at this position in the structure to maintain binary compatibility. */ unsigned int __flags; +# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, {0, 0, 0, 0, 0, 0, 0 } # define __PTHREAD_RWLOCK_INT_FLAGS_SHARED 1 } __data; # else @@ -204,7 +206,8 @@ typedef union binary compatibility. */ unsigned char __flags; unsigned char __shared; - unsigned char __pad1; + signed char __rwelision; +# define __PTHREAD_RWLOCK_ELISION_EXTRA 0 unsigned char __pad2; int __writer; } __data; diff --git a/sysdeps/x86/nptl/elide.h b/sysdeps/x86/nptl/elide.h new file mode 100644 index 0000000000..19f27e5438 --- /dev/null +++ b/sysdeps/x86/nptl/elide.h @@ -0,0 +1,109 @@ +/* elide.h: Generic lock elision support. + Copyright (C) 2014 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ +#ifndef ELIDE_H +#define ELIDE_H 1 + +#include <hle.h> +#include <elision-conf.h> + +#define ACCESS_ONCE(x) (* (volatile typeof(x) *) &(x)) + +/* Adapt elision with ADAPT_COUNT and STATUS and decide retries. */ + +static inline bool +elision_adapt(uint8_t *adapt_count, unsigned int status) +{ + if (status & _XABORT_RETRY) + return false; + if ((status & _XABORT_EXPLICIT) + && _XABORT_CODE (status) == _ABORT_LOCK_BUSY) + { + /* Right now we skip here. Better would be to wait a bit + and retry. This likely needs some spinning. Be careful + to avoid writing the lock. */ + if (*adapt_count != __elision_aconf.skip_lock_busy) + ACCESS_ONCE (*adapt_count) = __elision_aconf.skip_lock_busy; + } + /* Internal abort. There is no chance for retry. + Use the normal locking and next time use lock. + Be careful to avoid writing to the lock. */ + else if (*adapt_count != __elision_aconf.skip_lock_internal_abort) + ACCESS_ONCE (*adapt_count) = __elision_aconf.skip_lock_internal_abort; + return true; +} + +/* is_lock_free must be executed inside the transaction */ + +/* Returns true if lock defined by IS_LOCK_FREE was elided. + ADAPT_COUNT is a pointer to per-lock state variable. */ + +#define ELIDE_LOCK(adapt_count, is_lock_free) \ + ({ \ + int ret = 0; \ + \ + if ((adapt_count) <= 0) \ + { \ + for (int i = __elision_aconf.retry_try_xbegin; i > 0; i--) \ + { \ + unsigned int status; \ + if ((status = _xbegin ()) == _XBEGIN_STARTED) \ + { \ + if (is_lock_free) \ + { \ + ret = 1; \ + break; \ + } \ + _xabort (_ABORT_LOCK_BUSY); \ + } \ + if (!elision_adapt (&(adapt_count), status)) \ + break; \ + } \ + } \ + else \ + (adapt_count)--; /* missing updates ok */ \ + ret; \ + }) + +/* Returns true if lock defined by IS_LOCK_FREE was try-elided. + ADAPT_COUNT is a pointer to per-lock state variable. */ + +#define ELIDE_TRYLOCK(adapt_count, is_lock_free, write) ({ \ + int ret = 0; \ + if (__elision_aconf.retry_try_xbegin > 0) \ + { \ + if (write) \ + _xabort (_ABORT_NESTED_TRYLOCK); \ + ret = ELIDE_LOCK (adapt_count, is_lock_free); \ + } \ + ret; \ + }) + +/* Returns true if lock defined by IS_LOCK_FREE was elided. */ + +#define ELIDE_UNLOCK(is_lock_free) \ + ({ \ + int ret = 0; \ + if (is_lock_free) \ + { \ + _xend (); \ + ret = 1; \ + } \ + ret; \ + }) + +#endif |