diff options
Diffstat (limited to 'sysdeps/x86')
87 files changed, 3390 insertions, 854 deletions
diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile index 337b0b63dc..95182a508c 100644 --- a/sysdeps/x86/Makefile +++ b/sysdeps/x86/Makefile @@ -18,13 +18,19 @@ ifeq ($(enable-cet),yes) ifeq ($(subdir),elf) sysdep-dl-routines += dl-cet -tests += tst-cet-legacy-1 tst-cet-legacy-2 tst-cet-legacy-2a \ - tst-cet-legacy-3 tst-cet-legacy-4 +tests += tst-cet-legacy-1 tst-cet-legacy-1a tst-cet-legacy-2 \ + tst-cet-legacy-2a tst-cet-legacy-3 tst-cet-legacy-4 \ + tst-cet-legacy-5a tst-cet-legacy-6a +tst-cet-legacy-1a-ARGS = -- $(host-test-program-cmd) ifneq (no,$(have-tunables)) -tests += tst-cet-legacy-4a tst-cet-legacy-4b tst-cet-legacy-4c +tests += tst-cet-legacy-4a tst-cet-legacy-4b tst-cet-legacy-4c \ + tst-cet-legacy-5b tst-cet-legacy-6b endif modules-names += tst-cet-legacy-mod-1 tst-cet-legacy-mod-2 \ - tst-cet-legacy-mod-4 + tst-cet-legacy-mod-4 tst-cet-legacy-mod-5a \ + tst-cet-legacy-mod-5b tst-cet-legacy-mod-5c \ + tst-cet-legacy-mod-6a tst-cet-legacy-mod-6b \ + tst-cet-legacy-mod-6c CFLAGS-tst-cet-legacy-2.c += -fcf-protection=branch CFLAGS-tst-cet-legacy-2a.c += -fcf-protection @@ -35,25 +41,56 @@ CFLAGS-tst-cet-legacy-4.c += -fcf-protection=branch CFLAGS-tst-cet-legacy-4a.c += -fcf-protection CFLAGS-tst-cet-legacy-4b.c += -fcf-protection CFLAGS-tst-cet-legacy-mod-4.c += -fcf-protection=none +CFLAGS-tst-cet-legacy-5a.c += -fcf-protection +CFLAGS-tst-cet-legacy-5b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-5a.c += -fcf-protection=none +CFLAGS-tst-cet-legacy-mod-5b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-5c.c += -fcf-protection +CFLAGS-tst-cet-legacy-6a.c += -fcf-protection +CFLAGS-tst-cet-legacy-6b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-6a.c += -fcf-protection=none +CFLAGS-tst-cet-legacy-mod-6b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-6c.c += -fcf-protection $(objpfx)tst-cet-legacy-1: $(objpfx)tst-cet-legacy-mod-1.so \ $(objpfx)tst-cet-legacy-mod-2.so +$(objpfx)tst-cet-legacy-1a: $(objpfx)tst-cet-legacy-mod-1.so \ + $(objpfx)tst-cet-legacy-mod-2.so $(objpfx)tst-cet-legacy-2: $(objpfx)tst-cet-legacy-mod-2.so $(libdl) $(objpfx)tst-cet-legacy-2.out: $(objpfx)tst-cet-legacy-mod-1.so $(objpfx)tst-cet-legacy-2a: $(objpfx)tst-cet-legacy-mod-2.so $(libdl) $(objpfx)tst-cet-legacy-2a.out: $(objpfx)tst-cet-legacy-mod-1.so $(objpfx)tst-cet-legacy-4: $(libdl) $(objpfx)tst-cet-legacy-4.out: $(objpfx)tst-cet-legacy-mod-4.so +$(objpfx)tst-cet-legacy-5a: $(libdl) +$(objpfx)tst-cet-legacy-5a.out: $(objpfx)tst-cet-legacy-mod-5a.so \ + $(objpfx)tst-cet-legacy-mod-5b.so +$(objpfx)tst-cet-legacy-mod-5a.so: $(objpfx)tst-cet-legacy-mod-5c.so +$(objpfx)tst-cet-legacy-mod-5b.so: $(objpfx)tst-cet-legacy-mod-5c.so +$(objpfx)tst-cet-legacy-6a: $(libdl) +$(objpfx)tst-cet-legacy-6a.out: $(objpfx)tst-cet-legacy-mod-6a.so \ + $(objpfx)tst-cet-legacy-mod-6b.so +$(objpfx)tst-cet-legacy-mod-6a.so: $(objpfx)tst-cet-legacy-mod-6c.so +$(objpfx)tst-cet-legacy-mod-6b.so: $(objpfx)tst-cet-legacy-mod-6c.so +LDFLAGS-tst-cet-legacy-mod-6c.so = -Wl,--enable-new-dtags,-z,nodelete ifneq (no,$(have-tunables)) $(objpfx)tst-cet-legacy-4a: $(libdl) $(objpfx)tst-cet-legacy-4a.out: $(objpfx)tst-cet-legacy-mod-4.so -tst-cet-legacy-4a-ENV = GLIBC_TUNABLES=glibc.tune.x86_shstk=permissive +tst-cet-legacy-4a-ENV = GLIBC_TUNABLES=glibc.cpu.x86_shstk=permissive $(objpfx)tst-cet-legacy-4b: $(libdl) $(objpfx)tst-cet-legacy-4b.out: $(objpfx)tst-cet-legacy-mod-4.so -tst-cet-legacy-4b-ENV = GLIBC_TUNABLES=glibc.tune.x86_shstk=on +tst-cet-legacy-4b-ENV = GLIBC_TUNABLES=glibc.cpu.x86_shstk=on $(objpfx)tst-cet-legacy-4c: $(libdl) $(objpfx)tst-cet-legacy-4c.out: $(objpfx)tst-cet-legacy-mod-4.so -tst-cet-legacy-4c-ENV = GLIBC_TUNABLES=glibc.tune.x86_shstk=off +tst-cet-legacy-4c-ENV = GLIBC_TUNABLES=glibc.cpu.x86_shstk=off +$(objpfx)tst-cet-legacy-5b: $(libdl) +$(objpfx)tst-cet-legacy-5b.out: $(objpfx)tst-cet-legacy-mod-5a.so \ + $(objpfx)tst-cet-legacy-mod-5b.so +tst-cet-legacy-5b-ENV = GLIBC_TUNABLES=glibc.cpu.hwcaps=-IBT,-SHSTK +$(objpfx)tst-cet-legacy-6b: $(libdl) +$(objpfx)tst-cet-legacy-6b.out: $(objpfx)tst-cet-legacy-mod-6a.so \ + $(objpfx)tst-cet-legacy-mod-6b.so +tst-cet-legacy-6b-ENV = GLIBC_TUNABLES=glibc.cpu.hwcaps=-IBT,-SHSTK endif endif diff --git a/sysdeps/x86/__longjmp_cancel.S b/sysdeps/x86/__longjmp_cancel.S index b57dbfa376..197b4baa70 100644 --- a/sysdeps/x86/__longjmp_cancel.S +++ b/sysdeps/x86/__longjmp_cancel.S @@ -1,5 +1,5 @@ /* __longjmp_cancel for x86. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #define __longjmp __longjmp_cancel #include <__longjmp.S> diff --git a/sysdeps/x86/atomic-machine.h b/sysdeps/x86/atomic-machine.h new file mode 100644 index 0000000000..ea9a46a802 --- /dev/null +++ b/sysdeps/x86/atomic-machine.h @@ -0,0 +1,571 @@ +/* Atomic operations. X86 version. + Copyright (C) 2018-2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#ifndef _X86_ATOMIC_MACHINE_H +#define _X86_ATOMIC_MACHINE_H 1 + +#include <stdint.h> +#include <tls.h> /* For tcbhead_t. */ +#include <libc-pointer-arith.h> /* For cast_to_integer. */ + +typedef int8_t atomic8_t; +typedef uint8_t uatomic8_t; +typedef int_fast8_t atomic_fast8_t; +typedef uint_fast8_t uatomic_fast8_t; + +typedef int16_t atomic16_t; +typedef uint16_t uatomic16_t; +typedef int_fast16_t atomic_fast16_t; +typedef uint_fast16_t uatomic_fast16_t; + +typedef int32_t atomic32_t; +typedef uint32_t uatomic32_t; +typedef int_fast32_t atomic_fast32_t; +typedef uint_fast32_t uatomic_fast32_t; + +typedef int64_t atomic64_t; +typedef uint64_t uatomic64_t; +typedef int_fast64_t atomic_fast64_t; +typedef uint_fast64_t uatomic_fast64_t; + +typedef intptr_t atomicptr_t; +typedef uintptr_t uatomicptr_t; +typedef intmax_t atomic_max_t; +typedef uintmax_t uatomic_max_t; + + +#ifndef LOCK_PREFIX +# ifdef UP +# define LOCK_PREFIX /* nothing */ +# else +# define LOCK_PREFIX "lock;" +# endif +#endif + +#define USE_ATOMIC_COMPILER_BUILTINS 1 + +#ifdef __x86_64__ +# define __HAVE_64B_ATOMICS 1 +# define SP_REG "rsp" +# define SEG_REG "fs" +# define BR_CONSTRAINT "q" +# define IBR_CONSTRAINT "iq" +#else +# define __HAVE_64B_ATOMICS 0 +# define SP_REG "esp" +# define SEG_REG "gs" +# define BR_CONSTRAINT "r" +# define IBR_CONSTRAINT "ir" +#endif +#define ATOMIC_EXCHANGE_USES_CAS 0 + +#define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \ + __sync_val_compare_and_swap (mem, oldval, newval) +#define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ + (! __sync_bool_compare_and_swap (mem, oldval, newval)) + + +#define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \ + ({ __typeof (*mem) ret; \ + __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \ + "je 0f\n\t" \ + "lock\n" \ + "0:\tcmpxchgb %b2, %1" \ + : "=a" (ret), "=m" (*mem) \ + : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + ret; }) + +#define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \ + ({ __typeof (*mem) ret; \ + __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \ + "je 0f\n\t" \ + "lock\n" \ + "0:\tcmpxchgw %w2, %1" \ + : "=a" (ret), "=m" (*mem) \ + : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + ret; }) + +#define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \ + ({ __typeof (*mem) ret; \ + __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \ + "je 0f\n\t" \ + "lock\n" \ + "0:\tcmpxchgl %2, %1" \ + : "=a" (ret), "=m" (*mem) \ + : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + ret; }) + +#ifdef __x86_64__ +# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \ + ({ __typeof (*mem) ret; \ + __asm __volatile ("cmpl $0, %%fs:%P5\n\t" \ + "je 0f\n\t" \ + "lock\n" \ + "0:\tcmpxchgq %q2, %1" \ + : "=a" (ret), "=m" (*mem) \ + : "q" ((atomic64_t) cast_to_integer (newval)), \ + "m" (*mem), \ + "0" ((atomic64_t) cast_to_integer (oldval)), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + ret; }) +# define do_exchange_and_add_val_64_acq(pfx, mem, value) 0 +# define do_add_val_64_acq(pfx, mem, value) do { } while (0) +#else +/* XXX We do not really need 64-bit compare-and-exchange. At least + not in the moment. Using it would mean causing portability + problems since not many other 32-bit architectures have support for + such an operation. So don't define any code for now. If it is + really going to be used the code below can be used on Intel Pentium + and later, but NOT on i486. */ +# define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \ + ({ __typeof (*mem) ret = *(mem); \ + __atomic_link_error (); \ + ret = (newval); \ + ret = (oldval); \ + ret; }) + +# define __arch_compare_and_exchange_val_64_acq(mem, newval, oldval) \ + ({ __typeof (*mem) ret = *(mem); \ + __atomic_link_error (); \ + ret = (newval); \ + ret = (oldval); \ + ret; }) + +# define do_exchange_and_add_val_64_acq(pfx, mem, value) \ + ({ __typeof (value) __addval = (value); \ + __typeof (*mem) __result; \ + __typeof (mem) __memp = (mem); \ + __typeof (*mem) __tmpval; \ + __result = *__memp; \ + do \ + __tmpval = __result; \ + while ((__result = pfx##_compare_and_exchange_val_64_acq \ + (__memp, __result + __addval, __result)) == __tmpval); \ + __result; }) + +# define do_add_val_64_acq(pfx, mem, value) \ + { \ + __typeof (value) __addval = (value); \ + __typeof (mem) __memp = (mem); \ + __typeof (*mem) __oldval = *__memp; \ + __typeof (*mem) __tmpval; \ + do \ + __tmpval = __oldval; \ + while ((__oldval = pfx##_compare_and_exchange_val_64_acq \ + (__memp, __oldval + __addval, __oldval)) == __tmpval); \ + } +#endif + + +/* Note that we need no lock prefix. */ +#define atomic_exchange_acq(mem, newvalue) \ + ({ __typeof (*mem) result; \ + if (sizeof (*mem) == 1) \ + __asm __volatile ("xchgb %b0, %1" \ + : "=q" (result), "=m" (*mem) \ + : "0" (newvalue), "m" (*mem)); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile ("xchgw %w0, %1" \ + : "=r" (result), "=m" (*mem) \ + : "0" (newvalue), "m" (*mem)); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile ("xchgl %0, %1" \ + : "=r" (result), "=m" (*mem) \ + : "0" (newvalue), "m" (*mem)); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile ("xchgq %q0, %1" \ + : "=r" (result), "=m" (*mem) \ + : "0" ((atomic64_t) cast_to_integer (newvalue)), \ + "m" (*mem)); \ + else \ + { \ + result = 0; \ + __atomic_link_error (); \ + } \ + result; }) + + +#define __arch_exchange_and_add_body(lock, pfx, mem, value) \ + ({ __typeof (*mem) __result; \ + __typeof (value) __addval = (value); \ + if (sizeof (*mem) == 1) \ + __asm __volatile (lock "xaddb %b0, %1" \ + : "=q" (__result), "=m" (*mem) \ + : "0" (__addval), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (lock "xaddw %w0, %1" \ + : "=r" (__result), "=m" (*mem) \ + : "0" (__addval), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (lock "xaddl %0, %1" \ + : "=r" (__result), "=m" (*mem) \ + : "0" (__addval), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (lock "xaddq %q0, %1" \ + : "=r" (__result), "=m" (*mem) \ + : "0" ((atomic64_t) cast_to_integer (__addval)), \ + "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else \ + __result = do_exchange_and_add_val_64_acq (pfx, (mem), __addval); \ + __result; }) + +#define atomic_exchange_and_add(mem, value) \ + __sync_fetch_and_add (mem, value) + +#define __arch_exchange_and_add_cprefix \ + "cmpl $0, %%" SEG_REG ":%P4\n\tje 0f\n\tlock\n0:\t" + +#define catomic_exchange_and_add(mem, value) \ + __arch_exchange_and_add_body (__arch_exchange_and_add_cprefix, __arch_c, \ + mem, value) + + +#define __arch_add_body(lock, pfx, apfx, mem, value) \ + do { \ + if (__builtin_constant_p (value) && (value) == 1) \ + pfx##_increment (mem); \ + else if (__builtin_constant_p (value) && (value) == -1) \ + pfx##_decrement (mem); \ + else if (sizeof (*mem) == 1) \ + __asm __volatile (lock "addb %b1, %0" \ + : "=m" (*mem) \ + : IBR_CONSTRAINT (value), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (lock "addw %w1, %0" \ + : "=m" (*mem) \ + : "ir" (value), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (lock "addl %1, %0" \ + : "=m" (*mem) \ + : "ir" (value), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (lock "addq %q1, %0" \ + : "=m" (*mem) \ + : "ir" ((atomic64_t) cast_to_integer (value)), \ + "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else \ + do_add_val_64_acq (apfx, (mem), (value)); \ + } while (0) + +# define atomic_add(mem, value) \ + __arch_add_body (LOCK_PREFIX, atomic, __arch, mem, value) + +#define __arch_add_cprefix \ + "cmpl $0, %%" SEG_REG ":%P3\n\tje 0f\n\tlock\n0:\t" + +#define catomic_add(mem, value) \ + __arch_add_body (__arch_add_cprefix, atomic, __arch_c, mem, value) + + +#define atomic_add_negative(mem, value) \ + ({ unsigned char __result; \ + if (sizeof (*mem) == 1) \ + __asm __volatile (LOCK_PREFIX "addb %b2, %0; sets %1" \ + : "=m" (*mem), "=qm" (__result) \ + : IBR_CONSTRAINT (value), "m" (*mem)); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (LOCK_PREFIX "addw %w2, %0; sets %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "ir" (value), "m" (*mem)); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (LOCK_PREFIX "addl %2, %0; sets %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "ir" (value), "m" (*mem)); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (LOCK_PREFIX "addq %q2, %0; sets %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "ir" ((atomic64_t) cast_to_integer (value)), \ + "m" (*mem)); \ + else \ + __atomic_link_error (); \ + __result; }) + + +#define atomic_add_zero(mem, value) \ + ({ unsigned char __result; \ + if (sizeof (*mem) == 1) \ + __asm __volatile (LOCK_PREFIX "addb %b2, %0; setz %1" \ + : "=m" (*mem), "=qm" (__result) \ + : IBR_CONSTRAINT (value), "m" (*mem)); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (LOCK_PREFIX "addw %w2, %0; setz %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "ir" (value), "m" (*mem)); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (LOCK_PREFIX "addl %2, %0; setz %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "ir" (value), "m" (*mem)); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (LOCK_PREFIX "addq %q2, %0; setz %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "ir" ((atomic64_t) cast_to_integer (value)), \ + "m" (*mem)); \ + else \ + __atomic_link_error (); \ + __result; }) + + +#define __arch_increment_body(lock, pfx, mem) \ + do { \ + if (sizeof (*mem) == 1) \ + __asm __volatile (lock "incb %b0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (lock "incw %w0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (lock "incl %0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (lock "incq %q0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else \ + do_add_val_64_acq (pfx, mem, 1); \ + } while (0) + +#define atomic_increment(mem) __arch_increment_body (LOCK_PREFIX, __arch, mem) + +#define __arch_increment_cprefix \ + "cmpl $0, %%" SEG_REG ":%P2\n\tje 0f\n\tlock\n0:\t" + +#define catomic_increment(mem) \ + __arch_increment_body (__arch_increment_cprefix, __arch_c, mem) + + +#define atomic_increment_and_test(mem) \ + ({ unsigned char __result; \ + if (sizeof (*mem) == 1) \ + __asm __volatile (LOCK_PREFIX "incb %b0; sete %b1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (LOCK_PREFIX "incw %w0; sete %w1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (LOCK_PREFIX "incl %0; sete %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (LOCK_PREFIX "incq %q0; sete %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + else \ + __atomic_link_error (); \ + __result; }) + + +#define __arch_decrement_body(lock, pfx, mem) \ + do { \ + if (sizeof (*mem) == 1) \ + __asm __volatile (lock "decb %b0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (lock "decw %w0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (lock "decl %0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (lock "decq %q0" \ + : "=m" (*mem) \ + : "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else \ + do_add_val_64_acq (pfx, mem, -1); \ + } while (0) + +#define atomic_decrement(mem) __arch_decrement_body (LOCK_PREFIX, __arch, mem) + +#define __arch_decrement_cprefix \ + "cmpl $0, %%" SEG_REG ":%P2\n\tje 0f\n\tlock\n0:\t" + +#define catomic_decrement(mem) \ + __arch_decrement_body (__arch_decrement_cprefix, __arch_c, mem) + + +#define atomic_decrement_and_test(mem) \ + ({ unsigned char __result; \ + if (sizeof (*mem) == 1) \ + __asm __volatile (LOCK_PREFIX "decb %b0; sete %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (LOCK_PREFIX "decw %w0; sete %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (LOCK_PREFIX "decl %0; sete %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + else \ + __asm __volatile (LOCK_PREFIX "decq %q0; sete %1" \ + : "=m" (*mem), "=qm" (__result) \ + : "m" (*mem)); \ + __result; }) + + +#define atomic_bit_set(mem, bit) \ + do { \ + if (sizeof (*mem) == 1) \ + __asm __volatile (LOCK_PREFIX "orb %b2, %0" \ + : "=m" (*mem) \ + : "m" (*mem), IBR_CONSTRAINT (1L << (bit))); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (LOCK_PREFIX "orw %w2, %0" \ + : "=m" (*mem) \ + : "m" (*mem), "ir" (1L << (bit))); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (LOCK_PREFIX "orl %2, %0" \ + : "=m" (*mem) \ + : "m" (*mem), "ir" (1L << (bit))); \ + else if (__builtin_constant_p (bit) && (bit) < 32) \ + __asm __volatile (LOCK_PREFIX "orq %2, %0" \ + : "=m" (*mem) \ + : "m" (*mem), "i" (1L << (bit))); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (LOCK_PREFIX "orq %q2, %0" \ + : "=m" (*mem) \ + : "m" (*mem), "r" (1UL << (bit))); \ + else \ + __atomic_link_error (); \ + } while (0) + + +#define atomic_bit_test_set(mem, bit) \ + ({ unsigned char __result; \ + if (sizeof (*mem) == 1) \ + __asm __volatile (LOCK_PREFIX "btsb %3, %1; setc %0" \ + : "=q" (__result), "=m" (*mem) \ + : "m" (*mem), IBR_CONSTRAINT (bit)); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (LOCK_PREFIX "btsw %3, %1; setc %0" \ + : "=q" (__result), "=m" (*mem) \ + : "m" (*mem), "ir" (bit)); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (LOCK_PREFIX "btsl %3, %1; setc %0" \ + : "=q" (__result), "=m" (*mem) \ + : "m" (*mem), "ir" (bit)); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (LOCK_PREFIX "btsq %3, %1; setc %0" \ + : "=q" (__result), "=m" (*mem) \ + : "m" (*mem), "ir" (bit)); \ + else \ + __atomic_link_error (); \ + __result; }) + + +#define __arch_and_body(lock, mem, mask) \ + do { \ + if (sizeof (*mem) == 1) \ + __asm __volatile (lock "andb %b1, %0" \ + : "=m" (*mem) \ + : IBR_CONSTRAINT (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (lock "andw %w1, %0" \ + : "=m" (*mem) \ + : "ir" (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (lock "andl %1, %0" \ + : "=m" (*mem) \ + : "ir" (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (lock "andq %q1, %0" \ + : "=m" (*mem) \ + : "ir" (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else \ + __atomic_link_error (); \ + } while (0) + +#define __arch_cprefix \ + "cmpl $0, %%" SEG_REG ":%P3\n\tje 0f\n\tlock\n0:\t" + +#define atomic_and(mem, mask) __arch_and_body (LOCK_PREFIX, mem, mask) + +#define catomic_and(mem, mask) __arch_and_body (__arch_cprefix, mem, mask) + + +#define __arch_or_body(lock, mem, mask) \ + do { \ + if (sizeof (*mem) == 1) \ + __asm __volatile (lock "orb %b1, %0" \ + : "=m" (*mem) \ + : IBR_CONSTRAINT (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 2) \ + __asm __volatile (lock "orw %w1, %0" \ + : "=m" (*mem) \ + : "ir" (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (sizeof (*mem) == 4) \ + __asm __volatile (lock "orl %1, %0" \ + : "=m" (*mem) \ + : "ir" (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else if (__HAVE_64B_ATOMICS) \ + __asm __volatile (lock "orq %q1, %0" \ + : "=m" (*mem) \ + : "ir" (mask), "m" (*mem), \ + "i" (offsetof (tcbhead_t, multiple_threads))); \ + else \ + __atomic_link_error (); \ + } while (0) + +#define atomic_or(mem, mask) __arch_or_body (LOCK_PREFIX, mem, mask) + +#define catomic_or(mem, mask) __arch_or_body (__arch_cprefix, mem, mask) + +/* We don't use mfence because it is supposedly slower due to having to + provide stronger guarantees (e.g., regarding self-modifying code). */ +#define atomic_full_barrier() \ + __asm __volatile (LOCK_PREFIX "orl $0, (%%" SP_REG ")" ::: "memory") +#define atomic_read_barrier() __asm ("" ::: "memory") +#define atomic_write_barrier() __asm ("" ::: "memory") + +#define atomic_spin_nop() __asm ("pause") + +#endif /* atomic-machine.h */ diff --git a/sysdeps/x86/bits/endian.h b/sysdeps/x86/bits/endian.h deleted file mode 100644 index 5a56c726f7..0000000000 --- a/sysdeps/x86/bits/endian.h +++ /dev/null @@ -1,7 +0,0 @@ -/* i386/x86_64 are little-endian. */ - -#ifndef _ENDIAN_H -# error "Never use <bits/endian.h> directly; include <endian.h> instead." -#endif - -#define __BYTE_ORDER __LITTLE_ENDIAN diff --git a/sysdeps/x86/bits/endianness.h b/sysdeps/x86/bits/endianness.h new file mode 100644 index 0000000000..962a9ae4d6 --- /dev/null +++ b/sysdeps/x86/bits/endianness.h @@ -0,0 +1,11 @@ +#ifndef _BITS_ENDIANNESS_H +#define _BITS_ENDIANNESS_H 1 + +#ifndef _BITS_ENDIAN_H +# error "Never use <bits/endianness.h> directly; include <endian.h> instead." +#endif + +/* i386/x86_64 are little-endian. */ +#define __BYTE_ORDER __LITTLE_ENDIAN + +#endif /* bits/endianness.h */ diff --git a/sysdeps/x86/bits/floatn.h b/sysdeps/x86/bits/floatn.h index 49c75f26c5..66f2baf660 100644 --- a/sysdeps/x86/bits/floatn.h +++ b/sysdeps/x86/bits/floatn.h @@ -1,5 +1,5 @@ /* Macros to control TS 18661-3 glibc features on x86. - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _BITS_FLOATN_H #define _BITS_FLOATN_H diff --git a/sysdeps/x86/bits/flt-eval-method.h b/sysdeps/x86/bits/flt-eval-method.h index a6134a455f..3de048340c 100644 --- a/sysdeps/x86/bits/flt-eval-method.h +++ b/sysdeps/x86/bits/flt-eval-method.h @@ -1,5 +1,5 @@ /* Define __GLIBC_FLT_EVAL_METHOD. x86 version. - Copyright (C) 2016-2018 Free Software Foundation, Inc. + Copyright (C) 2016-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _MATH_H # error "Never use <bits/flt-eval-method.h> directly; include <math.h> instead." diff --git a/sysdeps/x86/bits/fp-logb.h b/sysdeps/x86/bits/fp-logb.h index 267c7ec1e1..295e7623f7 100644 --- a/sysdeps/x86/bits/fp-logb.h +++ b/sysdeps/x86/bits/fp-logb.h @@ -1,5 +1,5 @@ /* Define __FP_LOGB0_IS_MIN and __FP_LOGBNAN_IS_MIN. x86 version. - Copyright (C) 2016-2018 Free Software Foundation, Inc. + Copyright (C) 2016-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _MATH_H # error "Never use <bits/fp-logb.h> directly; include <math.h> instead." diff --git a/sysdeps/x86/bits/indirect-return.h b/sysdeps/x86/bits/indirect-return.h index d1acaca3b9..b8267233c5 100644 --- a/sysdeps/x86/bits/indirect-return.h +++ b/sysdeps/x86/bits/indirect-return.h @@ -1,5 +1,5 @@ /* Definition of __INDIRECT_RETURN. x86 version. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _UCONTEXT_H # error "Never include <bits/indirect-return.h> directly; use <ucontext.h> instead." diff --git a/sysdeps/x86/bits/link.h b/sysdeps/x86/bits/link.h index a97c41162c..b29574595d 100644 --- a/sysdeps/x86/bits/link.h +++ b/sysdeps/x86/bits/link.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2004-2018 Free Software Foundation, Inc. +/* Copyright (C) 2004-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _LINK_H # error "Never include <bits/link.h> directly; use <link.h> instead." diff --git a/sysdeps/x86/bits/select.h b/sysdeps/x86/bits/select.h index 2c0a2b5eb3..076adc0992 100644 --- a/sysdeps/x86/bits/select.h +++ b/sysdeps/x86/bits/select.h @@ -1,4 +1,4 @@ -/* Copyright (C) 1997-2018 Free Software Foundation, Inc. +/* Copyright (C) 1997-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _SYS_SELECT_H # error "Never use <bits/select.h> directly; include <sys/select.h> instead." diff --git a/sysdeps/x86/bits/semaphore.h b/sysdeps/x86/bits/semaphore.h index 1b8daf98be..c8c0e5c3fe 100644 --- a/sysdeps/x86/bits/semaphore.h +++ b/sysdeps/x86/bits/semaphore.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2002-2018 Free Software Foundation, Inc. +/* Copyright (C) 2002-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _SEMAPHORE_H # error "Never use <bits/semaphore.h> directly; include <semaphore.h> instead." diff --git a/sysdeps/x86/bits/setjmp.h b/sysdeps/x86/bits/setjmp.h index e0c22ac78f..4b50b04d03 100644 --- a/sysdeps/x86/bits/setjmp.h +++ b/sysdeps/x86/bits/setjmp.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2001-2018 Free Software Foundation, Inc. +/* Copyright (C) 2001-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ /* Define the machine-dependent type `jmp_buf'. x86-64 version. */ #ifndef _BITS_SETJMP_H diff --git a/sysdeps/x86/cacheinfo.c b/sysdeps/x86/cacheinfo.c index b9444ddd52..6b17482518 100644 --- a/sysdeps/x86/cacheinfo.c +++ b/sysdeps/x86/cacheinfo.c @@ -1,5 +1,5 @@ /* x86_64 cache info. - Copyright (C) 2003-2018 Free Software Foundation, Inc. + Copyright (C) 2003-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #if IS_IN (libc) @@ -205,8 +205,8 @@ intel_check_word (int name, unsigned int value, bool *has_level_2, /* Intel reused this value. For family 15, model 6 it specifies the 3rd level cache. Otherwise the 2nd level cache. */ - unsigned int family = cpu_features->family; - unsigned int model = cpu_features->model; + unsigned int family = cpu_features->basic.family; + unsigned int model = cpu_features->basic.model; if (family == 15 && model == 6) { @@ -258,7 +258,7 @@ intel_check_word (int name, unsigned int value, bool *has_level_2, static long int __attribute__ ((noinline)) handle_intel (int name, const struct cpu_features *cpu_features) { - unsigned int maxidx = cpu_features->max_cpuid; + unsigned int maxidx = cpu_features->basic.max_cpuid; /* Return -1 for older CPUs. */ if (maxidx < 2) @@ -443,10 +443,10 @@ __cache_sysconf (int name) { const struct cpu_features *cpu_features = __get_cpu_features (); - if (cpu_features->kind == arch_kind_intel) + if (cpu_features->basic.kind == arch_kind_intel) return handle_intel (name, cpu_features); - if (cpu_features->kind == arch_kind_amd) + if (cpu_features->basic.kind == arch_kind_amd) return handle_amd (name); // XXX Fill in more vendors. @@ -497,9 +497,9 @@ init_cacheinfo (void) unsigned int level; unsigned int threads = 0; const struct cpu_features *cpu_features = __get_cpu_features (); - int max_cpuid = cpu_features->max_cpuid; + int max_cpuid = cpu_features->basic.max_cpuid; - if (cpu_features->kind == arch_kind_intel) + if (cpu_features->basic.kind == arch_kind_intel) { data = handle_intel (_SC_LEVEL1_DCACHE_SIZE, cpu_features); @@ -538,8 +538,8 @@ init_cacheinfo (void) highest cache level. */ if (max_cpuid >= 4) { - unsigned int family = cpu_features->family; - unsigned int model = cpu_features->model; + unsigned int family = cpu_features->basic.family; + unsigned int model = cpu_features->basic.model; int i = 0; @@ -700,7 +700,7 @@ intel_bug_no_cache_info: shared += core; } } - else if (cpu_features->kind == arch_kind_amd) + else if (cpu_features->basic.kind == arch_kind_amd) { data = handle_amd (_SC_LEVEL1_DCACHE_SIZE); long int core = handle_amd (_SC_LEVEL2_CACHE_SIZE); diff --git a/sysdeps/x86/cet-tunables.h b/sysdeps/x86/cet-tunables.h index ca023053ee..5291fc81b3 100644 --- a/sysdeps/x86/cet-tunables.h +++ b/sysdeps/x86/cet-tunables.h @@ -1,6 +1,6 @@ /* x86 CET tuning. This file is part of the GNU C Library. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ /* Valid control values: 0: Enable CET features based on ELF property note. diff --git a/sysdeps/x86/check-cet.awk b/sysdeps/x86/check-cet.awk index 380d998caf..ae268fa995 100644 --- a/sysdeps/x86/check-cet.awk +++ b/sysdeps/x86/check-cet.awk @@ -1,5 +1,5 @@ # Verify that all shared objects contain the CET property. -# Copyright (C) 2018 Free Software Foundation, Inc. +# Copyright (C) 2018-2019 Free Software Foundation, Inc. # This file is part of the GNU C Library. # # The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Lesser General Public # License along with the GNU C Library; if not, see -# <http://www.gnu.org/licenses/>. +# <https://www.gnu.org/licenses/>. # This awk script expects to get command-line files that are each # the output of 'readelf -n' on a single shared object. diff --git a/sysdeps/x86/cpu-features-offsets.sym b/sysdeps/x86/cpu-features-offsets.sym index 33dd094e37..6d03cea8e8 100644 --- a/sysdeps/x86/cpu-features-offsets.sym +++ b/sysdeps/x86/cpu-features-offsets.sym @@ -2,23 +2,5 @@ #include <ldsodefs.h> -#define rtld_global_ro_offsetof(mem) offsetof (struct rtld_global_ro, mem) - -RTLD_GLOBAL_RO_DL_X86_CPU_FEATURES_OFFSET rtld_global_ro_offsetof (_dl_x86_cpu_features) - -CPU_FEATURES_SIZE sizeof (struct cpu_features) -CPUID_OFFSET offsetof (struct cpu_features, cpuid) -CPUID_SIZE sizeof (struct cpuid_registers) -CPUID_EAX_OFFSET offsetof (struct cpuid_registers, eax) -CPUID_EBX_OFFSET offsetof (struct cpuid_registers, ebx) -CPUID_ECX_OFFSET offsetof (struct cpuid_registers, ecx) -CPUID_EDX_OFFSET offsetof (struct cpuid_registers, edx) -FAMILY_OFFSET offsetof (struct cpu_features, family) -MODEL_OFFSET offsetof (struct cpu_features, model) +RTLD_GLOBAL_RO_DL_X86_CPU_FEATURES_OFFSET offsetof (struct rtld_global_ro, _dl_x86_cpu_features) XSAVE_STATE_SIZE_OFFSET offsetof (struct cpu_features, xsave_state_size) -FEATURE_OFFSET offsetof (struct cpu_features, feature) -FEATURE_SIZE sizeof (unsigned int) - -COMMON_CPUID_INDEX_1 -COMMON_CPUID_INDEX_7 -FEATURE_INDEX_1 diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c index ea0b64fdb9..0650786f01 100644 --- a/sysdeps/x86/cpu-features.c +++ b/sysdeps/x86/cpu-features.c @@ -1,6 +1,6 @@ /* Initialize CPU feature data. This file is part of the GNU C Library. - Copyright (C) 2008-2018 Free Software Foundation, Inc. + Copyright (C) 2008-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <cpuid.h> #include <cpu-features.h> @@ -22,7 +22,7 @@ #include <libc-pointer-arith.h> #if HAVE_TUNABLES -# define TUNABLE_NAMESPACE tune +# define TUNABLE_NAMESPACE cpu # include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */ # include <elf/dl-tunables.h> @@ -53,11 +53,22 @@ get_extended_indices (struct cpu_features *cpu_features) cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ebx, cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ecx, cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].edx); - + if (eax >= 0x80000007) + __cpuid (0x80000007, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000007].eax, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000007].ebx, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000007].ecx, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000007].edx); + if (eax >= 0x80000008) + __cpuid (0x80000008, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000008].eax, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000008].ebx, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000008].ecx, + cpu_features->cpuid[COMMON_CPUID_INDEX_80000008].edx); } static void -get_common_indeces (struct cpu_features *cpu_features, +get_common_indices (struct cpu_features *cpu_features, unsigned int *family, unsigned int *model, unsigned int *extended_model, unsigned int *stepping) { @@ -79,13 +90,20 @@ get_common_indeces (struct cpu_features *cpu_features, } } - if (cpu_features->max_cpuid >= 7) + if (cpu_features->basic.max_cpuid >= 7) __cpuid_count (7, 0, cpu_features->cpuid[COMMON_CPUID_INDEX_7].eax, cpu_features->cpuid[COMMON_CPUID_INDEX_7].ebx, cpu_features->cpuid[COMMON_CPUID_INDEX_7].ecx, cpu_features->cpuid[COMMON_CPUID_INDEX_7].edx); + if (cpu_features->basic.max_cpuid >= 0xd) + __cpuid_count (0xd, 1, + cpu_features->cpuid[COMMON_CPUID_INDEX_D_ECX_1].eax, + cpu_features->cpuid[COMMON_CPUID_INDEX_D_ECX_1].ebx, + cpu_features->cpuid[COMMON_CPUID_INDEX_D_ECX_1].ecx, + cpu_features->cpuid[COMMON_CPUID_INDEX_D_ECX_1].edx); + /* Can we call xgetbv? */ if (CPU_FEATURES_CPU_P (cpu_features, OSXSAVE)) { @@ -93,8 +111,8 @@ get_common_indeces (struct cpu_features *cpu_features, unsigned int xcrhigh; asm ("xgetbv" : "=a" (xcrlow), "=d" (xcrhigh) : "c" (0)); /* Is YMM and XMM state usable? */ - if ((xcrlow & (bit_YMM_state | bit_XMM_state)) == - (bit_YMM_state | bit_XMM_state)) + if ((xcrlow & (bit_YMM_state | bit_XMM_state)) + == (bit_YMM_state | bit_XMM_state)) { /* Determine if AVX is usable. */ if (CPU_FEATURES_CPU_P (cpu_features, AVX)) @@ -117,30 +135,94 @@ get_common_indeces (struct cpu_features *cpu_features, if (CPU_FEATURES_CPU_P (cpu_features, FMA)) cpu_features->feature[index_arch_FMA_Usable] |= bit_arch_FMA_Usable; + /* Determine if VAES is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, VAES)) + cpu_features->feature[index_arch_VAES_Usable] + |= bit_arch_VAES_Usable; + /* Determine if VPCLMULQDQ is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, VPCLMULQDQ)) + cpu_features->feature[index_arch_VPCLMULQDQ_Usable] + |= bit_arch_VPCLMULQDQ_Usable; + /* Determine if XOP is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, XOP)) + cpu_features->feature[index_arch_XOP_Usable] + |= bit_arch_XOP_Usable; } /* Check if OPMASK state, upper 256-bit of ZMM0-ZMM15 and ZMM16-ZMM31 state are enabled. */ if ((xcrlow & (bit_Opmask_state | bit_ZMM0_15_state - | bit_ZMM16_31_state)) == - (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state)) + | bit_ZMM16_31_state)) + == (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state)) { /* Determine if AVX512F is usable. */ if (CPU_FEATURES_CPU_P (cpu_features, AVX512F)) { cpu_features->feature[index_arch_AVX512F_Usable] |= bit_arch_AVX512F_Usable; + /* Determine if AVX512CD is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512CD)) + cpu_features->feature[index_arch_AVX512CD_Usable] + |= bit_arch_AVX512CD_Usable; + /* Determine if AVX512ER is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER)) + cpu_features->feature[index_arch_AVX512ER_Usable] + |= bit_arch_AVX512ER_Usable; + /* Determine if AVX512PF is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512PF)) + cpu_features->feature[index_arch_AVX512PF_Usable] + |= bit_arch_AVX512PF_Usable; + /* Determine if AVX512VL is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512VL)) + cpu_features->feature[index_arch_AVX512VL_Usable] + |= bit_arch_AVX512VL_Usable; /* Determine if AVX512DQ is usable. */ if (CPU_FEATURES_CPU_P (cpu_features, AVX512DQ)) cpu_features->feature[index_arch_AVX512DQ_Usable] |= bit_arch_AVX512DQ_Usable; + /* Determine if AVX512BW is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512BW)) + cpu_features->feature[index_arch_AVX512BW_Usable] + |= bit_arch_AVX512BW_Usable; + /* Determine if AVX512_4FMAPS is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_4FMAPS)) + cpu_features->feature[index_arch_AVX512_4FMAPS_Usable] + |= bit_arch_AVX512_4FMAPS_Usable; + /* Determine if AVX512_4VNNIW is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_4VNNIW)) + cpu_features->feature[index_arch_AVX512_4VNNIW_Usable] + |= bit_arch_AVX512_4VNNIW_Usable; + /* Determine if AVX512_BITALG is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_BITALG)) + cpu_features->feature[index_arch_AVX512_BITALG_Usable] + |= bit_arch_AVX512_BITALG_Usable; + /* Determine if AVX512_IFMA is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_IFMA)) + cpu_features->feature[index_arch_AVX512_IFMA_Usable] + |= bit_arch_AVX512_IFMA_Usable; + /* Determine if AVX512_VBMI is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_VBMI)) + cpu_features->feature[index_arch_AVX512_VBMI_Usable] + |= bit_arch_AVX512_VBMI_Usable; + /* Determine if AVX512_VBMI2 is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_VBMI2)) + cpu_features->feature[index_arch_AVX512_VBMI2_Usable] + |= bit_arch_AVX512_VBMI2_Usable; + /* Determine if is AVX512_VNNI usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_VNNI)) + cpu_features->feature[index_arch_AVX512_VNNI_Usable] + |= bit_arch_AVX512_VNNI_Usable; + /* Determine if AVX512_VPOPCNTDQ is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, AVX512_VPOPCNTDQ)) + cpu_features->feature[index_arch_AVX512_VPOPCNTDQ_Usable] + |= bit_arch_AVX512_VPOPCNTDQ_Usable; } } } /* For _dl_runtime_resolve, set xsave_state_size to xsave area size + integer register save size and align it to 64 bytes. */ - if (cpu_features->max_cpuid >= 0xd) + if (cpu_features->basic.max_cpuid >= 0xd) { unsigned int eax, ebx, ecx, edx; @@ -155,10 +237,8 @@ get_common_indeces (struct cpu_features *cpu_features, cpu_features->xsave_state_full_size = xsave_state_full_size; - __cpuid_count (0xd, 1, eax, ebx, ecx, edx); - /* Check if XSAVEC is available. */ - if ((eax & (1 << 1)) != 0) + if (CPU_FEATURES_CPU_P (cpu_features, XSAVEC)) { unsigned int xstate_comp_offsets[32]; unsigned int xstate_comp_sizes[32]; @@ -210,12 +290,25 @@ get_common_indeces (struct cpu_features *cpu_features, } } +_Static_assert (((index_arch_Fast_Unaligned_Load + == index_arch_Fast_Unaligned_Copy) + && (index_arch_Fast_Unaligned_Load + == index_arch_Prefer_PMINUB_for_stringop) + && (index_arch_Fast_Unaligned_Load + == index_arch_Slow_SSE4_2) + && (index_arch_Fast_Unaligned_Load + == index_arch_Fast_Rep_String) + && (index_arch_Fast_Unaligned_Load + == index_arch_Fast_Copy_Backward)), + "Incorrect index_arch_Fast_Unaligned_Load"); + static inline void init_cpu_features (struct cpu_features *cpu_features) { unsigned int ebx, ecx, edx; unsigned int family = 0; unsigned int model = 0; + unsigned int stepping = 0; enum cpu_features_kind kind; #if !HAS_CPUID @@ -226,16 +319,16 @@ init_cpu_features (struct cpu_features *cpu_features) } #endif - __cpuid (0, cpu_features->max_cpuid, ebx, ecx, edx); + __cpuid (0, cpu_features->basic.max_cpuid, ebx, ecx, edx); /* This spells out "GenuineIntel". */ if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69) { - unsigned int extended_model, stepping; + unsigned int extended_model; kind = arch_kind_intel; - get_common_indeces (cpu_features, &family, &model, &extended_model, + get_common_indices (cpu_features, &family, &model, &extended_model, &stepping); get_extended_indices (cpu_features); @@ -270,15 +363,6 @@ init_cpu_features (struct cpu_features *cpu_features) case 0x5d: /* Unaligned load versions are faster than SSSE3 on Silvermont. */ -#if index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop -# error index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop -#endif -#if index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2 -# error index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2 -#endif -#if index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy -# error index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy -#endif cpu_features->feature[index_arch_Fast_Unaligned_Load] |= (bit_arch_Fast_Unaligned_Load | bit_arch_Fast_Unaligned_Copy @@ -291,6 +375,7 @@ init_cpu_features (struct cpu_features *cpu_features) of Core i3/i5/i7 processors if AVX is available. */ if (!CPU_FEATURES_CPU_P (cpu_features, AVX)) break; + /* Fall through. */ case 0x1a: case 0x1e: @@ -301,26 +386,24 @@ init_cpu_features (struct cpu_features *cpu_features) case 0x2f: /* Rep string instructions, unaligned load, unaligned copy, and pminub are fast on Intel Core i3, i5 and i7. */ -#if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load -# error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load -#endif -#if index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop -# error index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop -#endif -#if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy -# error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy -#endif cpu_features->feature[index_arch_Fast_Rep_String] |= (bit_arch_Fast_Rep_String | bit_arch_Fast_Unaligned_Load | bit_arch_Fast_Unaligned_Copy | bit_arch_Prefer_PMINUB_for_stringop); break; + } + /* Disable TSX on some Haswell processors to avoid TSX on kernels that + weren't updated with the latest microcode package (which disables + broken feature by default). */ + switch (model) + { case 0x3f: /* Xeon E7 v3 with stepping >= 4 has working TSX. */ if (stepping >= 4) break; + /* Fall through. */ case 0x3c: case 0x45: case 0x46: @@ -344,14 +427,15 @@ init_cpu_features (struct cpu_features *cpu_features) cpu_features->feature[index_arch_Prefer_No_AVX512] |= bit_arch_Prefer_No_AVX512; } - /* This spells out "AuthenticAMD". */ - else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) + /* This spells out "AuthenticAMD" or "HygonGenuine". */ + else if ((ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) + || (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e)) { - unsigned int extended_model, stepping; + unsigned int extended_model; kind = arch_kind_amd; - get_common_indeces (cpu_features, &family, &model, &extended_model, + get_common_indices (cpu_features, &family, &model, &extended_model, &stepping); get_extended_indices (cpu_features); @@ -369,9 +453,6 @@ init_cpu_features (struct cpu_features *cpu_features) if (family == 0x15) { -#if index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward -# error index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward -#endif /* "Excavator" */ if (model >= 0x60 && model <= 0x7f) { @@ -388,7 +469,7 @@ init_cpu_features (struct cpu_features *cpu_features) else { kind = arch_kind_other; - get_common_indeces (cpu_features, NULL, NULL, NULL, NULL); + get_common_indices (cpu_features, NULL, NULL, NULL, NULL); } /* Support i586 if CX8 is available. */ @@ -403,9 +484,10 @@ init_cpu_features (struct cpu_features *cpu_features) no_cpuid: #endif - cpu_features->family = family; - cpu_features->model = model; - cpu_features->kind = kind; + cpu_features->basic.kind = kind; + cpu_features->basic.family = family; + cpu_features->basic.model = model; + cpu_features->basic.stepping = stepping; #if HAVE_TUNABLES TUNABLE_GET (hwcaps, tunable_val_t *, TUNABLE_CALLBACK (set_hwcaps)); @@ -419,14 +501,14 @@ no_cpuid: /* Reuse dl_platform, dl_hwcap and dl_hwcap_mask for x86. */ #if !HAVE_TUNABLES && defined SHARED - /* The glibc.tune.hwcap_mask tunable is initialized already, so no need to do + /* The glibc.cpu.hwcap_mask tunable is initialized already, so no need to do this. */ GLRO(dl_hwcap_mask) = HWCAP_IMPORTANT; #endif #ifdef __x86_64__ GLRO(dl_hwcap) = HWCAP_X86_64; - if (cpu_features->kind == arch_kind_intel) + if (cpu_features->basic.kind == arch_kind_intel) { const char *platform = NULL; @@ -494,7 +576,7 @@ no_cpuid: /* Disable IBT and/or SHSTK if they are enabled by kernel, but disabled by environment variable: - GLIBC_TUNABLES=glibc.tune.hwcaps=-IBT,-SHSTK + GLIBC_TUNABLES=glibc.cpu.hwcaps=-IBT,-SHSTK */ unsigned int cet_feature = 0; if (!HAS_CPU_FEATURE (IBT)) diff --git a/sysdeps/x86/cpu-features.h b/sysdeps/x86/cpu-features.h index 347a4b118d..e6d196c609 100644 --- a/sysdeps/x86/cpu-features.h +++ b/sysdeps/x86/cpu-features.h @@ -1,5 +1,5 @@ /* This file is part of the GNU C Library. - Copyright (C) 2008-2018 Free Software Foundation, Inc. + Copyright (C) 2008-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -13,125 +13,63 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef cpu_features_h #define cpu_features_h -#define bit_arch_Fast_Rep_String (1 << 0) -#define bit_arch_Fast_Copy_Backward (1 << 1) -#define bit_arch_Slow_BSF (1 << 2) -#define bit_arch_Fast_Unaligned_Load (1 << 4) -#define bit_arch_Prefer_PMINUB_for_stringop (1 << 5) -#define bit_arch_AVX_Usable (1 << 6) -#define bit_arch_FMA_Usable (1 << 7) -#define bit_arch_FMA4_Usable (1 << 8) -#define bit_arch_Slow_SSE4_2 (1 << 9) -#define bit_arch_AVX2_Usable (1 << 10) -#define bit_arch_AVX_Fast_Unaligned_Load (1 << 11) -#define bit_arch_AVX512F_Usable (1 << 12) -#define bit_arch_AVX512DQ_Usable (1 << 13) -#define bit_arch_I586 (1 << 14) -#define bit_arch_I686 (1 << 15) -#define bit_arch_Prefer_MAP_32BIT_EXEC (1 << 16) -#define bit_arch_Prefer_No_VZEROUPPER (1 << 17) -#define bit_arch_Fast_Unaligned_Copy (1 << 18) -#define bit_arch_Prefer_ERMS (1 << 19) -#define bit_arch_Prefer_No_AVX512 (1 << 20) -#define bit_arch_MathVec_Prefer_No_AVX512 (1 << 21) -#define bit_arch_XSAVEC_Usable (1 << 22) -#define bit_arch_Prefer_FSRM (1 << 23) - -/* CPUID Feature flags. */ +enum +{ + /* The integer bit array index for the first set of internal feature + bits. */ + FEATURE_INDEX_1 = 0, + FEATURE_INDEX_2, + /* The current maximum size of the feature integer bit array. */ + FEATURE_INDEX_MAX +}; -/* COMMON_CPUID_INDEX_1. */ -#define bit_cpu_CX8 (1 << 8) -#define bit_cpu_CMOV (1 << 15) -#define bit_cpu_SSE (1 << 25) -#define bit_cpu_SSE2 (1 << 26) -#define bit_cpu_SSSE3 (1 << 9) -#define bit_cpu_SSE4_1 (1 << 19) -#define bit_cpu_SSE4_2 (1 << 20) -#define bit_cpu_OSXSAVE (1 << 27) -#define bit_cpu_AVX (1 << 28) -#define bit_cpu_POPCOUNT (1 << 23) -#define bit_cpu_FMA (1 << 12) -#define bit_cpu_FMA4 (1 << 16) -#define bit_cpu_HTT (1 << 28) -#define bit_cpu_LZCNT (1 << 5) -#define bit_cpu_MOVBE (1 << 22) -#define bit_cpu_POPCNT (1 << 23) +enum +{ + COMMON_CPUID_INDEX_1 = 0, + COMMON_CPUID_INDEX_7, + COMMON_CPUID_INDEX_80000001, + COMMON_CPUID_INDEX_D_ECX_1, + COMMON_CPUID_INDEX_80000007, + COMMON_CPUID_INDEX_80000008, + /* Keep the following line at the end. */ + COMMON_CPUID_INDEX_MAX +}; -/* COMMON_CPUID_INDEX_7. */ -#define bit_cpu_BMI1 (1 << 3) -#define bit_cpu_BMI2 (1 << 8) -#define bit_cpu_ERMS (1 << 9) -#define bit_cpu_RTM (1 << 11) -#define bit_cpu_AVX2 (1 << 5) -#define bit_cpu_AVX512F (1 << 16) -#define bit_cpu_AVX512DQ (1 << 17) -#define bit_cpu_AVX512PF (1 << 26) -#define bit_cpu_AVX512ER (1 << 27) -#define bit_cpu_AVX512CD (1 << 28) -#define bit_cpu_AVX512BW (1 << 30) -#define bit_cpu_AVX512VL (1u << 31) -#define bit_cpu_IBT (1u << 20) -#define bit_cpu_SHSTK (1u << 7) -#define bit_cpu_FSRM (1 << 4) +struct cpuid_registers +{ + unsigned int eax; + unsigned int ebx; + unsigned int ecx; + unsigned int edx; +}; -/* XCR0 Feature flags. */ -#define bit_XMM_state (1 << 1) -#define bit_YMM_state (1 << 2) -#define bit_Opmask_state (1 << 5) -#define bit_ZMM0_15_state (1 << 6) -#define bit_ZMM16_31_state (1 << 7) - -/* The integer bit array index for the first set of internal feature bits. */ -#define FEATURE_INDEX_1 0 - -/* The current maximum size of the feature integer bit array. */ -#define FEATURE_INDEX_MAX 1 - -/* Offset for fxsave/xsave area used by _dl_runtime_resolve. Also need - space to preserve RCX, RDX, RSI, RDI, R8, R9 and RAX. It must be - aligned to 16 bytes for fxsave and 64 bytes for xsave. */ -#define STATE_SAVE_OFFSET (8 * 7 + 8) - -/* Save SSE, AVX, AVX512, mask and bound registers. */ -#define STATE_SAVE_MASK \ - ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) - -#ifdef __ASSEMBLER__ -# include <cpu-features-offsets.h> -#else /* __ASSEMBLER__ */ -enum - { - COMMON_CPUID_INDEX_1 = 0, - COMMON_CPUID_INDEX_7, - COMMON_CPUID_INDEX_80000001, - /* Keep the following line at the end. */ - COMMON_CPUID_INDEX_MAX - }; +enum cpu_features_kind +{ + arch_kind_unknown = 0, + arch_kind_intel, + arch_kind_amd, + arch_kind_other +}; -struct cpu_features +struct cpu_features_basic { - enum cpu_features_kind - { - arch_kind_unknown = 0, - arch_kind_intel, - arch_kind_amd, - arch_kind_other - } kind; + enum cpu_features_kind kind; int max_cpuid; - struct cpuid_registers - { - unsigned int eax; - unsigned int ebx; - unsigned int ecx; - unsigned int edx; - } cpuid[COMMON_CPUID_INDEX_MAX]; unsigned int family; unsigned int model; + unsigned int stepping; +}; + +struct cpu_features +{ + struct cpuid_registers cpuid[COMMON_CPUID_INDEX_MAX]; + unsigned int feature[FEATURE_INDEX_MAX]; + struct cpu_features_basic basic; /* The state size for XSAVEC or XSAVE. The type must be unsigned long int so that we use @@ -141,10 +79,9 @@ struct cpu_features unsigned long int xsave_state_size; /* The full state size for XSAVE when XSAVEC is disabled by - GLIBC_TUNABLES=glibc.tune.hwcaps=-XSAVEC_Usable + GLIBC_TUNABLES=glibc.cpu.hwcaps=-XSAVEC_Usable */ unsigned int xsave_state_full_size; - unsigned int feature[FEATURE_INDEX_MAX]; /* Data cache size for use in memory and string routines, typically L1 size. */ unsigned long int data_cache_size; @@ -160,141 +97,854 @@ struct cpu_features extern const struct cpu_features *__get_cpu_features (void) __attribute__ ((const)); -# if defined (_LIBC) && !IS_IN (nonlib) -/* Unused for x86. */ -# define INIT_ARCH() -# define __get_cpu_features() (&GLRO(dl_x86_cpu_features)) -# endif - - /* Only used directly in cpu-features.c. */ # define CPU_FEATURES_CPU_P(ptr, name) \ ((ptr->cpuid[index_cpu_##name].reg_##name & (bit_cpu_##name)) != 0) # define CPU_FEATURES_ARCH_P(ptr, name) \ ((ptr->feature[index_arch_##name] & (bit_arch_##name)) != 0) -/* HAS_* evaluates to true if we may use the feature at runtime. */ -# define HAS_CPU_FEATURE(name) \ - CPU_FEATURES_CPU_P (__get_cpu_features (), name) +/* HAS_CPU_FEATURE evaluates to true if CPU supports the feature. */ +#define HAS_CPU_FEATURE(name) \ + CPU_FEATURES_CPU_P (__get_cpu_features (), name) +/* HAS_ARCH_FEATURE evaluates to true if we may use the feature at + runtime. */ # define HAS_ARCH_FEATURE(name) \ - CPU_FEATURES_ARCH_P (__get_cpu_features (), name) - -# define index_cpu_CX8 COMMON_CPUID_INDEX_1 -# define index_cpu_CMOV COMMON_CPUID_INDEX_1 -# define index_cpu_SSE COMMON_CPUID_INDEX_1 -# define index_cpu_SSE2 COMMON_CPUID_INDEX_1 -# define index_cpu_SSSE3 COMMON_CPUID_INDEX_1 -# define index_cpu_SSE4_1 COMMON_CPUID_INDEX_1 -# define index_cpu_SSE4_2 COMMON_CPUID_INDEX_1 -# define index_cpu_AVX COMMON_CPUID_INDEX_1 -# define index_cpu_AVX2 COMMON_CPUID_INDEX_7 -# define index_cpu_AVX512F COMMON_CPUID_INDEX_7 -# define index_cpu_AVX512DQ COMMON_CPUID_INDEX_7 -# define index_cpu_AVX512PF COMMON_CPUID_INDEX_7 -# define index_cpu_AVX512ER COMMON_CPUID_INDEX_7 -# define index_cpu_AVX512CD COMMON_CPUID_INDEX_7 -# define index_cpu_AVX512BW COMMON_CPUID_INDEX_7 -# define index_cpu_AVX512VL COMMON_CPUID_INDEX_7 -# define index_cpu_ERMS COMMON_CPUID_INDEX_7 -# define index_cpu_RTM COMMON_CPUID_INDEX_7 -# define index_cpu_FMA COMMON_CPUID_INDEX_1 -# define index_cpu_FMA4 COMMON_CPUID_INDEX_80000001 -# define index_cpu_POPCOUNT COMMON_CPUID_INDEX_1 -# define index_cpu_OSXSAVE COMMON_CPUID_INDEX_1 -# define index_cpu_HTT COMMON_CPUID_INDEX_1 -# define index_cpu_BMI1 COMMON_CPUID_INDEX_7 -# define index_cpu_BMI2 COMMON_CPUID_INDEX_7 -# define index_cpu_LZCNT COMMON_CPUID_INDEX_80000001 -# define index_cpu_MOVBE COMMON_CPUID_INDEX_1 -# define index_cpu_POPCNT COMMON_CPUID_INDEX_1 -# define index_cpu_IBT COMMON_CPUID_INDEX_7 -# define index_cpu_SHSTK COMMON_CPUID_INDEX_7 -# define index_cpu_FSRM COMMON_CPUID_INDEX_7 - -# define reg_CX8 edx -# define reg_CMOV edx -# define reg_SSE edx -# define reg_SSE2 edx -# define reg_SSSE3 ecx -# define reg_SSE4_1 ecx -# define reg_SSE4_2 ecx -# define reg_AVX ecx -# define reg_AVX2 ebx -# define reg_AVX512F ebx -# define reg_AVX512DQ ebx -# define reg_AVX512PF ebx -# define reg_AVX512ER ebx -# define reg_AVX512CD ebx -# define reg_AVX512BW ebx -# define reg_AVX512VL ebx -# define reg_ERMS ebx -# define reg_RTM ebx -# define reg_FMA ecx -# define reg_FMA4 ecx -# define reg_POPCOUNT ecx -# define reg_OSXSAVE ecx -# define reg_HTT edx -# define reg_BMI1 ebx -# define reg_BMI2 ebx -# define reg_LZCNT ecx -# define reg_MOVBE ecx -# define reg_POPCNT ecx -# define reg_IBT edx -# define reg_SHSTK ecx -# define reg_FSRM edx - -# define index_arch_Fast_Rep_String FEATURE_INDEX_1 -# define index_arch_Fast_Copy_Backward FEATURE_INDEX_1 -# define index_arch_Slow_BSF FEATURE_INDEX_1 -# define index_arch_Fast_Unaligned_Load FEATURE_INDEX_1 -# define index_arch_Prefer_PMINUB_for_stringop FEATURE_INDEX_1 -# define index_arch_AVX_Usable FEATURE_INDEX_1 -# define index_arch_FMA_Usable FEATURE_INDEX_1 -# define index_arch_FMA4_Usable FEATURE_INDEX_1 -# define index_arch_Slow_SSE4_2 FEATURE_INDEX_1 -# define index_arch_AVX2_Usable FEATURE_INDEX_1 -# define index_arch_AVX_Fast_Unaligned_Load FEATURE_INDEX_1 -# define index_arch_AVX512F_Usable FEATURE_INDEX_1 -# define index_arch_AVX512DQ_Usable FEATURE_INDEX_1 -# define index_arch_I586 FEATURE_INDEX_1 -# define index_arch_I686 FEATURE_INDEX_1 -# define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1 -# define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_1 -# define index_arch_Fast_Unaligned_Copy FEATURE_INDEX_1 -# define index_arch_Prefer_ERMS FEATURE_INDEX_1 -# define index_arch_Prefer_No_AVX512 FEATURE_INDEX_1 -# define index_arch_MathVec_Prefer_No_AVX512 FEATURE_INDEX_1 -# define index_arch_XSAVEC_Usable FEATURE_INDEX_1 -# define index_arch_Prefer_FSRM FEATURE_INDEX_1 - -#endif /* !__ASSEMBLER__ */ + CPU_FEATURES_ARCH_P (__get_cpu_features (), name) +/* CPU_FEATURE_USABLE evaluates to true if the feature is usable. */ +#define CPU_FEATURE_USABLE(name) \ + ((need_arch_feature_##name && HAS_ARCH_FEATURE (name##_Usable)) \ + || (!need_arch_feature_##name && HAS_CPU_FEATURE(name))) + +/* Architecture features. */ + +/* FEATURE_INDEX_1. */ +#define bit_arch_AVX_Usable (1u << 0) +#define bit_arch_AVX2_Usable (1u << 1) +#define bit_arch_AVX512F_Usable (1u << 2) +#define bit_arch_AVX512CD_Usable (1u << 3) +#define bit_arch_AVX512ER_Usable (1u << 4) +#define bit_arch_AVX512PF_Usable (1u << 5) +#define bit_arch_AVX512VL_Usable (1u << 6) +#define bit_arch_AVX512DQ_Usable (1u << 7) +#define bit_arch_AVX512BW_Usable (1u << 8) +#define bit_arch_AVX512_4FMAPS_Usable (1u << 9) +#define bit_arch_AVX512_4VNNIW_Usable (1u << 10) +#define bit_arch_AVX512_BITALG_Usable (1u << 11) +#define bit_arch_AVX512_IFMA_Usable (1u << 12) +#define bit_arch_AVX512_VBMI_Usable (1u << 13) +#define bit_arch_AVX512_VBMI2_Usable (1u << 14) +#define bit_arch_AVX512_VNNI_Usable (1u << 15) +#define bit_arch_AVX512_VPOPCNTDQ_Usable (1u << 16) +#define bit_arch_FMA_Usable (1u << 17) +#define bit_arch_FMA4_Usable (1u << 18) +#define bit_arch_VAES_Usable (1u << 19) +#define bit_arch_VPCLMULQDQ_Usable (1u << 20) +#define bit_arch_XOP_Usable (1u << 21) +#define bit_arch_XSAVEC_Usable (1u << 22) + +#define index_arch_AVX_Usable FEATURE_INDEX_1 +#define index_arch_AVX2_Usable FEATURE_INDEX_1 +#define index_arch_AVX512F_Usable FEATURE_INDEX_1 +#define index_arch_AVX512CD_Usable FEATURE_INDEX_1 +#define index_arch_AVX512ER_Usable FEATURE_INDEX_1 +#define index_arch_AVX512PF_Usable FEATURE_INDEX_1 +#define index_arch_AVX512VL_Usable FEATURE_INDEX_1 +#define index_arch_AVX512BW_Usable FEATURE_INDEX_1 +#define index_arch_AVX512DQ_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_4FMAPS_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_4VNNIW_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_BITALG_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_IFMA_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_VBMI_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_VBMI2_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_VNNI_Usable FEATURE_INDEX_1 +#define index_arch_AVX512_VPOPCNTDQ_Usable FEATURE_INDEX_1 +#define index_arch_FMA_Usable FEATURE_INDEX_1 +#define index_arch_FMA4_Usable FEATURE_INDEX_1 +#define index_arch_VAES_Usable FEATURE_INDEX_1 +#define index_arch_VPCLMULQDQ_Usable FEATURE_INDEX_1 +#define index_arch_XOP_Usable FEATURE_INDEX_1 +#define index_arch_XSAVEC_Usable FEATURE_INDEX_1 + +/* Unused. Compiler will optimize them out. */ +#define bit_arch_SSE3_Usable (1u << 0) +#define bit_arch_PCLMULQDQ_Usable (1u << 0) +#define bit_arch_SSSE3_Usable (1u << 0) +#define bit_arch_CMPXCHG16B_Usable (1u << 0) +#define bit_arch_SSE4_1_Usable (1u << 0) +#define bit_arch_SSE4_2_Usable (1u << 0) +#define bit_arch_MOVBE_Usable (1u << 0) +#define bit_arch_POPCNT_Usable (1u << 0) +#define bit_arch_AES_Usable (1u << 0) +#define bit_arch_XSAVE_Usable (1u << 0) +#define bit_arch_OSXSAVE_Usable (1u << 0) +#define bit_arch_F16C_Usable (1u << 0) +#define bit_arch_RDRAND_Usable (1u << 0) +#define bit_arch_FPU_Usable (1u << 0) +#define bit_arch_TSC_Usable (1u << 0) +#define bit_arch_MSR_Usable (1u << 0) +#define bit_arch_CX8_Usable (1u << 0) +#define bit_arch_SEP_Usable (1u << 0) +#define bit_arch_CMOV_Usable (1u << 0) +#define bit_arch_CLFSH_Usable (1u << 0) +#define bit_arch_MMX_Usable (1u << 0) +#define bit_arch_FXSR_Usable (1u << 0) +#define bit_arch_SSE_Usable (1u << 0) +#define bit_arch_SSE2_Usable (1u << 0) +#define bit_arch_FSGSBASE_Usable (1u << 0) +#define bit_arch_BMI1_Usable (1u << 0) +#define bit_arch_HLE_Usable (1u << 0) +#define bit_arch_BMI2_Usable (1u << 0) +#define bit_arch_ERMS_Usable (1u << 0) +#define bit_arch_RTM_Usable (1u << 0) +#define bit_arch_RDSEED_Usable (1u << 0) +#define bit_arch_ADX_Usable (1u << 0) +#define bit_arch_CLFLUSHOPT_Usable (1u << 0) +#define bit_arch_CLWB_Usable (1u << 0) +#define bit_arch_SHA_Usable (1u << 0) +#define bit_arch_PREFETCHWT1_Usable (1u << 0) +#define bit_arch_GFNI_Usable (1u << 0) +#define bit_arch_RDPID_Usable (1u << 0) +#define bit_arch_CLDEMOTE_Usable (1u << 0) +#define bit_arch_MOVDIRI_Usable (1u << 0) +#define bit_arch_MOVDIR64B_Usable (1u << 0) +#define bit_arch_FSRM_Usable (1u << 0) +#define bit_arch_LAHF64_SAHF64_Usable (1u << 0) +#define bit_arch_SVM_Usable (1u << 0) +#define bit_arch_LZCNT_Usable (1u << 0) +#define bit_arch_SSE4A_Usable (1u << 0) +#define bit_arch_PREFETCHW_Usable (1u << 0) +#define bit_arch_TBM_Usable (1u << 0) +#define bit_arch_SYSCALL_SYSRET_Usable (1u << 0) +#define bit_arch_RDTSCP_Usable (1u << 0) +#define bit_arch_XSAVEOPT_Usable (1u << 0) +#define bit_arch_XGETBV_ECX_1_Usable (1u << 0) +#define bit_arch_XSAVES_Usable (1u << 0) +#define bit_arch_INVARIANT_TSC_Usable (1u << 0) +#define bit_arch_WBNOINVD_Usable (1u << 0) + +/* Unused. Compiler will optimize them out. */ +#define index_arch_SSE3_Usable FEATURE_INDEX_1 +#define index_arch_PCLMULQDQ_Usable FEATURE_INDEX_1 +#define index_arch_SSSE3_Usable FEATURE_INDEX_1 +#define index_arch_CMPXCHG16B_Usable FEATURE_INDEX_1 +#define index_arch_SSE4_1_Usable FEATURE_INDEX_1 +#define index_arch_SSE4_2_Usable FEATURE_INDEX_1 +#define index_arch_MOVBE_Usable FEATURE_INDEX_1 +#define index_arch_POPCNT_Usable FEATURE_INDEX_1 +#define index_arch_AES_Usable FEATURE_INDEX_1 +#define index_arch_XSAVE_Usable FEATURE_INDEX_1 +#define index_arch_OSXSAVE_Usable FEATURE_INDEX_1 +#define index_arch_F16C_Usable FEATURE_INDEX_1 +#define index_arch_RDRAND_Usable FEATURE_INDEX_1 +#define index_arch_FPU_Usable FEATURE_INDEX_1 +#define index_arch_TSC_Usable FEATURE_INDEX_1 +#define index_arch_MSR_Usable FEATURE_INDEX_1 +#define index_arch_CX8_Usable FEATURE_INDEX_1 +#define index_arch_SEP_Usable FEATURE_INDEX_1 +#define index_arch_CMOV_Usable FEATURE_INDEX_1 +#define index_arch_CLFSH_Usable FEATURE_INDEX_1 +#define index_arch_MMX_Usable FEATURE_INDEX_1 +#define index_arch_FXSR_Usable FEATURE_INDEX_1 +#define index_arch_SSE_Usable FEATURE_INDEX_1 +#define index_arch_SSE2_Usable FEATURE_INDEX_1 +#define index_arch_FSGSBASE_Usable FEATURE_INDEX_1 +#define index_arch_BMI1_Usable FEATURE_INDEX_1 +#define index_arch_HLE_Usable FEATURE_INDEX_1 +#define index_arch_BMI2_Usable FEATURE_INDEX_1 +#define index_arch_ERMS_Usable FEATURE_INDEX_1 +#define index_arch_RTM_Usable FEATURE_INDEX_1 +#define index_arch_RDSEED_Usable FEATURE_INDEX_1 +#define index_arch_ADX_Usable FEATURE_INDEX_1 +#define index_arch_CLFLUSHOPT_Usable FEATURE_INDEX_1 +#define index_arch_CLWB_Usable FEATURE_INDEX_1 +#define index_arch_SHA_Usable FEATURE_INDEX_1 +#define index_arch_PREFETCHWT1_Usable FEATURE_INDEX_1 +#define index_arch_GFNI_Usable FEATURE_INDEX_1 +#define index_arch_RDPID_Usable FEATURE_INDEX_1 +#define index_arch_CLDEMOTE_Usable FEATURE_INDEX_1 +#define index_arch_MOVDIRI_Usable FEATURE_INDEX_1 +#define index_arch_MOVDIR64B_Usable FEATURE_INDEX_1 +#define index_arch_FSRM_Usable FEATURE_INDEX_1 +#define index_arch_LAHF64_SAHF64_Usable FEATURE_INDEX_1 +#define index_arch_LZCNT_Usable FEATURE_INDEX_1 +#define index_arch_SSE4A_Usable FEATURE_INDEX_1 +#define index_arch_PREFETCHW_Usable FEATURE_INDEX_1 +#define index_arch_TBM_Usable FEATURE_INDEX_1 +#define index_arch_SYSCALL_SYSRET_Usable FEATURE_INDEX_1 +#define index_arch_RDTSCP_Usable FEATURE_INDEX_1 +#define index_arch_XSAVEOPT_Usable FEATURE_INDEX_1 +#define index_arch_XGETBV_ECX_1_Usable FEATURE_INDEX_1 +#define index_arch_XSAVES_Usable FEATURE_INDEX_1 +#define index_arch_INVARIANT_TSC_Usable FEATURE_INDEX_1 +#define index_arch_WBNOINVD_Usable FEATURE_INDEX_1 + +/* COMMON_CPUID_INDEX_1. */ + +/* ECX. */ +#define need_arch_feature_SSE3 0 +#define need_arch_feature_PCLMULQDQ 0 +#define need_arch_feature_SSSE3 0 +#define need_arch_feature_FMA 1 +#define need_arch_feature_CMPXCHG16B 0 +#define need_arch_feature_SSE4_1 0 +#define need_arch_feature_SSE4_2 0 +#define need_arch_feature_MOVBE 0 +#define need_arch_feature_POPCNT 0 +#define need_arch_feature_AES 0 +#define need_arch_feature_XSAVE 0 +#define need_arch_feature_OSXSAVE 0 +#define need_arch_feature_AVX 1 +#define need_arch_feature_F16C 0 +#define need_arch_feature_RDRAND 0 + +/* EDX. */ +#define need_arch_feature_FPU 0 +#define need_arch_feature_TSC 0 +#define need_arch_feature_MSR 0 +#define need_arch_feature_CX8 0 +#define need_arch_feature_SEP 0 +#define need_arch_feature_CMOV 0 +#define need_arch_feature_CLFSH 0 +#define need_arch_feature_MMX 0 +#define need_arch_feature_FXSR 0 +#define need_arch_feature_SSE 0 +#define need_arch_feature_SSE2 0 + +/* COMMON_CPUID_INDEX_7. */ + +/* EBX. */ +#define need_arch_feature_FSGSBASE 0 +#define need_arch_feature_BMI1 0 +#define need_arch_feature_HLE 0 +#define need_arch_feature_AVX2 1 +#define need_arch_feature_BMI2 0 +#define need_arch_feature_ERMS 0 +#define need_arch_feature_RTM 0 +#define need_arch_feature_AVX512F 1 +#define need_arch_feature_AVX512DQ 1 +#define need_arch_feature_RDSEED 0 +#define need_arch_feature_ADX 0 +#define need_arch_feature_AVX512_IFMA 1 +#define need_arch_feature_CLFLUSHOPT 0 +#define need_arch_feature_CLWB 0 +#define need_arch_feature_AVX512PF 1 +#define need_arch_feature_AVX512ER 1 +#define need_arch_feature_AVX512CD 1 +#define need_arch_feature_SHA 0 +#define need_arch_feature_AVX512BW 1 +#define need_arch_feature_AVX512VL 1 + +/* ECX. */ +#define need_arch_feature_PREFETCHWT1 0 +#define need_arch_feature_AVX512_VBMI 1 +#define need_arch_feature_AVX512_VBMI2 1 +#define need_arch_feature_GFNI 0 +#define need_arch_feature_VAES 1 +#define need_arch_feature_VPCLMULQDQ 1 +#define need_arch_feature_AVX512_VNNI 1 +#define need_arch_feature_AVX512_BITALG 1 +#define need_arch_feature_AVX512_VPOPCNTDQ 1 +#define need_arch_feature_RDPID 0 +#define need_arch_feature_CLDEMOTE 0 +#define need_arch_feature_MOVDIRI 0 +#define need_arch_feature_MOVDIR64B 0 + +/* EDX. */ +#define need_arch_feature_AVX512_4VNNIW 1 +#define need_arch_feature_AVX512_4FMAPS 1 +#define need_arch_feature_FSRM 0 + +/* COMMON_CPUID_INDEX_80000001. */ + +/* ECX. */ +#define need_arch_feature_LAHF64_SAHF64 0 +#define need_arch_feature_LZCNT 0 +#define need_arch_feature_SSE4A 0 +#define need_arch_feature_PREFETCHW 0 +#define need_arch_feature_XOP 1 +#define need_arch_feature_FMA4 1 +#define need_arch_feature_TBM 0 +#define need_arch_feature_SYSCALL_SYSRET 0 +#define need_arch_feature_RDTSCP 0 +#define need_arch_feature_XSAVEOPT 0 +#define need_arch_feature_XSAVEC 1 +#define need_arch_feature_XGETBV_ECX_1 0 +#define need_arch_feature_XSAVES 0 +#define need_arch_feature_INVARIANT_TSC 0 +#define need_arch_feature_WBNOINVD 0 + +/* CPU features. */ + +/* COMMON_CPUID_INDEX_1. */ + +/* ECX. */ +#define bit_cpu_SSE3 (1u << 0) +#define bit_cpu_PCLMULQDQ (1u << 1) +#define bit_cpu_DTES64 (1u << 2) +#define bit_cpu_MONITOR (1u << 3) +#define bit_cpu_DS_CPL (1u << 4) +#define bit_cpu_VMX (1u << 5) +#define bit_cpu_SMX (1u << 6) +#define bit_cpu_EST (1u << 7) +#define bit_cpu_TM2 (1u << 8) +#define bit_cpu_SSSE3 (1u << 9) +#define bit_cpu_CNXT_ID (1u << 10) +#define bit_cpu_SDBG (1u << 11) +#define bit_cpu_FMA (1u << 12) +#define bit_cpu_CMPXCHG16B (1u << 13) +#define bit_cpu_XTPRUPDCTRL (1u << 14) +#define bit_cpu_PDCM (1u << 15) +#define bit_cpu_PCID (1u << 17) +#define bit_cpu_DCA (1u << 18) +#define bit_cpu_SSE4_1 (1u << 19) +#define bit_cpu_SSE4_2 (1u << 20) +#define bit_cpu_X2APIC (1u << 21) +#define bit_cpu_MOVBE (1u << 22) +#define bit_cpu_POPCNT (1u << 23) +#define bit_cpu_TSC_DEADLINE (1u << 24) +#define bit_cpu_AES (1u << 25) +#define bit_cpu_XSAVE (1u << 26) +#define bit_cpu_OSXSAVE (1u << 27) +#define bit_cpu_AVX (1u << 28) +#define bit_cpu_F16C (1u << 29) +#define bit_cpu_RDRAND (1u << 30) + +/* EDX. */ +#define bit_cpu_FPU (1u << 0) +#define bit_cpu_VME (1u << 1) +#define bit_cpu_DE (1u << 2) +#define bit_cpu_PSE (1u << 3) +#define bit_cpu_TSC (1u << 4) +#define bit_cpu_MSR (1u << 5) +#define bit_cpu_PAE (1u << 6) +#define bit_cpu_MCE (1u << 7) +#define bit_cpu_CX8 (1u << 8) +#define bit_cpu_APIC (1u << 9) +#define bit_cpu_SEP (1u << 11) +#define bit_cpu_MTRR (1u << 12) +#define bit_cpu_PGE (1u << 13) +#define bit_cpu_MCA (1u << 14) +#define bit_cpu_CMOV (1u << 15) +#define bit_cpu_PAT (1u << 16) +#define bit_cpu_PSE_36 (1u << 17) +#define bit_cpu_PSN (1u << 18) +#define bit_cpu_CLFSH (1u << 20) +#define bit_cpu_DS (1u << 21) +#define bit_cpu_ACPI (1u << 22) +#define bit_cpu_MMX (1u << 23) +#define bit_cpu_FXSR (1u << 24) +#define bit_cpu_SSE (1u << 25) +#define bit_cpu_SSE2 (1u << 26) +#define bit_cpu_SS (1u << 27) +#define bit_cpu_HTT (1u << 28) +#define bit_cpu_TM (1u << 29) +#define bit_cpu_PBE (1u << 31) + +/* COMMON_CPUID_INDEX_7. */ + +/* EBX. */ +#define bit_cpu_FSGSBASE (1u << 0) +#define bit_cpu_TSC_ADJUST (1u << 1) +#define bit_cpu_SGX (1u << 2) +#define bit_cpu_BMI1 (1u << 3) +#define bit_cpu_HLE (1u << 4) +#define bit_cpu_AVX2 (1u << 5) +#define bit_cpu_SMEP (1u << 7) +#define bit_cpu_BMI2 (1u << 8) +#define bit_cpu_ERMS (1u << 9) +#define bit_cpu_INVPCID (1u << 10) +#define bit_cpu_RTM (1u << 11) +#define bit_cpu_PQM (1u << 12) +#define bit_cpu_MPX (1u << 14) +#define bit_cpu_PQE (1u << 15) +#define bit_cpu_AVX512F (1u << 16) +#define bit_cpu_AVX512DQ (1u << 17) +#define bit_cpu_RDSEED (1u << 18) +#define bit_cpu_ADX (1u << 19) +#define bit_cpu_SMAP (1u << 20) +#define bit_cpu_AVX512_IFMA (1u << 21) +#define bit_cpu_CLFLUSHOPT (1u << 22) +#define bit_cpu_CLWB (1u << 24) +#define bit_cpu_TRACE (1u << 25) +#define bit_cpu_AVX512PF (1u << 26) +#define bit_cpu_AVX512ER (1u << 27) +#define bit_cpu_AVX512CD (1u << 28) +#define bit_cpu_SHA (1u << 29) +#define bit_cpu_AVX512BW (1u << 30) +#define bit_cpu_AVX512VL (1u << 31) + +/* ECX. */ +#define bit_cpu_PREFETCHWT1 (1u << 0) +#define bit_cpu_AVX512_VBMI (1u << 1) +#define bit_cpu_UMIP (1u << 2) +#define bit_cpu_PKU (1u << 3) +#define bit_cpu_OSPKE (1u << 4) +#define bit_cpu_WAITPKG (1u << 5) +#define bit_cpu_AVX512_VBMI2 (1u << 6) +#define bit_cpu_SHSTK (1u << 7) +#define bit_cpu_GFNI (1u << 8) +#define bit_cpu_VAES (1u << 9) +#define bit_cpu_VPCLMULQDQ (1u << 10) +#define bit_cpu_AVX512_VNNI (1u << 11) +#define bit_cpu_AVX512_BITALG (1u << 12) +#define bit_cpu_AVX512_VPOPCNTDQ (1u << 14) +#define bit_cpu_RDPID (1u << 22) +#define bit_cpu_CLDEMOTE (1u << 25) +#define bit_cpu_MOVDIRI (1u << 27) +#define bit_cpu_MOVDIR64B (1u << 28) +#define bit_cpu_SGX_LC (1u << 30) + +/* EDX. */ +#define bit_cpu_AVX512_4VNNIW (1u << 2) +#define bit_cpu_AVX512_4FMAPS (1u << 3) +#define bit_cpu_FSRM (1u << 4) +#define bit_cpu_PCONFIG (1u << 18) +#define bit_cpu_IBT (1u << 20) +#define bit_cpu_IBRS_IBPB (1u << 26) +#define bit_cpu_STIBP (1u << 27) +#define bit_cpu_CAPABILITIES (1u << 29) +#define bit_cpu_SSBD (1u << 31) + +/* COMMON_CPUID_INDEX_80000001. */ + +/* ECX. */ +#define bit_cpu_LAHF64_SAHF64 (1u << 0) +#define bit_cpu_SVM (1u << 2) +#define bit_cpu_LZCNT (1u << 5) +#define bit_cpu_SSE4A (1u << 6) +#define bit_cpu_PREFETCHW (1u << 8) +#define bit_cpu_XOP (1u << 11) +#define bit_cpu_LWP (1u << 15) +#define bit_cpu_FMA4 (1u << 16) +#define bit_cpu_TBM (1u << 21) + +/* EDX. */ +#define bit_cpu_SYSCALL_SYSRET (1u << 11) +#define bit_cpu_NX (1u << 20) +#define bit_cpu_PAGE1GB (1u << 26) +#define bit_cpu_RDTSCP (1u << 27) +#define bit_cpu_LM (1u << 29) + +/* COMMON_CPUID_INDEX_D_ECX_1. */ + +/* EAX. */ +#define bit_cpu_XSAVEOPT (1u << 0) +#define bit_cpu_XSAVEC (1u << 1) +#define bit_cpu_XGETBV_ECX_1 (1u << 2) +#define bit_cpu_XSAVES (1u << 3) + +/* COMMON_CPUID_INDEX_80000007. */ + +/* EDX. */ +#define bit_cpu_INVARIANT_TSC (1u << 8) + +/* COMMON_CPUID_INDEX_80000008. */ + +/* EBX. */ +#define bit_cpu_WBNOINVD (1u << 9) + +/* COMMON_CPUID_INDEX_1. */ + +/* ECX. */ +#define index_cpu_SSE3 COMMON_CPUID_INDEX_1 +#define index_cpu_PCLMULQDQ COMMON_CPUID_INDEX_1 +#define index_cpu_DTES64 COMMON_CPUID_INDEX_1 +#define index_cpu_MONITOR COMMON_CPUID_INDEX_1 +#define index_cpu_DS_CPL COMMON_CPUID_INDEX_1 +#define index_cpu_VMX COMMON_CPUID_INDEX_1 +#define index_cpu_SMX COMMON_CPUID_INDEX_1 +#define index_cpu_EST COMMON_CPUID_INDEX_1 +#define index_cpu_TM2 COMMON_CPUID_INDEX_1 +#define index_cpu_SSSE3 COMMON_CPUID_INDEX_1 +#define index_cpu_CNXT_ID COMMON_CPUID_INDEX_1 +#define index_cpu_SDBG COMMON_CPUID_INDEX_1 +#define index_cpu_FMA COMMON_CPUID_INDEX_1 +#define index_cpu_CMPXCHG16B COMMON_CPUID_INDEX_1 +#define index_cpu_XTPRUPDCTRL COMMON_CPUID_INDEX_1 +#define index_cpu_PDCM COMMON_CPUID_INDEX_1 +#define index_cpu_PCID COMMON_CPUID_INDEX_1 +#define index_cpu_DCA COMMON_CPUID_INDEX_1 +#define index_cpu_SSE4_1 COMMON_CPUID_INDEX_1 +#define index_cpu_SSE4_2 COMMON_CPUID_INDEX_1 +#define index_cpu_X2APIC COMMON_CPUID_INDEX_1 +#define index_cpu_MOVBE COMMON_CPUID_INDEX_1 +#define index_cpu_POPCNT COMMON_CPUID_INDEX_1 +#define index_cpu_TSC_DEADLINE COMMON_CPUID_INDEX_1 +#define index_cpu_AES COMMON_CPUID_INDEX_1 +#define index_cpu_XSAVE COMMON_CPUID_INDEX_1 +#define index_cpu_OSXSAVE COMMON_CPUID_INDEX_1 +#define index_cpu_AVX COMMON_CPUID_INDEX_1 +#define index_cpu_F16C COMMON_CPUID_INDEX_1 +#define index_cpu_RDRAND COMMON_CPUID_INDEX_1 + +/* ECX. */ +#define index_cpu_FPU COMMON_CPUID_INDEX_1 +#define index_cpu_VME COMMON_CPUID_INDEX_1 +#define index_cpu_DE COMMON_CPUID_INDEX_1 +#define index_cpu_PSE COMMON_CPUID_INDEX_1 +#define index_cpu_TSC COMMON_CPUID_INDEX_1 +#define index_cpu_MSR COMMON_CPUID_INDEX_1 +#define index_cpu_PAE COMMON_CPUID_INDEX_1 +#define index_cpu_MCE COMMON_CPUID_INDEX_1 +#define index_cpu_CX8 COMMON_CPUID_INDEX_1 +#define index_cpu_APIC COMMON_CPUID_INDEX_1 +#define index_cpu_SEP COMMON_CPUID_INDEX_1 +#define index_cpu_MTRR COMMON_CPUID_INDEX_1 +#define index_cpu_PGE COMMON_CPUID_INDEX_1 +#define index_cpu_MCA COMMON_CPUID_INDEX_1 +#define index_cpu_CMOV COMMON_CPUID_INDEX_1 +#define index_cpu_PAT COMMON_CPUID_INDEX_1 +#define index_cpu_PSE_36 COMMON_CPUID_INDEX_1 +#define index_cpu_PSN COMMON_CPUID_INDEX_1 +#define index_cpu_CLFSH COMMON_CPUID_INDEX_1 +#define index_cpu_DS COMMON_CPUID_INDEX_1 +#define index_cpu_ACPI COMMON_CPUID_INDEX_1 +#define index_cpu_MMX COMMON_CPUID_INDEX_1 +#define index_cpu_FXSR COMMON_CPUID_INDEX_1 +#define index_cpu_SSE COMMON_CPUID_INDEX_1 +#define index_cpu_SSE2 COMMON_CPUID_INDEX_1 +#define index_cpu_SS COMMON_CPUID_INDEX_1 +#define index_cpu_HTT COMMON_CPUID_INDEX_1 +#define index_cpu_TM COMMON_CPUID_INDEX_1 +#define index_cpu_PBE COMMON_CPUID_INDEX_1 + +/* COMMON_CPUID_INDEX_7. */ + +/* EBX. */ +#define index_cpu_FSGSBASE COMMON_CPUID_INDEX_7 +#define index_cpu_TSC_ADJUST COMMON_CPUID_INDEX_7 +#define index_cpu_SGX COMMON_CPUID_INDEX_7 +#define index_cpu_BMI1 COMMON_CPUID_INDEX_7 +#define index_cpu_HLE COMMON_CPUID_INDEX_7 +#define index_cpu_AVX2 COMMON_CPUID_INDEX_7 +#define index_cpu_SMEP COMMON_CPUID_INDEX_7 +#define index_cpu_BMI2 COMMON_CPUID_INDEX_7 +#define index_cpu_ERMS COMMON_CPUID_INDEX_7 +#define index_cpu_INVPCID COMMON_CPUID_INDEX_7 +#define index_cpu_RTM COMMON_CPUID_INDEX_7 +#define index_cpu_PQM COMMON_CPUID_INDEX_7 +#define index_cpu_MPX COMMON_CPUID_INDEX_7 +#define index_cpu_PQE COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512F COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512DQ COMMON_CPUID_INDEX_7 +#define index_cpu_RDSEED COMMON_CPUID_INDEX_7 +#define index_cpu_ADX COMMON_CPUID_INDEX_7 +#define index_cpu_SMAP COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512_IFMA COMMON_CPUID_INDEX_7 +#define index_cpu_CLFLUSHOPT COMMON_CPUID_INDEX_7 +#define index_cpu_CLWB COMMON_CPUID_INDEX_7 +#define index_cpu_TRACE COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512PF COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512ER COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512CD COMMON_CPUID_INDEX_7 +#define index_cpu_SHA COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512BW COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512VL COMMON_CPUID_INDEX_7 + +/* ECX. */ +#define index_cpu_PREFETCHWT1 COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512_VBMI COMMON_CPUID_INDEX_7 +#define index_cpu_UMIP COMMON_CPUID_INDEX_7 +#define index_cpu_PKU COMMON_CPUID_INDEX_7 +#define index_cpu_OSPKE COMMON_CPUID_INDEX_7 +#define index_cpu_WAITPKG COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512_VBMI2 COMMON_CPUID_INDEX_7 +#define index_cpu_SHSTK COMMON_CPUID_INDEX_7 +#define index_cpu_GFNI COMMON_CPUID_INDEX_7 +#define index_cpu_VAES COMMON_CPUID_INDEX_7 +#define index_cpu_VPCLMULQDQ COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512_VNNI COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512_BITALG COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512_VPOPCNTDQ COMMON_CPUID_INDEX_7 +#define index_cpu_RDPID COMMON_CPUID_INDEX_7 +#define index_cpu_CLDEMOTE COMMON_CPUID_INDEX_7 +#define index_cpu_MOVDIRI COMMON_CPUID_INDEX_7 +#define index_cpu_MOVDIR64B COMMON_CPUID_INDEX_7 +#define index_cpu_SGX_LC COMMON_CPUID_INDEX_7 + +/* EDX. */ +#define index_cpu_AVX512_4VNNIW COMMON_CPUID_INDEX_7 +#define index_cpu_AVX512_4FMAPS COMMON_CPUID_INDEX_7 +#define index_cpu_FSRM COMMON_CPUID_INDEX_7 +#define index_cpu_PCONFIG COMMON_CPUID_INDEX_7 +#define index_cpu_IBT COMMON_CPUID_INDEX_7 +#define index_cpu_IBRS_IBPB COMMON_CPUID_INDEX_7 +#define index_cpu_STIBP COMMON_CPUID_INDEX_7 +#define index_cpu_CAPABILITIES COMMON_CPUID_INDEX_7 +#define index_cpu_SSBD COMMON_CPUID_INDEX_7 + +/* COMMON_CPUID_INDEX_80000001. */ + +/* ECX. */ +#define index_cpu_LAHF64_SAHF64 COMMON_CPUID_INDEX_80000001 +#define index_cpu_SVM COMMON_CPUID_INDEX_80000001 +#define index_cpu_LZCNT COMMON_CPUID_INDEX_80000001 +#define index_cpu_SSE4A COMMON_CPUID_INDEX_80000001 +#define index_cpu_PREFETCHW COMMON_CPUID_INDEX_80000001 +#define index_cpu_XOP COMMON_CPUID_INDEX_80000001 +#define index_cpu_LWP COMMON_CPUID_INDEX_80000001 +#define index_cpu_FMA4 COMMON_CPUID_INDEX_80000001 +#define index_cpu_TBM COMMON_CPUID_INDEX_80000001 + +/* EDX. */ +#define index_cpu_SYSCALL_SYSRET COMMON_CPUID_INDEX_80000001 +#define index_cpu_NX COMMON_CPUID_INDEX_80000001 +#define index_cpu_PAGE1GB COMMON_CPUID_INDEX_80000001 +#define index_cpu_RDTSCP COMMON_CPUID_INDEX_80000001 +#define index_cpu_LM COMMON_CPUID_INDEX_80000001 + +/* COMMON_CPUID_INDEX_D_ECX_1. */ + +/* EAX. */ +#define index_cpu_XSAVEOPT COMMON_CPUID_INDEX_D_ECX_1 +#define index_cpu_XSAVEC COMMON_CPUID_INDEX_D_ECX_1 +#define index_cpu_XGETBV_ECX_1 COMMON_CPUID_INDEX_D_ECX_1 +#define index_cpu_XSAVES COMMON_CPUID_INDEX_D_ECX_1 + +/* COMMON_CPUID_INDEX_80000007. */ + +/* EDX. */ +#define index_cpu_INVARIANT_TSC COMMON_CPUID_INDEX_80000007 + +/* COMMON_CPUID_INDEX_80000008. */ + +/* EBX. */ +#define index_cpu_WBNOINVD COMMON_CPUID_INDEX_80000008 + +/* COMMON_CPUID_INDEX_1. */ + +/* ECX. */ +#define reg_SSE3 ecx +#define reg_PCLMULQDQ ecx +#define reg_DTES64 ecx +#define reg_MONITOR ecx +#define reg_DS_CPL ecx +#define reg_VMX ecx +#define reg_SMX ecx +#define reg_EST ecx +#define reg_TM2 ecx +#define reg_SSSE3 ecx +#define reg_CNXT_ID ecx +#define reg_SDBG ecx +#define reg_FMA ecx +#define reg_CMPXCHG16B ecx +#define reg_XTPRUPDCTRL ecx +#define reg_PDCM ecx +#define reg_PCID ecx +#define reg_DCA ecx +#define reg_SSE4_1 ecx +#define reg_SSE4_2 ecx +#define reg_X2APIC ecx +#define reg_MOVBE ecx +#define reg_POPCNT ecx +#define reg_TSC_DEADLINE ecx +#define reg_AES ecx +#define reg_XSAVE ecx +#define reg_OSXSAVE ecx +#define reg_AVX ecx +#define reg_F16C ecx +#define reg_RDRAND ecx + +/* EDX. */ +#define reg_FPU edx +#define reg_VME edx +#define reg_DE edx +#define reg_PSE edx +#define reg_TSC edx +#define reg_MSR edx +#define reg_PAE edx +#define reg_MCE edx +#define reg_CX8 edx +#define reg_APIC edx +#define reg_SEP edx +#define reg_MTRR edx +#define reg_PGE edx +#define reg_MCA edx +#define reg_CMOV edx +#define reg_PAT edx +#define reg_PSE_36 edx +#define reg_PSN edx +#define reg_CLFSH edx +#define reg_DS edx +#define reg_ACPI edx +#define reg_MMX edx +#define reg_FXSR edx +#define reg_SSE edx +#define reg_SSE2 edx +#define reg_SS edx +#define reg_HTT edx +#define reg_TM edx +#define reg_PBE edx + +/* COMMON_CPUID_INDEX_7. */ + +/* EBX. */ +#define reg_FSGSBASE ebx +#define reg_TSC_ADJUST ebx +#define reg_SGX ebx +#define reg_BMI1 ebx +#define reg_HLE ebx +#define reg_BMI2 ebx +#define reg_AVX2 ebx +#define reg_SMEP ebx +#define reg_ERMS ebx +#define reg_INVPCID ebx +#define reg_RTM ebx +#define reg_PQM ebx +#define reg_MPX ebx +#define reg_PQE ebx +#define reg_AVX512F ebx +#define reg_AVX512DQ ebx +#define reg_RDSEED ebx +#define reg_ADX ebx +#define reg_SMAP ebx +#define reg_AVX512_IFMA ebx +#define reg_CLFLUSHOPT ebx +#define reg_CLWB ebx +#define reg_TRACE ebx +#define reg_AVX512PF ebx +#define reg_AVX512ER ebx +#define reg_AVX512CD ebx +#define reg_SHA ebx +#define reg_AVX512BW ebx +#define reg_AVX512VL ebx + +/* ECX. */ +#define reg_PREFETCHWT1 ecx +#define reg_AVX512_VBMI ecx +#define reg_UMIP ecx +#define reg_PKU ecx +#define reg_OSPKE ecx +#define reg_WAITPKG ecx +#define reg_AVX512_VBMI2 ecx +#define reg_SHSTK ecx +#define reg_GFNI ecx +#define reg_VAES ecx +#define reg_VPCLMULQDQ ecx +#define reg_AVX512_VNNI ecx +#define reg_AVX512_BITALG ecx +#define reg_AVX512_VPOPCNTDQ ecx +#define reg_RDPID ecx +#define reg_CLDEMOTE ecx +#define reg_MOVDIRI ecx +#define reg_MOVDIR64B ecx +#define reg_SGX_LC ecx + +/* EDX. */ +#define reg_AVX512_4VNNIW edx +#define reg_AVX512_4FMAPS edx +#define reg_FSRM edx +#define reg_PCONFIG edx +#define reg_IBT edx +#define reg_IBRS_IBPB edx +#define reg_STIBP edx +#define reg_CAPABILITIES edx +#define reg_SSBD edx + +/* COMMON_CPUID_INDEX_80000001. */ + +/* ECX. */ +#define reg_LAHF64_SAHF64 ecx +#define reg_SVM ecx +#define reg_LZCNT ecx +#define reg_SSE4A ecx +#define reg_PREFETCHW ecx +#define reg_XOP ecx +#define reg_LWP ecx +#define reg_FMA4 ecx +#define reg_TBM ecx + +/* EDX. */ +#define reg_SYSCALL_SYSRET edx +#define reg_NX edx +#define reg_PAGE1GB edx +#define reg_RDTSCP edx +#define reg_LM edx + +/* COMMON_CPUID_INDEX_D_ECX_1. */ + +/* EAX. */ +#define reg_XSAVEOPT eax +#define reg_XSAVEC eax +#define reg_XGETBV_ECX_1 eax +#define reg_XSAVES eax + +/* COMMON_CPUID_INDEX_80000007. */ + +/* EDX. */ +#define reg_INVARIANT_TSC edx + +/* COMMON_CPUID_INDEX_80000008. */ + +/* EBX. */ +#define reg_WBNOINVD ebx + +/* FEATURE_INDEX_2. */ +#define bit_arch_I586 (1u << 0) +#define bit_arch_I686 (1u << 1) +#define bit_arch_Fast_Rep_String (1u << 2) +#define bit_arch_Fast_Copy_Backward (1u << 3) +#define bit_arch_Fast_Unaligned_Load (1u << 4) +#define bit_arch_Fast_Unaligned_Copy (1u << 5) +#define bit_arch_Slow_BSF (1u << 6) +#define bit_arch_Slow_SSE4_2 (1u << 7) +#define bit_arch_AVX_Fast_Unaligned_Load (1u << 8) +#define bit_arch_Prefer_MAP_32BIT_EXEC (1u << 9) +#define bit_arch_Prefer_PMINUB_for_stringop (1u << 10) +#define bit_arch_Prefer_No_VZEROUPPER (1u << 11) +#define bit_arch_Prefer_ERMS (1u << 12) +#define bit_arch_Prefer_FSRM (1u << 13) +#define bit_arch_Prefer_No_AVX512 (1u << 14) +#define bit_arch_MathVec_Prefer_No_AVX512 (1u << 15) + +#define index_arch_Fast_Rep_String FEATURE_INDEX_2 +#define index_arch_Fast_Copy_Backward FEATURE_INDEX_2 +#define index_arch_Slow_BSF FEATURE_INDEX_2 +#define index_arch_Fast_Unaligned_Load FEATURE_INDEX_2 +#define index_arch_Prefer_PMINUB_for_stringop FEATURE_INDEX_2 +#define index_arch_Fast_Unaligned_Copy FEATURE_INDEX_2 +#define index_arch_I586 FEATURE_INDEX_2 +#define index_arch_I686 FEATURE_INDEX_2 +#define index_arch_Slow_SSE4_2 FEATURE_INDEX_2 +#define index_arch_AVX_Fast_Unaligned_Load FEATURE_INDEX_2 +#define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_2 +#define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_2 +#define index_arch_Prefer_ERMS FEATURE_INDEX_2 +#define index_arch_Prefer_No_AVX512 FEATURE_INDEX_2 +#define index_arch_MathVec_Prefer_No_AVX512 FEATURE_INDEX_2 +#define index_arch_Prefer_FSRM FEATURE_INDEX_2 + +/* XCR0 Feature flags. */ +#define bit_XMM_state (1u << 1) +#define bit_YMM_state (1u << 2) +#define bit_Opmask_state (1u << 5) +#define bit_ZMM0_15_state (1u << 6) +#define bit_ZMM16_31_state (1u << 7) + +# if defined (_LIBC) && !IS_IN (nonlib) +/* Unused for x86. */ +# define INIT_ARCH() +# define __get_cpu_features() (&GLRO(dl_x86_cpu_features)) +# define x86_get_cpuid_registers(i) \ + (&(GLRO(dl_x86_cpu_features).cpuid[i])) +# endif #ifdef __x86_64__ # define HAS_CPUID 1 -#elif defined __i586__ || defined __pentium__ +#elif (defined __i586__ || defined __pentium__ \ + || defined __geode__ || defined __k6__) # define HAS_CPUID 1 # define HAS_I586 1 # define HAS_I686 HAS_ARCH_FEATURE (I686) -#elif (defined __i686__ || defined __pentiumpro__ \ - || defined __pentium4__ || defined __nocona__ \ - || defined __atom__ || defined __core2__ \ - || defined __corei7__ || defined __corei7_avx__ \ - || defined __core_avx2__ || defined __nehalem__ \ - || defined __sandybridge__ || defined __haswell__ \ - || defined __knl__ || defined __bonnell__ \ - || defined __silvermont__ \ - || defined __k6__ || defined __k8__ \ - || defined __athlon__ || defined __amdfam10__ \ - || defined __bdver1__ || defined __bdver2__ \ - || defined __bdver3__ || defined __bdver4__ \ - || defined __btver1__ || defined __btver2__) -# define HAS_CPUID 1 -# define HAS_I586 1 -# define HAS_I686 1 -#else +#elif defined __i486__ # define HAS_CPUID 0 # define HAS_I586 HAS_ARCH_FEATURE (I586) # define HAS_I686 HAS_ARCH_FEATURE (I686) +#else +# define HAS_CPUID 1 +# define HAS_I586 1 +# define HAS_I686 1 #endif #endif /* cpu_features_h */ diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c index 69155a8f44..7445ec6161 100644 --- a/sysdeps/x86/cpu-tunables.c +++ b/sysdeps/x86/cpu-tunables.c @@ -1,6 +1,6 @@ /* x86 CPU feature tuning. This file is part of the GNU C Library. - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -14,10 +14,10 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #if HAVE_TUNABLES -# define TUNABLE_NAMESPACE tune +# define TUNABLE_NAMESPACE cpu # include <stdbool.h> # include <stdint.h> # include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */ @@ -116,7 +116,7 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) the hardware which wasn't available when the selection was made. The environment variable: - GLIBC_TUNABLES=glibc.tune.hwcaps=-xxx,yyy,-zzz,.... + GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,.... can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature yyy and zzz, where the feature name is case-sensitive and has to diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c index b82ba14e75..fa9aa63982 100644 --- a/sysdeps/x86/dl-cet.c +++ b/sysdeps/x86/dl-cet.c @@ -1,5 +1,5 @@ /* x86 CET initializers function. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <unistd.h> #include <errno.h> @@ -128,7 +128,7 @@ dl_cet_check (struct link_map *m, const char *program) /* Enable IBT and SHSTK only if they are enabled in executable. NB: IBT and SHSTK may be disabled by environment variable: - GLIBC_TUNABLES=glibc.tune.hwcaps=-IBT,-SHSTK + GLIBC_TUNABLES=glibc.cpu.hwcaps=-IBT,-SHSTK */ enable_ibt &= (HAS_CPU_FEATURE (IBT) && (enable_ibt_type == CET_ALWAYS_ON diff --git a/sysdeps/x86/dl-get-cpu-features.c b/sysdeps/x86/dl-get-cpu-features.c index 49593f19c6..559e97868b 100644 --- a/sysdeps/x86/dl-get-cpu-features.c +++ b/sysdeps/x86/dl-get-cpu-features.c @@ -1,5 +1,5 @@ /* This file is part of the GNU C Library. - Copyright (C) 2015-2018 Free Software Foundation, Inc. + Copyright (C) 2015-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <ldsodefs.h> diff --git a/sysdeps/x86/dl-hwcap.h b/sysdeps/x86/dl-hwcap.h index f5e9d542ca..0a83d93fd0 100644 --- a/sysdeps/x86/dl-hwcap.h +++ b/sysdeps/x86/dl-hwcap.h @@ -1,5 +1,5 @@ /* x86 version of hardware capability information handling macros. - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _DL_HWCAP_H #define _DL_HWCAP_H diff --git a/sysdeps/x86/dl-lookupcfg.h b/sysdeps/x86/dl-lookupcfg.h new file mode 100644 index 0000000000..682f0c547e --- /dev/null +++ b/sysdeps/x86/dl-lookupcfg.h @@ -0,0 +1,31 @@ +/* Configuration of lookup functions. + Copyright (C) 2005-2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#define DL_UNMAP_IS_SPECIAL + +#include_next <dl-lookupcfg.h> + +/* Address of protected data defined in the shared library may be + external due to copy relocation. */ +#define DL_EXTERN_PROTECTED_DATA + +struct link_map; + +extern void _dl_unmap (struct link_map *map) attribute_hidden; + +#define DL_UNMAP(map) _dl_unmap (map) diff --git a/sysdeps/x86/dl-procinfo.c b/sysdeps/x86/dl-procinfo.c index 4b0538ede8..0d834821f4 100644 --- a/sysdeps/x86/dl-procinfo.c +++ b/sysdeps/x86/dl-procinfo.c @@ -1,5 +1,5 @@ /* Data for x86 version of processor capability information. - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ /* This information must be kept in sync with the _DL_HWCAP_COUNT, HWCAP_PLATFORMS_START and HWCAP_PLATFORMS_COUNT definitions in diff --git a/sysdeps/x86/dl-procinfo.h b/sysdeps/x86/dl-procinfo.h index 55cafc26e2..5a39a9d0b0 100644 --- a/sysdeps/x86/dl-procinfo.h +++ b/sysdeps/x86/dl-procinfo.h @@ -1,5 +1,5 @@ /* x86 version of processor capability information handling macros. - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _DL_PROCINFO_H #define _DL_PROCINFO_H 1 diff --git a/sysdeps/x86/dl-procruntime.c b/sysdeps/x86/dl-procruntime.c index eddbde6a31..4787ffa62f 100644 --- a/sysdeps/x86/dl-procruntime.c +++ b/sysdeps/x86/dl-procruntime.c @@ -1,5 +1,5 @@ /* Data for processor runtime information. x86 version. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ /* This information must be kept in sync with the _DL_HWCAP_COUNT, HWCAP_PLATFORMS_START and HWCAP_PLATFORMS_COUNT definitions in diff --git a/sysdeps/x86/dl-prop.h b/sysdeps/x86/dl-prop.h index 26c3131ac5..fa5bd988ca 100644 --- a/sysdeps/x86/dl-prop.h +++ b/sysdeps/x86/dl-prop.h @@ -1,5 +1,5 @@ /* Support for GNU properties. x86 version. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _DL_PROP_H #define _DL_PROP_H @@ -49,6 +49,10 @@ _dl_process_cet_property_note (struct link_map *l, const ElfW(Addr) align) { #if CET_ENABLED + /* Skip if we have seen a NT_GNU_PROPERTY_TYPE_0 note before. */ + if (l->l_cet != lc_unknown) + return; + /* The NT_GNU_PROPERTY_TYPE_0 note must be aliged to 4 bytes in 32-bit objects and to 8 bytes in 64-bit objects. Skip notes with incorrect alignment. */ @@ -57,6 +61,9 @@ _dl_process_cet_property_note (struct link_map *l, const ElfW(Addr) start = (ElfW(Addr)) note; + unsigned int feature_1 = 0; + unsigned int last_type = 0; + while ((ElfW(Addr)) (note + 1) - start < size) { /* Find the NT_GNU_PROPERTY_TYPE_0 note. */ @@ -64,10 +71,18 @@ _dl_process_cet_property_note (struct link_map *l, && note->n_type == NT_GNU_PROPERTY_TYPE_0 && memcmp (note + 1, "GNU", 4) == 0) { + /* Stop if we see more than one GNU property note which may + be generated by the older linker. */ + if (l->l_cet != lc_unknown) + return; + + /* Check CET status now. */ + l->l_cet = lc_none; + /* Check for invalid property. */ if (note->n_descsz < 8 || (note->n_descsz % sizeof (ElfW(Addr))) != 0) - break; + return; /* Start and end of property array. */ unsigned char *ptr = (unsigned char *) (note + 1) + 4; @@ -78,9 +93,15 @@ _dl_process_cet_property_note (struct link_map *l, unsigned int type = *(unsigned int *) ptr; unsigned int datasz = *(unsigned int *) (ptr + 4); + /* Property type must be in ascending order. */ + if (type < last_type) + return; + ptr += 8; if ((ptr + datasz) > ptr_end) - break; + return; + + last_type = type; if (type == GNU_PROPERTY_X86_FEATURE_1_AND) { @@ -89,14 +110,18 @@ _dl_process_cet_property_note (struct link_map *l, we stop the search regardless if its size is correct or not. There is no point to continue if this note is ill-formed. */ - if (datasz == 4) - { - unsigned int feature_1 = *(unsigned int *) ptr; - if ((feature_1 & GNU_PROPERTY_X86_FEATURE_1_IBT)) - l->l_cet |= lc_ibt; - if ((feature_1 & GNU_PROPERTY_X86_FEATURE_1_SHSTK)) - l->l_cet |= lc_shstk; - } + if (datasz != 4) + return; + + feature_1 = *(unsigned int *) ptr; + + /* Keep searching for the next GNU property note + generated by the older linker. */ + break; + } + else if (type > GNU_PROPERTY_X86_FEATURE_1_AND) + { + /* Stop since property type is in ascending order. */ return; } @@ -112,6 +137,12 @@ _dl_process_cet_property_note (struct link_map *l, + ELF_NOTE_NEXT_OFFSET (note->n_namesz, note->n_descsz, align)); } + + /* We get here only if there is one or no GNU property note. */ + if ((feature_1 & GNU_PROPERTY_X86_FEATURE_1_IBT)) + l->l_cet |= lc_ibt; + if ((feature_1 & GNU_PROPERTY_X86_FEATURE_1_SHSTK)) + l->l_cet |= lc_shstk; #endif } @@ -136,8 +167,7 @@ _dl_process_pt_note (struct link_map *l, const ElfW(Phdr) *ph, note_malloced = malloc (size); note = note_malloced; } - __lseek (fd, ph->p_offset, SEEK_SET); - if (__read_nocancel (fd, (void *) note, size) != size) + if (__pread64_nocancel (fd, (void *) note, size, ph->p_offset) != size) { if (note_malloced) free (note_malloced); diff --git a/sysdeps/x86/dl-tunables.list b/sysdeps/x86/dl-tunables.list index 73886b1352..34b7bde8bb 100644 --- a/sysdeps/x86/dl-tunables.list +++ b/sysdeps/x86/dl-tunables.list @@ -1,5 +1,5 @@ # x86 specific tunables. -# Copyright (C) 2017-2018 Free Software Foundation, Inc. +# Copyright (C) 2017-2019 Free Software Foundation, Inc. # This file is part of the GNU C Library. # The GNU C Library is free software; you can redistribute it and/or @@ -14,10 +14,10 @@ # You should have received a copy of the GNU Lesser General Public # License along with the GNU C Library; if not, see -# <http://www.gnu.org/licenses/>. +# <https://www.gnu.org/licenses/>. glibc { - tune { + cpu { hwcaps { type: STRING } diff --git a/sysdeps/x86/elide.h b/sysdeps/x86/elide.h index 8d5589902f..5ad29fec9c 100644 --- a/sysdeps/x86/elide.h +++ b/sysdeps/x86/elide.h @@ -1,5 +1,5 @@ /* elide.h: Generic lock elision support. - Copyright (C) 2014-2018 Free Software Foundation, Inc. + Copyright (C) 2014-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef ELIDE_H #define ELIDE_H 1 diff --git a/sysdeps/x86/fpu/bits/fenv.h b/sysdeps/x86/fpu/bits/fenv.h index 4103982d8c..8953ea9937 100644 --- a/sysdeps/x86/fpu/bits/fenv.h +++ b/sysdeps/x86/fpu/bits/fenv.h @@ -1,4 +1,4 @@ -/* Copyright (C) 1997-2018 Free Software Foundation, Inc. +/* Copyright (C) 1997-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _FENV_H # error "Never use <bits/fenv.h> directly; include <fenv.h> instead." @@ -101,7 +101,7 @@ fenv_t; # define FE_NOMASK_ENV ((const fenv_t *) -2) #endif -#if __GLIBC_USE (IEC_60559_BFP_EXT) +#if __GLIBC_USE (IEC_60559_BFP_EXT_C2X) /* Type representing floating-point control modes. */ typedef struct { diff --git a/sysdeps/x86/fpu/bits/math-vector.h b/sysdeps/x86/fpu/bits/math-vector.h index 3d229d8705..90f00f396b 100644 --- a/sysdeps/x86/fpu/bits/math-vector.h +++ b/sysdeps/x86/fpu/bits/math-vector.h @@ -1,5 +1,5 @@ /* Platform-specific SIMD declarations of math functions. - Copyright (C) 2014-2018 Free Software Foundation, Inc. + Copyright (C) 2014-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _MATH_H # error "Never include <bits/math-vector.h> directly;\ diff --git a/sysdeps/x86/fpu/bits/mathinline.h b/sysdeps/x86/fpu/bits/mathinline.h deleted file mode 100644 index 91ece8dfb8..0000000000 --- a/sysdeps/x86/fpu/bits/mathinline.h +++ /dev/null @@ -1,328 +0,0 @@ -/* Inline math functions for i387 and SSE. - Copyright (C) 1995-2018 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#ifndef _MATH_H -# error "Never use <bits/mathinline.h> directly; include <math.h> instead." -#endif - -#ifndef __extern_always_inline -# define __MATH_INLINE __inline -#else -# define __MATH_INLINE __extern_always_inline -#endif - -/* Disable x87 inlines when -fpmath=sse is passed and also when we're building - on x86_64. Older gcc (gcc-3.2 for example) does not define __SSE2_MATH__ - for x86_64. */ -#if !defined __SSE2_MATH__ && !defined __x86_64__ -# if ((!defined __NO_MATH_INLINES || defined __LIBC_INTERNAL_MATH_INLINES) \ - && defined __OPTIMIZE__) - -/* The inline functions do not set errno or raise necessarily the - correct exceptions. */ -# undef math_errhandling - -/* A macro to define float, double, and long double versions of various - math functions for the ix87 FPU. FUNC is the function name (which will - be suffixed with f and l for the float and long double version, - respectively). OP is the name of the FPU operation. - We define two sets of macros. The set with the additional NP - doesn't add a prototype declaration. */ - -# ifdef __USE_ISOC99 -# define __inline_mathop(func, op) \ - __inline_mathop_ (double, func, op) \ - __inline_mathop_ (float, __CONCAT(func,f), op) \ - __inline_mathop_ (long double, __CONCAT(func,l), op) -# define __inline_mathopNP(func, op) \ - __inline_mathopNP_ (double, func, op) \ - __inline_mathopNP_ (float, __CONCAT(func,f), op) \ - __inline_mathopNP_ (long double, __CONCAT(func,l), op) -# else -# define __inline_mathop(func, op) \ - __inline_mathop_ (double, func, op) -# define __inline_mathopNP(func, op) \ - __inline_mathopNP_ (double, func, op) -# endif - -# define __inline_mathop_(float_type, func, op) \ - __inline_mathop_decl_ (float_type, func, op, "0" (__x)) -# define __inline_mathopNP_(float_type, func, op) \ - __inline_mathop_declNP_ (float_type, func, op, "0" (__x)) - - -# ifdef __USE_ISOC99 -# define __inline_mathop_decl(func, op, params...) \ - __inline_mathop_decl_ (double, func, op, params) \ - __inline_mathop_decl_ (float, __CONCAT(func,f), op, params) \ - __inline_mathop_decl_ (long double, __CONCAT(func,l), op, params) -# define __inline_mathop_declNP(func, op, params...) \ - __inline_mathop_declNP_ (double, func, op, params) \ - __inline_mathop_declNP_ (float, __CONCAT(func,f), op, params) \ - __inline_mathop_declNP_ (long double, __CONCAT(func,l), op, params) -# else -# define __inline_mathop_decl(func, op, params...) \ - __inline_mathop_decl_ (double, func, op, params) -# define __inline_mathop_declNP(func, op, params...) \ - __inline_mathop_declNP_ (double, func, op, params) -# endif - -# define __inline_mathop_decl_(float_type, func, op, params...) \ - __MATH_INLINE float_type func (float_type) __THROW; \ - __inline_mathop_declNP_ (float_type, func, op, params) - -# define __inline_mathop_declNP_(float_type, func, op, params...) \ - __MATH_INLINE float_type __NTH (func (float_type __x)) \ - { \ - register float_type __result; \ - __asm __volatile__ (op : "=t" (__result) : params); \ - return __result; \ - } - - -# ifdef __USE_ISOC99 -# define __inline_mathcode(func, arg, code) \ - __inline_mathcode_ (double, func, arg, code) \ - __inline_mathcode_ (float, __CONCAT(func,f), arg, code) \ - __inline_mathcode_ (long double, __CONCAT(func,l), arg, code) -# define __inline_mathcodeNP(func, arg, code) \ - __inline_mathcodeNP_ (double, func, arg, code) \ - __inline_mathcodeNP_ (float, __CONCAT(func,f), arg, code) \ - __inline_mathcodeNP_ (long double, __CONCAT(func,l), arg, code) -# define __inline_mathcode2(func, arg1, arg2, code) \ - __inline_mathcode2_ (double, func, arg1, arg2, code) \ - __inline_mathcode2_ (float, __CONCAT(func,f), arg1, arg2, code) \ - __inline_mathcode2_ (long double, __CONCAT(func,l), arg1, arg2, code) -# define __inline_mathcodeNP2(func, arg1, arg2, code) \ - __inline_mathcodeNP2_ (double, func, arg1, arg2, code) \ - __inline_mathcodeNP2_ (float, __CONCAT(func,f), arg1, arg2, code) \ - __inline_mathcodeNP2_ (long double, __CONCAT(func,l), arg1, arg2, code) -# define __inline_mathcode3(func, arg1, arg2, arg3, code) \ - __inline_mathcode3_ (double, func, arg1, arg2, arg3, code) \ - __inline_mathcode3_ (float, __CONCAT(func,f), arg1, arg2, arg3, code) \ - __inline_mathcode3_ (long double, __CONCAT(func,l), arg1, arg2, arg3, code) -# define __inline_mathcodeNP3(func, arg1, arg2, arg3, code) \ - __inline_mathcodeNP3_ (double, func, arg1, arg2, arg3, code) \ - __inline_mathcodeNP3_ (float, __CONCAT(func,f), arg1, arg2, arg3, code) \ - __inline_mathcodeNP3_ (long double, __CONCAT(func,l), arg1, arg2, arg3, code) -# else -# define __inline_mathcode(func, arg, code) \ - __inline_mathcode_ (double, func, (arg), code) -# define __inline_mathcodeNP(func, arg, code) \ - __inline_mathcodeNP_ (double, func, (arg), code) -# define __inline_mathcode2(func, arg1, arg2, code) \ - __inline_mathcode2_ (double, func, arg1, arg2, code) -# define __inline_mathcodeNP2(func, arg1, arg2, code) \ - __inline_mathcodeNP2_ (double, func, arg1, arg2, code) -# define __inline_mathcode3(func, arg1, arg2, arg3, code) \ - __inline_mathcode3_ (double, func, arg1, arg2, arg3, code) -# define __inline_mathcodeNP3(func, arg1, arg2, arg3, code) \ - __inline_mathcodeNP3_ (double, func, arg1, arg2, arg3, code) -# endif - -# define __inline_mathcode_(float_type, func, arg, code) \ - __MATH_INLINE float_type func (float_type) __THROW; \ - __inline_mathcodeNP_(float_type, func, arg, code) - -# define __inline_mathcodeNP_(float_type, func, arg, code) \ - __MATH_INLINE float_type __NTH (func (float_type arg)) \ - { \ - code; \ - } - - -# define __inline_mathcode2_(float_type, func, arg1, arg2, code) \ - __MATH_INLINE float_type func (float_type, float_type) __THROW; \ - __inline_mathcodeNP2_ (float_type, func, arg1, arg2, code) - -# define __inline_mathcodeNP2_(float_type, func, arg1, arg2, code) \ - __MATH_INLINE float_type __NTH (func (float_type arg1, float_type arg2)) \ - { \ - code; \ - } - -# define __inline_mathcode3_(float_type, func, arg1, arg2, arg3, code) \ - __MATH_INLINE float_type func (float_type, float_type, float_type) __THROW; \ - __inline_mathcodeNP3_(float_type, func, arg1, arg2, arg3, code) - -# define __inline_mathcodeNP3_(float_type, func, arg1, arg2, arg3, code) \ - __MATH_INLINE float_type __NTH (func (float_type arg1, float_type arg2, \ - float_type arg3)) \ - { \ - code; \ - } -# endif - - -# if !defined __NO_MATH_INLINES && defined __OPTIMIZE__ -/* Miscellaneous functions */ - -/* __FAST_MATH__ is defined by gcc -ffast-math. */ -# ifdef __FAST_MATH__ -/* Optimized inline implementation, sometimes with reduced precision - and/or argument range. */ - -# if __GNUC_PREREQ (3, 5) -# define __expm1_code \ - register long double __temp; \ - __temp = __builtin_expm1l (__x); \ - return __temp ? __temp : __x -# else -# define __expm1_code \ - register long double __value; \ - register long double __exponent; \ - register long double __temp; \ - __asm __volatile__ \ - ("fldl2e # e^x - 1 = 2^(x * log2(e)) - 1\n\t" \ - "fmul %%st(1) # x * log2(e)\n\t" \ - "fst %%st(1)\n\t" \ - "frndint # int(x * log2(e))\n\t" \ - "fxch\n\t" \ - "fsub %%st(1) # fract(x * log2(e))\n\t" \ - "f2xm1 # 2^(fract(x * log2(e))) - 1\n\t" \ - "fscale # 2^(x * log2(e)) - 2^(int(x * log2(e)))\n\t" \ - : "=t" (__value), "=u" (__exponent) : "0" (__x)); \ - __asm __volatile__ \ - ("fscale # 2^int(x * log2(e))\n\t" \ - : "=t" (__temp) : "0" (1.0), "u" (__exponent)); \ - __temp -= 1.0; \ - __temp += __value; \ - return __temp ? __temp : __x -# endif -__inline_mathcodeNP_ (long double, __expm1l, __x, __expm1_code) - -# if __GNUC_PREREQ (3, 4) -__inline_mathcodeNP_ (long double, __expl, __x, return __builtin_expl (__x)) -# else -# define __exp_code \ - register long double __value; \ - register long double __exponent; \ - __asm __volatile__ \ - ("fldl2e # e^x = 2^(x * log2(e))\n\t" \ - "fmul %%st(1) # x * log2(e)\n\t" \ - "fst %%st(1)\n\t" \ - "frndint # int(x * log2(e))\n\t" \ - "fxch\n\t" \ - "fsub %%st(1) # fract(x * log2(e))\n\t" \ - "f2xm1 # 2^(fract(x * log2(e))) - 1\n\t" \ - : "=t" (__value), "=u" (__exponent) : "0" (__x)); \ - __value += 1.0; \ - __asm __volatile__ \ - ("fscale" \ - : "=t" (__value) : "0" (__value), "u" (__exponent)); \ - return __value -__inline_mathcodeNP (exp, __x, __exp_code) -__inline_mathcodeNP_ (long double, __expl, __x, __exp_code) -# endif -# endif /* __FAST_MATH__ */ - - -# ifdef __FAST_MATH__ -# if !__GNUC_PREREQ (3,3) -__inline_mathopNP (sqrt, "fsqrt") -__inline_mathopNP_ (long double, __sqrtl, "fsqrt") -# define __libc_sqrtl(n) __sqrtl (n) -# else -# define __libc_sqrtl(n) __builtin_sqrtl (n) -# endif -# endif - -# if __GNUC_PREREQ (2, 8) -__inline_mathcodeNP_ (double, fabs, __x, return __builtin_fabs (__x)) -# ifdef __USE_ISOC99 -__inline_mathcodeNP_ (float, fabsf, __x, return __builtin_fabsf (__x)) -__inline_mathcodeNP_ (long double, fabsl, __x, return __builtin_fabsl (__x)) -# endif -__inline_mathcodeNP_ (long double, __fabsl, __x, return __builtin_fabsl (__x)) -# else -__inline_mathop (fabs, "fabs") -__inline_mathop_ (long double, __fabsl, "fabs") -# endif - -__inline_mathcode_ (long double, __sgn1l, __x, \ - __extension__ union { long double __xld; unsigned int __xi[3]; } __n = \ - { __xld: __x }; \ - __n.__xi[2] = (__n.__xi[2] & 0x8000) | 0x3fff; \ - __n.__xi[1] = 0x80000000; \ - __n.__xi[0] = 0; \ - return __n.__xld) - - -# ifdef __FAST_MATH__ -/* The argument range of the inline version of sinhl is slightly reduced. */ -__inline_mathcodeNP (sinh, __x, \ - register long double __exm1 = __expm1l (__fabsl (__x)); \ - return 0.5 * (__exm1 / (__exm1 + 1.0) + __exm1) * __sgn1l (__x)) - -__inline_mathcodeNP (cosh, __x, \ - register long double __ex = __expl (__x); \ - return 0.5 * (__ex + 1.0 / __ex)) - -__inline_mathcodeNP (tanh, __x, \ - register long double __exm1 = __expm1l (-__fabsl (__x + __x)); \ - return __exm1 / (__exm1 + 2.0) * __sgn1l (-__x)) -# endif - - -/* Optimized versions for some non-standardized functions. */ -# ifdef __USE_ISOC99 - -# ifdef __FAST_MATH__ -__inline_mathcodeNP (expm1, __x, __expm1_code) - -/* The argument range of the inline version of asinhl is slightly reduced. */ -__inline_mathcodeNP (asinh, __x, \ - register long double __y = __fabsl (__x); \ - return (log1pl (__y * __y / (__libc_sqrtl (__y * __y + 1.0) + 1.0) + __y) \ - * __sgn1l (__x))) - -__inline_mathcodeNP (acosh, __x, \ - return logl (__x + __libc_sqrtl (__x - 1.0) * __libc_sqrtl (__x + 1.0))) - -__inline_mathcodeNP (atanh, __x, \ - register long double __y = __fabsl (__x); \ - return -0.5 * log1pl (-(__y + __y) / (1.0 + __y)) * __sgn1l (__x)) - -/* The argument range of the inline version of hypotl is slightly reduced. */ -__inline_mathcodeNP2 (hypot, __x, __y, - return __libc_sqrtl (__x * __x + __y * __y)) - -# endif -# endif - - -/* Undefine some of the large macros which are not used anymore. */ -# ifdef __FAST_MATH__ -# undef __expm1_code -# undef __exp_code -# endif /* __FAST_MATH__ */ - -# endif /* __NO_MATH_INLINES */ - - -/* This code is used internally in the GNU libc. */ -# ifdef __LIBC_INTERNAL_MATH_INLINES -__inline_mathcode2_ (long double, __ieee754_atan2l, __y, __x, - register long double __value; - __asm __volatile__ ("fpatan\n\t" - : "=t" (__value) - : "0" (__x), "u" (__y) : "st(1)"); - return __value;) -# endif - -#endif /* !__SSE2_MATH__ && !__x86_64__ */ diff --git a/sysdeps/x86/fpu/e_sqrtf128.c b/sysdeps/x86/fpu/e_sqrtf128.c index cac5f63527..5c641a069c 100644 --- a/sysdeps/x86/fpu/e_sqrtf128.c +++ b/sysdeps/x86/fpu/e_sqrtf128.c @@ -1,6 +1,6 @@ /* soft-fp sqrt for _Float128 Return sqrt(a) - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -24,7 +24,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <soft-fp.h> #include <quad.h> diff --git a/sysdeps/x86/fpu/fenv_private.h b/sysdeps/x86/fpu/fenv_private.h new file mode 100644 index 0000000000..4b081e015b --- /dev/null +++ b/sysdeps/x86/fpu/fenv_private.h @@ -0,0 +1,497 @@ +#ifndef X86_FENV_PRIVATE_H +#define X86_FENV_PRIVATE_H 1 + +#include <bits/floatn.h> +#include <fenv.h> +#include <fpu_control.h> + +/* This file is used by both the 32- and 64-bit ports. The 64-bit port + has a field in the fenv_t for the mxcsr; the 32-bit port does not. + Instead, we (ab)use the only 32-bit field extant in the struct. */ +#ifndef __x86_64__ +# define __mxcsr __eip +#endif + + +/* All of these functions are private to libm, and are all used in pairs + to save+change the fp state and restore the original state. Thus we + need not care for both the 387 and the sse unit, only the one we're + actually using. */ + +#if defined __AVX__ || defined SSE2AVX +# define STMXCSR "vstmxcsr" +# define LDMXCSR "vldmxcsr" +#else +# define STMXCSR "stmxcsr" +# define LDMXCSR "ldmxcsr" +#endif + +static __always_inline void +libc_feholdexcept_sse (fenv_t *e) +{ + unsigned int mxcsr; + asm (STMXCSR " %0" : "=m" (*&mxcsr)); + e->__mxcsr = mxcsr; + mxcsr = (mxcsr | 0x1f80) & ~0x3f; + asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); +} + +static __always_inline void +libc_feholdexcept_387 (fenv_t *e) +{ + /* Recall that fnstenv has a side-effect of masking exceptions. + Clobber all of the fp registers so that the TOS field is 0. */ + asm volatile ("fnstenv %0; fnclex" + : "=m"(*e) + : : "st", "st(1)", "st(2)", "st(3)", + "st(4)", "st(5)", "st(6)", "st(7)"); +} + +static __always_inline void +libc_fesetround_sse (int r) +{ + unsigned int mxcsr; + asm (STMXCSR " %0" : "=m" (*&mxcsr)); + mxcsr = (mxcsr & ~0x6000) | (r << 3); + asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); +} + +static __always_inline void +libc_fesetround_387 (int r) +{ + fpu_control_t cw; + _FPU_GETCW (cw); + cw = (cw & ~0xc00) | r; + _FPU_SETCW (cw); +} + +static __always_inline void +libc_feholdexcept_setround_sse (fenv_t *e, int r) +{ + unsigned int mxcsr; + asm (STMXCSR " %0" : "=m" (*&mxcsr)); + e->__mxcsr = mxcsr; + mxcsr = ((mxcsr | 0x1f80) & ~0x603f) | (r << 3); + asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); +} + +/* Set both rounding mode and precision. A convenience function for use + by libc_feholdexcept_setround and libc_feholdexcept_setround_53bit. */ +static __always_inline void +libc_feholdexcept_setround_387_prec (fenv_t *e, int r) +{ + libc_feholdexcept_387 (e); + + fpu_control_t cw = e->__control_word; + cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); + cw |= r | 0x3f; + _FPU_SETCW (cw); +} + +static __always_inline void +libc_feholdexcept_setround_387 (fenv_t *e, int r) +{ + libc_feholdexcept_setround_387_prec (e, r | _FPU_EXTENDED); +} + +static __always_inline void +libc_feholdexcept_setround_387_53bit (fenv_t *e, int r) +{ + libc_feholdexcept_setround_387_prec (e, r | _FPU_DOUBLE); +} + +static __always_inline int +libc_fetestexcept_sse (int e) +{ + unsigned int mxcsr; + asm volatile (STMXCSR " %0" : "=m" (*&mxcsr)); + return mxcsr & e & FE_ALL_EXCEPT; +} + +static __always_inline int +libc_fetestexcept_387 (int ex) +{ + fexcept_t temp; + asm volatile ("fnstsw %0" : "=a" (temp)); + return temp & ex & FE_ALL_EXCEPT; +} + +static __always_inline void +libc_fesetenv_sse (fenv_t *e) +{ + asm volatile (LDMXCSR " %0" : : "m" (e->__mxcsr)); +} + +static __always_inline void +libc_fesetenv_387 (fenv_t *e) +{ + /* Clobber all fp registers so that the TOS value we saved earlier is + compatible with the current state of the compiler. */ + asm volatile ("fldenv %0" + : : "m" (*e) + : "st", "st(1)", "st(2)", "st(3)", + "st(4)", "st(5)", "st(6)", "st(7)"); +} + +static __always_inline int +libc_feupdateenv_test_sse (fenv_t *e, int ex) +{ + unsigned int mxcsr, old_mxcsr, cur_ex; + asm volatile (STMXCSR " %0" : "=m" (*&mxcsr)); + cur_ex = mxcsr & FE_ALL_EXCEPT; + + /* Merge current exceptions with the old environment. */ + old_mxcsr = e->__mxcsr; + mxcsr = old_mxcsr | cur_ex; + asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); + + /* Raise SIGFPE for any new exceptions since the hold. Expect that + the normal environment has all exceptions masked. */ + if (__glibc_unlikely (~(old_mxcsr >> 7) & cur_ex)) + __feraiseexcept (cur_ex); + + /* Test for exceptions raised since the hold. */ + return cur_ex & ex; +} + +static __always_inline int +libc_feupdateenv_test_387 (fenv_t *e, int ex) +{ + fexcept_t cur_ex; + + /* Save current exceptions. */ + asm volatile ("fnstsw %0" : "=a" (cur_ex)); + cur_ex &= FE_ALL_EXCEPT; + + /* Reload original environment. */ + libc_fesetenv_387 (e); + + /* Merge current exceptions. */ + __feraiseexcept (cur_ex); + + /* Test for exceptions raised since the hold. */ + return cur_ex & ex; +} + +static __always_inline void +libc_feupdateenv_sse (fenv_t *e) +{ + libc_feupdateenv_test_sse (e, 0); +} + +static __always_inline void +libc_feupdateenv_387 (fenv_t *e) +{ + libc_feupdateenv_test_387 (e, 0); +} + +static __always_inline void +libc_feholdsetround_sse (fenv_t *e, int r) +{ + unsigned int mxcsr; + asm (STMXCSR " %0" : "=m" (*&mxcsr)); + e->__mxcsr = mxcsr; + mxcsr = (mxcsr & ~0x6000) | (r << 3); + asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); +} + +static __always_inline void +libc_feholdsetround_387_prec (fenv_t *e, int r) +{ + fpu_control_t cw; + + _FPU_GETCW (cw); + e->__control_word = cw; + cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); + cw |= r; + _FPU_SETCW (cw); +} + +static __always_inline void +libc_feholdsetround_387 (fenv_t *e, int r) +{ + libc_feholdsetround_387_prec (e, r | _FPU_EXTENDED); +} + +static __always_inline void +libc_feholdsetround_387_53bit (fenv_t *e, int r) +{ + libc_feholdsetround_387_prec (e, r | _FPU_DOUBLE); +} + +static __always_inline void +libc_feresetround_sse (fenv_t *e) +{ + unsigned int mxcsr; + asm (STMXCSR " %0" : "=m" (*&mxcsr)); + mxcsr = (mxcsr & ~0x6000) | (e->__mxcsr & 0x6000); + asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); +} + +static __always_inline void +libc_feresetround_387 (fenv_t *e) +{ + _FPU_SETCW (e->__control_word); +} + +#ifdef __SSE_MATH__ +# define libc_feholdexceptf libc_feholdexcept_sse +# define libc_fesetroundf libc_fesetround_sse +# define libc_feholdexcept_setroundf libc_feholdexcept_setround_sse +# define libc_fetestexceptf libc_fetestexcept_sse +# define libc_fesetenvf libc_fesetenv_sse +# define libc_feupdateenv_testf libc_feupdateenv_test_sse +# define libc_feupdateenvf libc_feupdateenv_sse +# define libc_feholdsetroundf libc_feholdsetround_sse +# define libc_feresetroundf libc_feresetround_sse +#else +# define libc_feholdexceptf libc_feholdexcept_387 +# define libc_fesetroundf libc_fesetround_387 +# define libc_feholdexcept_setroundf libc_feholdexcept_setround_387 +# define libc_fetestexceptf libc_fetestexcept_387 +# define libc_fesetenvf libc_fesetenv_387 +# define libc_feupdateenv_testf libc_feupdateenv_test_387 +# define libc_feupdateenvf libc_feupdateenv_387 +# define libc_feholdsetroundf libc_feholdsetround_387 +# define libc_feresetroundf libc_feresetround_387 +#endif /* __SSE_MATH__ */ + +#ifdef __SSE2_MATH__ +# define libc_feholdexcept libc_feholdexcept_sse +# define libc_fesetround libc_fesetround_sse +# define libc_feholdexcept_setround libc_feholdexcept_setround_sse +# define libc_fetestexcept libc_fetestexcept_sse +# define libc_fesetenv libc_fesetenv_sse +# define libc_feupdateenv_test libc_feupdateenv_test_sse +# define libc_feupdateenv libc_feupdateenv_sse +# define libc_feholdsetround libc_feholdsetround_sse +# define libc_feresetround libc_feresetround_sse +#else +# define libc_feholdexcept libc_feholdexcept_387 +# define libc_fesetround libc_fesetround_387 +# define libc_feholdexcept_setround libc_feholdexcept_setround_387 +# define libc_fetestexcept libc_fetestexcept_387 +# define libc_fesetenv libc_fesetenv_387 +# define libc_feupdateenv_test libc_feupdateenv_test_387 +# define libc_feupdateenv libc_feupdateenv_387 +# define libc_feholdsetround libc_feholdsetround_387 +# define libc_feresetround libc_feresetround_387 +#endif /* __SSE2_MATH__ */ + +#define libc_feholdexceptl libc_feholdexcept_387 +#define libc_fesetroundl libc_fesetround_387 +#define libc_feholdexcept_setroundl libc_feholdexcept_setround_387 +#define libc_fetestexceptl libc_fetestexcept_387 +#define libc_fesetenvl libc_fesetenv_387 +#define libc_feupdateenv_testl libc_feupdateenv_test_387 +#define libc_feupdateenvl libc_feupdateenv_387 +#define libc_feholdsetroundl libc_feholdsetround_387 +#define libc_feresetroundl libc_feresetround_387 + +#ifndef __SSE2_MATH__ +# define libc_feholdexcept_setround_53bit libc_feholdexcept_setround_387_53bit +# define libc_feholdsetround_53bit libc_feholdsetround_387_53bit +#endif + +#ifdef __x86_64__ +/* The SSE rounding mode is used by soft-fp (libgcc and glibc) on + x86_64, so that must be set for float128 computations. */ +# define SET_RESTORE_ROUNDF128(RM) \ + SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_sse, libc_feresetround_sse) +# define libc_feholdexcept_setroundf128 libc_feholdexcept_setround_sse +# define libc_feupdateenv_testf128 libc_feupdateenv_test_sse +#else +/* The 387 rounding mode is used by soft-fp for 32-bit, but whether + 387 or SSE exceptions are used depends on whether libgcc was built + for SSE math, which is not known when glibc is being built. */ +# define libc_feholdexcept_setroundf128 default_libc_feholdexcept_setround +# define libc_feupdateenv_testf128 default_libc_feupdateenv_test +#endif + +/* We have support for rounding mode context. */ +#define HAVE_RM_CTX 1 + +static __always_inline void +libc_feholdexcept_setround_sse_ctx (struct rm_ctx *ctx, int r) +{ + unsigned int mxcsr, new_mxcsr; + asm (STMXCSR " %0" : "=m" (*&mxcsr)); + new_mxcsr = ((mxcsr | 0x1f80) & ~0x603f) | (r << 3); + + ctx->env.__mxcsr = mxcsr; + if (__glibc_unlikely (mxcsr != new_mxcsr)) + { + asm volatile (LDMXCSR " %0" : : "m" (*&new_mxcsr)); + ctx->updated_status = true; + } + else + ctx->updated_status = false; +} + +/* Unconditional since we want to overwrite any exceptions that occurred in the + context. This is also why all fehold* functions unconditionally write into + ctx->env. */ +static __always_inline void +libc_fesetenv_sse_ctx (struct rm_ctx *ctx) +{ + libc_fesetenv_sse (&ctx->env); +} + +static __always_inline void +libc_feupdateenv_sse_ctx (struct rm_ctx *ctx) +{ + if (__glibc_unlikely (ctx->updated_status)) + libc_feupdateenv_test_sse (&ctx->env, 0); +} + +static __always_inline void +libc_feholdexcept_setround_387_prec_ctx (struct rm_ctx *ctx, int r) +{ + libc_feholdexcept_387 (&ctx->env); + + fpu_control_t cw = ctx->env.__control_word; + fpu_control_t old_cw = cw; + cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); + cw |= r | 0x3f; + + if (__glibc_unlikely (old_cw != cw)) + { + _FPU_SETCW (cw); + ctx->updated_status = true; + } + else + ctx->updated_status = false; +} + +static __always_inline void +libc_feholdexcept_setround_387_ctx (struct rm_ctx *ctx, int r) +{ + libc_feholdexcept_setround_387_prec_ctx (ctx, r | _FPU_EXTENDED); +} + +static __always_inline void +libc_feholdexcept_setround_387_53bit_ctx (struct rm_ctx *ctx, int r) +{ + libc_feholdexcept_setround_387_prec_ctx (ctx, r | _FPU_DOUBLE); +} + +static __always_inline void +libc_feholdsetround_387_prec_ctx (struct rm_ctx *ctx, int r) +{ + fpu_control_t cw, new_cw; + + _FPU_GETCW (cw); + new_cw = cw; + new_cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); + new_cw |= r; + + ctx->env.__control_word = cw; + if (__glibc_unlikely (new_cw != cw)) + { + _FPU_SETCW (new_cw); + ctx->updated_status = true; + } + else + ctx->updated_status = false; +} + +static __always_inline void +libc_feholdsetround_387_ctx (struct rm_ctx *ctx, int r) +{ + libc_feholdsetround_387_prec_ctx (ctx, r | _FPU_EXTENDED); +} + +static __always_inline void +libc_feholdsetround_387_53bit_ctx (struct rm_ctx *ctx, int r) +{ + libc_feholdsetround_387_prec_ctx (ctx, r | _FPU_DOUBLE); +} + +static __always_inline void +libc_feholdsetround_sse_ctx (struct rm_ctx *ctx, int r) +{ + unsigned int mxcsr, new_mxcsr; + + asm (STMXCSR " %0" : "=m" (*&mxcsr)); + new_mxcsr = (mxcsr & ~0x6000) | (r << 3); + + ctx->env.__mxcsr = mxcsr; + if (__glibc_unlikely (new_mxcsr != mxcsr)) + { + asm volatile (LDMXCSR " %0" : : "m" (*&new_mxcsr)); + ctx->updated_status = true; + } + else + ctx->updated_status = false; +} + +static __always_inline void +libc_feresetround_sse_ctx (struct rm_ctx *ctx) +{ + if (__glibc_unlikely (ctx->updated_status)) + libc_feresetround_sse (&ctx->env); +} + +static __always_inline void +libc_feresetround_387_ctx (struct rm_ctx *ctx) +{ + if (__glibc_unlikely (ctx->updated_status)) + _FPU_SETCW (ctx->env.__control_word); +} + +static __always_inline void +libc_feupdateenv_387_ctx (struct rm_ctx *ctx) +{ + if (__glibc_unlikely (ctx->updated_status)) + libc_feupdateenv_test_387 (&ctx->env, 0); +} + +#ifdef __SSE_MATH__ +# define libc_feholdexcept_setroundf_ctx libc_feholdexcept_setround_sse_ctx +# define libc_fesetenvf_ctx libc_fesetenv_sse_ctx +# define libc_feupdateenvf_ctx libc_feupdateenv_sse_ctx +# define libc_feholdsetroundf_ctx libc_feholdsetround_sse_ctx +# define libc_feresetroundf_ctx libc_feresetround_sse_ctx +#else +# define libc_feholdexcept_setroundf_ctx libc_feholdexcept_setround_387_ctx +# define libc_feupdateenvf_ctx libc_feupdateenv_387_ctx +# define libc_feholdsetroundf_ctx libc_feholdsetround_387_ctx +# define libc_feresetroundf_ctx libc_feresetround_387_ctx +#endif /* __SSE_MATH__ */ + +#ifdef __SSE2_MATH__ +# if defined (__x86_64__) || !defined (MATH_SET_BOTH_ROUNDING_MODES) +# define libc_feholdexcept_setround_ctx libc_feholdexcept_setround_sse_ctx +# define libc_fesetenv_ctx libc_fesetenv_sse_ctx +# define libc_feupdateenv_ctx libc_feupdateenv_sse_ctx +# define libc_feholdsetround_ctx libc_feholdsetround_sse_ctx +# define libc_feresetround_ctx libc_feresetround_sse_ctx +# else +# define libc_feholdexcept_setround_ctx default_libc_feholdexcept_setround_ctx +# define libc_fesetenv_ctx default_libc_fesetenv_ctx +# define libc_feupdateenv_ctx default_libc_feupdateenv_ctx +# define libc_feholdsetround_ctx default_libc_feholdsetround_ctx +# define libc_feresetround_ctx default_libc_feresetround_ctx +# endif +#else +# define libc_feholdexcept_setround_ctx libc_feholdexcept_setround_387_ctx +# define libc_feupdateenv_ctx libc_feupdateenv_387_ctx +# define libc_feholdsetround_ctx libc_feholdsetround_387_ctx +# define libc_feresetround_ctx libc_feresetround_387_ctx +#endif /* __SSE2_MATH__ */ + +#define libc_feholdexcept_setroundl_ctx libc_feholdexcept_setround_387_ctx +#define libc_feupdateenvl_ctx libc_feupdateenv_387_ctx +#define libc_feholdsetroundl_ctx libc_feholdsetround_387_ctx +#define libc_feresetroundl_ctx libc_feresetround_387_ctx + +#ifndef __SSE2_MATH__ +# define libc_feholdsetround_53bit_ctx libc_feholdsetround_387_53bit_ctx +# define libc_feresetround_53bit_ctx libc_feresetround_387_ctx +#endif + +#undef __mxcsr + +#include_next <fenv_private.h> + +#endif /* X86_FENV_PRIVATE_H */ diff --git a/sysdeps/x86/fpu/finclude/math-vector-fortran.h b/sysdeps/x86/fpu/finclude/math-vector-fortran.h new file mode 100644 index 0000000000..8fa868346e --- /dev/null +++ b/sysdeps/x86/fpu/finclude/math-vector-fortran.h @@ -0,0 +1,43 @@ +! Platform-specific declarations of SIMD math functions for Fortran. -*- f90 -*- +! Copyright (C) 2019 Free Software Foundation, Inc. +! This file is part of the GNU C Library. +! +! The GNU C Library is free software; you can redistribute it and/or +! modify it under the terms of the GNU Lesser General Public +! License as published by the Free Software Foundation; either +! version 2.1 of the License, or (at your option) any later version. +! +! The GNU C Library is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +! Lesser General Public License for more details. +! +! You should have received a copy of the GNU Lesser General Public +! License along with the GNU C Library; if not, see +! <https://www.gnu.org/licenses/>. + +!GCC$ builtin (cos) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (cosf) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (sin) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (sinf) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (sincos) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (sincosf) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (log) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (logf) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (exp) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (expf) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (pow) attributes simd (notinbranch) if('x86_64') +!GCC$ builtin (powf) attributes simd (notinbranch) if('x86_64') + +!GCC$ builtin (cos) attributes simd (notinbranch) if('x32') +!GCC$ builtin (cosf) attributes simd (notinbranch) if('x32') +!GCC$ builtin (sin) attributes simd (notinbranch) if('x32') +!GCC$ builtin (sinf) attributes simd (notinbranch) if('x32') +!GCC$ builtin (sincos) attributes simd (notinbranch) if('x32') +!GCC$ builtin (sincosf) attributes simd (notinbranch) if('x32') +!GCC$ builtin (log) attributes simd (notinbranch) if('x32') +!GCC$ builtin (logf) attributes simd (notinbranch) if('x32') +!GCC$ builtin (exp) attributes simd (notinbranch) if('x32') +!GCC$ builtin (expf) attributes simd (notinbranch) if('x32') +!GCC$ builtin (pow) attributes simd (notinbranch) if('x32') +!GCC$ builtin (powf) attributes simd (notinbranch) if('x32') diff --git a/sysdeps/x86/fpu/fix-fp-int-compare-invalid.h b/sysdeps/x86/fpu/fix-fp-int-compare-invalid.h index 6bad27d0fa..2ab8cd3651 100644 --- a/sysdeps/x86/fpu/fix-fp-int-compare-invalid.h +++ b/sysdeps/x86/fpu/fix-fp-int-compare-invalid.h @@ -1,6 +1,6 @@ /* Fix for missing "invalid" exceptions from floating-point comparisons. x86 version. - Copyright (C) 2016-2018 Free Software Foundation, Inc. + Copyright (C) 2016-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef FIX_FP_INT_COMPARE_INVALID_H #define FIX_FP_INT_COMPARE_INVALID_H 1 diff --git a/sysdeps/x86/fpu/include/bits/fenv.h b/sysdeps/x86/fpu/include/bits/fenv.h index 3d2483b0bf..2ee656a533 100644 --- a/sysdeps/x86/fpu/include/bits/fenv.h +++ b/sysdeps/x86/fpu/include/bits/fenv.h @@ -1,5 +1,5 @@ /* Wrapper for x86 bits/fenv.h for use when building glibc. - Copyright (C) 1997-2018 Free Software Foundation, Inc. + Copyright (C) 1997-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _BITS_FENV_H diff --git a/sysdeps/x86/fpu/math-barriers.h b/sysdeps/x86/fpu/math-barriers.h index 1e1fabdb92..2e45378bd3 100644 --- a/sysdeps/x86/fpu/math-barriers.h +++ b/sysdeps/x86/fpu/math-barriers.h @@ -1,5 +1,5 @@ /* Control when floating-point expressions are evaluated. x86 version. - Copyright (C) 2007-2018 Free Software Foundation, Inc. + Copyright (C) 2007-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef X86_MATH_BARRIERS_H #define X86_MATH_BARRIERS_H 1 diff --git a/sysdeps/x86/bits/xtitypes.h b/sysdeps/x86/fpu/math_private.h index 4fd0fe788b..3d9763ae6e 100644 --- a/sysdeps/x86/bits/xtitypes.h +++ b/sysdeps/x86/fpu/math_private.h @@ -1,5 +1,5 @@ -/* bits/xtitypes.h -- Define some types used by <bits/stropts.h>. x86-64. - Copyright (C) 2002-2018 Free Software Foundation, Inc. +/* Private inline math functions for x86. + Copyright (C) 1995-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,20 +14,19 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ -#ifndef _STROPTS_H -# error "Never include <bits/xtitypes.h> directly; use <stropts.h> instead." -#endif - -#ifndef _BITS_XTITYPES_H -#define _BITS_XTITYPES_H 1 +#ifndef X86_MATH_PRIVATE_H +#define X86_MATH_PRIVATE_H 1 -#include <bits/types.h> +#include_next <math_private.h> -/* This type is used by some structs in <bits/stropts.h>. */ -typedef __SLONG32_TYPE __t_scalar_t; -typedef __ULONG32_TYPE __t_uscalar_t; +__extern_always_inline long double +__NTH (__ieee754_atan2l (long double y, long double x)) +{ + long double ret; + __asm__ __volatile__ ("fpatan" : "=t" (ret) : "0" (x), "u" (y) : "st(1)"); + return ret; +} - -#endif /* bits/xtitypes.h */ +#endif diff --git a/sysdeps/x86/fpu/powl_helper.c b/sysdeps/x86/fpu/powl_helper.c index 651eedd792..07897bf523 100644 --- a/sysdeps/x86/fpu/powl_helper.c +++ b/sysdeps/x86/fpu/powl_helper.c @@ -1,5 +1,5 @@ /* Implement powl for x86 using extra-precision log. - Copyright (C) 2012-2018 Free Software Foundation, Inc. + Copyright (C) 2012-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <math.h> #include <math_private.h> @@ -216,7 +216,7 @@ __powl_helper (long double x, long double y) /* Split the base-2 logarithm of the result into integer and fractional parts. */ - long double log2_res_int = __roundl (log2_res_hi); + long double log2_res_int = roundl (log2_res_hi); long double log2_res_frac = log2_res_hi - log2_res_int + log2_res_lo; /* If the integer part is very large, the computed fractional part may be outside the valid range for f2xm1. */ diff --git a/sysdeps/x86/fpu/s_sincosf_data.c b/sysdeps/x86/fpu/s_sincosf_data.c new file mode 100644 index 0000000000..b3bdd1cb3b --- /dev/null +++ b/sysdeps/x86/fpu/s_sincosf_data.c @@ -0,0 +1,68 @@ +/* Compute sine and cosine of argument. + Copyright (C) 2018-2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <stdint.h> +#include <math.h> +#include <sysdeps/ieee754/flt-32/math_config.h> +#include <s_sincosf.h> + +/* The constants and polynomials for sine and cosine. The 2nd entry + computes -cos (x) rather than cos (x) to get negation for free. */ +const sincos_t __sincosf_table[2] = +{ + { + { 1.0, -1.0, -1.0, 1.0 }, +#if TOINT_INTRINSICS + 0x1.45F306DC9C883p-1, +#else + 0x1.45F306DC9C883p+23, +#endif + 0x1.921FB54442D18p0, + 0x1p0, + -0x1.ffffffd0c621cp-2, + { -0x1.555545995a603p-3, 0x1.55553e1068f19p-5 }, + { 0x1.1107605230bc4p-7, -0x1.6c087e89a359dp-10 }, + { -0x1.994eb3774cf24p-13, 0x1.99343027bf8c3p-16 } + }, + { + { 1.0, -1.0, -1.0, 1.0 }, +#if TOINT_INTRINSICS + 0x1.45F306DC9C883p-1, +#else + 0x1.45F306DC9C883p+23, +#endif + 0x1.921FB54442D18p0, + -0x1p0, + 0x1.ffffffd0c621cp-2, + { -0x1.555545995a603p-3, -0x1.55553e1068f19p-5 }, + { 0x1.1107605230bc4p-7, 0x1.6c087e89a359dp-10 }, + { -0x1.994eb3774cf24p-13, -0x1.99343027bf8c3p-16 } + } +}; + +/* Table with 4/PI to 192 bit precision. To avoid unaligned accesses + only 8 new bits are added per entry, making the table 4 times larger. */ +const uint32_t __inv_pio4[24] = +{ + 0xa2, 0xa2f9, 0xa2f983, 0xa2f9836e, + 0xf9836e4e, 0x836e4e44, 0x6e4e4415, 0x4e441529, + 0x441529fc, 0x1529fc27, 0x29fc2757, 0xfc2757d1, + 0x2757d1f5, 0x57d1f534, 0xd1f534dd, 0xf534ddc0, + 0x34ddc0db, 0xddc0db62, 0xc0db6295, 0xdb629599, + 0x6295993c, 0x95993c43, 0x993c4390, 0x3c439041 +}; diff --git a/sysdeps/x86/fpu/sfp-machine.h b/sysdeps/x86/fpu/sfp-machine.h index df8906acb4..5892f4f5fe 100644 --- a/sysdeps/x86/fpu/sfp-machine.h +++ b/sysdeps/x86/fpu/sfp-machine.h @@ -15,7 +15,7 @@ typedef int __gcc_CMPtype __attribute__ ((mode (__libgcc_cmp_return__))); typedef int TItype __attribute__ ((mode (TI))); typedef unsigned int UTItype __attribute__ ((mode (TI))); -# define TI_BITS (__CHAR_BIT__ * (int)sizeof(TItype)) +# define TI_BITS (__CHAR_BIT__ * (int) sizeof (TItype)) # define _FP_MUL_MEAT_Q(R,X,Y) \ _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm) diff --git a/sysdeps/x86/fpu/sincosf_poly.h b/sysdeps/x86/fpu/sincosf_poly.h new file mode 100644 index 0000000000..ba18d17526 --- /dev/null +++ b/sysdeps/x86/fpu/sincosf_poly.h @@ -0,0 +1,111 @@ +/* Used by sinf, cosf and sincosf functions. X86-64 version. + Copyright (C) 2018-2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +typedef double v2df_t __attribute__ ((vector_size (2 * sizeof (double)))); + +#ifdef __SSE2_MATH__ +typedef float v4sf_t __attribute__ ((vector_size (4 * sizeof (float)))); + +static inline void +v2df_to_sf (v2df_t v2df, float *f0p, float *f1p) +{ + v4sf_t v4sf = __builtin_ia32_cvtpd2ps (v2df); + *f0p = v4sf[0]; + *f1p = v4sf[1]; +} +#else +static inline void +v2df_to_sf (v2df_t v2df, float *f0p, float *f1p) +{ + *f0p = (float) v2df[0]; + *f1p = (float) v2df[1]; +} +#endif + +/* The constants and polynomials for sine and cosine. */ +typedef struct +{ + double sign[4]; /* Sign of sine in quadrants 0..3. */ + double hpi_inv; /* 2 / PI ( * 2^24 if !TOINT_INTRINSICS). */ + double hpi; /* PI / 2. */ + /* Cosine polynomial: c0, c1, c2, c3, c4. + Sine polynomial: s1, s2, s3. */ + double c0, c1; + v2df_t s1c2, s2c3, s3c4; +} sincos_t; + +/* Compute the sine and cosine of inputs X and X2 (X squared), using the + polynomial P and store the results in SINP and COSP. N is the quadrant, + if odd the cosine and sine polynomials are swapped. */ +static inline void +sincosf_poly (double x, double x2, const sincos_t *p, int n, float *sinp, + float *cosp) +{ + v2df_t vx2x2 = { x2, x2 }; + v2df_t vxx2 = { x, x2 }; + v2df_t vx3x4, vs1c2; + + vx3x4 = vx2x2 * vxx2; + vs1c2 = p->s2c3 + vx2x2 * p->s3c4; + + /* Swap sin/cos result based on quadrant. */ + if (n & 1) + { + float *tmp = cosp; + cosp = sinp; + sinp = tmp; + } + + double c1 = p->c0 + x2 * p->c1; + v2df_t vxc1 = { x, c1 }; + v2df_t vx5x6 = vx3x4 * vx2x2; + + v2df_t vsincos = vxc1 + vx3x4 * p->s1c2; + vsincos = vsincos + vx5x6 * vs1c2; + v2df_to_sf (vsincos, sinp, cosp); +} + +/* Return the sine of inputs X and X2 (X squared) using the polynomial P. + N is the quadrant, and if odd the cosine polynomial is used. */ +static inline float +sinf_poly (double x, double x2, const sincos_t *p, int n) +{ + double x3, x4, x6, x7, s, c, c1, c2, s1; + + if ((n & 1) == 0) + { + x3 = x * x2; + s1 = p->s2c3[0] + x2 * p->s3c4[0]; + + x7 = x3 * x2; + s = x + x3 * p->s1c2[0]; + + return s + x7 * s1; + } + else + { + x4 = x2 * x2; + c2 = p->s2c3[1] + x2 * p->s3c4[1]; + c1 = p->c0 + x2 * p->c1; + + x6 = x4 * x2; + c = c1 + x4 * p->s1c2[1]; + + return c + x6 * c2; + } +} diff --git a/sysdeps/x86/fpu/test-fenv-clear-sse.c b/sysdeps/x86/fpu/test-fenv-clear-sse.c index c67a3ba7c9..9d5b762e3a 100644 --- a/sysdeps/x86/fpu/test-fenv-clear-sse.c +++ b/sysdeps/x86/fpu/test-fenv-clear-sse.c @@ -1,6 +1,6 @@ /* Test fesetenv (FE_DFL_ENV) and fesetenv (FE_NOMASK_ENV) clear exceptions (bug 19181). SSE version. - Copyright (C) 2015-2018 Free Software Foundation, Inc. + Copyright (C) 2015-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <cpuid.h> #include <stdbool.h> diff --git a/sysdeps/x86/fpu/test-fenv-sse-2.c b/sysdeps/x86/fpu/test-fenv-sse-2.c index fcb1011555..56dfbc3c5e 100644 --- a/sysdeps/x86/fpu/test-fenv-sse-2.c +++ b/sysdeps/x86/fpu/test-fenv-sse-2.c @@ -1,5 +1,5 @@ /* Test x86-specific floating-point environment (bug 16068): SSE part. - Copyright (C) 2015-2018 Free Software Foundation, Inc. + Copyright (C) 2015-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <cpuid.h> #include <fenv.h> diff --git a/sysdeps/x86/fpu/test-fenv-sse.c b/sysdeps/x86/fpu/test-fenv-sse.c index 5462315811..d6ee7dc9ee 100644 --- a/sysdeps/x86/fpu/test-fenv-sse.c +++ b/sysdeps/x86/fpu/test-fenv-sse.c @@ -1,5 +1,5 @@ /* Test floating-point environment includes SSE state (bug 16064). - Copyright (C) 2014-2018 Free Software Foundation, Inc. + Copyright (C) 2014-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <cpuid.h> #include <fenv.h> diff --git a/sysdeps/x86/fpu/test-fenv-x87.c b/sysdeps/x86/fpu/test-fenv-x87.c index c9cd84e9c7..bd8cc6fdd7 100644 --- a/sysdeps/x86/fpu/test-fenv-x87.c +++ b/sysdeps/x86/fpu/test-fenv-x87.c @@ -1,5 +1,5 @@ /* Test x86-specific floating-point environment (bug 16068): x87 part. - Copyright (C) 2015-2018 Free Software Foundation, Inc. + Copyright (C) 2015-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <fenv.h> #include <float.h> diff --git a/sysdeps/x86/fpu/test-math-vector-sincos.h b/sysdeps/x86/fpu/test-math-vector-sincos.h index d422ffa4a7..c095133a9d 100644 --- a/sysdeps/x86/fpu/test-math-vector-sincos.h +++ b/sysdeps/x86/fpu/test-math-vector-sincos.h @@ -1,6 +1,6 @@ /* Wrappers definitions for tests of ABI of vector sincos/sincosf having vector declaration "#pragma omp declare simd notinbranch". - Copyright (C) 2016-2018 Free Software Foundation, Inc. + Copyright (C) 2016-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #define INIT_VEC_PTRS_LOOP(vec, val, len) \ do \ diff --git a/sysdeps/x86/fpu_control.h b/sysdeps/x86/fpu_control.h index 4cb98c5679..bb52bd5522 100644 --- a/sysdeps/x86/fpu_control.h +++ b/sysdeps/x86/fpu_control.h @@ -1,5 +1,5 @@ /* FPU control word bits. x86 version. - Copyright (C) 1993-2018 Free Software Foundation, Inc. + Copyright (C) 1993-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Olaf Flebbe. @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _FPU_CONTROL_H #define _FPU_CONTROL_H 1 diff --git a/sysdeps/x86/hp-timing.h b/sysdeps/x86/hp-timing.h new file mode 100644 index 0000000000..a230732baf --- /dev/null +++ b/sysdeps/x86/hp-timing.h @@ -0,0 +1,61 @@ +/* High precision, low overhead timing functions. x86 version. + Copyright (C) 2018-2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#ifndef _HP_TIMING_H +#define _HP_TIMING_H 1 + +#include <isa.h> + +#if MINIMUM_ISA == 686 || MINIMUM_ISA == 8664 +/* We indeed have inlined functions. */ +# define HP_TIMING_INLINE (1) + +/* We use 64bit values for the times. */ +typedef unsigned long long int hp_timing_t; + +/* That's quite simple. Use the `rdtsc' instruction. Note that the value + might not be 100% accurate since there might be some more instructions + running in this moment. This could be changed by using a barrier like + 'cpuid' right before the `rdtsc' instruciton. But we are not interested + in accurate clock cycles here so we don't do this. + + NB: Use __builtin_ia32_rdtsc directly since including <x86intrin.h> + makes building glibc very slow. */ +# ifdef USE_RDTSCP +/* RDTSCP waits until all previous instructions have executed and all + previous loads are globally visible before reading the counter. + RDTSC doesn't wait until all previous instructions have been executed + before reading the counter. */ +# define HP_TIMING_NOW(Var) \ + (__extension__ ({ \ + unsigned int __aux; \ + (Var) = __builtin_ia32_rdtscp (&__aux); \ + })) +# else +# define HP_TIMING_NOW(Var) ((Var) = __builtin_ia32_rdtsc ()) +# endif + +# include <hp-timing-common.h> +#else +/* NB: Undefine _HP_TIMING_H so that <sysdeps/generic/hp-timing.h> will + be included. */ +# undef _HP_TIMING_H +# include <sysdeps/generic/hp-timing.h> +#endif + +#endif /* hp-timing.h */ diff --git a/sysdeps/x86/init-arch.h b/sysdeps/x86/init-arch.h index a81ca8a4eb..a6c2fe28b7 100644 --- a/sysdeps/x86/init-arch.h +++ b/sysdeps/x86/init-arch.h @@ -1,5 +1,5 @@ /* This file is part of the GNU C Library. - Copyright (C) 2008-2018 Free Software Foundation, Inc. + Copyright (C) 2008-2019 Free Software Foundation, Inc. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifdef __ASSEMBLER__ # include <cpu-features.h> @@ -21,6 +21,7 @@ # include <ldsodefs.h> #endif #include <ifunc-init.h> +#include <isa.h> #ifndef __x86_64__ /* Due to the reordering and the other nifty extensions in i686, it is diff --git a/sysdeps/x86/ldsodefs.h b/sysdeps/x86/ldsodefs.h index 0616215b7a..57877e8b08 100644 --- a/sysdeps/x86/ldsodefs.h +++ b/sysdeps/x86/ldsodefs.h @@ -1,6 +1,6 @@ /* Run-time dynamic linker data structures for loaded ELF shared objects. X86 version. - Copyright (C) 1995-2018 Free Software Foundation, Inc. + Copyright (C) 1995-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _X86_LDSODEFS_H #define _X86_LDSODEFS_H 1 diff --git a/sysdeps/x86/libc-start.c b/sysdeps/x86/libc-start.c index eb5335c154..9df877ea3b 100644 --- a/sysdeps/x86/libc-start.c +++ b/sysdeps/x86/libc-start.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2018 Free Software Foundation, Inc. +/* Copyright (C) 2015-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef SHARED /* Define I386_USE_SYSENTER to support syscall during startup in static diff --git a/sysdeps/x86/libc-start.h b/sysdeps/x86/libc-start.h index 6f44262bf4..b3b0cb7ea5 100644 --- a/sysdeps/x86/libc-start.h +++ b/sysdeps/x86/libc-start.h @@ -1,5 +1,5 @@ /* X86 definitions for libc main startup. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef SHARED # define ARCH_SETUP_IREL() apply_irel () diff --git a/sysdeps/x86/link_map.h b/sysdeps/x86/link_map.h index ef1206a9d2..38dcc7fc26 100644 --- a/sysdeps/x86/link_map.h +++ b/sysdeps/x86/link_map.h @@ -1,5 +1,5 @@ /* Additional fields in struct link_map. Linux/x86 version. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,13 +14,14 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ /* If this object is enabled with CET. */ enum { - lc_none = 0, /* Not enabled with CET. */ - lc_ibt = 1 << 0, /* Enabled with IBT. */ - lc_shstk = 1 << 1, /* Enabled with STSHK. */ + lc_unknown = 0, /* Unknown CET status. */ + lc_none = 1 << 0, /* Not enabled with CET. */ + lc_ibt = 1 << 1, /* Enabled with IBT. */ + lc_shstk = 1 << 2, /* Enabled with STSHK. */ lc_ibt_and_shstk = lc_ibt | lc_shstk /* Enabled with both. */ - } l_cet:2; + } l_cet:3; diff --git a/sysdeps/x86/longjmp.c b/sysdeps/x86/longjmp.c index a53f31e1dd..5cb6c9f7c4 100644 --- a/sysdeps/x86/longjmp.c +++ b/sysdeps/x86/longjmp.c @@ -1,5 +1,5 @@ /* __libc_siglongjmp for x86. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #define __libc_longjmp __redirect___libc_longjmp #include <setjmp/longjmp.c> diff --git a/sysdeps/x86/nptl/bits/pthreadtypes-arch.h b/sysdeps/x86/nptl/bits/pthreadtypes-arch.h index 290f2f4640..426a20751d 100644 --- a/sysdeps/x86/nptl/bits/pthreadtypes-arch.h +++ b/sysdeps/x86/nptl/bits/pthreadtypes-arch.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2002-2018 Free Software Foundation, Inc. +/* Copyright (C) 2002-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -13,7 +13,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _BITS_PTHREADTYPES_ARCH_H #define _BITS_PTHREADTYPES_ARCH_H 1 @@ -47,57 +47,9 @@ #define __SIZEOF_PTHREAD_RWLOCKATTR_T 8 #define __SIZEOF_PTHREAD_BARRIERATTR_T 4 -/* Definitions for internal mutex struct. */ -#define __PTHREAD_COMPAT_PADDING_MID -#define __PTHREAD_COMPAT_PADDING_END -#define __PTHREAD_MUTEX_LOCK_ELISION 1 -#ifdef __x86_64__ -# define __PTHREAD_MUTEX_NUSERS_AFTER_KIND 0 -# define __PTHREAD_MUTEX_USE_UNION 0 -#else -# define __PTHREAD_MUTEX_NUSERS_AFTER_KIND 1 -# define __PTHREAD_MUTEX_USE_UNION 1 -#endif - #define __LOCK_ALIGNMENT #define __ONCE_ALIGNMENT -struct __pthread_rwlock_arch_t -{ - unsigned int __readers; - unsigned int __writers; - unsigned int __wrphase_futex; - unsigned int __writers_futex; - unsigned int __pad3; - unsigned int __pad4; -#ifdef __x86_64__ - int __cur_writer; - int __shared; - signed char __rwelision; -# ifdef __ILP32__ - unsigned char __pad1[3]; -# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, { 0, 0, 0 } -# else - unsigned char __pad1[7]; -# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, { 0, 0, 0, 0, 0, 0, 0 } -# endif - unsigned long int __pad2; - /* FLAGS must stay at this position in the structure to maintain - binary compatibility. */ - unsigned int __flags; -# define __PTHREAD_RWLOCK_INT_FLAGS_SHARED 1 -#else - /* FLAGS must stay at this position in the structure to maintain - binary compatibility. */ - unsigned char __flags; - unsigned char __shared; - signed char __rwelision; -# define __PTHREAD_RWLOCK_ELISION_EXTRA 0 - unsigned char __pad2; - int __cur_writer; -#endif -}; - #ifndef __x86_64__ /* Extra attributes for the cleanup functions. */ # define __cleanup_fct_attribute __attribute__ ((__regparm__ (1))) diff --git a/sysdeps/x86/nptl/bits/struct_mutex.h b/sysdeps/x86/nptl/bits/struct_mutex.h new file mode 100644 index 0000000000..09384487f9 --- /dev/null +++ b/sysdeps/x86/nptl/bits/struct_mutex.h @@ -0,0 +1,63 @@ +/* x86 internal mutex struct definitions. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef _THREAD_MUTEX_INTERNAL_H +#define _THREAD_MUTEX_INTERNAL_H 1 + +struct __pthread_mutex_s +{ + int __lock; + unsigned int __count; + int __owner; +#ifdef __x86_64__ + unsigned int __nusers; +#endif + /* KIND must stay at this position in the structure to maintain + binary compatibility with static initializers. */ + int __kind; +#ifdef __x86_64__ + short __spins; + short __elision; + __pthread_list_t __list; +# define __PTHREAD_MUTEX_HAVE_PREV 1 +#else + unsigned int __nusers; + __extension__ union + { + struct + { + short __espins; + short __eelision; +# define __spins __elision_data.__espins +# define __elision __elision_data.__eelision + } __elision_data; + __pthread_slist_t __list; + }; +# define __PTHREAD_MUTEX_HAVE_PREV 0 +#endif +}; + +#ifdef __x86_64__ +# define __PTHREAD_MUTEX_INITIALIZER(__kind) \ + 0, 0, 0, 0, __kind, 0, 0, { 0, 0 } +#else +# define __PTHREAD_MUTEX_INITIALIZER(__kind) \ + 0, 0, 0, __kind, 0, { { 0, 0 } } +#endif + +#endif diff --git a/sysdeps/x86/nptl/bits/struct_rwlock.h b/sysdeps/x86/nptl/bits/struct_rwlock.h new file mode 100644 index 0000000000..7f4728ec84 --- /dev/null +++ b/sysdeps/x86/nptl/bits/struct_rwlock.h @@ -0,0 +1,65 @@ +/* x86 internal rwlock struct definitions. + Copyright (C) 2019 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef _RWLOCK_INTERNAL_H +#define _RWLOCK_INTERNAL_H + +struct __pthread_rwlock_arch_t +{ + unsigned int __readers; + unsigned int __writers; + unsigned int __wrphase_futex; + unsigned int __writers_futex; + unsigned int __pad3; + unsigned int __pad4; +#ifdef __x86_64__ + int __cur_writer; + int __shared; + signed char __rwelision; +# ifdef __ILP32__ + unsigned char __pad1[3]; +# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, { 0, 0, 0 } +# else + unsigned char __pad1[7]; +# define __PTHREAD_RWLOCK_ELISION_EXTRA 0, { 0, 0, 0, 0, 0, 0, 0 } +# endif + unsigned long int __pad2; + /* FLAGS must stay at this position in the structure to maintain + binary compatibility. */ + unsigned int __flags; +#else /* __x86_64__ */ + /* FLAGS must stay at this position in the structure to maintain + binary compatibility. */ + unsigned char __flags; + unsigned char __shared; + signed char __rwelision; + unsigned char __pad2; + int __cur_writer; +#endif +}; + +#ifdef __x86_64__ +# define __PTHREAD_RWLOCK_INITIALIZER(__flags) \ + 0, 0, 0, 0, 0, 0, 0, 0, __PTHREAD_RWLOCK_ELISION_EXTRA, 0, __flags +#else +# define __PTHREAD_RWLOCK_INITIALIZER(__flags) \ + 0, 0, 0, 0, 0, 0, __flags, 0, 0, 0, 0 +#endif + +#endif diff --git a/sysdeps/x86/nptl/pt-longjmp.c b/sysdeps/x86/nptl/pt-longjmp.c index 6165c7d4a7..2293571b6f 100644 --- a/sysdeps/x86/nptl/pt-longjmp.c +++ b/sysdeps/x86/nptl/pt-longjmp.c @@ -1,6 +1,6 @@ /* ABI compatibility for 'longjmp' and 'siglongjmp' symbols in libpthread ABI. X86 version. - Copyright (C) 18 Free Software Foundation, Inc. + Copyright (C) 1918-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <pthreadP.h> #include <jmp_buf-ssp.h> diff --git a/sysdeps/x86/nptl/tls-setup.h b/sysdeps/x86/nptl/tls-setup.h index ef5a4df78c..c855a19ab6 100644 --- a/sysdeps/x86/nptl/tls-setup.h +++ b/sysdeps/x86/nptl/tls-setup.h @@ -1,5 +1,5 @@ /* Definitions to set up thread-local data. x86 version. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ static inline void __attribute__ ((always_inline)) tls_setup_tcbhead (struct pthread *pd) diff --git a/sysdeps/x86/string_private.h b/sysdeps/x86/string_private.h index 4bc45f63d8..f18553f1bb 100644 --- a/sysdeps/x86/string_private.h +++ b/sysdeps/x86/string_private.h @@ -1,5 +1,5 @@ /* Define _STRING_ARCH_unaligned. i486/x86-64 version. - Copyright (C) 2016-2018 Free Software Foundation, Inc. + Copyright (C) 2016-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ /* The ix86 processors can access unaligned multi-byte variables. */ #define _STRING_ARCH_unaligned 1 diff --git a/sysdeps/x86/sysdep.h b/sysdeps/x86/sysdep.h index 8776ad8374..9140e667b0 100644 --- a/sysdeps/x86/sysdep.h +++ b/sysdeps/x86/sysdep.h @@ -1,5 +1,5 @@ /* Assembler macros for x86. - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,7 +14,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #ifndef _X86_SYSDEP_H #define _X86_SYSDEP_H 1 @@ -48,6 +48,15 @@ enum cf_protection_level # define SHSTK_ENABLED 0 #endif +/* Offset for fxsave/xsave area used by _dl_runtime_resolve. Also need + space to preserve RCX, RDX, RSI, RDI, R8, R9 and RAX. It must be + aligned to 16 bytes for fxsave and 64 bytes for xsave. */ +#define STATE_SAVE_OFFSET (8 * 7 + 8) + +/* Save SSE, AVX, AVX512, mask and bound registers. */ +#define STATE_SAVE_MASK \ + ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) + #ifdef __ASSEMBLER__ /* Syntactic details of assembler. */ diff --git a/sysdeps/x86/tst-cet-legacy-1.c b/sysdeps/x86/tst-cet-legacy-1.c index 861c09a26e..0938306e42 100644 --- a/sysdeps/x86/tst-cet-legacy-1.c +++ b/sysdeps/x86/tst-cet-legacy-1.c @@ -1,6 +1,6 @@ /* Check compatibility of CET-enabled executable linked with legacy shared object. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <stdio.h> #include <stdlib.h> diff --git a/sysdeps/x86/tst-cet-legacy-1a.c b/sysdeps/x86/tst-cet-legacy-1a.c new file mode 100644 index 0000000000..8e77290e20 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-1a.c @@ -0,0 +1,81 @@ +/* Test for re-exec with legacy bitmap. + Copyright (C) 2018-2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <getopt.h> +#include <support/check.h> + +/* Nonzero if the program gets called via `exec'. */ +static int restart; + +#define CMDLINE_OPTIONS \ + { "restart", no_argument, &restart, 1 }, + +extern int do_test (int argc, char *argv[]); + +extern int in_dso_1 (void); +extern int in_dso_2 (void); + +static int +check (void) +{ + if (in_dso_1 () != 0x1234678) + { + puts ("in_dso_1 () != 0x1234678"); + exit (1); + } + + if (in_dso_2 () != 0xbadbeef) + { + puts ("in_dso_2 () != 0xbadbeef"); + exit (1); + } + + return 0; +} + +int +do_test (int argc, char *argv[]) +{ + /* We must have + - one or four parameters left if called initially + + path for ld.so optional + + "--library-path" optional + + the library path optional + + the application name + */ + + if (restart) + return check (); + + if (argc != 2 && argc != 5) + FAIL_EXIT1 ("wrong number of arguments (%d)", argc); + + if (argc == 5) + execl (argv[1], argv[1], argv[2], argv[3], argv[4], "--direct", + "--restart", NULL); + else + execl (argv[1], argv[1], "--direct", "--restart", NULL); + + return -1; +} + +#define TEST_FUNCTION_ARGV do_test +#include <support/test-driver.c> diff --git a/sysdeps/x86/tst-cet-legacy-2.c b/sysdeps/x86/tst-cet-legacy-2.c index e039a16797..e70d289e4f 100644 --- a/sysdeps/x86/tst-cet-legacy-2.c +++ b/sysdeps/x86/tst-cet-legacy-2.c @@ -1,6 +1,6 @@ /* Check compatibility of CET-enabled executable with dlopened legacy shared object. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <dlfcn.h> #include <stdio.h> diff --git a/sysdeps/x86/tst-cet-legacy-3.c b/sysdeps/x86/tst-cet-legacy-3.c index bab9faa8b0..1850220437 100644 --- a/sysdeps/x86/tst-cet-legacy-3.c +++ b/sysdeps/x86/tst-cet-legacy-3.c @@ -1,6 +1,6 @@ /* Check compatibility of CET-enabled executable with dlopened legacy shared object. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <stdio.h> diff --git a/sysdeps/x86/tst-cet-legacy-4.c b/sysdeps/x86/tst-cet-legacy-4.c index 3ead63dd24..6550ac863b 100644 --- a/sysdeps/x86/tst-cet-legacy-4.c +++ b/sysdeps/x86/tst-cet-legacy-4.c @@ -1,6 +1,6 @@ /* Check compatibility of CET-enabled executable with dlopened legacy shared object. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <dlfcn.h> #include <stdio.h> diff --git a/sysdeps/x86/tst-cet-legacy-5.c b/sysdeps/x86/tst-cet-legacy-5.c new file mode 100644 index 0000000000..47c327b7ab --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-5.c @@ -0,0 +1,76 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <dlfcn.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> + +static void +do_test_1 (const char *modname, bool fail) +{ + int (*fp) (void); + void *h; + + h = dlopen (modname, RTLD_LAZY); + if (h == NULL) + { + if (fail) + { + const char *err = dlerror (); + if (strstr (err, "shadow stack isn't enabled") == NULL) + { + printf ("incorrect dlopen '%s' error: %s\n", modname, + dlerror ()); + exit (1); + } + + return; + } + + printf ("cannot open '%s': %s\n", modname, dlerror ()); + exit (1); + } + + fp = dlsym (h, "test"); + if (fp == NULL) + { + printf ("cannot get symbol 'test': %s\n", dlerror ()); + exit (1); + } + + if (fp () != 0) + { + puts ("test () != 0"); + exit (1); + } + + dlclose (h); +} + +static int +do_test (void) +{ + do_test_1 ("tst-cet-legacy-mod-5a.so", true); + do_test_1 ("tst-cet-legacy-mod-5b.so", false); + return 0; +} + +#include <support/test-driver.c> diff --git a/sysdeps/x86/tst-cet-legacy-5a.c b/sysdeps/x86/tst-cet-legacy-5a.c new file mode 100644 index 0000000000..fc5a609dff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-5a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-5b.c b/sysdeps/x86/tst-cet-legacy-5b.c new file mode 100644 index 0000000000..fc5a609dff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-5b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-6.c b/sysdeps/x86/tst-cet-legacy-6.c new file mode 100644 index 0000000000..719e5d706b --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-6.c @@ -0,0 +1,76 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <dlfcn.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> + +static void +do_test_1 (const char *modname, bool fail) +{ + int (*fp) (void); + void *h; + + h = dlopen (modname, RTLD_LAZY); + if (h == NULL) + { + if (fail) + { + const char *err = dlerror (); + if (strstr (err, "shadow stack isn't enabled") == NULL) + { + printf ("incorrect dlopen '%s' error: %s\n", modname, + dlerror ()); + exit (1); + } + + return; + } + + printf ("cannot open '%s': %s\n", modname, dlerror ()); + exit (1); + } + + fp = dlsym (h, "test"); + if (fp == NULL) + { + printf ("cannot get symbol 'test': %s\n", dlerror ()); + exit (1); + } + + if (fp () != 0) + { + puts ("test () != 0"); + exit (1); + } + + dlclose (h); +} + +static int +do_test (void) +{ + do_test_1 ("tst-cet-legacy-mod-6a.so", true); + do_test_1 ("tst-cet-legacy-mod-6b.so", false); + return 0; +} + +#include <support/test-driver.c> diff --git a/sysdeps/x86/tst-cet-legacy-6a.c b/sysdeps/x86/tst-cet-legacy-6a.c new file mode 100644 index 0000000000..2d1546d36b --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-6a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-6b.c b/sysdeps/x86/tst-cet-legacy-6b.c new file mode 100644 index 0000000000..2d1546d36b --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-6b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-1.c b/sysdeps/x86/tst-cet-legacy-mod-1.c index 09762bce13..cdc8767390 100644 --- a/sysdeps/x86/tst-cet-legacy-mod-1.c +++ b/sysdeps/x86/tst-cet-legacy-mod-1.c @@ -1,6 +1,6 @@ /* Check compatibility of CET-enabled executable with legacy shared object. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ int in_dso_1 (void) diff --git a/sysdeps/x86/tst-cet-legacy-mod-2.c b/sysdeps/x86/tst-cet-legacy-mod-2.c index 1c8de443f6..e7469bf391 100644 --- a/sysdeps/x86/tst-cet-legacy-mod-2.c +++ b/sysdeps/x86/tst-cet-legacy-mod-2.c @@ -1,6 +1,6 @@ /* Check compatibility of CET-enabled executable with legacy shared object. - Copyright (C) 2018 Free Software Foundation, Inc. + Copyright (C) 2018-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -15,7 +15,7 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ int in_dso_2 (void) diff --git a/sysdeps/x86/math-tests.h b/sysdeps/x86/tst-cet-legacy-mod-5.c index 43c7ebe337..fb64b45d62 100644 --- a/sysdeps/x86/math-tests.h +++ b/sysdeps/x86/tst-cet-legacy-mod-5.c @@ -1,5 +1,6 @@ -/* Configuration for math tests. x86 version. - Copyright (C) 2017-2018 Free Software Foundation, Inc. +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,12 +15,17 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ -/* Before GCC 7, there is no built-in function to provide a __float128 - sNaN, so most sNaN tests for this type cannot work. */ -#if !__GNUC_PREREQ (7, 0) -# define SNAN_TESTS_float128 0 -#endif +#include <error.h> +#include <stdio.h> +#include <stdlib.h> -#include_next <math-tests.h> +extern void foo (void); + +int +test (void) +{ + foo (); + return 0; +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-5a.c b/sysdeps/x86/tst-cet-legacy-mod-5a.c new file mode 100644 index 0000000000..daa43e4e8d --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-5a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-5b.c b/sysdeps/x86/tst-cet-legacy-mod-5b.c new file mode 100644 index 0000000000..daa43e4e8d --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-5b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-5c.c b/sysdeps/x86/tst-cet-legacy-mod-5c.c new file mode 100644 index 0000000000..604717c85c --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-5c.c @@ -0,0 +1,36 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <stdlib.h> + +static int called = 0; + +static void +__attribute__ ((constructor)) +init (void) +{ + called = 1; +} + +void +foo (void) +{ + if (!called) + abort (); +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-6.c b/sysdeps/x86/tst-cet-legacy-mod-6.c new file mode 100644 index 0000000000..fb64b45d62 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6.c @@ -0,0 +1,31 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <error.h> +#include <stdio.h> +#include <stdlib.h> + +extern void foo (void); + +int +test (void) +{ + foo (); + return 0; +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-6a.c b/sysdeps/x86/tst-cet-legacy-mod-6a.c new file mode 100644 index 0000000000..c89b8fe8ff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-6b.c b/sysdeps/x86/tst-cet-legacy-mod-6b.c new file mode 100644 index 0000000000..c89b8fe8ff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-6c.c b/sysdeps/x86/tst-cet-legacy-mod-6c.c new file mode 100644 index 0000000000..604717c85c --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6c.c @@ -0,0 +1,36 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <stdlib.h> + +static int called = 0; + +static void +__attribute__ ((constructor)) +init (void) +{ + called = 1; +} + +void +foo (void) +{ + if (!called) + abort (); +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-6d.c b/sysdeps/x86/tst-cet-legacy-mod-6d.c new file mode 100644 index 0000000000..eb233a1d10 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6d.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-6c.c" diff --git a/sysdeps/x86/tst-get-cpu-features.c b/sysdeps/x86/tst-get-cpu-features.c index b2fac197da..3b24e20d9c 100644 --- a/sysdeps/x86/tst-get-cpu-features.c +++ b/sysdeps/x86/tst-get-cpu-features.c @@ -1,5 +1,5 @@ /* Test case for x86 __get_cpu_features interface - Copyright (C) 2015-2018 Free Software Foundation, Inc. + Copyright (C) 2015-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -14,18 +14,274 @@ You should have received a copy of the GNU Lesser General Public License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ + <https://www.gnu.org/licenses/>. */ #include <stdlib.h> +#include <stdio.h> #include <cpu-features.h> +#include <support/check.h> + +#define CHECK_CPU_FEATURE(name) \ + { \ + if (HAS_CPU_FEATURE (name)) \ + printf (" " #name "\n"); \ + } + +#define CHECK_CPU_FEATURE_USABLE(name) \ + { \ + if (CPU_FEATURE_USABLE(name)) \ + printf (" " #name "\n"); \ + } + +static const char * const cpu_kinds[] = +{ + "Unknown", + "Intel", + "AMD", + "Other", +}; static int do_test (void) { - if (__get_cpu_features ()->kind == arch_kind_unknown) - abort (); + const struct cpu_features *cpu_features = __get_cpu_features (); + + switch (cpu_features->basic.kind) + { + case arch_kind_intel: + case arch_kind_amd: + case arch_kind_other: + printf ("Vendor: %s\n", cpu_kinds[cpu_features->basic.kind]); + printf ("Family: 0x%x\n", cpu_features->basic.family); + printf ("Model: 0x%x\n", cpu_features->basic.model); + printf ("Stepping: 0x%x\n", cpu_features->basic.stepping); + break; + + default: + abort (); + } + +#ifdef __SSE2__ + TEST_VERIFY_EXIT (HAS_CPU_FEATURE (SSE2)); +#endif + + printf ("CPU features:\n"); + CHECK_CPU_FEATURE (SSE3); + CHECK_CPU_FEATURE (PCLMULQDQ); + CHECK_CPU_FEATURE (DTES64); + CHECK_CPU_FEATURE (MONITOR); + CHECK_CPU_FEATURE (DS_CPL); + CHECK_CPU_FEATURE (VMX); + CHECK_CPU_FEATURE (SMX); + CHECK_CPU_FEATURE (EST); + CHECK_CPU_FEATURE (TM2); + CHECK_CPU_FEATURE (SSSE3); + CHECK_CPU_FEATURE (CNXT_ID); + CHECK_CPU_FEATURE (SDBG); + CHECK_CPU_FEATURE (FMA); + CHECK_CPU_FEATURE (CMPXCHG16B); + CHECK_CPU_FEATURE (XTPRUPDCTRL); + CHECK_CPU_FEATURE (PDCM); + CHECK_CPU_FEATURE (PCID); + CHECK_CPU_FEATURE (DCA); + CHECK_CPU_FEATURE (SSE4_1); + CHECK_CPU_FEATURE (SSE4_2); + CHECK_CPU_FEATURE (X2APIC); + CHECK_CPU_FEATURE (MOVBE); + CHECK_CPU_FEATURE (POPCNT); + CHECK_CPU_FEATURE (TSC_DEADLINE); + CHECK_CPU_FEATURE (AES); + CHECK_CPU_FEATURE (XSAVE); + CHECK_CPU_FEATURE (OSXSAVE); + CHECK_CPU_FEATURE (AVX); + CHECK_CPU_FEATURE (F16C); + CHECK_CPU_FEATURE (RDRAND); + CHECK_CPU_FEATURE (FPU); + CHECK_CPU_FEATURE (VME); + CHECK_CPU_FEATURE (DE); + CHECK_CPU_FEATURE (PSE); + CHECK_CPU_FEATURE (TSC); + CHECK_CPU_FEATURE (MSR); + CHECK_CPU_FEATURE (PAE); + CHECK_CPU_FEATURE (MCE); + CHECK_CPU_FEATURE (CX8); + CHECK_CPU_FEATURE (APIC); + CHECK_CPU_FEATURE (SEP); + CHECK_CPU_FEATURE (MTRR); + CHECK_CPU_FEATURE (PGE); + CHECK_CPU_FEATURE (MCA); + CHECK_CPU_FEATURE (CMOV); + CHECK_CPU_FEATURE (PAT); + CHECK_CPU_FEATURE (PSE_36); + CHECK_CPU_FEATURE (PSN); + CHECK_CPU_FEATURE (CLFSH); + CHECK_CPU_FEATURE (DS); + CHECK_CPU_FEATURE (ACPI); + CHECK_CPU_FEATURE (MMX); + CHECK_CPU_FEATURE (FXSR); + CHECK_CPU_FEATURE (SSE); + CHECK_CPU_FEATURE (SSE2); + CHECK_CPU_FEATURE (SS); + CHECK_CPU_FEATURE (HTT); + CHECK_CPU_FEATURE (TM); + CHECK_CPU_FEATURE (PBE); + CHECK_CPU_FEATURE (FSGSBASE); + CHECK_CPU_FEATURE (TSC_ADJUST); + CHECK_CPU_FEATURE (SGX); + CHECK_CPU_FEATURE (BMI1); + CHECK_CPU_FEATURE (HLE); + CHECK_CPU_FEATURE (AVX2); + CHECK_CPU_FEATURE (SMEP); + CHECK_CPU_FEATURE (BMI2); + CHECK_CPU_FEATURE (ERMS); + CHECK_CPU_FEATURE (INVPCID); + CHECK_CPU_FEATURE (RTM); + CHECK_CPU_FEATURE (PQM); + CHECK_CPU_FEATURE (MPX); + CHECK_CPU_FEATURE (PQE); + CHECK_CPU_FEATURE (AVX512F); + CHECK_CPU_FEATURE (AVX512DQ); + CHECK_CPU_FEATURE (RDSEED); + CHECK_CPU_FEATURE (ADX); + CHECK_CPU_FEATURE (SMAP); + CHECK_CPU_FEATURE (AVX512_IFMA); + CHECK_CPU_FEATURE (CLFLUSHOPT); + CHECK_CPU_FEATURE (CLWB); + CHECK_CPU_FEATURE (TRACE); + CHECK_CPU_FEATURE (AVX512PF); + CHECK_CPU_FEATURE (AVX512ER); + CHECK_CPU_FEATURE (AVX512CD); + CHECK_CPU_FEATURE (SHA); + CHECK_CPU_FEATURE (AVX512BW); + CHECK_CPU_FEATURE (AVX512VL); + CHECK_CPU_FEATURE (PREFETCHWT1); + CHECK_CPU_FEATURE (AVX512_VBMI); + CHECK_CPU_FEATURE (UMIP); + CHECK_CPU_FEATURE (PKU); + CHECK_CPU_FEATURE (OSPKE); + CHECK_CPU_FEATURE (WAITPKG); + CHECK_CPU_FEATURE (AVX512_VBMI2); + CHECK_CPU_FEATURE (SHSTK); + CHECK_CPU_FEATURE (GFNI); + CHECK_CPU_FEATURE (VAES); + CHECK_CPU_FEATURE (VPCLMULQDQ); + CHECK_CPU_FEATURE (AVX512_VNNI); + CHECK_CPU_FEATURE (AVX512_BITALG); + CHECK_CPU_FEATURE (AVX512_VPOPCNTDQ); + CHECK_CPU_FEATURE (RDPID); + CHECK_CPU_FEATURE (CLDEMOTE); + CHECK_CPU_FEATURE (MOVDIRI); + CHECK_CPU_FEATURE (MOVDIR64B); + CHECK_CPU_FEATURE (SGX_LC); + CHECK_CPU_FEATURE (AVX512_4VNNIW); + CHECK_CPU_FEATURE (AVX512_4FMAPS); + CHECK_CPU_FEATURE (FSRM); + CHECK_CPU_FEATURE (PCONFIG); + CHECK_CPU_FEATURE (IBT); + CHECK_CPU_FEATURE (IBRS_IBPB); + CHECK_CPU_FEATURE (STIBP); + CHECK_CPU_FEATURE (CAPABILITIES); + CHECK_CPU_FEATURE (SSBD); + CHECK_CPU_FEATURE (LAHF64_SAHF64); + CHECK_CPU_FEATURE (SVM); + CHECK_CPU_FEATURE (LZCNT); + CHECK_CPU_FEATURE (SSE4A); + CHECK_CPU_FEATURE (PREFETCHW); + CHECK_CPU_FEATURE (XOP); + CHECK_CPU_FEATURE (LWP); + CHECK_CPU_FEATURE (FMA4); + CHECK_CPU_FEATURE (TBM); + CHECK_CPU_FEATURE (SYSCALL_SYSRET); + CHECK_CPU_FEATURE (NX); + CHECK_CPU_FEATURE (PAGE1GB); + CHECK_CPU_FEATURE (RDTSCP); + CHECK_CPU_FEATURE (LM); + CHECK_CPU_FEATURE (XSAVEOPT); + CHECK_CPU_FEATURE (XSAVEC); + CHECK_CPU_FEATURE (XGETBV_ECX_1); + CHECK_CPU_FEATURE (XSAVES); + CHECK_CPU_FEATURE (INVARIANT_TSC); + CHECK_CPU_FEATURE (WBNOINVD); + + printf ("Usable CPU features:\n"); + CHECK_CPU_FEATURE_USABLE (SSE3); + CHECK_CPU_FEATURE_USABLE (PCLMULQDQ); + CHECK_CPU_FEATURE_USABLE (SSSE3); + CHECK_CPU_FEATURE_USABLE (FMA); + CHECK_CPU_FEATURE_USABLE (CMPXCHG16B); + CHECK_CPU_FEATURE_USABLE (SSE4_1); + CHECK_CPU_FEATURE_USABLE (SSE4_2); + CHECK_CPU_FEATURE_USABLE (MOVBE); + CHECK_CPU_FEATURE_USABLE (POPCNT); + CHECK_CPU_FEATURE_USABLE (AES); + CHECK_CPU_FEATURE_USABLE (XSAVE); + CHECK_CPU_FEATURE_USABLE (OSXSAVE); + CHECK_CPU_FEATURE_USABLE (AVX); + CHECK_CPU_FEATURE_USABLE (F16C); + CHECK_CPU_FEATURE_USABLE (RDRAND); + CHECK_CPU_FEATURE_USABLE (FPU); + CHECK_CPU_FEATURE_USABLE (TSC); + CHECK_CPU_FEATURE_USABLE (MSR); + CHECK_CPU_FEATURE_USABLE (CX8); + CHECK_CPU_FEATURE_USABLE (SEP); + CHECK_CPU_FEATURE_USABLE (CMOV); + CHECK_CPU_FEATURE_USABLE (CLFSH); + CHECK_CPU_FEATURE_USABLE (MMX); + CHECK_CPU_FEATURE_USABLE (FXSR); + CHECK_CPU_FEATURE_USABLE (SSE); + CHECK_CPU_FEATURE_USABLE (SSE2); + CHECK_CPU_FEATURE_USABLE (FSGSBASE); + CHECK_CPU_FEATURE_USABLE (BMI1); + CHECK_CPU_FEATURE_USABLE (HLE); + CHECK_CPU_FEATURE_USABLE (AVX2); + CHECK_CPU_FEATURE_USABLE (BMI2); + CHECK_CPU_FEATURE_USABLE (ERMS); + CHECK_CPU_FEATURE_USABLE (AVX512F); + CHECK_CPU_FEATURE_USABLE (AVX512DQ); + CHECK_CPU_FEATURE_USABLE (RDSEED); + CHECK_CPU_FEATURE_USABLE (ADX); + CHECK_CPU_FEATURE_USABLE (AVX512_IFMA); + CHECK_CPU_FEATURE_USABLE (CLFLUSHOPT); + CHECK_CPU_FEATURE_USABLE (CLWB); + CHECK_CPU_FEATURE_USABLE (AVX512PF); + CHECK_CPU_FEATURE_USABLE (AVX512ER); + CHECK_CPU_FEATURE_USABLE (AVX512CD); + CHECK_CPU_FEATURE_USABLE (SHA); + CHECK_CPU_FEATURE_USABLE (AVX512BW); + CHECK_CPU_FEATURE_USABLE (AVX512VL); + CHECK_CPU_FEATURE_USABLE (PREFETCHWT1); + CHECK_CPU_FEATURE_USABLE (AVX512_VBMI); + CHECK_CPU_FEATURE_USABLE (AVX512_VBMI2); + CHECK_CPU_FEATURE_USABLE (GFNI); + CHECK_CPU_FEATURE_USABLE (VAES); + CHECK_CPU_FEATURE_USABLE (VPCLMULQDQ); + CHECK_CPU_FEATURE_USABLE (AVX512_VNNI); + CHECK_CPU_FEATURE_USABLE (AVX512_BITALG); + CHECK_CPU_FEATURE_USABLE (AVX512_VPOPCNTDQ); + CHECK_CPU_FEATURE_USABLE (RDPID); + CHECK_CPU_FEATURE_USABLE (CLDEMOTE); + CHECK_CPU_FEATURE_USABLE (MOVDIRI); + CHECK_CPU_FEATURE_USABLE (MOVDIR64B); + CHECK_CPU_FEATURE_USABLE (AVX512_4VNNIW); + CHECK_CPU_FEATURE_USABLE (AVX512_4FMAPS); + CHECK_CPU_FEATURE_USABLE (FSRM); + CHECK_CPU_FEATURE_USABLE (LAHF64_SAHF64); + CHECK_CPU_FEATURE_USABLE (LZCNT); + CHECK_CPU_FEATURE_USABLE (SSE4A); + CHECK_CPU_FEATURE_USABLE (PREFETCHW); + CHECK_CPU_FEATURE_USABLE (XOP); + CHECK_CPU_FEATURE_USABLE (FMA4); + CHECK_CPU_FEATURE_USABLE (TBM); + CHECK_CPU_FEATURE_USABLE (SYSCALL_SYSRET); + CHECK_CPU_FEATURE_USABLE (RDTSCP); + CHECK_CPU_FEATURE_USABLE (XSAVEOPT); + CHECK_CPU_FEATURE_USABLE (XSAVEC); + CHECK_CPU_FEATURE_USABLE (XGETBV_ECX_1); + CHECK_CPU_FEATURE_USABLE (XSAVES); + CHECK_CPU_FEATURE_USABLE (INVARIANT_TSC); + CHECK_CPU_FEATURE_USABLE (WBNOINVD); + return 0; } -#define TEST_FUNCTION do_test () -#include "../../test-skeleton.c" +#include <support/test-driver.c> |