diff options
Diffstat (limited to 'arch/x86/machine/atomic.h')
-rw-r--r-- | arch/x86/machine/atomic.h | 109 |
1 files changed, 50 insertions, 59 deletions
diff --git a/arch/x86/machine/atomic.h b/arch/x86/machine/atomic.h index 36f0067..e10d301 100644 --- a/arch/x86/machine/atomic.h +++ b/arch/x86/machine/atomic.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2017 Richard Braun. + * Copyright (c) 2012-2018 Richard Braun. * Copyright (c) 2017 Agustina Arzille. * * This program is free software: you can redistribute it and/or modify @@ -27,78 +27,69 @@ #endif #include <stdbool.h> -#include <stdint.h> +#include <kern/atomic_types.h> #include <kern/macros.h> -#ifndef __LP64__ +#ifdef __LP64__ -/* - * On i386, the compiler generates either an FP-stack read/write, or an SSE2 - * store/load to implement these 64-bit atomic operations. Since that's not - * feasible in the kernel, fall back to cmpxchg8b. Note that, in this case, - * loading becomes a potentially mutating operation, but it's not expected - * to be a problem since atomic operations are normally not used on read-only - * memory. Also note that this assumes the processor is at least an i586. - */ +/* Report that 64-bits operations are supported */ +#define ATOMIC_HAVE_64B_OPS -/* - * Temporarily discard qualifiers when loading 64-bits values with a - * compare-and-swap operation. - */ -#define atomic_load_64(ptr, mo) \ -MACRO_BEGIN \ - uint64_t ret_ = 0; \ - \ - __atomic_compare_exchange_n((uint64_t *)(ptr), &ret_, 0, \ - false, mo, __ATOMIC_RELAXED); \ - ret_; \ -MACRO_END - -#define atomic_load(ptr, mo) \ - (typeof(*(ptr)))__builtin_choose_expr(sizeof(*(ptr)) == 8, \ - atomic_load_64(ptr, mo), \ - __atomic_load_n(ptr, mo)) - -#define atomic_store(ptr, val, mo) \ -MACRO_BEGIN \ - if (sizeof(*(ptr)) != 8) { \ - __atomic_store_n(ptr, val, mo); \ - } else { \ - typeof(*(ptr)) oval_, nval_; \ - bool done_; \ - \ - oval_ = *(ptr); \ - nval_ = (val); \ - \ - do { \ - done_ = __atomic_compare_exchange_n(ptr, &oval_, nval_, \ - false, mo, \ - __ATOMIC_RELAXED); \ - } while (!done_); \ - \ - } \ -MACRO_END +#else /* __LP64__ */ /* - * Report that load and store are architecture-specific. + * XXX Clang doesn't provide any __atomic_xxx_8 functions on i386. */ -#define ATOMIC_ARCH_SPECIFIC_LOAD -#define ATOMIC_ARCH_SPECIFIC_STORE +#ifndef __clang__ -#endif /* __LP64__ */ +/* Report that 64-bits operations are supported */ +#define ATOMIC_HAVE_64B_OPS /* - * XXX Clang seems to have trouble with 64-bits operations on 32-bits - * processors. + * On i386, GCC generates either an FP-stack read/write, or an SSE2 + * store/load to implement these 64-bit atomic operations. Since that's not + * feasible in the kernel, fall back to cmpxchg8b. + * + * XXX Note that, in this case, loading becomes a potentially mutating + * operation, but it's not expected to be a problem since atomic operations + * are normally not used on read-only memory. + * + * Also note that this assumes the processor is at least an i586. */ -#if defined(__LP64__) || !defined(__clang__) -/* - * Report that 64-bits operations are supported. - */ -#define ATOMIC_HAVE_64B_OPS +static inline unsigned long long +atomic_i386_load_64(union atomic_constptr_64 ptr, int memorder) +{ + unsigned long long prev; + + prev = 0; + __atomic_compare_exchange_n((unsigned long long *)(ptr.ull_ptr), + &prev, 0, false, memorder, __ATOMIC_RELAXED); + return prev; +} + +#define atomic_load_64 atomic_i386_load_64 + +static inline void +atomic_i386_store_64(union atomic_ptr_64 ptr, union atomic_val_64 val, + int memorder) +{ + unsigned long long prev; + bool done; + + prev = *ptr.ull_ptr; + + do { + done = __atomic_compare_exchange_n(ptr.ull_ptr, &prev, val.ull, + false, memorder, __ATOMIC_RELAXED); + } while (!done); +} + +#define atomic_store_64 atomic_i386_store_64 #endif /* __clang__ */ +#endif /* __LP64__ */ + #endif /* X86_ATOMIC_H */ |