summaryrefslogtreecommitdiff
path: root/kern/atomic_i.h
diff options
context:
space:
mode:
Diffstat (limited to 'kern/atomic_i.h')
-rw-r--r--kern/atomic_i.h535
1 files changed, 535 insertions, 0 deletions
diff --git a/kern/atomic_i.h b/kern/atomic_i.h
new file mode 100644
index 0000000..03b5a32
--- /dev/null
+++ b/kern/atomic_i.h
@@ -0,0 +1,535 @@
+/*
+ * Copyright (c) 2018 Richard Braun.
+ * Copyright (c) 2017 Agustina Arzille.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Architecture-specific code may override any of the type-specific
+ * functions by defining the macro of the same name.
+ */
+
+#ifndef KERN_ATOMIC_I_H
+#define KERN_ATOMIC_I_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include <kern/atomic_types.h>
+#include <kern/macros.h>
+#include <machine/atomic.h>
+
+#define ATOMIC_ALIGN(ptr) MIN(sizeof(*(ptr)), sizeof(ptr))
+#define atomic_ptr_aligned(ptr) P2ALIGNED((uintptr_t)(ptr), ATOMIC_ALIGN(ptr))
+
+/*
+ * This macro is used to select the appropriate function for the given
+ * operation. The default expression is selected for pointer types.
+ * In order to avoid confusing errors, all built-in types are explicitely
+ * listed, so that unsupported ones don't select pointer operations.
+ * Instead, they select a function with an explicit name indicating
+ * an invalid type.
+ */
+#define atomic_select(ptr, op) \
+_Generic(*(ptr), \
+ float: atomic_invalid_type, \
+ double: atomic_invalid_type, \
+ long double: atomic_invalid_type, \
+ bool: atomic_invalid_type, \
+ char: atomic_invalid_type, \
+ signed char: atomic_invalid_type, \
+ unsigned char: atomic_invalid_type, \
+ short: atomic_invalid_type, \
+ unsigned short: atomic_invalid_type, \
+ int: atomic_ ## op ## _32, \
+ unsigned int: atomic_ ## op ## _32, \
+ long: atomic_ ## op ## _ul, \
+ unsigned long: atomic_ ## op ## _ul, \
+ long long: atomic_ ## op ## _64, \
+ unsigned long long: atomic_ ## op ## _64, \
+ default: atomic_ ## op ## _ptr)
+
+void atomic_invalid_type(void);
+
+/* atomic_load */
+
+#ifndef atomic_load_32
+static inline unsigned int
+atomic_load_32(union atomic_constptr_32 ptr, int memorder)
+{
+ return __atomic_load_n(ptr.ui_ptr, memorder);
+}
+#endif /* atomic_load_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_load_64
+static inline unsigned long long
+atomic_load_64(union atomic_constptr_64 ptr, int memorder)
+{
+ return __atomic_load_n(ptr.ull_ptr, memorder);
+}
+#endif /* atomic_load_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_load_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_load_ul atomic_load_64
+#else /* __LP64__ */
+#define atomic_load_ul atomic_load_32
+#endif /* __LP64__ */
+
+#define atomic_load_ptr atomic_load_ul
+
+/* atomic_store */
+
+#ifndef atomic_store_32
+static inline void
+atomic_store_32(union atomic_ptr_32 ptr, union atomic_val32 val, int memorder)
+{
+ return __atomic_store_n(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_store_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_store_64
+static inline void
+atomic_store_64(union atomic_ptr_64 ptr, union atomic_val_64 val, int memorder)
+{
+ return __atomic_store_n(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_store_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_store_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_store_ul atomic_store_64
+#else /* __LP64__ */
+#define atomic_store_ul atomic_store_32
+#endif /* __LP64__ */
+
+#define atomic_store_ptr atomic_store_ul
+
+/* atomic_cas */
+
+#define atomic_cas_n(ptr, oval, nval, memorder) \
+MACRO_BEGIN \
+ typeof(*(ptr)) oval_; \
+ \
+ oval_ = (oval); \
+ __atomic_compare_exchange_n(ptr, &oval_, (nval), false, \
+ memorder, __ATOMIC_RELAXED); \
+ oval_; \
+MACRO_END
+
+#ifndef atomic_cas_32
+static inline unsigned int
+atomic_cas_32(union atomic_ptr_32 ptr, union atomic_val32 oval,
+ union atomic_val32 nval, int memorder)
+{
+ return atomic_cas_n(ptr.ui_ptr, oval.ui, nval.ui, memorder);
+}
+#endif /* atomic_cas_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_cas_64
+static inline unsigned long long
+atomic_cas_64(union atomic_ptr_64 ptr, union atomic_val_64 oval,
+ union atomic_val_64 nval, int memorder)
+{
+ return atomic_cas_n(ptr.ull_ptr, oval.ull, nval.ull, memorder);
+}
+#endif /* atomic_cas_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_cas_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_cas_ul atomic_cas_64
+#else /* __LP64__ */
+#define atomic_cas_ul atomic_cas_32
+#endif /* __LP64__ */
+
+#define atomic_cas_ptr atomic_cas_ul
+
+/* atomic_swap */
+
+#ifndef atomic_swap_32
+static inline unsigned int
+atomic_swap_32(union atomic_ptr_32 ptr, union atomic_val32 val, int memorder)
+{
+ return __atomic_exchange_n(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_swap_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_swap_64
+static inline unsigned long long
+atomic_swap_64(union atomic_ptr_64 ptr, union atomic_val_64 val, int memorder)
+{
+ return __atomic_exchange_n(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_swap_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_swap_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_swap_ul atomic_swap_64
+#else /* __LP64__ */
+#define atomic_swap_ul atomic_swap_32
+#endif /* __LP64__ */
+
+#define atomic_swap_ptr atomic_swap_ul
+
+/* atomic_fetch_add */
+
+#ifndef atomic_fetch_add_32
+static inline unsigned int
+atomic_fetch_add_32(union atomic_ptr_32 ptr, union atomic_val32 val,
+ int memorder)
+{
+ return __atomic_fetch_add(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_fetch_add_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_fetch_add_64
+static inline unsigned long long
+atomic_fetch_add_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
+ int memorder)
+{
+ return __atomic_fetch_add(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_fetch_add_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_fetch_add_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_fetch_add_ul atomic_fetch_add_64
+#else /* __LP64__ */
+#define atomic_fetch_add_ul atomic_fetch_add_32
+#endif /* __LP64__ */
+
+#define atomic_fetch_add_ptr atomic_fetch_add_ul
+
+/* atomic_fetch_sub */
+
+#ifndef atomic_fetch_sub_32
+static inline unsigned int
+atomic_fetch_sub_32(union atomic_ptr_32 ptr, union atomic_val32 val,
+ int memorder)
+{
+ return __atomic_fetch_sub(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_fetch_sub_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_fetch_sub_64
+static inline unsigned long long
+atomic_fetch_sub_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
+ int memorder)
+{
+ return __atomic_fetch_sub(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_fetch_sub_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_fetch_sub_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_fetch_sub_ul atomic_fetch_sub_64
+#else /* __LP64__ */
+#define atomic_fetch_sub_ul atomic_fetch_sub_32
+#endif /* __LP64__ */
+
+#define atomic_fetch_sub_ptr atomic_fetch_sub_ul
+
+/* atomic_fetch_and */
+
+#ifndef atomic_fetch_and_32
+static inline unsigned int
+atomic_fetch_and_32(union atomic_ptr_32 ptr, union atomic_val32 val,
+ int memorder)
+{
+ return __atomic_fetch_and(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_fetch_and_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_fetch_and_64
+static inline unsigned long long
+atomic_fetch_and_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
+ int memorder)
+{
+ return __atomic_fetch_and(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_fetch_and_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_fetch_and_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_fetch_and_ul atomic_fetch_and_64
+#else /* __LP64__ */
+#define atomic_fetch_and_ul atomic_fetch_and_32
+#endif /* __LP64__ */
+
+#define atomic_fetch_and_ptr atomic_fetch_and_ul
+
+/* atomic_fetch_or */
+
+#ifndef atomic_fetch_or_32
+static inline unsigned int
+atomic_fetch_or_32(union atomic_ptr_32 ptr, union atomic_val32 val,
+ int memorder)
+{
+ return __atomic_fetch_or(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_fetch_or_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_fetch_or_64
+static inline unsigned long long
+atomic_fetch_or_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
+ int memorder)
+{
+ return __atomic_fetch_or(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_fetch_or_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_or_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_fetch_or_ul atomic_fetch_or_64
+#else /* __LP64__ */
+#define atomic_fetch_or_ul atomic_fetch_or_32
+#endif /* __LP64__ */
+
+#define atomic_fetch_or_ptr atomic_fetch_or_ul
+
+/* atomic_fetch_xor */
+
+#ifndef atomic_fetch_xor_32
+static inline unsigned int
+atomic_fetch_xor_32(union atomic_ptr_32 ptr, union atomic_val32 val,
+ int memorder)
+{
+ return __atomic_fetch_xor(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_fetch_xor_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_fetch_xor_64
+static inline unsigned long long
+atomic_fetch_xor_64(union atomic_ptr_64 ptr, union atomic_val_64 val,
+ int memorder)
+{
+ return __atomic_fetch_xor(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_fetch_xor_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_xor_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_fetch_xor_ul atomic_fetch_xor_64
+#else /* __LP64__ */
+#define atomic_fetch_xor_ul atomic_fetch_xor_32
+#endif /* __LP64__ */
+
+#define atomic_fetch_xor_ptr atomic_fetch_xor_ul
+
+/* atomic_add */
+
+#ifndef atomic_add_32
+static inline void
+atomic_add_32(union atomic_ptr_32 ptr, union atomic_val32 val, int memorder)
+{
+ __atomic_add_fetch(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_add_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_add_64
+static inline void
+atomic_add_64(union atomic_ptr_64 ptr, union atomic_val_64 val, int memorder)
+{
+ __atomic_add_fetch(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_add_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_add_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_add_ul atomic_add_64
+#else /* __LP64__ */
+#define atomic_add_ul atomic_add_32
+#endif /* __LP64__ */
+
+#define atomic_add_ptr atomic_add_ul
+
+/* atomic_sub */
+
+#ifndef atomic_sub_32
+static inline void
+atomic_sub_32(union atomic_ptr_32 ptr, union atomic_val32 val, int memorder)
+{
+ __atomic_sub_fetch(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_sub_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_sub_64
+static inline void
+atomic_sub_64(union atomic_ptr_64 ptr, union atomic_val_64 val, int memorder)
+{
+ __atomic_sub_fetch(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_sub_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_sub_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_sub_ul atomic_sub_64
+#else /* __LP64__ */
+#define atomic_sub_ul atomic_sub_32
+#endif /* __LP64__ */
+
+#define atomic_sub_ptr atomic_sub_ul
+
+/* atomic_and */
+
+#ifndef atomic_and_32
+static inline void
+atomic_and_32(union atomic_ptr_32 ptr, union atomic_val32 val, int memorder)
+{
+ __atomic_and_fetch(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_and_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_and_64
+static inline void
+atomic_and_64(union atomic_ptr_64 ptr, union atomic_val_64 val, int memorder)
+{
+ __atomic_and_fetch(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_and_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_and_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_and_ul atomic_and_64
+#else /* __LP64__ */
+#define atomic_and_ul atomic_and_32
+#endif /* __LP64__ */
+
+#define atomic_and_ptr atomic_and_ul
+
+/* atomic_or */
+
+#ifndef atomic_or_32
+static inline void
+atomic_or_32(union atomic_ptr_32 ptr, union atomic_val32 val, int memorder)
+{
+ __atomic_or_fetch(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_or_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_or_64
+static inline void
+atomic_or_64(union atomic_ptr_64 ptr, union atomic_val_64 val, int memorder)
+{
+ __atomic_or_fetch(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_or_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_or_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_or_ul atomic_or_64
+#else /* __LP64__ */
+#define atomic_or_ul atomic_or_32
+#endif /* __LP64__ */
+
+#define atomic_or_ptr atomic_or_ul
+
+/* atomic_xor */
+
+#ifndef atomic_xor_32
+static inline void
+atomic_xor_32(union atomic_ptr_32 ptr, union atomic_val32 val, int memorder)
+{
+ __atomic_xor_fetch(ptr.ui_ptr, val.ui, memorder);
+}
+#endif /* atomic_xor_32 */
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+#ifndef atomic_xor_64
+static inline void
+atomic_xor_64(union atomic_ptr_64 ptr, union atomic_val_64 val, int memorder)
+{
+ __atomic_xor_fetch(ptr.ull_ptr, val.ull, memorder);
+}
+#endif /* atomic_xor_64 */
+
+#else /* ATOMIC_HAVE_64B_OPS */
+#define atomic_xor_64 atomic_invalid_type
+#endif /* ATOMIC_HAVE_64B_OPS */
+
+#ifdef __LP64__
+#define atomic_xor_ul atomic_xor_64
+#else /* __LP64__ */
+#define atomic_xor_ul atomic_xor_32
+#endif /* __LP64__ */
+
+#define atomic_xor_ptr atomic_xor_ul
+
+#endif /* KERN_ATOMIC_I_H */
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-24 21:55:32 +0000