kern/mutex: new adaptive spinning mutex implementation

3 files changed, 293 insertions, 0 deletions

diff --git a/kern/mutex/mutex_adaptive.c b/kern/mutex/mutex_adaptive.c
new file mode 100644
index 00000000..f4274ce1
--- /dev/null
+++ b/kern/mutex/mutex_adaptive.c
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2017 Agustina Arzille.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <kern/atomic.h>
+#include <kern/mutex.h>
+#include <kern/mutex_types.h>
+#include <kern/sleepq.h>
+#include <kern/thread.h>
+#include <machine/cpu.h>
+
+static struct thread *
+mutex_adaptive_get_thread(uintptr_t owner)
+{
+    return (struct thread *)(owner & ~MUTEX_ADAPTIVE_CONTENDED);
+}
+
+static void
+mutex_adaptive_set_contended(struct mutex *mutex)
+{
+    atomic_or_acq_rel(&mutex->owner, MUTEX_ADAPTIVE_CONTENDED);
+}
+
+static inline bool
+mutex_adaptive_is_owner(struct mutex *mutex, uintptr_t owner)
+{
+    uintptr_t prev;
+
+    prev = atomic_load(&mutex->owner, ATOMIC_RELAXED);
+    return mutex_adaptive_get_thread(prev) == mutex_adaptive_get_thread(owner);
+}
+
+void
+mutex_adaptive_lock_slow(struct mutex *mutex)
+{
+    uintptr_t self, owner;
+    struct sleepq *sleepq;
+    unsigned long flags;
+
+    self = (uintptr_t)thread_self();
+
+    sleepq = sleepq_lend(mutex, false, &flags);
+
+    mutex_adaptive_set_contended(mutex);
+
+    for (;;) {
+        owner = atomic_cas_acquire(&mutex->owner, MUTEX_ADAPTIVE_CONTENDED,
+                                   self | MUTEX_ADAPTIVE_CONTENDED);
+        assert(owner & MUTEX_ADAPTIVE_CONTENDED);
+
+        if (mutex_adaptive_get_thread(owner) == NULL) {
+            break;
+        }
+
+        /*
+         * The owner may not return from the unlock function if a thread is
+         * spinning on it.
+         */
+        while (mutex_adaptive_is_owner(mutex, owner)) {
+            if (thread_is_running(mutex_adaptive_get_thread(owner))) {
+                cpu_pause();
+            } else {
+                sleepq_wait(sleepq, "mutex");
+            }
+        }
+    }
+
+    /*
+     * A potentially spinning thread wouldn't be accounted in the sleep queue,
+     * but the only potentially spinning thread is the new owner.
+     */
+    if (sleepq_empty(sleepq)) {
+        atomic_store(&mutex->owner, self, ATOMIC_RELAXED);
+    }
+
+    sleepq_return(sleepq, flags);
+}
+
+void
+mutex_adaptive_unlock_slow(struct mutex *mutex)
+{
+    uintptr_t owner;
+    struct sleepq *sleepq;
+    unsigned long flags;
+
+    atomic_store(&mutex->owner, MUTEX_ADAPTIVE_CONTENDED, ATOMIC_RELEASE);
+
+    for (;;) {
+        owner = atomic_load(&mutex->owner, ATOMIC_RELAXED);
+
+        /*
+         * This only happens if another thread was able to become the new
+         * owner, in which case that thread isn't spinning on the current
+         * thread, i.e. there is no need for an additional reference.
+         */
+        if (owner != MUTEX_ADAPTIVE_CONTENDED) {
+            break;
+        }
+
+        /*
+         * Avoid contending with incoming threads that are about to spin/wait
+         * on the mutex. This is particularly expensive with queued locks.
+         *
+         * Also, this call returns NULL if another thread is currently spinning
+         * on the current thread, in which case the latter doesn't return,
+         * averting the need for an additional reference.
+         */
+        sleepq = sleepq_tryacquire(mutex, false, &flags);
+
+        if (sleepq != NULL) {
+            sleepq_signal(sleepq);
+            sleepq_release(sleepq, flags);
+            break;
+        }
+
+        /*
+         * Acquiring the sleep queue may fail because of contention on
+         * unrelated objects. Retry.
+         */
+    }
+}
diff --git a/kern/mutex/mutex_adaptive_i.h b/kern/mutex/mutex_adaptive_i.h
new file mode 100644
index 00000000..2dd06792
--- /dev/null
+++ b/kern/mutex/mutex_adaptive_i.h
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2017 Agustina Arzille.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _KERN_MUTEX_ADAPTIVE_I_H
+#define _KERN_MUTEX_ADAPTIVE_I_H
+
+#ifndef _KERN_MUTEX_H
+#error "don't include <kern/mutex/mutex_adaptive_i.h> directly," \
+       " use <kern/mutex.h> instead"
+#endif
+
+#include <stdint.h>
+
+#include <kern/assert.h>
+#include <kern/atomic.h>
+#include <kern/error.h>
+#include <kern/macros.h>
+#include <kern/mutex_types.h>
+#include <kern/thread.h>
+
+/*
+ * Mutex flags.
+ *
+ * The "contended" flag indicates that threads are waiting for the mutex
+ * to be unlocked, potentially spinning on the owner. It forces threads
+ * trying to lock the mutex as well as the owner to take the slow path.
+ */
+#define MUTEX_ADAPTIVE_CONTENDED 0x1
+
+static inline void
+mutex_adaptive_init(struct mutex *mutex)
+{
+    mutex->owner = 0;
+}
+
+#define mutex_adaptive_assert_locked(mutex) assert((mutex)->owner != 0)
+
+static inline int
+mutex_adaptive_lock_fast(struct mutex *mutex)
+{
+    uintptr_t owner;
+
+    owner = atomic_cas_acquire(&mutex->owner, 0, (uintptr_t)thread_self());
+
+    if (unlikely(owner != 0)) {
+        return ERROR_BUSY;
+    }
+
+    return 0;
+}
+
+static inline int
+mutex_adaptive_unlock_fast(struct mutex *mutex)
+{
+    uintptr_t owner;
+
+    owner = atomic_cas_release(&mutex->owner, (uintptr_t)thread_self(), 0);
+
+    if (unlikely(owner & MUTEX_ADAPTIVE_CONTENDED)) {
+        return ERROR_BUSY;
+    }
+
+    return 0;
+}
+
+void mutex_adaptive_lock_slow(struct mutex *mutex);
+void mutex_adaptive_unlock_slow(struct mutex *mutex);
+
+/*
+ * Interface exported to the public mutex header.
+ */
+
+#define mutex_impl_init             mutex_adaptive_init
+#define mutex_impl_assert_locked    mutex_adaptive_assert_locked
+
+static inline int
+mutex_impl_trylock(struct mutex *mutex)
+{
+    return mutex_adaptive_lock_fast(mutex);
+}
+
+static inline void
+mutex_impl_lock(struct mutex *mutex)
+{
+    int error;
+
+    error = mutex_adaptive_lock_fast(mutex);
+
+    if (unlikely(error)) {
+        mutex_adaptive_lock_slow(mutex);
+    }
+}
+
+static inline void
+mutex_impl_unlock(struct mutex *mutex)
+{
+    int error;
+
+    error = mutex_adaptive_unlock_fast(mutex);
+
+    if (unlikely(error)) {
+        mutex_adaptive_unlock_slow(mutex);
+    }
+}
+
+#endif /* _KERN_MUTEX_ADAPTIVE_I_H */
diff --git a/kern/mutex/mutex_adaptive_types.h b/kern/mutex/mutex_adaptive_types.h
new file mode 100644
index 00000000..efbbf218
--- /dev/null
+++ b/kern/mutex/mutex_adaptive_types.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2017 Agustina Arzille.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Isolated type definition used to avoid inclusion circular dependencies.
+ */
+
+#ifndef _KERN_MUTEX_ADAPTIVE_TYPES_H
+#define _KERN_MUTEX_ADAPTIVE_TYPES_H
+
+#ifndef _KERN_MUTEX_TYPES_H
+#error "don't include <kern/mutex/mutex_adaptive_types.h> directly," \
+       " use <kern/mutex_types.h> instead"
+#endif
+
+#include <stdint.h>
+
+struct mutex {
+    uintptr_t owner;
+};
+
+#endif /* _KERN_MUTEX_ADAPTIVE_TYPES_H */