kern/sleepq: also disable interrupts during critical sections

5 files changed, 42 insertions, 30 deletions

diff --git a/kern/condition.c b/kern/condition.c
index 7440fa36..4a837f4e 100644
--- a/kern/condition.c
+++ b/kern/condition.c
@@ -73,6 +73,7 @@ condition_wait(struct condition *condition, struct mutex *mutex)
 {
     struct condition *last_cond;
     struct sleepq *sleepq;
+    unsigned long flags;
 
     mutex_assert_locked(mutex);
 
@@ -94,7 +95,7 @@ condition_wait(struct condition *condition, struct mutex *mutex)
      */
     last_cond = thread_pull_last_cond();
 
-    sleepq = sleepq_lend(condition, true);
+    sleepq = sleepq_lend(condition, true, &flags);
 
     mutex_unlock(mutex);
 
@@ -114,7 +115,7 @@ condition_wait(struct condition *condition, struct mutex *mutex)
         thread_set_last_cond(condition);
     }
 
-    sleepq_return(sleepq);
+    sleepq_return(sleepq, flags);
 
     mutex_lock(mutex);
 }
@@ -123,8 +124,9 @@ void
 condition_signal(struct condition *condition)
 {
     struct sleepq *sleepq;
+    unsigned long flags;
 
-    sleepq = sleepq_acquire(condition, true);
+    sleepq = sleepq_acquire(condition, true, &flags);
 
     if (sleepq == NULL) {
         return;
@@ -140,15 +142,16 @@ condition_signal(struct condition *condition)
     condition_inc_nr_pending_waiters(condition);
 
 out:
-    sleepq_release(sleepq);
+    sleepq_release(sleepq, flags);
 }
 
 void
 condition_broadcast(struct condition *condition)
 {
     struct sleepq *sleepq;
+    unsigned long flags;
 
-    sleepq = sleepq_acquire(condition, true);
+    sleepq = sleepq_acquire(condition, true, &flags);
 
     if (sleepq == NULL) {
         return;
@@ -163,15 +166,16 @@ condition_broadcast(struct condition *condition)
     condition_move_waiters(condition);
 
 out:
-    sleepq_release(sleepq);
+    sleepq_release(sleepq, flags);
 }
 
 void
 condition_wakeup(struct condition *condition)
 {
     struct sleepq *sleepq;
+    unsigned long flags;
 
-    sleepq = sleepq_acquire(condition, true);
+    sleepq = sleepq_acquire(condition, true, &flags);
 
     if (sleepq == NULL) {
         return;
@@ -189,5 +193,5 @@ condition_wakeup(struct condition *condition)
     sleepq_signal(sleepq);
 
 out:
-    sleepq_release(sleepq);
+    sleepq_release(sleepq, flags);
 }
diff --git a/kern/mutex.c b/kern/mutex.c
index b193aedb..353d94a7 100644
--- a/kern/mutex.c
+++ b/kern/mutex.c
@@ -28,9 +28,10 @@ void
 mutex_lock_slow(struct mutex *mutex)
 {
     struct sleepq *sleepq;
+    unsigned long flags;
     unsigned int state;
 
-    sleepq = sleepq_lend(mutex, false);
+    sleepq = sleepq_lend(mutex, false, &flags);
 
     for (;;) {
         state = atomic_swap_uint(&mutex->state, MUTEX_CONTENDED);
@@ -47,15 +48,16 @@ mutex_lock_slow(struct mutex *mutex)
         assert(state == MUTEX_CONTENDED);
     }
 
-    sleepq_return(sleepq);
+    sleepq_return(sleepq, flags);
 }
 
 void
 mutex_unlock_slow(struct mutex *mutex)
 {
     struct sleepq *sleepq;
+    unsigned long flags;
 
-    sleepq = sleepq_acquire(mutex, false);
+    sleepq = sleepq_acquire(mutex, false, &flags);
 
     if (sleepq == NULL) {
         return;
@@ -63,7 +65,7 @@ mutex_unlock_slow(struct mutex *mutex)
 
     sleepq_signal(sleepq);
 
-    sleepq_release(sleepq);
+    sleepq_release(sleepq, flags);
 }
 
 #endif /* X15_MUTEX_PI */
diff --git a/kern/semaphore.c b/kern/semaphore.c
index e41e2c2f..7e94dafd 100644
--- a/kern/semaphore.c
+++ b/kern/semaphore.c
@@ -26,9 +26,10 @@ void
 semaphore_wait_slow(struct semaphore *semaphore)
 {
     struct sleepq *sleepq;
+    unsigned long flags;
     unsigned int prev;
 
-    sleepq = sleepq_lend(semaphore, false);
+    sleepq = sleepq_lend(semaphore, false, &flags);
 
     for (;;) {
         prev = semaphore_dec(semaphore);
@@ -40,15 +41,16 @@ semaphore_wait_slow(struct semaphore *semaphore)
         sleepq_wait(sleepq, "sem");
     }
 
-    sleepq_return(sleepq);
+    sleepq_return(sleepq, flags);
 }
 
 void
 semaphore_post_slow(struct semaphore *semaphore)
 {
     struct sleepq *sleepq;
+    unsigned long flags;
 
-    sleepq = sleepq_acquire(semaphore, false);
+    sleepq = sleepq_acquire(semaphore, false, &flags);
 
     if (sleepq == NULL) {
         return;
@@ -56,5 +58,5 @@ semaphore_post_slow(struct semaphore *semaphore)
 
     sleepq_signal(sleepq);
 
-    sleepq_release(sleepq);
+    sleepq_release(sleepq, flags);
 }
diff --git a/kern/sleepq.c b/kern/sleepq.c
index bd310d04..c4946532 100644
--- a/kern/sleepq.c
+++ b/kern/sleepq.c
@@ -236,7 +236,7 @@ sleepq_destroy(struct sleepq *sleepq)
 }
 
 struct sleepq *
-sleepq_acquire(const void *sync_obj, bool condition)
+sleepq_acquire(const void *sync_obj, bool condition, unsigned long *flags)
 {
     struct sleepq_bucket *bucket;
     struct sleepq *sleepq;
@@ -245,12 +245,12 @@ sleepq_acquire(const void *sync_obj, bool condition)
 
     bucket = sleepq_bucket_get(sync_obj, condition);
 
-    spinlock_lock(&bucket->lock);
+    spinlock_lock_intr_save(&bucket->lock, flags);
 
     sleepq = sleepq_bucket_lookup(bucket, sync_obj);
 
     if (sleepq == NULL) {
-        spinlock_unlock(&bucket->lock);
+        spinlock_unlock_intr_restore(&bucket->lock, *flags);
         return NULL;
     }
 
@@ -258,9 +258,9 @@ sleepq_acquire(const void *sync_obj, bool condition)
 }
 
 void
-sleepq_release(struct sleepq *sleepq)
+sleepq_release(struct sleepq *sleepq, unsigned long flags)
 {
-    spinlock_unlock(&sleepq->bucket->lock);
+    spinlock_unlock_intr_restore(&sleepq->bucket->lock, flags);
 }
 
 static void
@@ -288,7 +288,7 @@ sleepq_pop_free(struct sleepq *sleepq)
 }
 
 struct sleepq *
-sleepq_lend(const void *sync_obj, bool condition)
+sleepq_lend(const void *sync_obj, bool condition, unsigned long *flags)
 {
     struct sleepq_bucket *bucket;
     struct sleepq *sleepq, *prev;
@@ -300,7 +300,7 @@ sleepq_lend(const void *sync_obj, bool condition)
 
     bucket = sleepq_bucket_get(sync_obj, condition);
 
-    spinlock_lock(&bucket->lock);
+    spinlock_lock_intr_save(&bucket->lock, flags);
 
     prev = sleepq_bucket_lookup(bucket, sync_obj);
 
@@ -316,7 +316,7 @@ sleepq_lend(const void *sync_obj, bool condition)
 }
 
 void
-sleepq_return(struct sleepq *sleepq)
+sleepq_return(struct sleepq *sleepq, unsigned long flags)
 {
     struct sleepq_bucket *bucket;
     struct sleepq *free_sleepq;
@@ -332,7 +332,7 @@ sleepq_return(struct sleepq *sleepq)
         free_sleepq = sleepq;
     }
 
-    spinlock_unlock(&bucket->lock);
+    spinlock_unlock_intr_restore(&bucket->lock, flags);
 
     sleepq_assert_init_state(free_sleepq);
     thread_sleepq_return(free_sleepq);
diff --git a/kern/sleepq.h b/kern/sleepq.h
index aeeb94b7..e1f7863b 100644
--- a/kern/sleepq.h
+++ b/kern/sleepq.h
@@ -63,13 +63,15 @@ void sleepq_destroy(struct sleepq *sleepq);
 /*
  * Acquire/release a sleep queue.
  *
- * Acquiring a sleep queue serializes all access and disables preemption.
+ * Acquiring a sleep queue serializes all access and disables both
+ * preemption and interrupts.
  *
  * The condition argument must be true if the synchronization object
  * is a condition variable.
  */
-struct sleepq * sleepq_acquire(const void *sync_obj, bool condition);
-void sleepq_release(struct sleepq *sleepq);
+struct sleepq * sleepq_acquire(const void *sync_obj, bool condition,
+                               unsigned long *flags);
+void sleepq_release(struct sleepq *sleepq, unsigned long flags);
 
 /*
  * Lend/return a sleep queue.
@@ -90,8 +92,9 @@ void sleepq_release(struct sleepq *sleepq);
  * The condition argument must be true if the synchronization object
  * is a condition variable.
  */
-struct sleepq * sleepq_lend(const void *sync_obj, bool condition);
-void sleepq_return(struct sleepq *sleepq);
+struct sleepq * sleepq_lend(const void *sync_obj, bool condition,
+                            unsigned long *flags);
+void sleepq_return(struct sleepq *sleepq, unsigned long flags);
 
 /*
  * Return true if the given sleep queue has no waiters.
@@ -120,6 +123,7 @@ void sleepq_wait(struct sleepq *sleepq, const char *wchan);
  * Wake up a thread waiting on the given sleep queue, if any.
  *
  * The sleep queue must be acquired when calling this function.
+ * A sleep queue may be signalled from interrupt context.
  *
  * Since a sleep queue must be lent (and in turn is automatically
  * acquired) when waiting, and acquired in order to signal it,