kern/clock: new module

5 files changed, 273 insertions, 22 deletions

diff --git a/kern/clock.c b/kern/clock.c
new file mode 100644
index 00000000..d8af3f4c
--- /dev/null
+++ b/kern/clock.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2017 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+
+#include <kern/atomic.h>
+#include <kern/clock.h>
+#include <kern/clock_i.h>
+#include <kern/init.h>
+#include <kern/llsync.h>
+#include <kern/percpu.h>
+#include <kern/sref.h>
+#include <kern/syscnt.h>
+#include <kern/thread.h>
+#include <kern/work.h>
+#include <machine/boot.h>
+#include <machine/cpu.h>
+
+struct clock_cpu_data {
+    struct syscnt sc_tick_intrs;
+};
+
+static struct clock_cpu_data clock_cpu_data __percpu;
+
+union clock_global_time clock_global_time;
+
+static inline void __init
+clock_cpu_data_init(struct clock_cpu_data *cpu_data, unsigned int cpu)
+{
+    char name[SYSCNT_NAME_SIZE];
+
+    snprintf(name, sizeof(name), "clock_tick_intrs/%u", cpu);
+    syscnt_register(&cpu_data->sc_tick_intrs, name);
+}
+
+static int __init
+clock_setup(void)
+{
+    for (unsigned int cpu = 0; cpu < cpu_count(); cpu++) {
+        clock_cpu_data_init(percpu_ptr(clock_cpu_data, cpu), cpu);
+    }
+
+    return 0;
+}
+
+INIT_OP_DEFINE(clock_setup,
+               INIT_OP_DEP(boot_setup_intr, true),
+               INIT_OP_DEP(cpu_mp_probe, true),
+               INIT_OP_DEP(syscnt_setup, true));
+
+void clock_tick_intr(void)
+{
+    struct clock_cpu_data *cpu_data;
+
+    assert(!cpu_intr_enabled());
+    assert(!thread_preempt_enabled());
+
+    if (cpu_id() == 0) {
+#ifdef ATOMIC_HAVE_64B_OPS
+
+        atomic_add(&clock_global_time.ticks, 1, ATOMIC_RELAXED);
+
+#else /* ATOMIC_HAVE_64B_OPS */
+
+        union clock_global_time t;
+
+        t.ticks = clock_global_time.ticks;
+        t.ticks++;
+
+        atomic_store(&clock_global_time.high2, t.high1, ATOMIC_RELAXED);
+        atomic_store_release(&clock_global_time.low, t.low);
+        atomic_store_release(&clock_global_time.high1, t.high1);
+
+#endif /* ATOMIC_HAVE_64B_OPS */
+    }
+
+    llsync_report_periodic_event();
+    sref_report_periodic_event();
+    work_report_periodic_event();
+    thread_report_periodic_event();
+
+    cpu_data = cpu_local_ptr(clock_cpu_data);
+    syscnt_inc(&cpu_data->sc_tick_intrs);
+}
diff --git a/kern/clock.h b/kern/clock.h
new file mode 100644
index 00000000..30db0a82
--- /dev/null
+++ b/kern/clock.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2017 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Timekeeping module.
+ */
+
+#ifndef _KERN_CLOCK_H
+#define _KERN_CLOCK_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include <kern/atomic.h>
+#include <kern/clock_i.h>
+#include <kern/init.h>
+#include <kern/macros.h>
+
+/*
+ * Clock frequency.
+ *
+ * TODO Clock frequency selection.
+ */
+#define CLOCK_FREQ 200
+
+#if (1000 % CLOCK_FREQ) != 0
+#error "invalid clock frequency"
+#endif /* (1000 % CLOCK_FREQ) != 0 */
+
+/*
+ * Arbitrary value used to determine if a time is in the past or the future.
+ *
+ * Time is represented as 64-bits unsigned integers counting ticks. The
+ * global time currently starts from 0 but this isn't a strong assumption
+ * users should rely on. Instead, all time checks involve a time reference
+ * against which to compare. The result of that comparison, done by
+ * substraction, is either in the future, i.e. the difference is less
+ * than the expire threshold, or in the past, i.e. the difference is
+ * greater (keep in mind the result is unsigned). The threshold must be
+ * large enough to allow both a wide range of possible times in the future,
+ * but also enough time in the past for reliable timeout detection. Note
+ * that using signed integers would be equivalent to dividing the range
+ * in two (almost) equal past and future halves.
+ */
+#define CLOCK_EXPIRE_THRESHOLD (-(1ULL << 60))
+
+static inline uint64_t
+clock_get_time(void)
+{
+    extern union clock_global_time clock_global_time;
+
+#ifdef ATOMIC_HAVE_64B_OPS
+
+    /*
+     * Don't enforce a stronger memory order, since :
+     *  1/ it's useless as long as the reader remains on the same processor
+     *  2/ thread migration enforces sequential consistency
+     */
+    return atomic_load(&clock_global_time.ticks, ATOMIC_RELAXED);
+
+#else /* ATOMIC_HAVE_64B_OPS */
+
+    uint32_t high1, low, high2;
+
+    /*
+     * For machines with no 64-bits atomic accessors, this implementation uses
+     * a variant of the two-digit monotonic-clock algorithm, described in the
+     * paper "Concurrent Reading and Writing of Clocks" by Leslie Lamport.
+     */
+
+    do {
+        high1 = atomic_load_acquire(&clock_global_time.high1);
+        low = atomic_load_acquire(&clock_global_time.low);
+        high2 = atomic_load(&clock_global_time.high2, ATOMIC_RELAXED);
+    } while (high1 != high2);
+
+    return ((uint64_t)high2 << 32) | low;
+
+#endif /* ATOMIC_HAVE_64B_OPS */
+}
+
+static inline uint64_t
+clock_ticks_to_ms(uint64_t ticks)
+{
+    return ticks * (1000 / CLOCK_FREQ);
+}
+
+static inline uint64_t
+clock_ticks_from_ms(uint64_t ms)
+{
+    return DIV_CEIL(ms, (1000 / CLOCK_FREQ));
+}
+
+static inline bool
+clock_time_expired(uint64_t t, uint64_t ref)
+{
+    return (t - ref) > CLOCK_EXPIRE_THRESHOLD;
+}
+
+static inline bool
+clock_time_occurred(uint64_t t, uint64_t ref)
+{
+    return (t == ref) || clock_time_expired(t, ref);
+}
+
+void clock_tick_intr(void);
+
+INIT_OP_DECLARE(clock_setup);
+
+#endif /* _KERN_CLOCK_H */
diff --git a/kern/clock_i.h b/kern/clock_i.h
new file mode 100644
index 00000000..830f48c5
--- /dev/null
+++ b/kern/clock_i.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2017 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _KERN_CLOCK_I_H
+#define _KERN_CLOCK_I_H
+
+#include <stdint.h>
+
+#include <machine/cpu.h>
+
+union clock_global_time {
+    alignas(CPU_L1_SIZE) uint64_t ticks;
+
+#ifndef ATOMIC_HAVE_64B_OPS
+    struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+        uint32_t high1;
+        uint32_t low;
+#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */
+        uint32_t low;
+        uint32_t high1;
+#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */
+        uint32_t high2;
+    };
+#endif /* ATOMIC_HAVE_64B_OPS */
+};
+
+#endif /* _KERN_CLOCK_I_H */
diff --git a/kern/thread.c b/kern/thread.c
index fba95e06..2a1e124e 100644
--- a/kern/thread.c
+++ b/kern/thread.c
@@ -91,6 +91,7 @@
 #include <string.h>
 
 #include <kern/atomic.h>
+#include <kern/clock.h>
 #include <kern/condition.h>
 #include <kern/cpumap.h>
 #include <kern/error.h>
@@ -160,7 +161,7 @@
 /*
  * Default time slice for real-time round-robin scheduling.
  */
-#define THREAD_DEFAULT_RR_TIME_SLICE (THREAD_TICK_FREQ / 10)
+#define THREAD_DEFAULT_RR_TIME_SLICE (CLOCK_FREQ / 10)
 
 /*
  * Maximum number of threads which can be pulled from a remote run queue
@@ -171,7 +172,7 @@
 /*
  * Delay (in ticks) between two balance attempts when a run queue is idle.
  */
-#define THREAD_IDLE_BALANCE_TICKS (THREAD_TICK_FREQ / 2)
+#define THREAD_IDLE_BALANCE_TICKS (CLOCK_FREQ / 2)
 
 /*
  * Run queue properties for real-time threads.
@@ -191,7 +192,7 @@ struct thread_rt_runq {
 /*
  * Round slice base unit for fair-scheduling threads.
  */
-#define THREAD_FS_ROUND_SLICE_BASE (THREAD_TICK_FREQ / 10)
+#define THREAD_FS_ROUND_SLICE_BASE (CLOCK_FREQ / 10)
 
 /*
  * Group of threads sharing the same weight.
@@ -257,7 +258,6 @@ struct thread_runq {
     unsigned int idle_balance_ticks;
 
     struct syscnt sc_schedule_intrs;
-    struct syscnt sc_tick_intrs;
     struct syscnt sc_boosts;
 };
 
@@ -451,8 +451,6 @@ thread_runq_init(struct thread_runq *runq, unsigned int cpu,
     runq->idle_balance_ticks = (unsigned int)-1;
     snprintf(name, sizeof(name), "thread_schedule_intrs/%u", cpu);
     syscnt_register(&runq->sc_schedule_intrs, name);
-    snprintf(name, sizeof(name), "thread_tick_intrs/%u", cpu);
-    syscnt_register(&runq->sc_tick_intrs, name);
     snprintf(name, sizeof(name), "thread_boosts/%u", cpu);
     syscnt_register(&runq->sc_boosts, name);
 }
@@ -2570,7 +2568,7 @@ thread_schedule_intr(void)
 }
 
 void
-thread_tick_intr(void)
+thread_report_periodic_event(void)
 {
     const struct thread_sched_ops *ops;
     struct thread_runq *runq;
@@ -2579,10 +2577,6 @@ thread_tick_intr(void)
     thread_assert_interrupted();
 
     runq = thread_runq_local();
-    syscnt_inc(&runq->sc_tick_intrs);
-    llsync_report_periodic_event();
-    sref_report_periodic_event();
-    work_report_periodic_event();
     thread = thread_self();
 
     spinlock_lock(&runq->lock);
diff --git a/kern/thread.h b/kern/thread.h
index 1c052364..b3bf93f0 100644
--- a/kern/thread.h
+++ b/kern/thread.h
@@ -48,14 +48,6 @@
 #include <machine/tcb.h>
 
 /*
- * Scheduler tick frequency.
- *
- * The selected value of 200 translates to a period of 5ms, small enough to
- * provide low latency, and is practical as both a dividend and divisor.
- */
-#define THREAD_TICK_FREQ 200
-
-/*
  * Thread structure.
  */
 struct thread;
@@ -253,10 +245,11 @@ void thread_yield(void);
 void thread_schedule_intr(void);
 
 /*
- * Report a periodic timer interrupt on the thread currently running on
- * the local processor.
+ * Report a periodic event on the current processor.
+ *
+ * Interrupts and preemption must be disabled when calling this function.
  */
-void thread_tick_intr(void);
+void thread_report_periodic_event(void);
 
 /*
  * Set thread scheduling parameters.