kern/percpu: introduce per-CPU operations

3 files changed, 63 insertions, 0 deletions

diff --git a/arch/x86/machine/boot.c b/arch/x86/machine/boot.c
index 6934896e..6a26a90d 100644
--- a/arch/x86/machine/boot.c
+++ b/arch/x86/machine/boot.c
@@ -56,6 +56,7 @@
 #include <kern/log.h>
 #include <kern/macros.h>
 #include <kern/panic.h>
+#include <kern/percpu.h>
 #include <kern/thread.h>
 #include <machine/acpi.h>
 #include <machine/atcons.h>
@@ -504,6 +505,7 @@ boot_ap_main(void)
     cpu_ap_setup();
     thread_ap_setup();
     pmap_ap_setup();
+    percpu_ap_setup();
     kernel_ap_main();
 
     /* Never reached */
diff --git a/kern/percpu.c b/kern/percpu.c
index 53861a30..f344bd70 100644
--- a/kern/percpu.c
+++ b/kern/percpu.c
@@ -26,6 +26,7 @@
 #include <kern/macros.h>
 #include <kern/panic.h>
 #include <kern/percpu.h>
+#include <kern/slist.h>
 #include <machine/cpu.h>
 #include <vm/vm_kmem.h>
 #include <vm/vm_page.h>
@@ -36,6 +37,14 @@ static void *percpu_area_content __initdata;
 static size_t percpu_area_size __initdata;
 static int percpu_skip_warning __initdata;
 
+static struct slist percpu_ops __initdata;
+
+static void __init
+percpu_op_run(const struct percpu_op *op)
+{
+    op->fn();
+}
+
 static int __init
 percpu_bootstrap(void)
 {
@@ -51,6 +60,8 @@ percpu_setup(void)
     struct vm_page *page;
     unsigned int order;
 
+    slist_init(&percpu_ops);
+
     percpu_area_size = &_percpu_end - &_percpu;
     log_info("percpu: max_cpus: %u, section size: %zuk", CONFIG_MAX_CPUS,
              percpu_area_size >> 10);
@@ -76,6 +87,15 @@ INIT_OP_DEFINE(percpu_setup,
                INIT_OP_DEP(percpu_bootstrap, true),
                INIT_OP_DEP(vm_page_setup, true));
 
+void __init
+percpu_register_op(struct percpu_op *op)
+{
+    slist_insert_tail(&percpu_ops, &op->node);
+
+    /* Run on BSP */
+    percpu_op_run(op);
+}
+
 int __init
 percpu_add(unsigned int cpu)
 {
@@ -116,6 +136,16 @@ out:
     return 0;
 }
 
+void __init
+percpu_ap_setup(void)
+{
+    struct percpu_op *op;
+
+    slist_for_each_entry(&percpu_ops, op, node) {
+        percpu_op_run(op);
+    }
+}
+
 static int __init
 percpu_cleanup(void)
 {
diff --git a/kern/percpu.h b/kern/percpu.h
index 96f706ea..f77e7fd8 100644
--- a/kern/percpu.h
+++ b/kern/percpu.h
@@ -59,10 +59,26 @@
 
 #include <kern/init.h>
 #include <kern/macros.h>
+#include <kern/slist_types.h>
 
 #define PERCPU_SECTION .percpu
 #define __percpu __section(QUOTE(PERCPU_SECTION))
 
+typedef void (*percpu_op_fn_t)(void);
+
+/*
+ * Per-CPU operation.
+ *
+ * These operations allow initialization code to register functions to be run
+ * on APs when they're started.
+ */
+struct percpu_op {
+    struct slist_node node;
+    percpu_op_fn_t fn;
+};
+
+#define PERCPU_OP_INITIALIZER(op_fn) { .fn = op_fn }
+
 /*
  * Boundaries of the percpu section.
  *
@@ -96,6 +112,15 @@ percpu_area(unsigned int cpu)
 }
 
 /*
+ * Register a percpu operation to be run on all processors when
+ * they're started.
+ *
+ * The operation is run on the BSP when it's registered. It's run as late as
+ * possible on APs, normally right before scheduling is enabled.
+ */
+void percpu_register_op(struct percpu_op *op);
+
+/*
  * Register a processor.
  *
  * This function creates a percpu area from kernel virtual memory for the
@@ -105,6 +130,11 @@ percpu_area(unsigned int cpu)
 int percpu_add(unsigned int cpu);
 
 /*
+ * Run registered percpu operations on an AP.
+ */
+void percpu_ap_setup(void);
+
+/*
  * This init operation provides :
  *  - access to percpu variables on processor 0
  */
@@ -112,6 +142,7 @@ INIT_OP_DECLARE(percpu_bootstrap);
 
 /*
  * This init operation provides :
+ *  - percpu operations can be registered
  *  - new percpu areas can be created
  *
  * The dependency that provides access to percpu variables on all processors