kern/xcall: new module

Provide cross-processor function calls.

11 files changed, 349 insertions, 1 deletions

diff --git a/Makefrag.am b/Makefrag.am
index 9fac955e..6610ce24 100644
--- a/Makefrag.am
+++ b/Makefrag.am
@@ -56,7 +56,9 @@ x15_SOURCES +=              \
 	kern/thread.h       \
 	kern/types.h        \
 	kern/work.c         \
-	kern/work.h
+	kern/work.h         \
+	kern/xcall.c        \
+	kern/xcall.h
 
 x15_SOURCES +=              \
 	vm/vm_adv.h         \
@@ -93,3 +95,7 @@ endif TEST_SREF_NOREF
 if TEST_X86_PMAP_REMOVE_PTP
 x15_SOURCES += test/test_x86_pmap_remove_ptp.c
 endif TEST_X86_PMAP_REMOVE_PTP
+
+if TEST_XCALL
+x15_SOURCES += test/test_xcall.c
+endif TEST_XCALL
diff --git a/arch/x86/machine/cpu.c b/arch/x86/machine/cpu.c
index a1bb93d1..1f4ac05a 100644
--- a/arch/x86/machine/cpu.c
+++ b/arch/x86/machine/cpu.c
@@ -25,6 +25,7 @@
 #include <kern/stddef.h>
 #include <kern/stdint.h>
 #include <kern/string.h>
+#include <kern/xcall.h>
 #include <machine/acpimp.h>
 #include <machine/biosmem.h>
 #include <machine/boot.h>
@@ -714,6 +715,16 @@ cpu_halt_intr(struct trap_frame *frame)
 }
 
 void
+cpu_xcall_intr(struct trap_frame *frame)
+{
+    (void)frame;
+
+    lapic_eoi();
+
+    xcall_intr();
+}
+
+void
 cpu_thread_schedule_intr(struct trap_frame *frame)
 {
     (void)frame;
diff --git a/arch/x86/machine/cpu.h b/arch/x86/machine/cpu.h
index 322e30dd..88d2a270 100644
--- a/arch/x86/machine/cpu.h
+++ b/arch/x86/machine/cpu.h
@@ -563,6 +563,20 @@ void cpu_mp_setup(void);
 void cpu_ap_setup(void);
 
 /*
+ * Send a cross-call interrupt to a remote processor.
+ */
+static inline void
+cpu_send_xcall(unsigned int cpu)
+{
+    lapic_ipi_send(cpu_from_id(cpu)->apic_id, TRAP_XCALL);
+}
+
+/*
+ * Interrupt handler for cross-calls.
+ */
+void cpu_xcall_intr(struct trap_frame *frame);
+
+/*
  * Send a scheduling interrupt to a remote processor.
  */
 static inline void
diff --git a/arch/x86/machine/trap.c b/arch/x86/machine/trap.c
index e028a8f3..1dcd239c 100644
--- a/arch/x86/machine/trap.c
+++ b/arch/x86/machine/trap.c
@@ -75,6 +75,7 @@ void trap_isr_machine_check(void);
 void trap_isr_simd_fp_exception(void);
 void trap_isr_pic_int7(void);
 void trap_isr_pic_int15(void);
+void trap_isr_xcall(void);
 void trap_isr_thread_schedule(void);
 void trap_isr_cpu_halt(void);
 void trap_isr_lapic_timer(void);
@@ -201,6 +202,8 @@ trap_setup(void)
                  trap_isr_pic_int15, pic_spurious_intr);
 
     /* System defined traps */
+    trap_install(TRAP_XCALL, TRAP_HF_NOPREEMPT,
+                 trap_isr_xcall, cpu_xcall_intr);
     trap_install(TRAP_THREAD_SCHEDULE, TRAP_HF_NOPREEMPT,
                  trap_isr_thread_schedule, cpu_thread_schedule_intr);
     trap_install(TRAP_CPU_HALT, TRAP_HF_NOPREEMPT,
diff --git a/arch/x86/machine/trap.h b/arch/x86/machine/trap.h
index 99e23896..656bc50d 100644
--- a/arch/x86/machine/trap.h
+++ b/arch/x86/machine/trap.h
@@ -53,6 +53,7 @@
  *
  * The local APIC assigns one priority every 16 vectors.
  */
+#define TRAP_XCALL              238
 #define TRAP_THREAD_SCHEDULE    239
 #define TRAP_CPU_HALT           240
 #define TRAP_LAPIC_TIMER        253
diff --git a/arch/x86/machine/trap_asm.S b/arch/x86/machine/trap_asm.S
index bb70f46e..07639cd8 100644
--- a/arch/x86/machine/trap_asm.S
+++ b/arch/x86/machine/trap_asm.S
@@ -157,6 +157,7 @@ TRAP(TRAP_PIC_BASE + 7, trap_isr_pic_int7)
 TRAP(TRAP_PIC_BASE + 15, trap_isr_pic_int15)
 
 /* System defined traps */
+TRAP(TRAP_XCALL, trap_isr_xcall)
 TRAP(TRAP_THREAD_SCHEDULE, trap_isr_thread_schedule)
 TRAP(TRAP_CPU_HALT, trap_isr_cpu_halt)
 TRAP(TRAP_LAPIC_TIMER, trap_isr_lapic_timer)
diff --git a/configure.ac b/configure.ac
index 0a399a21..988ff0a7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -52,6 +52,7 @@ m4_define([ENABLE_TEST_MODULE],
                        [test_x86_pmap_remove_ptp=yes
                         AC_MSG_NOTICE([forcing max_cpus to 1])
                         opt_max_cpus=1],
+                   [xcall], [test_xcall=yes],
                    [AC_MSG_ERROR([invalid test module])])
           AC_DEFINE([RUN_TEST_MODULE], [1],
                     [run test module instead of booting])
@@ -68,6 +69,8 @@ AM_CONDITIONAL([TEST_SREF_NOREF],
                [test x"$test_sref_noref" = xyes])
 AM_CONDITIONAL([TEST_X86_PMAP_REMOVE_PTP],
                [test x"$test_x86_pmap_remove_ptp" = xyes])
+AM_CONDITIONAL([TEST_XCALL],
+               [test x"$test_xcall" = xyes])
 
 AC_DEFINE_UNQUOTED([MAX_CPUS], [$opt_max_cpus],
                    [maximum number of supported processors])
diff --git a/kern/kernel.c b/kern/kernel.c
index 462fa9d9..b63be5d2 100644
--- a/kern/kernel.c
+++ b/kern/kernel.c
@@ -24,6 +24,7 @@
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/work.h>
+#include <kern/xcall.h>
 #include <machine/cpu.h>
 #include <vm/vm_page.h>
 
@@ -38,6 +39,7 @@ kernel_main(void)
 
     percpu_cleanup();
     cpumap_setup();
+    xcall_setup();
     task_setup();
     thread_setup();
     work_setup();
diff --git a/kern/xcall.c b/kern/xcall.c
new file mode 100644
index 00000000..7ea299db
--- /dev/null
+++ b/kern/xcall.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <kern/assert.h>
+#include <kern/macros.h>
+#include <kern/param.h>
+#include <kern/percpu.h>
+#include <kern/spinlock.h>
+#include <kern/stddef.h>
+#include <kern/thread.h>
+#include <kern/xcall.h>
+#include <machine/mb.h>
+#include <machine/cpu.h>
+
+struct xcall {
+    xcall_fn_t fn;
+    void *arg;
+} __aligned(CPU_L1_SIZE);
+
+/*
+ * Per-CPU data.
+ *
+ * Send calls are sent to remote processors. Their access is synchronized
+ * by disabling preemption.
+ *
+ * The received call points to either NULL if there is no call to process,
+ * or a remote send call otherwise. The lock serializes the complete
+ * inter-processor operation, i.e. setting the received call pointer,
+ * communication through an IPI, and waiting for the processor to
+ * acknowledge execution. By serializing interrupts, it is certain that
+ * there is a 1:1 mapping between interrupts and cross-calls, allowing
+ * the handler to process only one cross-call instead of iterating over
+ * a queue. This way, interrupts with higher priority can be handled
+ * between multiple cross-calls.
+ */
+struct xcall_cpu_data {
+    struct xcall send_calls[MAX_CPUS];
+
+    struct xcall *recv_call;
+    struct spinlock lock;
+} __aligned(CPU_L1_SIZE);
+
+static struct xcall_cpu_data xcall_cpu_data __percpu;
+
+static inline void
+xcall_set(struct xcall *call, xcall_fn_t fn, void *arg)
+{
+    call->fn = fn;
+    call->arg = arg;
+}
+
+static void
+xcall_cpu_data_init(struct xcall_cpu_data *cpu_data)
+{
+    cpu_data->recv_call = NULL;
+    spinlock_init(&cpu_data->lock);
+}
+
+static struct xcall_cpu_data *
+xcall_cpu_data_get(void)
+{
+    assert(!thread_preempt_enabled());
+    return cpu_local_ptr(xcall_cpu_data);
+}
+
+static struct xcall *
+xcall_cpu_data_get_send_call(struct xcall_cpu_data *cpu_data, unsigned int cpu)
+{
+    assert(cpu < ARRAY_SIZE(cpu_data->send_calls));
+    return &cpu_data->send_calls[cpu];
+}
+
+static struct xcall *
+xcall_cpu_data_get_recv_call(struct xcall_cpu_data *cpu_data)
+{
+    return cpu_data->recv_call;
+}
+
+static void
+xcall_cpu_data_clear_recv_call(struct xcall_cpu_data *cpu_data)
+{
+    cpu_data->recv_call = NULL;
+}
+
+void
+xcall_setup(void)
+{
+    unsigned int i;
+
+    for (i = 0; i < cpu_count(); i++)
+        xcall_cpu_data_init(percpu_ptr(xcall_cpu_data, i));
+}
+
+void
+xcall_call(xcall_fn_t fn, void *arg, unsigned int cpu)
+{
+    struct xcall_cpu_data *local_data, *remote_data;
+    struct xcall *call;
+
+    assert(fn != NULL);
+
+    remote_data = percpu_ptr(xcall_cpu_data, cpu);
+
+    thread_preempt_disable();
+
+    if (cpu == cpu_id()) {
+        unsigned long flags;
+
+        cpu_intr_save(&flags);
+        fn(arg);
+        cpu_intr_restore(flags);
+        goto out;
+    }
+
+    local_data = xcall_cpu_data_get();
+    call = xcall_cpu_data_get_send_call(local_data, cpu);
+    xcall_set(call, fn, arg);
+
+    spinlock_lock(&remote_data->lock);
+
+    remote_data->recv_call = call;
+
+    /* This barrier pairs with the one implied by the received IPI */
+    mb_store();
+
+    cpu_send_xcall(cpu);
+
+    while (remote_data->recv_call != NULL)
+        cpu_pause();
+
+    spinlock_unlock(&remote_data->lock);
+
+    /* This barrier pairs with the one in the interrupt handler */
+    mb_load();
+
+out:
+    thread_preempt_enable();
+}
+
+void
+xcall_intr(void)
+{
+    struct xcall_cpu_data *cpu_data;
+    struct xcall *call;
+
+    assert(!cpu_intr_enabled());
+    assert(!thread_preempt_enabled());
+
+    cpu_data = xcall_cpu_data_get();
+    call = xcall_cpu_data_get_recv_call(cpu_data);
+    call->fn(call->arg);
+    mb_store();
+    xcall_cpu_data_clear_recv_call(cpu_data);
+}
diff --git a/kern/xcall.h b/kern/xcall.h
new file mode 100644
index 00000000..37c85867
--- /dev/null
+++ b/kern/xcall.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Cross-processor function calls.
+ *
+ * This module provides the ability to run functions, called cross-calls,
+ * on specific processors.
+ */
+
+#ifndef _KERN_XCALL_H
+#define _KERN_XCALL_H
+
+/*
+ * Type for cross-call functions.
+ */
+typedef void (*xcall_fn_t)(void *arg);
+
+/*
+ * Initialize the xcall module.
+ */
+void xcall_setup(void);
+
+/*
+ * Run the given cross-call function on a specific processor.
+ *
+ * The operation is completely synchronous, returning only when the function
+ * has finished running on the target processor, with the side effects of
+ * the function visible.
+ *
+ * The function is run in interrupt context.
+ */
+void xcall_call(xcall_fn_t fn, void *arg, unsigned int cpu);
+
+/*
+ * Report a cross-call interrupt from a remote processor.
+ *
+ * Called from interrupt context.
+ */
+void xcall_intr(void);
+
+#endif /* _KERN_XCALL_H */
diff --git a/test/test_xcall.c b/test/test_xcall.c
new file mode 100644
index 00000000..9da6a151
--- /dev/null
+++ b/test/test_xcall.c
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * This is a simple test of the cross-call functionality. One thread is
+ * created and bound to CPU 0. It makes two cross-calls, one on its local
+ * processor, and another on a remote processor.
+ */
+
+#include <kern/error.h>
+#include <kern/cpumap.h>
+#include <kern/printk.h>
+#include <kern/stddef.h>
+#include <kern/thread.h>
+#include <kern/xcall.h>
+#include <test/test.h>
+
+static int test_done;
+
+static void
+test_fn(void *arg)
+{
+    (void)arg;
+
+    printk("function called, running on cpu%u\n", cpu_id());
+    test_done = 1;
+}
+
+static void
+test_once(unsigned int cpu)
+{
+    test_done = 0;
+
+    printk("cross-call on cpu%u:\n", cpu);
+    xcall_call(test_fn, NULL, cpu);
+
+    if (!test_done)
+        panic("test_done false");
+}
+
+static void
+test_run(void *arg)
+{
+    (void)arg;
+
+    test_once(0);
+    test_once(1);
+    printk("done\n");
+}
+
+void
+test_setup(void)
+{
+    struct thread_attr attr;
+    struct thread *thread;
+    struct cpumap *cpumap;
+    int error;
+
+    error = cpumap_create(&cpumap);
+    error_check(error, "cpumap_create");
+    cpumap_zero(cpumap);
+    cpumap_set(cpumap, 0);
+
+    thread_attr_init(&attr, "x15_test_run");
+    thread_attr_set_detached(&attr);
+    thread_attr_set_cpumap(&attr, cpumap);
+    error = thread_create(&thread, &attr, test_run, NULL);
+    error_check(error, "thread_create");
+
+    cpumap_destroy(cpumap);
+}