diff options
| -rw-r--r-- | Makefrag.am | 8 | ||||
| -rw-r--r-- | arch/x86/machine/cpu.c | 11 | ||||
| -rw-r--r-- | arch/x86/machine/cpu.h | 14 | ||||
| -rw-r--r-- | arch/x86/machine/trap.c | 3 | ||||
| -rw-r--r-- | arch/x86/machine/trap.h | 1 | ||||
| -rw-r--r-- | arch/x86/machine/trap_asm.S | 1 | ||||
| -rw-r--r-- | configure.ac | 3 | ||||
| -rw-r--r-- | kern/kernel.c | 2 | ||||
| -rw-r--r-- | kern/xcall.c | 168 | ||||
| -rw-r--r-- | kern/xcall.h | 55 | ||||
| -rw-r--r-- | test/test_xcall.c | 84 |
11 files changed, 349 insertions, 1 deletions
diff --git a/Makefrag.am b/Makefrag.am index 9fac955e..6610ce24 100644 --- a/Makefrag.am +++ b/Makefrag.am @@ -56,7 +56,9 @@ x15_SOURCES += \ kern/thread.h \ kern/types.h \ kern/work.c \ - kern/work.h + kern/work.h \ + kern/xcall.c \ + kern/xcall.h x15_SOURCES += \ vm/vm_adv.h \ @@ -93,3 +95,7 @@ endif TEST_SREF_NOREF if TEST_X86_PMAP_REMOVE_PTP x15_SOURCES += test/test_x86_pmap_remove_ptp.c endif TEST_X86_PMAP_REMOVE_PTP + +if TEST_XCALL +x15_SOURCES += test/test_xcall.c +endif TEST_XCALL diff --git a/arch/x86/machine/cpu.c b/arch/x86/machine/cpu.c index a1bb93d1..1f4ac05a 100644 --- a/arch/x86/machine/cpu.c +++ b/arch/x86/machine/cpu.c @@ -25,6 +25,7 @@ #include <kern/stddef.h> #include <kern/stdint.h> #include <kern/string.h> +#include <kern/xcall.h> #include <machine/acpimp.h> #include <machine/biosmem.h> #include <machine/boot.h> @@ -714,6 +715,16 @@ cpu_halt_intr(struct trap_frame *frame) } void +cpu_xcall_intr(struct trap_frame *frame) +{ + (void)frame; + + lapic_eoi(); + + xcall_intr(); +} + +void cpu_thread_schedule_intr(struct trap_frame *frame) { (void)frame; diff --git a/arch/x86/machine/cpu.h b/arch/x86/machine/cpu.h index 322e30dd..88d2a270 100644 --- a/arch/x86/machine/cpu.h +++ b/arch/x86/machine/cpu.h @@ -563,6 +563,20 @@ void cpu_mp_setup(void); void cpu_ap_setup(void); /* + * Send a cross-call interrupt to a remote processor. + */ +static inline void +cpu_send_xcall(unsigned int cpu) +{ + lapic_ipi_send(cpu_from_id(cpu)->apic_id, TRAP_XCALL); +} + +/* + * Interrupt handler for cross-calls. + */ +void cpu_xcall_intr(struct trap_frame *frame); + +/* * Send a scheduling interrupt to a remote processor. */ static inline void diff --git a/arch/x86/machine/trap.c b/arch/x86/machine/trap.c index e028a8f3..1dcd239c 100644 --- a/arch/x86/machine/trap.c +++ b/arch/x86/machine/trap.c @@ -75,6 +75,7 @@ void trap_isr_machine_check(void); void trap_isr_simd_fp_exception(void); void trap_isr_pic_int7(void); void trap_isr_pic_int15(void); +void trap_isr_xcall(void); void trap_isr_thread_schedule(void); void trap_isr_cpu_halt(void); void trap_isr_lapic_timer(void); @@ -201,6 +202,8 @@ trap_setup(void) trap_isr_pic_int15, pic_spurious_intr); /* System defined traps */ + trap_install(TRAP_XCALL, TRAP_HF_NOPREEMPT, + trap_isr_xcall, cpu_xcall_intr); trap_install(TRAP_THREAD_SCHEDULE, TRAP_HF_NOPREEMPT, trap_isr_thread_schedule, cpu_thread_schedule_intr); trap_install(TRAP_CPU_HALT, TRAP_HF_NOPREEMPT, diff --git a/arch/x86/machine/trap.h b/arch/x86/machine/trap.h index 99e23896..656bc50d 100644 --- a/arch/x86/machine/trap.h +++ b/arch/x86/machine/trap.h @@ -53,6 +53,7 @@ * * The local APIC assigns one priority every 16 vectors. */ +#define TRAP_XCALL 238 #define TRAP_THREAD_SCHEDULE 239 #define TRAP_CPU_HALT 240 #define TRAP_LAPIC_TIMER 253 diff --git a/arch/x86/machine/trap_asm.S b/arch/x86/machine/trap_asm.S index bb70f46e..07639cd8 100644 --- a/arch/x86/machine/trap_asm.S +++ b/arch/x86/machine/trap_asm.S @@ -157,6 +157,7 @@ TRAP(TRAP_PIC_BASE + 7, trap_isr_pic_int7) TRAP(TRAP_PIC_BASE + 15, trap_isr_pic_int15) /* System defined traps */ +TRAP(TRAP_XCALL, trap_isr_xcall) TRAP(TRAP_THREAD_SCHEDULE, trap_isr_thread_schedule) TRAP(TRAP_CPU_HALT, trap_isr_cpu_halt) TRAP(TRAP_LAPIC_TIMER, trap_isr_lapic_timer) diff --git a/configure.ac b/configure.ac index 0a399a21..988ff0a7 100644 --- a/configure.ac +++ b/configure.ac @@ -52,6 +52,7 @@ m4_define([ENABLE_TEST_MODULE], [test_x86_pmap_remove_ptp=yes AC_MSG_NOTICE([forcing max_cpus to 1]) opt_max_cpus=1], + [xcall], [test_xcall=yes], [AC_MSG_ERROR([invalid test module])]) AC_DEFINE([RUN_TEST_MODULE], [1], [run test module instead of booting]) @@ -68,6 +69,8 @@ AM_CONDITIONAL([TEST_SREF_NOREF], [test x"$test_sref_noref" = xyes]) AM_CONDITIONAL([TEST_X86_PMAP_REMOVE_PTP], [test x"$test_x86_pmap_remove_ptp" = xyes]) +AM_CONDITIONAL([TEST_XCALL], + [test x"$test_xcall" = xyes]) AC_DEFINE_UNQUOTED([MAX_CPUS], [$opt_max_cpus], [maximum number of supported processors]) diff --git a/kern/kernel.c b/kern/kernel.c index 462fa9d9..b63be5d2 100644 --- a/kern/kernel.c +++ b/kern/kernel.c @@ -24,6 +24,7 @@ #include <kern/task.h> #include <kern/thread.h> #include <kern/work.h> +#include <kern/xcall.h> #include <machine/cpu.h> #include <vm/vm_page.h> @@ -38,6 +39,7 @@ kernel_main(void) percpu_cleanup(); cpumap_setup(); + xcall_setup(); task_setup(); thread_setup(); work_setup(); diff --git a/kern/xcall.c b/kern/xcall.c new file mode 100644 index 00000000..7ea299db --- /dev/null +++ b/kern/xcall.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2014 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <kern/assert.h> +#include <kern/macros.h> +#include <kern/param.h> +#include <kern/percpu.h> +#include <kern/spinlock.h> +#include <kern/stddef.h> +#include <kern/thread.h> +#include <kern/xcall.h> +#include <machine/mb.h> +#include <machine/cpu.h> + +struct xcall { + xcall_fn_t fn; + void *arg; +} __aligned(CPU_L1_SIZE); + +/* + * Per-CPU data. + * + * Send calls are sent to remote processors. Their access is synchronized + * by disabling preemption. + * + * The received call points to either NULL if there is no call to process, + * or a remote send call otherwise. The lock serializes the complete + * inter-processor operation, i.e. setting the received call pointer, + * communication through an IPI, and waiting for the processor to + * acknowledge execution. By serializing interrupts, it is certain that + * there is a 1:1 mapping between interrupts and cross-calls, allowing + * the handler to process only one cross-call instead of iterating over + * a queue. This way, interrupts with higher priority can be handled + * between multiple cross-calls. + */ +struct xcall_cpu_data { + struct xcall send_calls[MAX_CPUS]; + + struct xcall *recv_call; + struct spinlock lock; +} __aligned(CPU_L1_SIZE); + +static struct xcall_cpu_data xcall_cpu_data __percpu; + +static inline void +xcall_set(struct xcall *call, xcall_fn_t fn, void *arg) +{ + call->fn = fn; + call->arg = arg; +} + +static void +xcall_cpu_data_init(struct xcall_cpu_data *cpu_data) +{ + cpu_data->recv_call = NULL; + spinlock_init(&cpu_data->lock); +} + +static struct xcall_cpu_data * +xcall_cpu_data_get(void) +{ + assert(!thread_preempt_enabled()); + return cpu_local_ptr(xcall_cpu_data); +} + +static struct xcall * +xcall_cpu_data_get_send_call(struct xcall_cpu_data *cpu_data, unsigned int cpu) +{ + assert(cpu < ARRAY_SIZE(cpu_data->send_calls)); + return &cpu_data->send_calls[cpu]; +} + +static struct xcall * +xcall_cpu_data_get_recv_call(struct xcall_cpu_data *cpu_data) +{ + return cpu_data->recv_call; +} + +static void +xcall_cpu_data_clear_recv_call(struct xcall_cpu_data *cpu_data) +{ + cpu_data->recv_call = NULL; +} + +void +xcall_setup(void) +{ + unsigned int i; + + for (i = 0; i < cpu_count(); i++) + xcall_cpu_data_init(percpu_ptr(xcall_cpu_data, i)); +} + +void +xcall_call(xcall_fn_t fn, void *arg, unsigned int cpu) +{ + struct xcall_cpu_data *local_data, *remote_data; + struct xcall *call; + + assert(fn != NULL); + + remote_data = percpu_ptr(xcall_cpu_data, cpu); + + thread_preempt_disable(); + + if (cpu == cpu_id()) { + unsigned long flags; + + cpu_intr_save(&flags); + fn(arg); + cpu_intr_restore(flags); + goto out; + } + + local_data = xcall_cpu_data_get(); + call = xcall_cpu_data_get_send_call(local_data, cpu); + xcall_set(call, fn, arg); + + spinlock_lock(&remote_data->lock); + + remote_data->recv_call = call; + + /* This barrier pairs with the one implied by the received IPI */ + mb_store(); + + cpu_send_xcall(cpu); + + while (remote_data->recv_call != NULL) + cpu_pause(); + + spinlock_unlock(&remote_data->lock); + + /* This barrier pairs with the one in the interrupt handler */ + mb_load(); + +out: + thread_preempt_enable(); +} + +void +xcall_intr(void) +{ + struct xcall_cpu_data *cpu_data; + struct xcall *call; + + assert(!cpu_intr_enabled()); + assert(!thread_preempt_enabled()); + + cpu_data = xcall_cpu_data_get(); + call = xcall_cpu_data_get_recv_call(cpu_data); + call->fn(call->arg); + mb_store(); + xcall_cpu_data_clear_recv_call(cpu_data); +} diff --git a/kern/xcall.h b/kern/xcall.h new file mode 100644 index 00000000..37c85867 --- /dev/null +++ b/kern/xcall.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Cross-processor function calls. + * + * This module provides the ability to run functions, called cross-calls, + * on specific processors. + */ + +#ifndef _KERN_XCALL_H +#define _KERN_XCALL_H + +/* + * Type for cross-call functions. + */ +typedef void (*xcall_fn_t)(void *arg); + +/* + * Initialize the xcall module. + */ +void xcall_setup(void); + +/* + * Run the given cross-call function on a specific processor. + * + * The operation is completely synchronous, returning only when the function + * has finished running on the target processor, with the side effects of + * the function visible. + * + * The function is run in interrupt context. + */ +void xcall_call(xcall_fn_t fn, void *arg, unsigned int cpu); + +/* + * Report a cross-call interrupt from a remote processor. + * + * Called from interrupt context. + */ +void xcall_intr(void); + +#endif /* _KERN_XCALL_H */ diff --git a/test/test_xcall.c b/test/test_xcall.c new file mode 100644 index 00000000..9da6a151 --- /dev/null +++ b/test/test_xcall.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2014 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * This is a simple test of the cross-call functionality. One thread is + * created and bound to CPU 0. It makes two cross-calls, one on its local + * processor, and another on a remote processor. + */ + +#include <kern/error.h> +#include <kern/cpumap.h> +#include <kern/printk.h> +#include <kern/stddef.h> +#include <kern/thread.h> +#include <kern/xcall.h> +#include <test/test.h> + +static int test_done; + +static void +test_fn(void *arg) +{ + (void)arg; + + printk("function called, running on cpu%u\n", cpu_id()); + test_done = 1; +} + +static void +test_once(unsigned int cpu) +{ + test_done = 0; + + printk("cross-call on cpu%u:\n", cpu); + xcall_call(test_fn, NULL, cpu); + + if (!test_done) + panic("test_done false"); +} + +static void +test_run(void *arg) +{ + (void)arg; + + test_once(0); + test_once(1); + printk("done\n"); +} + +void +test_setup(void) +{ + struct thread_attr attr; + struct thread *thread; + struct cpumap *cpumap; + int error; + + error = cpumap_create(&cpumap); + error_check(error, "cpumap_create"); + cpumap_zero(cpumap); + cpumap_set(cpumap, 0); + + thread_attr_init(&attr, "x15_test_run"); + thread_attr_set_detached(&attr); + thread_attr_set_cpumap(&attr, cpumap); + error = thread_create(&thread, &attr, test_run, NULL); + error_check(error, "thread_create"); + + cpumap_destroy(cpumap); +} |