kern/percpu: fix setup of BSP percpu area

Relocating percpu data is actually tricky, for the same reason relocating
memory in general is. There may be pointers to such variables which then
become invalid. Instead, keep using the percpu section as the percpu area
for the BSP and store the content of the percpu section in dedicated
kernel virtual memory. In addition, remove the notification kludge from
the x86 cpu module.

5 files changed, 21 insertions, 37 deletions

diff --git a/arch/x86/machine/cpu.c b/arch/x86/machine/cpu.c
index 0f362527..7e729d72 100644
--- a/arch/x86/machine/cpu.c
+++ b/arch/x86/machine/cpu.c
@@ -495,31 +495,6 @@ cpu_info(const struct cpu *cpu)
 }
 
 void __init
-cpu_fixup_bsp_percpu_area(void)
-{
-    struct cpu_pseudo_desc gdtr;
-    struct cpu *cpu;
-    void *pcpu_area;
-
-    /*
-     * It's important to use the percpu interface here, and not the cpu_local
-     * accessors : this function updates the GDTR (and the GDT on i386), as a
-     * result it must reference the future version of the GDT from the newly
-     * allocated percpu area.
-     */
-    cpu = percpu_ptr(cpu_desc, 0);
-    pcpu_area = percpu_area(0);
-
-#ifndef __LP64__
-    cpu_seg_set_data(cpu->gdt, CPU_GDT_SEL_PERCPU, (unsigned long)pcpu_area);
-#endif /* __LP64__ */
-
-    cpu_init_gdtr(&gdtr, cpu);
-    cpu_load_gdt(&gdtr);
-    cpu_set_percpu_area(cpu, pcpu_area);
-}
-
-void __init
 cpu_mp_register_lapic(unsigned int apic_id, int is_bsp)
 {
     struct cpu *cpu;
diff --git a/arch/x86/machine/cpu.h b/arch/x86/machine/cpu.h
index 52a2af03..9b6b028e 100644
--- a/arch/x86/machine/cpu.h
+++ b/arch/x86/machine/cpu.h
@@ -593,12 +593,6 @@ void cpu_check(const struct cpu *cpu);
 void cpu_info(const struct cpu *cpu);
 
 /*
- * Notify the cpu module that the true percpu area for the BSP has been
- * created.
- */
-void cpu_fixup_bsp_percpu_area(void);
-
-/*
  * Register the presence of a local APIC.
  */
 void cpu_mp_register_lapic(unsigned int apic_id, int is_bsp);
diff --git a/kern/kernel.c b/kern/kernel.c
index e5469079..069649de 100644
--- a/kern/kernel.c
+++ b/kern/kernel.c
@@ -19,6 +19,7 @@
 #include <kern/init.h>
 #include <kern/kernel.h>
 #include <kern/llsync.h>
+#include <kern/percpu.h>
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/work.h>
@@ -34,6 +35,7 @@ kernel_main(void)
 {
     assert(!cpu_intr_enabled());
 
+    percpu_cleanup();
     cpumap_setup();
     task_setup();
     thread_setup();
diff --git a/kern/percpu.c b/kern/percpu.c
index aab9d064..9a0bdf31 100644
--- a/kern/percpu.c
+++ b/kern/percpu.c
@@ -30,6 +30,7 @@
 
 void *percpu_areas[MAX_CPUS] __read_mostly;
 
+static void *percpu_area_content __initdata;
 static size_t percpu_size __initdata;
 static int percpu_skip_warning __initdata;
 
@@ -55,11 +56,10 @@ percpu_setup(void)
     va = vm_kmem_alloc(percpu_size);
 
     if (va == 0)
-        panic("percpu: unable to allocate percpu area for BSP");
+        panic("percpu: unable to allocate memory for percpu area content");
 
-    percpu_areas[0] = (void *)va;
-    memcpy(percpu_area(0), &_percpu, percpu_size);
-    cpu_fixup_bsp_percpu_area();
+    percpu_area_content = (void *)va;
+    memcpy(percpu_area_content, &_percpu, percpu_size);
 }
 
 int __init
@@ -93,8 +93,14 @@ percpu_add(unsigned int cpu)
     }
 
     percpu_areas[cpu] = (void *)va;
-    memcpy(percpu_area(cpu), &_percpu, percpu_size);
+    memcpy(percpu_area(cpu), percpu_area_content, percpu_size);
 
 out:
     return 0;
 }
+
+void
+percpu_cleanup(void)
+{
+    vm_kmem_free((unsigned long)percpu_area_content, percpu_size);
+}
diff --git a/kern/percpu.h b/kern/percpu.h
index 3492c6d8..46d33ed7 100644
--- a/kern/percpu.h
+++ b/kern/percpu.h
@@ -101,7 +101,9 @@ void percpu_bootstrap(void);
 /*
  * Complete initialization of the percpu module.
  *
- * Stop using the percpu section as the percpu area of the BSP.
+ * The BSP keeps using the percpu section, but its content is copied to a
+ * dedicated block of memory used as a template for subsequently added
+ * processors.
  */
 void percpu_setup(void);
 
@@ -114,4 +116,9 @@ void percpu_setup(void);
  */
 int percpu_add(unsigned int cpu);
 
+/*
+ * Release init data allocated for setup.
+ */
+void percpu_cleanup(void);
+
 #endif /* _KERN_PERCPU_H */