x86/pmap: inter-processor TLB updates

Scheduling is temporarily disabled until the thread module is able to
cope with multiple processors.

11 files changed, 125 insertions, 3 deletions

diff --git a/arch/x86/machine/boot.c b/arch/x86/machine/boot.c
index 9a0821e9..f25606a0 100644
--- a/arch/x86/machine/boot.c
+++ b/arch/x86/machine/boot.c
@@ -291,6 +291,8 @@ boot_ap_main(void)
     cpu_ap_setup();
     pmap_ap_bootstrap();
 
+    cpu_intr_enable();
+
     for (;;)
         cpu_idle();
 
diff --git a/arch/x86/machine/cga.c b/arch/x86/machine/cga.c
index 6e4ffc0d..d2d0005d 100644
--- a/arch/x86/machine/cga.c
+++ b/arch/x86/machine/cga.c
@@ -17,6 +17,7 @@
 
 #include <kern/init.h>
 #include <kern/macros.h>
+#include <kern/param.h>
 #include <kern/stdint.h>
 #include <kern/string.h>
 #include <machine/io.h>
@@ -107,6 +108,7 @@ cga_setup(void)
 
     va = pmap_bootalloc(1);
     pmap_kenter(va, CGA_MEMORY);
+    pmap_kupdate(va, va + PAGE_SIZE);
     cga_memory = (uint8_t *)va;
 
     /*
diff --git a/arch/x86/machine/cpu.c b/arch/x86/machine/cpu.c
index 7976f81d..0f395fc3 100644
--- a/arch/x86/machine/cpu.c
+++ b/arch/x86/machine/cpu.c
@@ -30,6 +30,7 @@
 #include <machine/cpu.h>
 #include <machine/io.h>
 #include <machine/lapic.h>
+#include <machine/pmap.h>
 #include <machine/trap.h>
 #include <vm/vm_kmem.h>
 #include <vm/vm_page.h>
@@ -427,6 +428,7 @@ cpu_mp_setup(void)
 {
     acpimp_setup();
     cpu_mp_start_aps();
+    pmap_mp_setup();
     cpu_mp_info();
 }
 
diff --git a/arch/x86/machine/lapic.c b/arch/x86/machine/lapic.c
index 01c13c64..e65d5a3b 100644
--- a/arch/x86/machine/lapic.c
+++ b/arch/x86/machine/lapic.c
@@ -340,7 +340,9 @@ lapic_intr_timer(struct trap_frame *frame)
 {
     (void)frame;
 
+#if 0
     thread_tick();
+#endif
     lapic_eoi();
 }
 
diff --git a/arch/x86/machine/pmap.c b/arch/x86/machine/pmap.c
index 49d62988..786a235c 100644
--- a/arch/x86/machine/pmap.c
+++ b/arch/x86/machine/pmap.c
@@ -13,6 +13,9 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * TODO Check TLB flushes on the recursive mapping.
  */
 
 #include <kern/assert.h>
@@ -23,10 +26,14 @@
 #include <kern/stddef.h>
 #include <kern/string.h>
 #include <kern/types.h>
+#include <machine/atomic.h>
 #include <machine/biosmem.h>
 #include <machine/boot.h>
 #include <machine/cpu.h>
+#include <machine/lapic.h>
+#include <machine/mb.h>
 #include <machine/pmap.h>
+#include <machine/trap.h>
 #include <vm/vm_kmem.h>
 #include <vm/vm_page.h>
 #include <vm/vm_prot.h>
@@ -136,6 +143,24 @@ static pmap_pte_t pmap_prot_table[8];
  */
 static unsigned long pmap_zero_va;
 
+/*
+ * True if running on multiple processors (TLB flushes must be propagated).
+ */
+static int pmap_mp_mode;
+
+/*
+ * Shared variables used by the inter-processor update functions.
+ */
+static unsigned long pmap_update_start;
+static unsigned long pmap_update_end;
+
+/*
+ * There is strong bouncing on this counter so give it its own cache line.
+ */
+static struct {
+    volatile unsigned long count __aligned(CPU_L1_SIZE);
+} pmap_nr_updates;
+
 static void __init
 pmap_boot_enter(pmap_pte_t *root_pt, unsigned long va, phys_addr_t pa)
 {
@@ -371,8 +396,10 @@ static void
 pmap_zero_page(phys_addr_t pa)
 {
     pmap_kenter(pmap_zero_va, pa);
+    cpu_tlb_flush_va(pmap_zero_va);
     memset((void *)pmap_zero_va, 0, PAGE_SIZE);
     pmap_kremove(pmap_zero_va, pmap_zero_va + PAGE_SIZE);
+    cpu_tlb_flush_va(pmap_zero_va);
 }
 
 void
@@ -427,7 +454,6 @@ pmap_kenter(unsigned long va, phys_addr_t pa)
     pte = PMAP_PTEMAP_BASE + PMAP_PTEMAP_INDEX(va, PMAP_L1_SHIFT);
     *pte = ((pa & PMAP_PA_MASK) | PMAP_PTE_G | PMAP_PTE_RW | PMAP_PTE_P)
            & pmap_pt_levels[0].mask;
-    cpu_tlb_flush_va(va);
 }
 
 void
@@ -438,7 +464,6 @@ pmap_kremove(unsigned long start, unsigned long end)
     while (start < end) {
         pte = PMAP_PTEMAP_BASE + PMAP_PTEMAP_INDEX(start, PMAP_L1_SHIFT);
         *pte = 0;
-        cpu_tlb_flush_va(start);
         start += PAGE_SIZE;
     }
 }
@@ -453,11 +478,50 @@ pmap_kprotect(unsigned long start, unsigned long end, int prot)
     while (start < end) {
         pte = PMAP_PTEMAP_BASE + PMAP_PTEMAP_INDEX(start, PMAP_L1_SHIFT);
         *pte = (*pte & ~PMAP_PTE_PROT_MASK) | flags;
+        start += PAGE_SIZE;
+    }
+}
+
+static void
+pmap_kupdate_local(unsigned long start, unsigned long end)
+{
+    while (start < end) {
         cpu_tlb_flush_va(start);
         start += PAGE_SIZE;
     }
 }
 
+void
+pmap_kupdate(unsigned long start, unsigned long end)
+{
+    unsigned int nr_cpus;
+
+    if (pmap_mp_mode)
+        nr_cpus = cpu_count();
+    else
+        nr_cpus = 1;
+
+    if (nr_cpus == 1) {
+        pmap_kupdate_local(start, end);
+        return;
+    }
+
+    pmap_update_start = start;
+    pmap_update_end = end;
+    pmap_nr_updates.count = nr_cpus - 1;
+    mb_store();
+    lapic_ipi_broadcast(TRAP_PMAP_UPDATE);
+
+    /*
+     * Perform the local update now so that some time is given to the other
+     * processors, which slightly reduces contention on the update counter.
+     */
+    pmap_kupdate_local(start, end);
+
+    while (pmap_nr_updates.count != 0)
+        cpu_pause();
+}
+
 phys_addr_t
 pmap_kextract(unsigned long va)
 {
@@ -475,3 +539,21 @@ pmap_kextract(unsigned long va)
 
     return *pte & PMAP_PA_MASK;
 }
+
+void
+pmap_mp_setup(void)
+{
+    pmap_mp_mode = 1;
+}
+
+void
+pmap_update_intr(struct trap_frame *frame)
+{
+    (void)frame;
+
+    lapic_eoi();
+
+    /* Interrupts are serializing events, no memory barrier required */
+    pmap_kupdate_local(pmap_update_start, pmap_update_end);
+    atomic_add(&pmap_nr_updates.count, -1);
+}
diff --git a/arch/x86/machine/pmap.h b/arch/x86/machine/pmap.h
index 5b1c2bb9..cbaf6597 100644
--- a/arch/x86/machine/pmap.h
+++ b/arch/x86/machine/pmap.h
@@ -102,6 +102,7 @@
 
 #include <kern/stdint.h>
 #include <kern/types.h>
+#include <machine/trap.h>
 
 #ifdef X86_PAE
 typedef uint64_t pmap_pte_t;
@@ -165,13 +166,25 @@ void pmap_growkernel(unsigned long va);
 /*
  * Kernel specific mapping functions.
  *
- * Resources for the new mappings must be preallocated.
+ * Resources for the new mappings must be preallocated. The only function
+ * which actually flushes the TLB is pmap_kupdate.
  */
 void pmap_kenter(unsigned long va, phys_addr_t pa);
 void pmap_kremove(unsigned long start, unsigned long end);
 void pmap_kprotect(unsigned long start, unsigned long end, int prot);
+void pmap_kupdate(unsigned long start, unsigned long end);
 phys_addr_t pmap_kextract(unsigned long va);
 
+/*
+ * Prepare the pmap module for a multiprocessor environment.
+ */
+void pmap_mp_setup(void);
+
+/*
+ * Interrupt handler for inter-processor update requests.
+ */
+void pmap_update_intr(struct trap_frame *frame);
+
 #endif /* __ASSEMBLER__ */
 
 #endif /* _X86_PMAP_H */
diff --git a/arch/x86/machine/trap.c b/arch/x86/machine/trap.c
index 27f35f5b..a26212a2 100644
--- a/arch/x86/machine/trap.c
+++ b/arch/x86/machine/trap.c
@@ -24,6 +24,7 @@
 #include <machine/cpu.h>
 #include <machine/lapic.h>
 #include <machine/pic.h>
+#include <machine/pmap.h>
 #include <machine/trap.h>
 
 /*
@@ -63,6 +64,7 @@ void trap_isr_machine_check(void);
 void trap_isr_simd_fp_exception(void);
 void trap_isr_pic_int7(void);
 void trap_isr_pic_int15(void);
+void trap_isr_pmap_update(void);
 void trap_isr_lapic_timer(void);
 void trap_isr_lapic_error(void);
 void trap_isr_lapic_spurious(void);
@@ -134,6 +136,7 @@ trap_setup(void)
     trap_install(TRAP_PIC_BASE + 15, trap_isr_pic_int15, pic_intr_spurious);
 
     /* System defined traps */
+    trap_install(TRAP_PMAP_UPDATE, trap_isr_pmap_update, pmap_update_intr);
     trap_install(TRAP_LAPIC_TIMER, trap_isr_lapic_timer, lapic_intr_timer);
     trap_install(TRAP_LAPIC_ERROR, trap_isr_lapic_error, lapic_intr_error);
     trap_install(TRAP_LAPIC_SPURIOUS, trap_isr_lapic_spurious,
@@ -147,7 +150,10 @@ trap_main(struct trap_frame *frame)
 {
     assert(frame->vector < ARRAY_SIZE(trap_handlers));
     trap_handlers[frame->vector].fn(frame);
+
+#if 0
     thread_reschedule();
+#endif
 }
 
 #ifdef __LP64__
diff --git a/arch/x86/machine/trap.h b/arch/x86/machine/trap.h
index aa2e09a5..778769c1 100644
--- a/arch/x86/machine/trap.h
+++ b/arch/x86/machine/trap.h
@@ -50,7 +50,10 @@
 
 /*
  * System defined traps.
+ *
+ * The local APIC assigns one priority every 16 vectors.
  */
+#define TRAP_PMAP_UPDATE    240
 #define TRAP_LAPIC_TIMER    253
 #define TRAP_LAPIC_ERROR    254
 #define TRAP_LAPIC_SPURIOUS 255
diff --git a/arch/x86/machine/trap_asm.S b/arch/x86/machine/trap_asm.S
index 255cabde..0fb28962 100644
--- a/arch/x86/machine/trap_asm.S
+++ b/arch/x86/machine/trap_asm.S
@@ -169,6 +169,7 @@ TRAP(TRAP_PIC_BASE + 7, pic_int7)
 TRAP(TRAP_PIC_BASE + 15, pic_int15)
 
 /* System defined traps */
+TRAP(TRAP_PMAP_UPDATE, pmap_update)
 TRAP(TRAP_LAPIC_TIMER, lapic_timer)
 TRAP(TRAP_LAPIC_ERROR, lapic_error)
 TRAP(TRAP_LAPIC_SPURIOUS, lapic_spurious)
diff --git a/vm/vm_kmem.c b/vm/vm_kmem.c
index 5cc7374a..0ba97038 100644
--- a/vm/vm_kmem.c
+++ b/vm/vm_kmem.c
@@ -70,6 +70,7 @@ vm_kmem_bootalloc(size_t size)
         pmap_kenter(va, pa);
     }
 
+    pmap_kupdate(start, vm_kmem_boot_start);
     return start;
 }
 
@@ -137,6 +138,7 @@ vm_kmem_free_va(unsigned long addr, size_t size)
 
     end = addr + vm_page_round(size);
     pmap_kremove(addr, end);
+    pmap_kupdate(addr, end);
     vm_map_remove(kernel_map, addr, end);
 }
 
@@ -162,6 +164,7 @@ vm_kmem_alloc(size_t size)
         pmap_kenter(start, vm_page_to_pa(page));
     }
 
+    pmap_kupdate(start, end);
     return va;
 
 error_page:
@@ -215,6 +218,8 @@ vm_kmem_map_pa(phys_addr_t addr, size_t size, unsigned long *map_addrp,
     for (offset = 0; offset < map_size; offset += PAGE_SIZE)
         pmap_kenter(map_addr + offset, start + offset);
 
+    pmap_kupdate(map_addr, map_addr + map_size);
+
     if (map_addrp != NULL)
         *map_addrp = map_addr;
 
diff --git a/vm/vm_map.c b/vm/vm_map.c
index 58ff2cda..3ae930f0 100644
--- a/vm/vm_map.c
+++ b/vm/vm_map.c
@@ -198,6 +198,7 @@ vm_map_kentry_alloc(size_t slab_size)
         pmap_kenter(va + i, vm_page_to_pa(page));
     }
 
+    pmap_kupdate(va, va + slab_size);
     return va;
 }
 
@@ -222,6 +223,7 @@ vm_map_kentry_free(unsigned long va, size_t slab_size)
     }
 
     pmap_kremove(va, va + slab_size);
+    pmap_kupdate(va, va + slab_size);
     vm_map_kentry_free_va(va, slab_size);
 }
 
@@ -273,6 +275,8 @@ vm_map_kentry_setup(void)
         pmap_kenter(table_va + (i * PAGE_SIZE), vm_page_to_pa(page));
     }
 
+    pmap_kupdate(table_va, table_va + (nr_pages * PAGE_SIZE));
+
     slabs = (struct vm_map_kentry_slab *)table_va;
     vm_map_kentry_free_slabs = &slabs[nr_slabs - 1];
     vm_map_kentry_free_slabs->next = NULL;