x86/pmap: use percpu variables

1 files changed, 44 insertions, 35 deletions

diff --git a/arch/x86/machine/pmap.c b/arch/x86/machine/pmap.c
index 064f2440..f840f0b2 100644
--- a/arch/x86/machine/pmap.c
+++ b/arch/x86/machine/pmap.c
@@ -27,6 +27,7 @@
 #include <kern/mutex.h>
 #include <kern/panic.h>
 #include <kern/param.h>
+#include <kern/percpu.h>
 #include <kern/spinlock.h>
 #include <kern/sprintf.h>
 #include <kern/stddef.h>
@@ -108,8 +109,10 @@ struct pmap_tmp_mapping {
     unsigned long va;
 } __aligned(CPU_L1_SIZE);
 
-static struct pmap_tmp_mapping pmap_zero_mappings[MAX_CPUS];
-static struct pmap_tmp_mapping pmap_ptp_mappings[MAX_CPUS];
+static struct pmap_tmp_mapping pmap_zero_mapping __percpu;
+static struct pmap_tmp_mapping pmap_ptp_mapping __percpu;
+static unsigned long pmap_zero_va __read_mostly;
+static unsigned long pmap_ptp_va __read_mostly;
 
 /*
  * Reserved pages of virtual memory available for early allocation.
@@ -143,6 +146,12 @@ struct pmap {
 
 static struct pmap kernel_pmap_store __read_mostly;
 struct pmap *kernel_pmap __read_mostly = &kernel_pmap_store;
+
+/*
+ * The kernel per-CPU page tables are used early enough during bootstrap
+ * that using a percpu variable would actually become ugly. This array
+ * is rather small anyway.
+ */
 static struct pmap_cpu_table kernel_pmap_cpu_tables[MAX_CPUS] __read_mostly;
 
 struct pmap *pmap_current_ptr __percpu;
@@ -155,6 +164,8 @@ struct pmap *pmap_current_ptr __percpu;
 
 /*
  * "Hidden" kernel root page tables for PAE mode.
+ *
+ * Don't use a percpu variable so that PDPTs are kept low in physical memory.
  */
 static pmap_pte_t pmap_cpu_kpdpts[MAX_CPUS][PMAP_NR_RPTPS] __read_mostly
     __aligned(PMAP_PDPT_ALIGN);
@@ -272,7 +283,7 @@ struct pmap_syncer {
 
 static void pmap_sync(void *arg);
 
-static struct pmap_syncer pmap_syncers[MAX_CPUS];
+static struct pmap_syncer pmap_syncer __percpu;
 
 /*
  * Maximum number of mappings for which individual TLB invalidations can be
@@ -307,7 +318,7 @@ struct pmap_update_request_array {
     struct mutex lock;
 } __aligned(CPU_L1_SIZE);
 
-static struct pmap_update_request_array pmap_update_request_arrays[MAX_CPUS];
+static struct pmap_update_request_array pmap_update_request_array __percpu;
 
 static int pmap_do_remote_updates __read_mostly;
 
@@ -462,6 +473,13 @@ pmap_ap_setup_paging(void)
     return root_ptp;
 }
 
+static void __init
+pmap_tmp_mapping_init(struct pmap_tmp_mapping *mapping, unsigned long va)
+{
+    mutex_init(&mapping->lock);
+    mapping->va = va;
+}
+
 /*
  * Helper function for initialization procedures that require post-fixing
  * page properties.
@@ -671,7 +689,7 @@ pmap_update_request_array_acquire(void)
     struct pmap_update_request_array *array;
 
     thread_pin();
-    array = &pmap_update_request_arrays[cpu_id()];
+    array = cpu_local_ptr(pmap_update_request_array);
     mutex_lock(&array->lock);
     return array;
 }
@@ -684,13 +702,11 @@ pmap_update_request_array_release(struct pmap_update_request_array *array)
 }
 
 static void __init
-pmap_syncer_init(struct pmap_syncer *syncer)
+pmap_syncer_init(struct pmap_syncer *syncer, unsigned int cpu)
 {
     char name[EVCNT_NAME_SIZE];
     struct pmap_update_queue *queue;
-    unsigned int cpu;
 
-    cpu = syncer - pmap_syncers;
     queue = &syncer->queue;
     mutex_init(&queue->lock);
     condition_init(&queue->cond);
@@ -733,24 +749,15 @@ pmap_bootstrap(void)
     pmap_prot_table[VM_PROT_EXECUTE | VM_PROT_WRITE] = PMAP_PTE_RW;
     pmap_prot_table[VM_PROT_ALL] = PMAP_PTE_RW;
 
-    va = pmap_bootalloc(1);
-
-    for (i = 0; i < ARRAY_SIZE(pmap_zero_mappings); i++) {
-        mutex_init(&pmap_zero_mappings[i].lock);
-        pmap_zero_mappings[i].va = va;
-    }
-
-    va = pmap_bootalloc(PMAP_NR_RPTPS);
+    pmap_zero_va = pmap_bootalloc(1);
+    pmap_tmp_mapping_init(cpu_local_ptr(pmap_zero_mapping), pmap_zero_va);
 
-    for (i = 0; i < ARRAY_SIZE(pmap_ptp_mappings); i++) {
-        mutex_init(&pmap_ptp_mappings[i].lock);
-        pmap_ptp_mappings[i].va = va;
-    }
+    pmap_ptp_va = pmap_bootalloc(PMAP_NR_RPTPS);
+    pmap_tmp_mapping_init(cpu_local_ptr(pmap_ptp_mapping), pmap_ptp_va);
 
-    for (i = 0; i < ARRAY_SIZE(pmap_update_request_arrays); i++)
-        pmap_update_request_array_init(&pmap_update_request_arrays[i]);
+    pmap_update_request_array_init(cpu_local_ptr(pmap_update_request_array));
 
-    pmap_syncer_init(&pmap_syncers[0]);
+    pmap_syncer_init(cpu_local_ptr(pmap_syncer), 0);
 
     pmap_update_oplist_ctor(&pmap_booter_oplist);
     thread_key_create(&pmap_oplist_tsd_key, pmap_update_oplist_destroy);
@@ -781,6 +788,11 @@ pmap_ap_bootstrap(void)
 {
     cpu_local_assign(pmap_current_ptr, kernel_pmap);
 
+    pmap_tmp_mapping_init(cpu_local_ptr(pmap_zero_mapping), pmap_zero_va);
+    pmap_tmp_mapping_init(cpu_local_ptr(pmap_ptp_mapping), pmap_ptp_va);
+    pmap_update_request_array_init(cpu_local_ptr(pmap_update_request_array));
+    pmap_syncer_init(cpu_local_ptr(pmap_syncer), cpu_id());
+
     if (cpu_has_global_pages())
         cpu_enable_global_pages();
     else
@@ -869,7 +881,7 @@ pmap_zero_page(phys_addr_t pa)
     unsigned long va;
 
     thread_pin();
-    zero_mapping = &pmap_zero_mappings[cpu_id()];
+    zero_mapping = cpu_local_ptr(pmap_zero_mapping);
     mutex_lock(&zero_mapping->lock);
     va = zero_mapping->va;
     pmap_kenter(va, pa, VM_PROT_WRITE);
@@ -895,7 +907,7 @@ pmap_map_ptp(phys_addr_t pa, unsigned int nr_pages)
 #endif
 
     thread_pin();
-    ptp_mapping  = &pmap_ptp_mappings[cpu_id()];
+    ptp_mapping  = cpu_local_ptr(pmap_ptp_mapping);
     mutex_lock(&ptp_mapping->lock);
 
     for (i = 0; i < nr_pages; i++) {
@@ -1095,16 +1107,13 @@ pmap_mp_setup(void)
     unsigned int cpu;
     int error;
 
-    for (cpu = 1; cpu < cpu_count(); cpu++)
-        pmap_syncer_init(&pmap_syncers[cpu]);
-
     error = cpumap_create(&cpumap);
 
     if (error)
         panic("pmap: unable to create syncer cpumap");
 
     for (cpu = 0; cpu < cpu_count(); cpu++) {
-        syncer = &pmap_syncers[cpu];
+        syncer = percpu_ptr(pmap_syncer, cpu);
         snprintf(name, sizeof(name), "x15_pmap_sync/%u", cpu);
         cpumap_zero(cpumap);
         cpumap_set(cpumap, cpu);
@@ -1528,18 +1537,17 @@ pmap_update_local(const struct pmap_update_oplist *oplist,
 {
     const struct pmap_update_op *op;
     struct pmap_syncer *syncer;
-    unsigned int i, cpu;
     int flush_tlb_entries;
+    unsigned int i;
 
-    cpu = cpu_id();
-    syncer = &pmap_syncers[cpu];
+    syncer = cpu_local_ptr(pmap_syncer);
     evcnt_inc(&syncer->ev_update);
     flush_tlb_entries = (nr_mappings <= PMAP_UPDATE_MAX_MAPPINGS);
 
     for (i = 0; i < oplist->nr_ops; i++) {
         op = &oplist->ops[i];
 
-        if (!cpumap_test(&op->cpumap, cpu))
+        if (!cpumap_test(&op->cpumap, cpu_id()))
             continue;
 
         switch (op->operation) {
@@ -1574,6 +1582,7 @@ pmap_update(struct pmap *pmap)
     struct pmap_update_request_array *array;
     struct pmap_update_request *request;
     struct pmap_update_queue *queue;
+    struct pmap_syncer *syncer;
     unsigned int nr_mappings;
     int cpu;
 
@@ -1593,9 +1602,9 @@ pmap_update(struct pmap *pmap)
     array = pmap_update_request_array_acquire();
 
     cpumap_for_each(&oplist->cpumap, cpu) {
+        syncer = percpu_ptr(pmap_syncer, cpu);
+        queue = &syncer->queue;
         request = &array->requests[cpu];
-        queue = &pmap_syncers[cpu].queue;
-
         request->oplist = oplist;
         request->nr_mappings = pmap_update_oplist_count_mappings(oplist, cpu);
         request->done = 0;