Improve page table creation at boot time

9 files changed, 51 insertions, 36 deletions

diff --git a/arch/arm/machine/boot.c b/arch/arm/machine/boot.c
index 0bc1b9cf..9a0f33a4 100644
--- a/arch/arm/machine/boot.c
+++ b/arch/arm/machine/boot.c
@@ -33,6 +33,8 @@ alignas(CPU_DATA_ALIGN) char boot_stack[BOOT_STACK_SIZE] __bootdata;
 
 pmap_pte_t * boot_setup_paging(void);
 
+void boot_main(void);
+
 void __boot
 boot_panic(const char *s)
 {
@@ -61,6 +63,12 @@ boot_log_info(void)
 {
 }
 
+void __init
+boot_main(void)
+{
+    for (;;);
+}
+
 /*
  * Init operation aliases.
  */
diff --git a/arch/arm/machine/boot.h b/arch/arm/machine/boot.h
index b1f00458..4d3d95e3 100644
--- a/arch/arm/machine/boot.h
+++ b/arch/arm/machine/boot.h
@@ -38,6 +38,7 @@
 #define BOOT_VTOL(addr) ((addr) - PMAP_START_KERNEL_ADDRESS)
 
 #define BOOT_VTOP(addr) ((addr) - BOOT_KERNEL_OFFSET)
+#define BOOT_PTOV(addr) ((addr) + BOOT_KERNEL_OFFSET)
 
 #define BOOT_MEM_BLOCK_BITS     10
 #define BOOT_MEM_NR_FREE_LISTS  5
diff --git a/arch/arm/machine/boot_asm.S b/arch/arm/machine/boot_asm.S
index 897b5526..b54e26e7 100644
--- a/arch/arm/machine/boot_asm.S
+++ b/arch/arm/machine/boot_asm.S
@@ -99,6 +99,8 @@ ASM_FUNC(boot_start_ram):
  orr %r0, %r0, #1                       /* Set the M bit */
  mcr p15, 0, %r0, c1, c0, 0             /* Write CP15 control register */
 
+ blx boot_main
+
  /* Never reached */
  b .
 ASM_END(boot_start_ram)
diff --git a/arch/arm/machine/pmap.c b/arch/arm/machine/pmap.c
index 4943286f..ad077f64 100644
--- a/arch/arm/machine/pmap.c
+++ b/arch/arm/machine/pmap.c
@@ -53,7 +53,7 @@
 #define PMAP_PTE_B              0x00000004
 #define PMAP_PTE_C              0x00000008
 
-#define PMAP_PTE_L0_RW          0x00000030
+#define PMAP_PTE_L0_RW          0x000000f0
 #define PMAP_PTE_L1_RW          0x00000c00
 
 /*
@@ -144,20 +144,27 @@ static struct kmem_cache pmap_cache;
 static char pmap_panic_directmap_msg[] __bootdata
     = "vm_ptable: invalid direct physical mapping";
 
-static unsigned long __boot
-pmap_boot_get_large_pgsize(void)
+unsigned int __boot
+pmap_boot_get_last_level(size_t page_size)
+{
+    switch (page_size) {
+    case (1 << PMAP_L1_SKIP):
+        return 1;
+    default:
+        return 0;
+    }
+}
+
+static size_t __boot
+pmap_boot_get_large_page_size(void)
 {
-#if 0
     return (1 << PMAP_L1_SKIP);
-#else
-    return PAGE_SIZE;
-#endif
 }
 
 pmap_pte_t * __boot
 pmap_setup_paging(void)
 {
-    unsigned long i, size, pgsize;
+    size_t size, page_size;
     phys_addr_t pa, directmap_end;
     struct vm_ptable *ptable;
     struct pmap *kernel_pmap;
@@ -166,26 +173,26 @@ pmap_setup_paging(void)
     kernel_pmap = (void *)BOOT_VTOP((uintptr_t)&pmap_kernel_pmap);
 
     /* Use large pages for the direct physical mapping when possible */
-    pgsize = pmap_boot_get_large_pgsize();
+    page_size = pmap_boot_get_large_page_size();
 
     /* TODO LPAE */
 
     vm_ptable_bootstrap(pmap_boot_pt_levels, ARRAY_SIZE(pmap_boot_pt_levels));
 
     ptable = &kernel_pmap->ptable;
-    vm_ptable_build(ptable);
+    vm_ptable_boot_build(ptable);
 
     /*
      * Create the initial mappings. The first is for the .boot section
      * and acts as the mandatory identity mapping. The second is the
-     * direct physical mapping of physical memory.
+     * direct mapping of physical memory.
      */
 
     va = vm_page_trunc((uintptr_t)&_boot);
     pa = va;
     size = vm_page_round((uintptr_t)&_boot_end) - va;
 
-    for (i = 0; i < size; i += PAGE_SIZE) {
+    for (size_t i = 0; i < size; i += PAGE_SIZE) {
         vm_ptable_boot_enter(ptable, va, pa, PAGE_SIZE);
         va += PAGE_SIZE;
         pa += PAGE_SIZE;
@@ -201,10 +208,10 @@ pmap_setup_paging(void)
     va = PMAP_START_DIRECTMAP_ADDRESS;
     pa = PMEM_RAM_START;
 
-    for (i = PMEM_RAM_START; i < directmap_end; i += pgsize) {
-        vm_ptable_boot_enter(ptable, va, pa, pgsize);
-        va += pgsize;
-        pa += pgsize;
+    for (size_t i = PMEM_RAM_START; i < directmap_end; i += page_size) {
+        vm_ptable_boot_enter(ptable, va, pa, page_size);
+        va += page_size;
+        pa += page_size;
     }
 
     return vm_ptable_boot_root(ptable);
@@ -216,10 +223,10 @@ pmap_ap_setup_paging(void)
 {
     struct pmap_cpu_table *cpu_table;
     struct pmap *pmap;
-    unsigned long pgsize;
+    unsigned long page_size;
 
-    pgsize = pmap_boot_get_pgsize();
-    pmap_boot_enable_pgext(pgsize);
+    page_size = pmap_boot_get_page_size();
+    pmap_boot_enable_pgext(page_size);
 
     pmap = (void *)BOOT_VTOP((uintptr_t)&pmap_kernel_pmap);
     cpu_table = (void *)BOOT_VTOP((uintptr_t)pmap->cpu_tables[boot_ap_id]);
diff --git a/arch/arm/machine/pmap.h b/arch/arm/machine/pmap.h
index d61eb00c..3f55ab91 100644
--- a/arch/arm/machine/pmap.h
+++ b/arch/arm/machine/pmap.h
@@ -107,6 +107,8 @@ pmap_get_kernel_pmap(void)
     return &pmap_kernel_pmap;
 }
 
+unsigned int pmap_boot_get_last_level(size_t page_size);
+
 /*
  * Early initialization of the MMU.
  *
diff --git a/kern/bootmem.c b/kern/bootmem.c
index 446bf421..0bf92e19 100644
--- a/kern/bootmem.c
+++ b/kern/bootmem.c
@@ -774,7 +774,7 @@ bootmem_alloc(size_t size)
     return (void *)block->phys_addr;
 }
 
-phys_addr_t __boot
+size_t __boot
 bootmem_directmap_end(void)
 {
     if (bootmem_zone_size(bootmem_get_zone(PMEM_ZONE_DIRECTMAP)) != 0) {
diff --git a/kern/bootmem.h b/kern/bootmem.h
index 26ac2582..69a62115 100644
--- a/kern/bootmem.h
+++ b/kern/bootmem.h
@@ -93,6 +93,6 @@ void * bootmem_alloc(size_t size);
 /*
  * Return the end address of the direct physical mapping.
  */
-phys_addr_t bootmem_directmap_end(void);
+size_t bootmem_directmap_end(void);
 
 #endif /* _KERN_BOOTMEM_H */
diff --git a/vm/vm_ptable.c b/vm/vm_ptable.c
index 73ed18bf..f5b02e98 100644
--- a/vm/vm_ptable.c
+++ b/vm/vm_ptable.c
@@ -247,14 +247,16 @@ vm_ptable_pa_aligned(phys_addr_t pa)
 }
 
 void __boot
-vm_ptable_build(struct vm_ptable *ptable)
+vm_ptable_boot_build(struct vm_ptable *ptable)
 {
     const struct vm_ptable_level *pt_level;
     struct vm_ptable_cpu_pt *pt;
+    void *ptr;
 
     pt_level = vm_ptable_boot_get_pt_level(vm_ptable_boot_nr_levels - 1);
     pt = &vm_ptable_boot_cpu_pt;
-    pt->root = bootmem_alloc(pt_level->ptes_per_pt * sizeof(pmap_pte_t));
+    ptr = bootmem_alloc(pt_level->ptes_per_pt * sizeof(pmap_pte_t));
+    pt->root = (void *)BOOT_PTOV((uintptr_t)ptr);
     ptable->cpu_pts[0] = pt;
 
     for (size_t i = 1; i < ARRAY_SIZE(ptable->cpu_pts); i++) {
@@ -264,7 +266,7 @@ vm_ptable_build(struct vm_ptable *ptable)
 
 void __boot
 vm_ptable_boot_enter(struct vm_ptable *ptable, uintptr_t va,
-                     phys_addr_t pa, size_t pgsize)
+                     phys_addr_t pa, size_t page_size)
 {
     const struct vm_ptable_level *pt_level;
     unsigned int level, last_level;
@@ -275,15 +277,8 @@ vm_ptable_boot_enter(struct vm_ptable *ptable, uintptr_t va,
         boot_panic(vm_ptable_panic_inval_msg);
     }
 
-#if 0
-    switch (pgsize) {
-    case (1 << PMAP_L1_SKIP):
-        last_level = 1;
-        break;
-    default:
-#endif
-    last_level = 0;
-    pt = ptable->cpu_pts[0]->root;
+    last_level = pmap_boot_get_last_level(page_size);
+    pt = (void *)BOOT_VTOP((uintptr_t)ptable->cpu_pts[0]->root);
 
     for (level = vm_ptable_boot_nr_levels - 1; level != last_level; level--) {
         pt_level = vm_ptable_boot_get_pt_level(level);
@@ -293,7 +288,7 @@ vm_ptable_boot_enter(struct vm_ptable *ptable, uintptr_t va,
             mask = vm_ptable_level_pa_mask(pt_level);
             next_pt = (void *)(uintptr_t)(*pte & mask);
         } else {
-            next_pt = bootmem_alloc(pt_level->ptes_per_pt * sizeof(pmap_pte_t));
+            next_pt = bootmem_alloc(pt_level[-1].ptes_per_pt * sizeof(pmap_pte_t));
             *pte = pt_level->make_pte_fn((uintptr_t)next_pt, VM_PROT_ALL);
         }
 
@@ -308,5 +303,5 @@ vm_ptable_boot_enter(struct vm_ptable *ptable, uintptr_t va,
 pmap_pte_t * __boot
 vm_ptable_boot_root(const struct vm_ptable *ptable)
 {
-    return ptable->cpu_pts[0]->root;
+    return (void *)BOOT_VTOP((uintptr_t)ptable->cpu_pts[0]->root);
 }
diff --git a/vm/vm_ptable.h b/vm/vm_ptable.h
index a6ba5902..748b1b61 100644
--- a/vm/vm_ptable.h
+++ b/vm/vm_ptable.h
@@ -48,7 +48,7 @@ struct vm_ptable {
 void vm_ptable_bootstrap(const struct vm_ptable_level *pt_levels,
                          unsigned int nr_levels);
 
-void vm_ptable_build(struct vm_ptable *ptable);
+void vm_ptable_boot_build(struct vm_ptable *ptable);
 
 void vm_ptable_boot_enter(struct vm_ptable *ptable, uintptr_t va,
                           phys_addr_t pa, size_t pgsize);