x86: initialize long mode on amd64

The boot procedure creates an identity mapping for the first 4 GiB of
physical memory, then switches to long mode so that 64-bit code can be
run to initialize the kernel.

6 files changed, 203 insertions, 123 deletions

diff --git a/arch/x86/machine/asm.h b/arch/x86/machine/asm.h
index e427b51e..3d2da755 100644
--- a/arch/x86/machine/asm.h
+++ b/arch/x86/machine/asm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011 Richard Braun.
+ * Copyright (c) 2011, 2012 Richard Braun.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -18,24 +18,25 @@
 #ifndef _X86_ASM_H
 #define _X86_ASM_H
 
-#ifdef __ASSEMBLY__
+#include <machine/param.h>
 
-#define TEXT_ALIGN  4
-#define DATA_ALIGN  2
+#ifdef __ASSEMBLY__
 
-#define ENTRY(x)            \
-.p2align TEXT_ALIGN, 0x90;  \
-.global x;                  \
-.type x, STT_FUNC;          \
+#define ENTRY(x)        \
+.align TEXT_ALIGN;      \
+.global x;              \
+.type x, STT_FUNC;      \
 x:
 
 #define DATA(x)         \
-.p2align DATA_ALIGN;    \
+.align DATA_ALIGN;      \
 .global x;              \
 .type x, STT_OBJECT;    \
 x:
 
-#define END(x) .size x, . - x;
+#define END(x)  \
+.size x, . - x; \
+x ## _end:
 
 #endif /* __ASSEMBLY__ */
 
diff --git a/arch/x86/machine/boot.c b/arch/x86/machine/boot.c
index 9df4451d..a7b276e0 100644
--- a/arch/x86/machine/boot.c
+++ b/arch/x86/machine/boot.c
@@ -13,6 +13,32 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Early initialization procedure for x86.
+ *
+ * This module is separated in assembly and C code. The former is where
+ * the first instructions are run, and where actions that aren't possible,
+ * easy or clean in C are performed.
+ *
+ * When the boot loader passes control to the kernel, the main processor is
+ * in protected mode, paging is disabled, and some boot data are availabe
+ * outside the kernel. This module first sets up a basic physical memory
+ * allocator so that it can allocate page tables without corrupting the
+ * boot data. The .init section is linked at physical addresses, so that
+ * it can run with and without paging enabled. The page tables must properly
+ * configure an identity mapping so that this remains true as long as
+ * initialization code and data are used. Once the VM system is available,
+ * boot data are copied in kernel allocated buffers and their original pages
+ * are freed.
+ *
+ * On amd64, 64-bit code cannot run in legacy or compatibility mode. In order
+ * to walk the boot data structures, the kernel must either run 32-bit code
+ * (e.g. converting ELF32 to ELF64 objects before linking them) or establish
+ * a temporary identity mapping for the first 4 GiB of physical memory. As a
+ * way to simplify development, and make it possible to use 64-bit code
+ * almost everywhere, the latter solution is implemented (a small part of
+ * 32-bit code is required until the identity mapping is in place).
  */
 
 #include <kern/init.h>
@@ -42,11 +68,17 @@
 #define INIT_VGACHARS   (80 * 25)
 #define INIT_VGACOLOR   0x7
 
-char boot_stack[BOOT_STACK_SIZE] __aligned(8) __initdata;
-char boot_ap_stack[BOOT_STACK_SIZE] __aligned(8) __initdata;
+char boot_stack[BOOT_STACK_SIZE] __aligned(DATA_ALIGN) __initdata;
+char boot_ap_stack[BOOT_STACK_SIZE] __aligned(DATA_ALIGN) __initdata;
 unsigned long boot_ap_id __initdata;
 unsigned long boot_ap_stack_addr __initdata;
 
+#ifdef __LP64__
+pmap_pte_t boot_pml4[PMAP_PTE_PER_PT] __aligned(PAGE_SIZE) __initdata;
+pmap_pte_t boot_pdpt[PMAP_PTE_PER_PT] __aligned(PAGE_SIZE) __initdata;
+pmap_pte_t boot_pdir[4 * PMAP_PTE_PER_PT] __aligned(PAGE_SIZE) __initdata;
+#endif /* __LP64__ */
+
 /*
  * Copy of the multiboot data passed by the boot loader.
  */
@@ -61,7 +93,7 @@ boot_panic(const char *msg)
     ptr = INIT_VGAMEM;
     end = ptr + INIT_VGACHARS;
 
-    s = (const char *)BOOT_VTOP("boot panic: ");
+    s = (const char *)BOOT_VTOP("panic: ");
 
     while ((ptr < end) && (*s != '\0'))
         *ptr++ = (INIT_VGACOLOR << 8) | *s++;
@@ -91,6 +123,10 @@ boot_setup_paging(uint32_t eax, const struct multiboot_info *mbi)
     if (!(mbi->flags & MULTIBOOT_LOADER_MEMORY))
         boot_panic("missing basic memory information");
 
+#ifdef __LP64__
+    boot_panic("64-bit long mode successfully enabled");
+#endif
+
     /*
      * Save the multiboot data passed by the boot loader and initialize the
      * bootstrap allocator.
diff --git a/arch/x86/machine/boot_asm.S b/arch/x86/machine/boot_asm.S
index e46eedb5..4387aa77 100644
--- a/arch/x86/machine/boot_asm.S
+++ b/arch/x86/machine/boot_asm.S
@@ -21,6 +21,8 @@
 #include <machine/cpu.h>
 #include <machine/boot.h>
 #include <machine/multiboot.h>
+#include <machine/param.h>
+#include <machine/pmap.h>
 
 /*
  * Convert a physical address in the .boot section to its real address in
@@ -29,12 +31,12 @@
 #define BOOT_MP_ADDR_PTOT(addr) (BOOT_MP_TRAMPOLINE_ADDR + (addr) \
                                  - boot_mp_trampoline)
 
-.section .init.hdr, "wax"
+.section .init.hdr, "awx"
+.code32
 
  /*
   * Multiboot header.
   */
-.align 4
 DATA(boot_header)
  .long MULTIBOOT_OS_MAGIC
  .long MULTIBOOT_OS_FLAGS
@@ -45,40 +47,35 @@ END(boot_header)
  * Entry point.
  */
 ENTRY(_start)
- .code32
- /*
-  * Set up a simple GDT to conform to the multiboot specification.
-  */
  lgdt boot_gdtr
 
- /*
-  * Keep %eax and %ebx.
-  */
- movw $0x10, %cx
- movw %cx, %ds
- movw %cx, %es
- movw %cx, %ss
- xorw %cx, %cx
- movw %cx, %fs
- movw %cx, %gs
+ /* Keep %eax and %ebx */
+ movl $0x10, %ecx
+ movl %ecx, %ds
+ movl %ecx, %es
+ movl %ecx, %ss
+ xorl %ecx, %ecx
+ movl %ecx, %fs
+ movl %ecx, %gs
  ljmp $8, $1f
 
 1:
+ movl $(boot_stack + BOOT_STACK_SIZE), %esp
+ movl %esp, %ebp
 
-/* XXX For now */
 #ifdef __LP64__
- hlt
-#endif
+ call boot_setup_long_mode
 
  /*
-  * Set up the boot stack.
+  * At this point, the processor runs in long mode, but still uses the compatibility
+  * mode code segment. Switch to 64-bit mode with a far return.
   */
- movl $(boot_stack + BOOT_STACK_SIZE), %esp
- movl %esp, %ebp
+ pushl $0x18
+ pushl $1f
+ lret
+1:
+#endif
 
- /*
-  * Enable paging.
-  */
  pushl %ebx
  pushl %eax
  call boot_setup_paging
@@ -92,29 +89,74 @@ ENTRY(_start)
 #ifdef __LP64__
  hlt
 #else /* __LP64__ */
- /*
-  * Reset the stack, use high addresses.
-  */
- movl $(boot_stack + BOOT_STACK_SIZE + KERNEL_OFFSET), %esp
- movl %esp, %ebp
-
- /*
-  * Prevent stack tracing from searching previous frames.
-  */
+ /* Prevent stack tracing from searching previous frames */
  pushl $0
  jmp boot_main
 #endif /* __LP64__ */
 
- /*
-  * Never reached.
-  */
+ /* Never reached */
 END(_start)
 
 DATA(boot_gdtr)
- .word (8 * 3)
+ .word boot_gdt_end - boot_gdt
  .long boot_gdt
 END(boot_gdtr)
 
+#ifdef __LP64__
+/*
+ * The %eax and %ebx registers must be preserved.
+ */
+ENTRY(boot_setup_long_mode)
+ /* Set PML4[0] */
+ movl $boot_pdpt, %edx
+ orl $(PMAP_PTE_WRITE | PMAP_PTE_PRESENT), %edx
+ movl %edx, boot_pml4
+
+ /* Set PDPT[0] through PDPT[3] */
+ movl $boot_pdir, %edx
+ orl $(PMAP_PTE_WRITE | PMAP_PTE_PRESENT), %edx
+ movl $boot_pdpt, %edi
+ movl $4, %ecx
+
+1:
+ movl %edx, (%edi)
+ addl $PAGE_SIZE, %edx
+ addl $8, %edi
+ loop 1b
+
+ /* Set PDIR[0] through PDIR[2047] */
+ movl $(PMAP_PTE_PAGE_SIZE | PMAP_PTE_WRITE | PMAP_PTE_PRESENT), %edx
+ movl $boot_pdir, %edi
+ movl $2048, %ecx
+
+1:
+ movl %edx, (%edi)
+ addl $(1 << PMAP_PDE_SHIFT), %edx
+ addl $8, %edi
+ loop 1b
+
+ /* Switch to long mode */
+ movl %eax, %edi
+ movl %cr4, %eax
+ orl $CPU_CR4_PAE, %eax
+ movl %eax, %cr4
+ movl $boot_pml4, %eax
+ movl %eax, %cr3
+ movl $CPU_MSR_EFER, %ecx
+ rdmsr
+ orl $CPU_EFER_LME, %eax
+ wrmsr
+ movl %cr0, %eax
+ orl $CPU_CR0_PG, %eax
+ movl %eax, %cr0
+ ljmp $8, $1f
+
+1:
+ movl %edi, %eax
+ ret
+END(boot_setup_long_mode)
+#endif /* __LP64__ */
+
 /*
  * This is where an AP runs after leaving the trampoline code.
  */
@@ -125,25 +167,18 @@ ENTRY(boot_ap_start32)
   * is enabled.
   */
  lgdt boot_gdtr
- movw $0x10, %ax
- movw %ax, %ds
- movw %ax, %es
- movw %ax, %ss
- xorw %ax, %ax
- movw %ax, %fs
- movw %ax, %gs
+ movl $0x10, %eax
+ movl %eax, %ds
+ movl %eax, %es
+ movl %eax, %ss
+ xorl %eax, %eax
+ movl %eax, %fs
+ movl %eax, %gs
  ljmp $8, $1f
 
 1:
- /*
-  * Set up the boot stack.
-  */
  movl $(boot_ap_stack + BOOT_STACK_SIZE), %esp
  movl %esp, %ebp
-
- /*
-  * Enable paging.
-  */
  call boot_ap_setup_paging
  movl %eax, %cr3
  movl %cr0, %eax
@@ -152,22 +187,16 @@ ENTRY(boot_ap_start32)
  ljmp $8, $1f
 
 1:
- /*
-  * Switch to the boot stack preallocated for this AP by the BSP.
-  */
+ /* Switch to the boot stack preallocated for this AP by the BSP */
  movl boot_ap_stack_addr, %esp
  addl $BOOT_STACK_SIZE, %esp
  movl %esp, %ebp
 
- /*
-  * Prevent stack tracing from searching previous frames.
-  */
+ /* Prevent stack tracing from searching previous frames */
  pushl $0
  jmp boot_ap
 
- /*
-  * Never reached.
-  */
+ /* Never reached */
 END(boot_ap_start32)
 
 /*
@@ -175,7 +204,6 @@ END(boot_ap_start32)
  * on startup. It is copied at a fixed location in the first segment and
  * must enable protected mode to jump back into the kernel.
  */
-.align 8
 ENTRY(boot_mp_trampoline)
  .code16
  cli
@@ -194,46 +222,29 @@ ENTRY(boot_mp_trampoline)
 .align 4
 1:
  .code32
- movw $0x10, %ax
- movw %ax, %ds
- movw %ax, %es
- movw %ax, %ss
- xorw %ax, %ax
- movw %ax, %fs
- movw %ax, %gs
+ movl $0x10, %eax
+ movl %eax, %ds
+ movl %eax, %es
+ movl %eax, %ss
+ xorl %eax, %eax
+ movl %eax, %fs
+ movl %eax, %gs
  ljmp $8, $boot_ap_start32
 END(boot_mp_trampoline)
 
 DATA(boot_ap_gdtr)
- .word (8 * 3)
+ .word boot_gdt_end - boot_gdt
  .long BOOT_MP_ADDR_PTOT(boot_gdt)
 END(boot_ap_gdtr)
 
-.align 8
 DATA(boot_gdt)
- /*
-  * Null selector.
-  */
- .word 0x0000
- .word 0x0000
- .word 0x0000
- .word 0x0000
-
- /*
-  * Code segment selector.
-  */
- .word 0xffff
- .word 0x0000
- .word 0x9a00
- .word 0x00cf
+ .quad 0x0000000000000000   /* Null selector */
+ .quad 0x00cf9a000000ffff   /* Code segment selector */
+ .quad 0x00cf92000000ffff   /* Data segment selector */
 
- /*
-  * Data segment selector.
-  */
- .word 0xffff
- .word 0x0000
- .word 0x9200
- .word 0x00cf
+#ifdef __LP64__
+ .quad 0x00209a0000000000   /* 64-bit code segment selector */
+#endif /* __LP64__ */
 END(boot_gdt)
 
 DATA(boot_mp_trampoline_size)
diff --git a/arch/x86/machine/cpu.h b/arch/x86/machine/cpu.h
index 4f451590..db847c68 100644
--- a/arch/x86/machine/cpu.h
+++ b/arch/x86/machine/cpu.h
@@ -64,6 +64,16 @@
 #define CPU_FEATURE2_APIC   0x00000200
 #define CPU_FEATURE2_PGE    0x00002000
 
+/*
+ * Model specific registers.
+ */
+#define CPU_MSR_EFER    0xc0000080
+
+/*
+ * EFER MSR flags.
+ */
+#define CPU_EFER_LME    0x00000100
+
 #ifndef __ASSEMBLY__
 
 #include <kern/param.h>
diff --git a/arch/x86/machine/param.h b/arch/x86/machine/param.h
index d806b192..c2936fa8 100644
--- a/arch/x86/machine/param.h
+++ b/arch/x86/machine/param.h
@@ -30,6 +30,12 @@
 #define CPU_L1_SIZE 64
 
 /*
+ * Code/data alignment.
+ */
+#define TEXT_ALIGN  16
+#define DATA_ALIGN  8
+
+/*
  * System timer frequency.
  */
 #define HZ 100
diff --git a/arch/x86/machine/pmap.h b/arch/x86/machine/pmap.h
index 89237c14..d90f52d2 100644
--- a/arch/x86/machine/pmap.h
+++ b/arch/x86/machine/pmap.h
@@ -18,39 +18,35 @@
 #ifndef _X86_PMAP_H
 #define _X86_PMAP_H
 
-#include <kern/param.h>
-#include <kern/types.h>
-#include <lib/stdint.h>
+#include <lib/macros.h>
 
 /*
  * Page directory/table properties.
+ *
+ * TODO amd64
  */
+#ifdef __LP64__
+#define PMAP_PDE_SHIFT  21
+#define PMAP_PT_SHIFT   9
+#define PMAP_NR_PDT     4
+#define PMAP_PTE_PMASK  DECL_CONST(0x000ffffffffff000, ULL)
+#else /* __LP64__ */
 #ifdef PAE
 #define PMAP_PDE_SHIFT  21
 #define PMAP_PT_SHIFT   9
 #define PMAP_NR_PDT     4
-#define PMAP_PTE_PMASK  0x0000000ffffff000ULL
+#define PMAP_PTE_PMASK  DECL_CONST(0x0000000ffffff000, ULL)
 #else /* PAE */
 #define PMAP_PDE_SHIFT  22
 #define PMAP_PT_SHIFT   10
 #define PMAP_NR_PDT     1
-#define PMAP_PTE_PMASK  (~PAGE_MASK)
+#define PMAP_PTE_PMASK  DECL_CONST(0xfffff000, UL)
 #endif /* PAE */
+#endif /* __LP64__ */
 
 #define PMAP_PTE_SHIFT 12
 
 /*
- * The amount of virtual memory described by a page directory/table entry.
- */
-#define PMAP_PDE_MAPSIZE    (1 << PMAP_PDE_SHIFT)
-#define PMAP_PTE_MAPSIZE    (1 << PMAP_PTE_SHIFT)
-
-/*
- * Number of entries in a page directory/table.
- */
-#define PMAP_PTE_PER_PT (1 << PMAP_PT_SHIFT)
-
-/*
  * PDE/PTE flags.
  */
 #define PMAP_PTE_PRESENT        0x001
@@ -60,11 +56,18 @@
 #define PMAP_PTE_CACHE_DISABLE  0x010
 #define PMAP_PTE_ACCESSED       0x020
 #define PMAP_PTE_DIRTY          0x040
+#define PMAP_PTE_PAGE_SIZE      0x080
 #define PMAP_PTE_GLOBAL         0x100
 #define PMAP_PTE_AVAIL1         0x200
 #define PMAP_PTE_AVAIL2         0x400
 #define PMAP_PTE_AVAIL3         0x800
 
+#ifndef __ASSEMBLY__
+
+#include <kern/param.h>
+#include <kern/types.h>
+#include <lib/stdint.h>
+
 /*
  * Flags related to page protection.
  */
@@ -87,10 +90,21 @@
 #ifdef PAE
 typedef uint64_t pmap_pte_t;
 #else /* PAE */
-typedef uint32_t pmap_pte_t;
+typedef unsigned long pmap_pte_t;
 #endif /* PAE */
 
 /*
+ * The amount of virtual memory described by a page directory/table entry.
+ */
+#define PMAP_PDE_MAPSIZE    (1 << PMAP_PDE_SHIFT)
+#define PMAP_PTE_MAPSIZE    (1 << PMAP_PTE_SHIFT)
+
+/*
+ * Number of entries in a page directory/table.
+ */
+#define PMAP_PTE_PER_PT (1 << PMAP_PT_SHIFT)
+
+/*
  * Base virtual address of the linear mapping of PTEs.
  */
 #define PMAP_PTE_BASE   ((pmap_pte_t *)(PMAP_PDE_PTE << PMAP_PDE_SHIFT))
@@ -194,4 +208,6 @@ vm_phys_t pmap_kextract(unsigned long va);
  */
 void pmap_zero_page(vm_phys_t pa);
 
+#endif /* __ASSEMBLY__ */
+
 #endif /* _X86_PMAP_H */