diff options
39 files changed, 4150 insertions, 85 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig new file mode 100644 index 00000000..86b6478e --- /dev/null +++ b/arch/arm/Kconfig @@ -0,0 +1,34 @@ +menu "Architecture-specific options" + +config ARM + def_bool y + +config ARMV7 + bool + +config ARMV7A + bool + select ARMV7 + +config ARM_ARCH + int + default 7 if ARMV7 + +config SUBARCH + string + default "armv7a" if ARMV7A + +choice + prompt "Board" + default BOARD_SELECT_QEMU_VIRT_2_8 + +config BOARD_SELECT_QEMU_VIRT_2_8 + bool "QEMU 2.8 virtual machine" + ---help--- + This virtual board is used by the tools/qemu_arm.sh script. + +endchoice + +source "arch/arm/board-qemu-virt28/Kconfig" + +endmenu diff --git a/arch/arm/Makefile b/arch/arm/Makefile new file mode 100644 index 00000000..95a2e503 --- /dev/null +++ b/arch/arm/Makefile @@ -0,0 +1,12 @@ +KCONFIG_DEFCONFIG := qemu_virt28_defconfig + +include arch/arm/board-qemu-virt28/Makefile + +x15_SOURCES-y += \ + arch/arm/machine/boot_asm.S \ + arch/arm/machine/boot.c \ + arch/arm/machine/cpu.c \ + arch/arm/machine/pmap.c \ + arch/arm/machine/strace.c \ + arch/arm/machine/tcb_asm.S \ + arch/arm/machine/tcb.c diff --git a/arch/arm/board-qemu-virt28/Kconfig b/arch/arm/board-qemu-virt28/Kconfig new file mode 100644 index 00000000..1ecb40b7 --- /dev/null +++ b/arch/arm/board-qemu-virt28/Kconfig @@ -0,0 +1,7 @@ +if BOARD_SELECT_QEMU_VIRT_2_8 + +config BOARD_QEMU_VIRT_2_8 + def_bool y + select ARMV7A + +endif diff --git a/arch/arm/board-qemu-virt28/Makefile b/arch/arm/board-qemu-virt28/Makefile new file mode 100644 index 00000000..194ee6ee --- /dev/null +++ b/arch/arm/board-qemu-virt28/Makefile @@ -0,0 +1,5 @@ +ifeq ($(CONFIG_BOARD_QEMU_VIRT_2_8),y) + +XBUILD_CFLAGS += -mcpu=cortex-a15 + +endif diff --git a/arch/arm/configs/qemu_virt28_defconfig b/arch/arm/configs/qemu_virt28_defconfig new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/arch/arm/configs/qemu_virt28_defconfig diff --git a/arch/arm/machine/asm.h b/arch/arm/machine/asm.h new file mode 100644 index 00000000..6474b225 --- /dev/null +++ b/arch/arm/machine/asm.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011, 2012 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _X86_ASM_H +#define _X86_ASM_H + +#ifndef __ASSEMBLER__ +#warning "asm.h included from a C file" +#endif /* __ASSEMBLER__ */ + +#include <machine/cpu.h> + +#define ASM_FUNC(x) \ +.align CPU_TEXT_SHIFT; \ +.global x; \ +.type x, STT_FUNC; \ +x + +#define ASM_DATA(x) \ +.align CPU_DATA_SHIFT; \ +.global x; \ +.type x, STT_OBJECT; \ +x + +#define ASM_END(x) \ +.size x, . - x + +#endif /* _X86_ASM_H */ diff --git a/arch/arm/machine/atomic.h b/arch/arm/machine/atomic.h new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/arch/arm/machine/atomic.h diff --git a/arch/arm/machine/boot.c b/arch/arm/machine/boot.c new file mode 100644 index 00000000..52882119 --- /dev/null +++ b/arch/arm/machine/boot.c @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <stdalign.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <kern/init.h> +#include <kern/bootmem.h> +#include <machine/boot.h> +#include <machine/cpu.h> +#include <machine/pmap.h> +#include <machine/pmem.h> +#include <vm/vm_kmem.h> + +#define BOOT_UART_DATA_REG 0x9000000 + +alignas(CPU_DATA_ALIGN) char boot_stack[BOOT_STACK_SIZE] __bootdata; + +pmap_pte_t * boot_setup_paging(void); + +void boot_main(void); + +void __boot +boot_panic(const char *s) +{ + volatile unsigned long *uart_data_reg; + + uart_data_reg = (volatile unsigned long *)BOOT_UART_DATA_REG; + + while (*s != '\0') { + *uart_data_reg = *s; + s++; + } + + for (;;); +} + +pmap_pte_t * __boot +boot_setup_paging(void) +{ + bootmem_register_zone(PMEM_ZONE_DMA, true, PMEM_RAM_START, PMEM_DMA_LIMIT); + bootmem_setup(); + return pmap_setup_paging(); +} + +void __init +boot_log_info(void) +{ +} + +static void __init +boot_clear_bss(void) +{ + memset(&_bss, 0, &_end - &_bss); +} + +void __init +boot_main(void) +{ + boot_clear_bss(); + kernel_main(); + + /* Never reached */ +} + +/* + * Init operation aliases. + */ + +static int __init +boot_bootstrap_console(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_bootstrap_console); + +static int __init +boot_setup_console(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_setup_console); + +static int __init +boot_load_vm_page_zones(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_load_vm_page_zones); + +static int __init +boot_setup_intr(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_setup_intr); + +static int __init +boot_setup_shutdown(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_setup_shutdown); diff --git a/arch/arm/machine/boot.h b/arch/arm/machine/boot.h new file mode 100644 index 00000000..e7c3c8f2 --- /dev/null +++ b/arch/arm/machine/boot.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _ARM_BOOT_H +#define _ARM_BOOT_H + +#include <kern/macros.h> +#include <machine/page.h> +#include <machine/pmap.h> +#include <machine/pmem.h> + +/* + * Size of the stack used when booting a processor. + */ +#define BOOT_STACK_SIZE PAGE_SIZE + +#define BOOT_LOAD_SECTION .boot.load +#define BOOT_TEXT_SECTION .boot.text +#define BOOT_DATA_SECTION .boot.data + +#define BOOT_KERNEL_OFFSET (PMAP_START_KERNEL_ADDRESS - PMEM_RAM_START) + +#define BOOT_RTOL(addr) ((addr) - PMEM_RAM_START) +#define BOOT_VTOL(addr) ((addr) - PMAP_START_KERNEL_ADDRESS) + +#define BOOT_VTOP(addr) ((addr) - BOOT_KERNEL_OFFSET) +#define BOOT_PTOV(addr) ((addr) + BOOT_KERNEL_OFFSET) + +#define BOOT_MEM_BLOCK_BITS 10 +#define BOOT_MEM_NR_FREE_LISTS 5 + +#ifndef __ASSEMBLER__ + +#include <stdnoreturn.h> + +#include <kern/init.h> + +/* + * Macros for boot code and data, respectively. + * + * Note that all boot data end up in the same section, which means they + * can't have conflicting qualifiers. As a result, boot data may not be + * declared const. + */ +#define __boot __section(QUOTE(BOOT_TEXT_SECTION)) +#define __bootdata __section(QUOTE(BOOT_DATA_SECTION)) __attribute__((used)) + +/* + * Boundaries of the .boot section. + */ +extern char _boot; +extern char _boot_end; + +noreturn void boot_panic(const char *s); + +/* + * Log kernel version and other architecture-specific information. + */ +void boot_log_info(void); + +/* + * This init operation provides : + * - all console devices are bootstrapped + */ +INIT_OP_DECLARE(boot_bootstrap_console); + +/* + * This init operation provides : + * - all console devices are fully initialized + */ +INIT_OP_DECLARE(boot_setup_console); + +/* + * This init operation provides : + * - physical memory has been loaded to the VM system + */ +INIT_OP_DECLARE(boot_load_vm_page_zones); + +/* + * This init operation provides : + * - all interrupt controllers have been registered + */ +INIT_OP_DECLARE(boot_setup_intr); + +/* + * This init operation provides : + * - all shutdown operations have been registered + */ +INIT_OP_DECLARE(boot_setup_shutdown); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ARM_BOOT_H */ diff --git a/arch/arm/machine/boot_asm.S b/arch/arm/machine/boot_asm.S new file mode 100644 index 00000000..ed5e2ccc --- /dev/null +++ b/arch/arm/machine/boot_asm.S @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <machine/asm.h> +#include <machine/boot.h> +#include <machine/pmap.h> +#include <machine/pmem.h> + +.section BOOT_LOAD_SECTION, "awx" + +ASM_DATA(boot_exception_table): + .long _start /* Reset */ + .long 0 /* Undefined instruction */ + .long 0 /* Software interrupt */ + .long 0 /* Prefetch abort */ + .long 0 /* Data abort */ + .long 0 /* IRQ */ + .long 0 /* FIQ */ +ASM_END(boot_exception_table) + +ASM_FUNC(_start): + /* + * Assume the code runs from flash. For the sake of simplicity, make + * the kernel run from RAM. This requires relocating it, and since the + * RAM size isn't known, we can't use it for a stack. As a result, + * perform the relocation in assembly without using a stack. + */ + ldr %r5, =(PMEM_RAM_START) /* Load RAM address in %r5 */ + mov %r6, #0 /* Load kernel address in %r6 */ + ldr %r0, boot_kernel_end /* Load kernel end virtual address + in %r0 */ + ldr %r1, =(PMAP_START_KERNEL_ADDRESS) /* Load kernel virtual address in %r1 */ + sub %r7, %r0, %r1 /* Compute kernel size in bytes */ + lsr %r7, %r7, #2 /* Compute kernel size in words */ + + sub %r5, %r5, #4 /* Prepare for auto-increment */ + sub %r6, %r6, #4 + +1: + ldr %r0, [%r6, #4]! /* Load word with auto-increment */ + str %r0, [%r5, #4]! /* Store word with auto-increment */ + sub %r7, %r7, #1 /* Account for the written word */ + cmp %r7, #0 /* Check the number of words left */ + beq 1f /* Break if copy complete */ + b 1b /* Continue otherwise */ + +1: + /* + * Relocation is done. Make sure that the processor actually runs kernel + * code by flushing all the relevant caches. + * + * Also, in preparation for paging, disable all caches. + * + * The reference manual only suggests disabling the instruction cache + * (B4.2.3 Enabling and disabling the MMU) but the instruction cache may + * be unified. + */ + mrc p15, 0, %r0, c1, c0, 0 /* Read CP15 control register */ + bic %r0, %r0, #0x2 /* Clear the C bit (D-cache) */ + bic %r0, %r0, #0x4 /* Clear the W bit (write buffers) */ + bic %r0, %r0, #0x1000 /* Clear the I bit (I-cache) */ + mcr p15, 0, %r0, c1, c0, 0 /* Write CP15 control register */ + mov %r0, #0 + mcr p15, 0, %r0, c7, c7, 0 /* Clean all caches */ + mcr p15, 0, %r0, c7, c10, 5 /* Data synchronization barrier */ + mcr p15, 0, %r0, c7, c5, 4 /* Flush prefetch buffer */ + + b boot_start_ram /* Branch to kernel code in RAM */ +ASM_END(_start) + +boot_kernel_end: + .long _end + +.section BOOT_TEXT_SECTION, "awx" + +ASM_FUNC(boot_start_ram): + ldr %r13, boot_stack_addr /* Set up the boot stack */ + add %r13, %r13, #BOOT_STACK_SIZE + + blx boot_setup_paging + + orr %r0, %r0, #0x01 /* Set the C bit (cacheable) */ + orr %r0, %r0, #0x02 /* Set the S bit (shareable) */ + orr %r0, %r0, #0x18 /* Set the RGN bits to write-back */ + mcr p15, 0, %r0, c2, c0, 0 /* Load the root page table address */ + mov %r0, #1 + mcr p15, 0, %r0, c3, c0, 0 /* Set domain 0 access to client */ + mrc p15, 0, %r0, c1, c0, 0 /* Read CP15 control register */ + orr %r0, %r0, #0x1 /* Set the M bit (MMU enabled) */ + orr %r0, %r0, #0x2 /* Set the A bit (check alignment) */ + orr %r0, %r0, #0x4 /* Set the C bit (D-cache) */ + orr %r0, %r0, #0x8 /* Set the W bit (write buffers) */ + orr %r0, %r0, #0x1000 /* Set the I bit (I-cache) */ + mcr p15, 0, %r0, c1, c0, 0 /* Write CP15 control register */ + + mov %r11, #0 + blx boot_main + + /* Never reached */ + b . +ASM_END(boot_start_ram) + +boot_stack_addr: + .long boot_stack diff --git a/arch/arm/machine/config.h b/arch/arm/machine/config.h new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/arch/arm/machine/config.h diff --git a/arch/arm/machine/cpu.c b/arch/arm/machine/cpu.c new file mode 100644 index 00000000..a79200fe --- /dev/null +++ b/arch/arm/machine/cpu.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <kern/init.h> +#include <machine/cpu.h> + +void cpu_halt_broadcast(void) +{ +} + +void cpu_log_info(const struct cpu *cpu) +{ + (void)cpu; +} + +void cpu_mp_setup(void) +{ +} + +static int __init +cpu_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(cpu_setup); + +static int __init +cpu_mp_probe(void) +{ + return 0; +} + +INIT_OP_DEFINE(cpu_mp_probe); diff --git a/arch/arm/machine/cpu.h b/arch/arm/machine/cpu.h new file mode 100644 index 00000000..c2124ad1 --- /dev/null +++ b/arch/arm/machine/cpu.h @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _ARM_CPU_H +#define _ARM_CPU_H + +#include <limits.h> + +#include <machine/cpu_armv6.h> + +/* + * L1 cache line size. + * + * XXX Use this value until processor selection is available. + */ +#define CPU_L1_SIZE 32 + +/* + * Data alignment, normally the word size. + * + * TODO Check. + */ +#define CPU_DATA_SHIFT 2 +#define CPU_DATA_ALIGN (1 << CPU_DATA_SHIFT) + +/* + * Function alignment. + * + * Aligning functions improves instruction fetching. + * + * Used for assembly functions only. + * + * XXX Use this value until processor selection is available. + */ +#define CPU_TEXT_SHIFT 4 +#define CPU_TEXT_ALIGN (1 << CPU_TEXT_SHIFT) + +/* + * PSR flags. + */ +#define CPU_PSR_I 0x00000080 + +#ifndef __ASSEMBLER__ + +#include <stdbool.h> +#include <stdnoreturn.h> + +#include <kern/init.h> + +struct cpu { +}; + +/* + * Return the content of the CPSR register. + * + * Implies a compiler barrier. + */ +static __always_inline unsigned long +cpu_get_cpsr(void) +{ + unsigned long cpsr; + + asm volatile("mrs %0, cpsr" + : "=r" (cpsr) + : : "memory"); + + return cpsr; +} + +/* + * Restore the content of the CPSR register, possibly enabling interrupts. + * + * Implies a compiler barrier. + */ +static __always_inline void +cpu_intr_restore(unsigned long flags) +{ + asm volatile("msr cpsr_c, %0" + : : "r" (flags) + : "memory"); +} + +/* + * Disable local interrupts, returning the previous content of the CPSR + * register. + * + * Implies a compiler barrier. + */ +static __always_inline void +cpu_intr_save(unsigned long *flags) +{ + *flags = cpu_get_cpsr(); + cpu_intr_disable(); +} + +static __always_inline bool +cpu_intr_enabled(void) +{ + unsigned long cpsr; + + cpsr = cpu_get_cpsr(); + return !(cpsr & CPU_PSR_I); +} + +void cpu_halt_broadcast(void); + +#define cpu_local_ptr(var) (&(var)) + +static inline struct cpu * +cpu_current(void) +{ + return NULL; +} + +static inline unsigned int +cpu_id(void) +{ + return 0; +} + +static inline unsigned int +cpu_count(void) +{ + return 1; +} + +/* + * Log processor information. + */ +void cpu_log_info(const struct cpu *cpu); + +void cpu_mp_setup(void); + +static inline void +cpu_send_xcall(unsigned int cpu) +{ + (void)cpu; +} + +static inline void +cpu_send_thread_schedule(unsigned int cpu) +{ + (void)cpu; +} + +/* + * This init operation provides : + * - initialization of the BSP structure. + * - cpu_delay() + * - cpu_local_ptr() and cpu_local_var() + */ +INIT_OP_DECLARE(cpu_setup); + +/* + * This init operation provides : + * - cpu_count() + */ +INIT_OP_DECLARE(cpu_mp_probe); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ARM_CPU_H */ diff --git a/arch/arm/machine/cpu_armv6.h b/arch/arm/machine/cpu_armv6.h new file mode 100644 index 00000000..7520f4b1 --- /dev/null +++ b/arch/arm/machine/cpu_armv6.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _ARM_CPU_ARMV6_H +#define _ARM_CPU_ARMV6_H + +#if CONFIG_ARM_ARCH >= 6 + +#ifndef __ASSEMBLER__ + +#include <stdbool.h> +#include <stdnoreturn.h> + +#include <kern/macros.h> + +/* + * Enable local interrupts. + * + * Implies a compiler barrier. + */ +static __always_inline void +cpu_intr_enable(void) +{ + asm volatile("cpsie i" : : : "memory", "cc"); +} + +/* + * Disable local interrupts. + * + * Implies a compiler barrier. + */ +static __always_inline void +cpu_intr_disable(void) +{ + asm volatile("cpsid i" : : : "memory", "cc"); +} + +static __always_inline void +cpu_pause(void) +{ + for (;;); +} + +/* + * Make the CPU idle until the next interrupt. + * + * Implies a compiler barrier. + */ +static __always_inline void +cpu_idle(void) +{ + asm volatile("wfi" : : : "memory"); +} + +noreturn static __always_inline void +cpu_halt(void) +{ + cpu_intr_disable(); + + for (;;) { + cpu_idle(); + } +} + +#endif /* __ASSEMBLER__ */ + +#endif /* ARM_ARCH >= 6 */ + +#endif /* _ARM_CPU_ARMV6_H */ diff --git a/arch/arm/machine/page.h b/arch/arm/machine/page.h new file mode 100644 index 00000000..bea2efd6 --- /dev/null +++ b/arch/arm/machine/page.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * This file is a top header in the inclusion hierarchy, and shouldn't include + * other headers that may cause circular dependencies. + */ + +#ifndef _ARM_PAGE_H +#define _ARM_PAGE_H + +#define PAGE_BITS 12 +#define PAGE_SHIFT PAGE_BITS +#define PAGE_SIZE (1 << PAGE_BITS) +#define PAGE_MASK (PAGE_BITS - 1) + +#endif /* _ARM_PAGE_H */ diff --git a/arch/arm/machine/pmap.c b/arch/arm/machine/pmap.c new file mode 100644 index 00000000..9b864e6d --- /dev/null +++ b/arch/arm/machine/pmap.c @@ -0,0 +1,1403 @@ +/* + * Copyright (c) 2010-2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * TODO Review locking. + */ + +#include <assert.h> +#include <stdalign.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <kern/bootmem.h> +#include <kern/cpumap.h> +#include <kern/error.h> +#include <kern/init.h> +#include <kern/kmem.h> +#include <kern/list.h> +#include <kern/log.h> +#include <kern/macros.h> +#include <kern/mutex.h> +#include <kern/panic.h> +#include <kern/percpu.h> +#include <kern/spinlock.h> +#include <kern/syscnt.h> +#include <kern/thread.h> +#include <machine/boot.h> +#include <machine/cpu.h> +#include <machine/page.h> +#include <machine/pmap.h> +#include <machine/tcb.h> +#include <machine/trap.h> +#include <machine/types.h> +#include <vm/vm_kmem.h> +#include <vm/vm_page.h> +#include <vm/vm_ptable.h> +#include <vm/vm_prot.h> + +#define PMAP_PTE_B 0x00000004 +#define PMAP_PTE_C 0x00000008 +#define PMAP_PTE_L0_RW 0x000000f0 +#define PMAP_PTE_L1_RW 0x00000c00 + +#define PMAP_NR_LEVELS 2 +#define PMAP_L0_BITS 8 +#define PMAP_L1_BITS 12 + +#define PMAP_VA_MASK 0xffffffff + +#define PMAP_L0_PA_MASK 0xfffff000 +#define PMAP_L1_PA_MASK 0xfffffc00 + +#define PMAP_L0_SKIP 12 +#define PMAP_L1_SKIP (PMAP_L0_SKIP + PMAP_L0_BITS) + +#define PMAP_L0_PTES_PER_PT (1 << PMAP_L0_BITS) +#define PMAP_L1_PTES_PER_PT (1 << PMAP_L1_BITS) + +static pmap_pte_t __boot +pmap_make_coarse_pte(phys_addr_t pa, int prot) +{ + (void)prot; + + assert((pa & PMAP_L1_PA_MASK) == pa); + return pa | PMAP_PTE_TYPE_COARSE; +} + +static pmap_pte_t __boot +pmap_make_small_page_pte(phys_addr_t pa, int prot) +{ + (void)prot; + + assert((pa & PMAP_L0_PA_MASK) == pa); + return pa | PMAP_PTE_L0_RW | PMAP_PTE_C | PMAP_PTE_B | PMAP_PTE_TYPE_SMALL; +} + +static pmap_pte_t __boot +pmap_make_section_pte(phys_addr_t pa, int prot) +{ + (void)prot; + + assert((pa & 0xfff00000) == pa); + return pa | PMAP_PTE_L1_RW | PMAP_PTE_C | PMAP_PTE_B | PMAP_PTE_TYPE_SECTION; +} + +/* + * Physical translation level properties. + */ +static struct vm_ptable_tlp pmap_boot_tlps[] __bootdata = { + { + PMAP_L0_SKIP, + PMAP_L0_BITS, + PMAP_L0_PTES_PER_PT, + NULL, + pmap_make_small_page_pte, + }, + { + PMAP_L1_SKIP, + PMAP_L1_BITS, + PMAP_L1_PTES_PER_PT, + pmap_make_coarse_pte, + pmap_make_section_pte, + }, +}; + +struct pmap { + struct vm_ptable ptable; +}; + +struct pmap pmap_kernel_pmap; + +/* + * Flags related to page protection. + */ +#define PMAP_PTE_PROT_MASK PMAP_PTE_RW + +/* + * Table used to convert machine independent protection flags to architecture + * specific PTE bits. + */ +static pmap_pte_t pmap_prot_table[VM_PROT_ALL + 1] __read_mostly; + +static struct kmem_cache pmap_cache; + +static char pmap_panic_directmap_msg[] __bootdata + = "vm_ptable: invalid direct physical mapping"; + +unsigned int __boot +pmap_boot_get_last_level(size_t page_size) +{ + switch (page_size) { + case (1 << PMAP_L1_SKIP): + return 1; + default: + return 0; + } +} + +static size_t __boot +pmap_boot_get_large_page_size(void) +{ + return (1 << PMAP_L1_SKIP); +} + +pmap_pte_t * __boot +pmap_setup_paging(void) +{ + size_t size, page_size; + phys_addr_t pa, directmap_end; + struct vm_ptable *ptable; + struct pmap *kernel_pmap; + uintptr_t va; + + kernel_pmap = (void *)BOOT_VTOP((uintptr_t)&pmap_kernel_pmap); + + /* Use large pages for the direct physical mapping when possible */ + page_size = pmap_boot_get_large_page_size(); + + /* TODO LPAE */ + + vm_ptable_bootstrap(pmap_boot_tlps, ARRAY_SIZE(pmap_boot_tlps)); + + ptable = &kernel_pmap->ptable; + vm_ptable_boot_build(ptable); + + /* + * Create the initial mappings. The first is for the .boot section + * and acts as the mandatory identity mapping. The second is the + * direct mapping of physical memory. + */ + + va = vm_page_trunc((uintptr_t)&_boot); + pa = va; + size = vm_page_round((uintptr_t)&_boot_end) - va; + + for (size_t i = 0; i < size; i += PAGE_SIZE) { + vm_ptable_boot_enter(ptable, va, pa, PAGE_SIZE); + va += PAGE_SIZE; + pa += PAGE_SIZE; + } + + directmap_end = bootmem_directmap_end(); + size = directmap_end - PMEM_RAM_START; + + if (size > (PMAP_END_DIRECTMAP_ADDRESS - PMAP_START_DIRECTMAP_ADDRESS)) { + boot_panic(pmap_panic_directmap_msg); + } + + va = PMAP_START_DIRECTMAP_ADDRESS; + pa = PMEM_RAM_START; + + for (size_t i = PMEM_RAM_START; i < directmap_end; i += page_size) { + vm_ptable_boot_enter(ptable, va, pa, page_size); + va += page_size; + pa += page_size; + } + + return vm_ptable_boot_root(ptable); +} + +#if 0 +pmap_pte_t * __boot +pmap_ap_setup_paging(void) +{ + struct pmap_cpu_table *cpu_table; + struct pmap *pmap; + unsigned long page_size; + + page_size = pmap_boot_get_page_size(); + pmap_boot_enable_pgext(page_size); + + pmap = (void *)BOOT_VTOP((uintptr_t)&pmap_kernel_pmap); + cpu_table = (void *)BOOT_VTOP((uintptr_t)pmap->cpu_tables[boot_ap_id]); + + return (void *)cpu_table->root_ptp_pa; +} + +/* + * Check address range with regard to physical map. + */ +#define pmap_assert_range(pmap, start, end) \ +MACRO_BEGIN \ + assert((start) < (end)); \ + assert(((end) <= PMAP_START_DIRECTMAP_ADDRESS) \ + || ((start) >= PMAP_END_DIRECTMAP_ADDRESS)); \ + \ + if ((pmap) == pmap_get_kernel_pmap()) { \ + assert(((start) >= PMAP_START_KMEM_ADDRESS) \ + && ((end) <= PMAP_END_KMEM_ADDRESS)); \ + } else { \ + assert((end) <= PMAP_END_ADDRESS); \ + } \ +MACRO_END + +static inline pmap_pte_t * +pmap_ptp_from_pa(phys_addr_t pa) +{ + uintptr_t va; + + va = vm_page_direct_va(pa); + return (pmap_pte_t *)va; +} + +static void +pmap_ptp_clear(pmap_pte_t *ptp) +{ + memset(ptp, 0, PAGE_SIZE); +} + +static inline void +pmap_pte_set(pmap_pte_t *pte, phys_addr_t pa, pmap_pte_t pte_bits, + const struct pmap_pt_level *pt_level) +{ + *pte = ((pa & PMAP_PA_MASK) | PMAP_PTE_P | pte_bits) & pt_level->mask; +} + +static inline void +pmap_pte_clear(pmap_pte_t *pte) +{ + *pte = 0; +} + +static inline int +pmap_pte_valid(pmap_pte_t pte) +{ + return (pte != 0); +} + +static inline int +pmap_pte_large(pmap_pte_t pte) +{ + return ((pte & PMAP_PTE_PS) != 0); +} + +static inline pmap_pte_t * +pmap_pte_next(pmap_pte_t pte) +{ + assert(pmap_pte_valid(pte)); + return pmap_ptp_from_pa(pte & PMAP_PA_MASK); +} + +/* + * Helper function for initialization procedures that require post-fixing + * page properties. + */ +static void __init +pmap_walk_vas(uintptr_t start, uintptr_t end, pmap_walk_fn_t walk_fn) +{ + const struct pmap_pt_level *pt_level; + phys_addr_t root_ptp_pa, ptp_pa; + pmap_pte_t *ptp, *pte; + unsigned int index, level; + uintptr_t va; + + assert(vm_page_aligned(start)); + assert(start < end); +#ifdef __LP64__ + assert((start < PMAP_END_ADDRESS) || (start >= PMAP_START_KERNEL_ADDRESS)); +#endif /* __LP64__ */ + + va = start; + root_ptp_pa = pmap_get_kernel_pmap()->cpu_tables[cpu_id()]->root_ptp_pa; + + do { +#ifdef __LP64__ + /* Handle long mode canonical form */ + if (va == PMAP_END_ADDRESS) { + va = PMAP_START_KERNEL_ADDRESS; + } +#endif /* __LP64__ */ + + level = PMAP_NR_LEVELS - 1; + ptp_pa = root_ptp_pa; + ptp = pmap_ptp_from_pa(ptp_pa); + + for (;;) { + pt_level = &pmap_pt_levels[level]; + index = pmap_pte_index(va, pt_level); + pte = &ptp[index]; + + if (!pmap_pte_valid(*pte)) { + break; + } + + walk_fn(ptp_pa, index, level); + + if ((level == 0) || pmap_pte_large(*pte)) { + break; + } + + level--; + ptp_pa = *pte & PMAP_PA_MASK; + ptp = pmap_ptp_from_pa(ptp_pa); + } + + va = P2END(va, 1UL << pt_level->skip); + } while ((va > start) && (va < end)); +} + +static void __init +pmap_setup_global_page(phys_addr_t ptp_pa, unsigned int index, + unsigned int level) +{ + pmap_pte_t *pte; + + pte = &pmap_ptp_from_pa(ptp_pa)[index]; + + if ((level == 0) || pmap_pte_large(*pte)) { + *pte |= PMAP_PTE_G; + } +} + +static void __init +pmap_setup_global_pages(void) +{ + pmap_walk_vas(PMAP_START_KERNEL_ADDRESS, PMAP_END_KERNEL_ADDRESS, + pmap_setup_global_page); + pmap_pt_levels[0].mask |= PMAP_PTE_G; + cpu_enable_global_pages(); +} + +static void +pmap_update_oplist_ctor(void *arg) +{ + struct pmap_update_oplist *oplist; + + oplist = arg; + cpumap_zero(&oplist->cpumap); + oplist->pmap = NULL; + oplist->nr_ops = 0; +} + +static int +pmap_update_oplist_create(struct pmap_update_oplist **oplistp) +{ + struct pmap_update_oplist *oplist; + + oplist = kmem_cache_alloc(&pmap_update_oplist_cache); + + if (oplist == NULL) { + return ERROR_NOMEM; + } + + *oplistp = oplist; + return 0; +} + +static void +pmap_update_oplist_destroy(struct pmap_update_oplist *oplist) +{ + kmem_cache_free(&pmap_update_oplist_cache, oplist); +} + +static struct pmap_update_oplist * +pmap_update_oplist_get(void) +{ + struct pmap_update_oplist *oplist; + + oplist = tcb_get_pmap_update_oplist(tcb_current()); + assert(oplist != NULL); + return oplist; +} + +static int +pmap_update_oplist_prepare(struct pmap_update_oplist *oplist, + struct pmap *pmap) +{ + int error; + + if (oplist->pmap != pmap) { + assert(oplist->pmap == NULL); + oplist->pmap = pmap; + } else if (oplist->nr_ops == ARRAY_SIZE(oplist->ops)) { + error = pmap_update(pmap); + oplist->pmap = pmap; + return error; + } + + return 0; +} + +static struct pmap_update_op * +pmap_update_oplist_prev_op(struct pmap_update_oplist *oplist) +{ + if (oplist->nr_ops == 0) { + return NULL; + } + + return &oplist->ops[oplist->nr_ops - 1]; +} + +static struct pmap_update_op * +pmap_update_oplist_prepare_op(struct pmap_update_oplist *oplist) +{ + assert(oplist->nr_ops < ARRAY_SIZE(oplist->ops)); + return &oplist->ops[oplist->nr_ops]; +} + +static void +pmap_update_oplist_finish_op(struct pmap_update_oplist *oplist) +{ + struct pmap_update_op *op; + + assert(oplist->nr_ops < ARRAY_SIZE(oplist->ops)); + op = &oplist->ops[oplist->nr_ops]; + cpumap_or(&oplist->cpumap, &op->cpumap); + oplist->nr_ops++; +} + +static unsigned int +pmap_update_oplist_count_mappings(const struct pmap_update_oplist *oplist, + unsigned int cpu) +{ + const struct pmap_update_op *op; + unsigned int i, nr_mappings; + + nr_mappings = 0; + + for (i = 0; i < oplist->nr_ops; i++) { + op = &oplist->ops[i]; + + if (!cpumap_test(&op->cpumap, cpu)) { + continue; + } + + switch (op->operation) { + case PMAP_UPDATE_OP_ENTER: + nr_mappings++; + break; + case PMAP_UPDATE_OP_REMOVE: + nr_mappings += (op->remove_args.end - op->remove_args.start) + / PAGE_SIZE; + break; + case PMAP_UPDATE_OP_PROTECT: + nr_mappings += (op->protect_args.end - op->protect_args.start) + / PAGE_SIZE; + break; + default: + assert(!"invalid update operation"); + } + } + + assert(nr_mappings != 0); + return nr_mappings; +} + +static void +pmap_update_request_array_init(struct pmap_update_request_array *array) +{ + struct pmap_update_request *request; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(array->requests); i++) { + request = &array->requests[i]; + spinlock_init(&request->lock); + } + + mutex_init(&array->lock); +} + +static struct pmap_update_request_array * +pmap_update_request_array_acquire(void) +{ + struct pmap_update_request_array *array; + + thread_pin(); + array = cpu_local_ptr(pmap_update_request_array); + mutex_lock(&array->lock); + return array; +} + +static void +pmap_update_request_array_release(struct pmap_update_request_array *array) +{ + mutex_unlock(&array->lock); + thread_unpin(); +} + +static void __init +pmap_syncer_init(struct pmap_syncer *syncer, unsigned int cpu) +{ + char name[SYSCNT_NAME_SIZE]; + struct pmap_update_queue *queue; + + queue = &syncer->queue; + spinlock_init(&queue->lock); + list_init(&queue->requests); + snprintf(name, sizeof(name), "pmap_updates/%u", cpu); + syscnt_register(&syncer->sc_updates, name); + snprintf(name, sizeof(name), "pmap_update_enters/%u", cpu); + syscnt_register(&syncer->sc_update_enters, name); + snprintf(name, sizeof(name), "pmap_update_removes/%u", cpu); + syscnt_register(&syncer->sc_update_removes, name); + snprintf(name, sizeof(name), "pmap_update_protects/%u", cpu); + syscnt_register(&syncer->sc_update_protects, name); +} +#endif + +static int __init +pmap_bootstrap(void) +{ +#if 0 + struct pmap_cpu_table *cpu_table; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(pmap_get_kernel_pmap()->cpu_tables); i++) { + cpu_table = &pmap_kernel_cpu_tables[i]; + pmap_get_kernel_pmap()->cpu_tables[i] = cpu_table; + } + + cpu_local_assign(pmap_current_ptr, pmap_get_kernel_pmap()); + + pmap_prot_table[VM_PROT_NONE] = 0; + pmap_prot_table[VM_PROT_READ] = 0; + pmap_prot_table[VM_PROT_WRITE] = PMAP_PTE_RW; + pmap_prot_table[VM_PROT_WRITE | VM_PROT_READ] = PMAP_PTE_RW; + pmap_prot_table[VM_PROT_EXECUTE] = 0; + pmap_prot_table[VM_PROT_EXECUTE | VM_PROT_READ] = 0; + pmap_prot_table[VM_PROT_EXECUTE | VM_PROT_WRITE] = PMAP_PTE_RW; + pmap_prot_table[VM_PROT_ALL] = PMAP_PTE_RW; + + pmap_update_request_array_init(cpu_local_ptr(pmap_update_request_array)); + + pmap_syncer_init(cpu_local_ptr(pmap_syncer), 0); + + pmap_update_oplist_ctor(&pmap_booter_oplist); + tcb_set_pmap_update_oplist(tcb_current(), &pmap_booter_oplist); + + cpumap_zero(&pmap_booter_cpumap); + cpumap_set(&pmap_booter_cpumap, 0); + + if (cpu_has_global_pages()) { + pmap_setup_global_pages(); + } + +#endif + return 0; +} + +INIT_OP_DEFINE(pmap_bootstrap, + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_bootstrap, true)); + +#if 0 +static void __init +pmap_setup_set_ptp_type(phys_addr_t ptp_pa, unsigned int index, + unsigned int level) +{ + struct vm_page *page; + + (void)index; + + if (level == 0) { + return; + } + + page = vm_page_lookup(ptp_pa); + assert(page != NULL); + + if (vm_page_type(page) != VM_PAGE_PMAP) { + assert(vm_page_type(page) == VM_PAGE_RESERVED); + vm_page_set_type(page, 0, VM_PAGE_PMAP); + } +} + +static void __init +pmap_setup_fix_ptps(void) +{ + pmap_walk_vas(PMAP_START_ADDRESS, PMAP_END_KERNEL_ADDRESS, + pmap_setup_set_ptp_type); +} +#endif + +static int __init +pmap_setup(void) +{ +#if 0 + pmap_setup_fix_ptps(); + kmem_cache_init(&pmap_cache, "pmap", sizeof(struct pmap), 0, NULL, 0); + kmem_cache_init(&pmap_update_oplist_cache, "pmap_update_oplist", + sizeof(struct pmap_update_oplist), CPU_L1_SIZE, + pmap_update_oplist_ctor, 0); +#else + return 0; +#endif +} + +INIT_OP_DEFINE(pmap_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(pmap_bootstrap, true), + INIT_OP_DEP(vm_page_setup, true)); + +#if 0 +void __init +pmap_ap_setup(void) +{ + cpu_local_assign(pmap_current_ptr, pmap_get_kernel_pmap()); + + if (cpu_has_global_pages()) { + cpu_enable_global_pages(); + } else { + cpu_tlb_flush(); + } +} + +static void __init +pmap_copy_cpu_table_page(const pmap_pte_t *sptp, unsigned int level, + struct vm_page *page) +{ + const struct pmap_pt_level *pt_level; + pmap_pte_t *dptp; + + pt_level = &pmap_pt_levels[level]; + dptp = vm_page_direct_ptr(page); + memcpy(dptp, sptp, pt_level->ptes_per_pt * sizeof(pmap_pte_t)); +} + +static void __init +pmap_copy_cpu_table_recursive(const pmap_pte_t *sptp, unsigned int level, + pmap_pte_t *dptp, uintptr_t start_va) +{ + const struct pmap_pt_level *pt_level; + struct vm_page *page; + phys_addr_t pa; + unsigned int i; + uintptr_t va; + + assert(level != 0); + + pt_level = &pmap_pt_levels[level]; + memset(dptp, 0, pt_level->ptes_per_pt * sizeof(pmap_pte_t)); + + for (i = 0, va = start_va; + i < pt_level->ptes_per_pt; + i++, va = P2END(va, 1UL << pt_level->skip)) { +#ifdef __LP64__ + /* Handle long mode canonical form */ + if (va == PMAP_END_ADDRESS) { + va = PMAP_START_KERNEL_ADDRESS; + } +#endif /* __LP64__ */ + + if (!pmap_pte_valid(sptp[i])) { + continue; + } else if (pmap_pte_large(sptp[i])) { + dptp[i] = sptp[i]; + continue; + } + + page = vm_page_alloc(0, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_PMAP); + + if (page == NULL) { + panic("pmap: unable to allocate page table page copy"); + } + + pa = vm_page_to_pa(page); + dptp[i] = (sptp[i] & ~PMAP_PA_MASK) | (pa & PMAP_PA_MASK); + + if (((level - 1) == 0) || pmap_pte_large(sptp[i])) { + pmap_copy_cpu_table_page(pmap_pte_next(sptp[i]), level - 1, page); + } else { + pmap_copy_cpu_table_recursive(pmap_pte_next(sptp[i]), level - 1, + vm_page_direct_ptr(page), va); + } + } +} + +static void __init +pmap_copy_cpu_table(unsigned int cpu) +{ + struct pmap_cpu_table *cpu_table; + struct pmap *kernel_pmap; + unsigned int level; + const pmap_pte_t *sptp; + pmap_pte_t *dptp; + + assert(cpu != 0); + + kernel_pmap = pmap_get_kernel_pmap(); + assert(cpu < ARRAY_SIZE(kernel_pmap->cpu_tables)); + cpu_table = kernel_pmap->cpu_tables[cpu]; + level = PMAP_NR_LEVELS - 1; + sptp = pmap_ptp_from_pa(kernel_pmap->cpu_tables[cpu_id()]->root_ptp_pa); + + struct vm_page *page; + + page = vm_page_alloc(0, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_PMAP); + + if (page == NULL) { + panic("pmap: unable to allocate page table root page copy"); + } + + cpu_table->root_ptp_pa = vm_page_to_pa(page); + dptp = vm_page_direct_ptr(page); + + pmap_copy_cpu_table_recursive(sptp, level, dptp, PMAP_START_ADDRESS); +} + +void __init +pmap_mp_setup(void) +{ + char name[THREAD_NAME_SIZE]; + struct pmap_update_oplist *oplist; + struct thread_attr attr; + struct pmap_syncer *syncer; + struct cpumap *cpumap; + struct tcb *tcb; + unsigned int cpu; + int error; + + error = cpumap_create(&cpumap); + + if (error) { + panic("pmap: unable to create syncer cpumap"); + } + + for (cpu = 1; cpu < cpu_count(); cpu++) { + pmap_update_request_array_init(percpu_ptr(pmap_update_request_array, + cpu)); + pmap_syncer_init(percpu_ptr(pmap_syncer, cpu), cpu); + } + + for (cpu = 0; cpu < cpu_count(); cpu++) { + syncer = percpu_ptr(pmap_syncer, cpu); + snprintf(name, sizeof(name), THREAD_KERNEL_PREFIX "pmap_sync/%u", cpu); + cpumap_zero(cpumap); + cpumap_set(cpumap, cpu); + thread_attr_init(&attr, name); + thread_attr_set_cpumap(&attr, cpumap); + thread_attr_set_priority(&attr, THREAD_SCHED_FS_PRIO_MAX); + error = thread_create(&syncer->thread, &attr, pmap_sync, syncer); + + if (error) { + panic("pmap: unable to create syncer thread"); + } + + tcb = thread_get_tcb(syncer->thread); + oplist = tcb_get_pmap_update_oplist(tcb); + tcb_set_pmap_update_oplist(tcb, NULL); + kmem_cache_free(&pmap_update_oplist_cache, oplist); + } + + cpumap_destroy(cpumap); + + for (cpu = 1; cpu < cpu_count(); cpu++) { + pmap_copy_cpu_table(cpu); + } + + pmap_do_remote_updates = 1; +} + +int +pmap_thread_build(struct thread *thread) +{ + struct pmap_update_oplist *oplist; + int error; + + error = pmap_update_oplist_create(&oplist); + + if (error) { + return error; + } + + tcb_set_pmap_update_oplist(thread_get_tcb(thread), oplist); + return 0; +} + +void +pmap_thread_cleanup(struct thread *thread) +{ + struct pmap_update_oplist *oplist; + + oplist = tcb_get_pmap_update_oplist(thread_get_tcb(thread)); + + if (oplist) { + pmap_update_oplist_destroy(oplist); + } +} +#endif + +int +pmap_kextract(uintptr_t va, phys_addr_t *pap) +{ +#if 0 + const struct pmap_pt_level *pt_level; + struct pmap *kernel_pmap; + pmap_pte_t *ptp, *pte; + unsigned int level; + + level = PMAP_NR_LEVELS - 1; + kernel_pmap = pmap_get_kernel_pmap(); + ptp = pmap_ptp_from_pa(kernel_pmap->cpu_tables[cpu_id()]->root_ptp_pa); + + for (;;) { + pt_level = &pmap_pt_levels[level]; + pte = &ptp[pmap_pte_index(va, pt_level)]; + + if (!pmap_pte_valid(*pte)) { + return ERROR_FAULT; + } + + if ((level == 0) || pmap_pte_large(*pte)) { + break; + } + + level--; + ptp = pmap_pte_next(*pte); + } + + *pap = (*pte & PMAP_PA_MASK); + return 0; +#else + (void)va; + (void)pap; + return ERROR_AGAIN; +#endif +} + +int +pmap_create(struct pmap **pmapp) +{ +#if 0 + struct pmap *pmap; + unsigned int i; + + pmap = kmem_cache_alloc(&pmap_cache); + + if (pmap == NULL) { + return ERROR_NOMEM; + } + + for (i = 0; i < ARRAY_SIZE(pmap->cpu_tables); i++) { + pmap->cpu_tables[i] = NULL; + } + + *pmapp = pmap; + return 0; +#else + (void)pmapp; + return ERROR_AGAIN; +#endif +} + +#if 0 +static int +pmap_enter_local(struct pmap *pmap, uintptr_t va, phys_addr_t pa, + int prot, int flags) +{ + const struct pmap_pt_level *pt_level; + struct vm_page *page; + phys_addr_t ptp_pa; + pmap_pte_t *ptp, *pte, pte_bits; + unsigned int level; + + /* TODO Page attributes */ + (void)flags; + + pte_bits = PMAP_PTE_RW; + + if (pmap != pmap_get_kernel_pmap()) { + pte_bits |= PMAP_PTE_US; + } + + level = PMAP_NR_LEVELS - 1; + ptp = pmap_ptp_from_pa(pmap->cpu_tables[cpu_id()]->root_ptp_pa); + + for (;;) { + pt_level = &pmap_pt_levels[level]; + pte = &ptp[pmap_pte_index(va, pt_level)]; + + if (level == 0) { + break; + } + + if (pmap_pte_valid(*pte)) { + ptp = pmap_pte_next(*pte); + } else { + page = vm_page_alloc(0, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_PMAP); + + if (page == NULL) { + log_warning("pmap: page table page allocation failure"); + return ERROR_NOMEM; + } + + ptp_pa = vm_page_to_pa(page); + ptp = pmap_ptp_from_pa(ptp_pa); + pmap_ptp_clear(ptp); + pmap_pte_set(pte, ptp_pa, pte_bits, pt_level); + } + + level--; + } + + assert(!pmap_pte_valid(*pte)); + pte_bits = ((pmap == pmap_get_kernel_pmap()) ? PMAP_PTE_G : PMAP_PTE_US) + | pmap_prot_table[prot & VM_PROT_ALL]; + pmap_pte_set(pte, pa, pte_bits, pt_level); + return 0; +} +#endif + +int +pmap_enter(struct pmap *pmap, uintptr_t va, phys_addr_t pa, + int prot, int flags) +{ +#if 0 + struct pmap_update_oplist *oplist; + struct pmap_update_op *op; + int error; + + va = vm_page_trunc(va); + pa = vm_page_trunc(pa); + pmap_assert_range(pmap, va, va + PAGE_SIZE); + + oplist = pmap_update_oplist_get(); + error = pmap_update_oplist_prepare(oplist, pmap); + + if (error) { + return error; + } + + op = pmap_update_oplist_prepare_op(oplist); + + if (flags & PMAP_PEF_GLOBAL) { + cpumap_copy(&op->cpumap, cpumap_all()); + } else { + cpumap_zero(&op->cpumap); + cpumap_set(&op->cpumap, cpu_id()); + } + + op->operation = PMAP_UPDATE_OP_ENTER; + op->enter_args.va = va; + op->enter_args.pa = pa; + op->enter_args.prot = prot; + op->enter_args.flags = flags & ~PMAP_PEF_GLOBAL; + pmap_update_oplist_finish_op(oplist); + return 0; +#else + (void)pmap; + (void)va; + (void)pa; + (void)prot; + (void)flags; + return ERROR_AGAIN; +#endif +} + +#if 0 +static void +pmap_remove_local_single(struct pmap *pmap, uintptr_t va) +{ + const struct pmap_pt_level *pt_level; + pmap_pte_t *ptp, *pte; + unsigned int level; + + level = PMAP_NR_LEVELS - 1; + ptp = pmap_ptp_from_pa(pmap->cpu_tables[cpu_id()]->root_ptp_pa); + + for (;;) { + pt_level = &pmap_pt_levels[level]; + pte = &ptp[pmap_pte_index(va, pt_level)]; + + if (!pmap_pte_valid(*pte)) { + return; + } + + if (level == 0) { + break; + } + + level--; + ptp = pmap_pte_next(*pte); + } + + pmap_pte_clear(pte); +} + +static void +pmap_remove_local(struct pmap *pmap, uintptr_t start, uintptr_t end) +{ + while (start < end) { + pmap_remove_local_single(pmap, start); + start += PAGE_SIZE; + } +} +#endif + +int +pmap_remove(struct pmap *pmap, uintptr_t va, const struct cpumap *cpumap) +{ +#if 0 + struct pmap_update_oplist *oplist; + struct pmap_update_op *op; + int error; + + va = vm_page_trunc(va); + pmap_assert_range(pmap, va, va + PAGE_SIZE); + + oplist = pmap_update_oplist_get(); + error = pmap_update_oplist_prepare(oplist, pmap); + + if (error) { + return error; + } + + /* Attempt naive merge with previous operation */ + op = pmap_update_oplist_prev_op(oplist); + + if ((op != NULL) + && (op->operation == PMAP_UPDATE_OP_REMOVE) + && (op->remove_args.end == va) + && (cpumap_cmp(&op->cpumap, cpumap) == 0)) { + op->remove_args.end = va + PAGE_SIZE; + return 0; + } + + op = pmap_update_oplist_prepare_op(oplist); + cpumap_copy(&op->cpumap, cpumap); + op->operation = PMAP_UPDATE_OP_REMOVE; + op->remove_args.start = va; + op->remove_args.end = va + PAGE_SIZE; + pmap_update_oplist_finish_op(oplist); + return 0; +#else + (void)pmap; + (void)va; + (void)cpumap; + return ERROR_AGAIN; +#endif +} + +#if 0 +static void +pmap_protect_local(struct pmap *pmap, uintptr_t start, + uintptr_t end, int prot) +{ + (void)pmap; + (void)start; + (void)end; + (void)prot; + + /* TODO Implement */ + panic("pmap: pmap_protect not implemented"); +} + +int +pmap_protect(struct pmap *pmap, uintptr_t va, int prot, + const struct cpumap *cpumap) +{ + struct pmap_update_oplist *oplist; + struct pmap_update_op *op; + int error; + + va = vm_page_trunc(va); + pmap_assert_range(pmap, va, va + PAGE_SIZE); + + oplist = pmap_update_oplist_get(); + error = pmap_update_oplist_prepare(oplist, pmap); + + if (error) { + return error; + } + + /* Attempt naive merge with previous operation */ + op = pmap_update_oplist_prev_op(oplist); + + if ((op != NULL) + && (op->operation == PMAP_UPDATE_OP_PROTECT) + && (op->protect_args.end == va) + && (op->protect_args.prot == prot) + && (cpumap_cmp(&op->cpumap, cpumap) == 0)) { + op->protect_args.end = va + PAGE_SIZE; + return 0; + } + + op = pmap_update_oplist_prepare_op(oplist); + cpumap_copy(&op->cpumap, cpumap); + op->operation = PMAP_UPDATE_OP_PROTECT; + op->protect_args.start = va; + op->protect_args.end = va + PAGE_SIZE; + op->protect_args.prot = prot; + pmap_update_oplist_finish_op(oplist); + return 0; +} + +static void +pmap_flush_tlb(struct pmap *pmap, uintptr_t start, uintptr_t end) +{ + if ((pmap != pmap_current()) && (pmap != pmap_get_kernel_pmap())) { + return; + } + + while (start < end) { + cpu_tlb_flush_va(start); + start += PAGE_SIZE; + } +} + +static void +pmap_flush_tlb_all(struct pmap *pmap) +{ + if ((pmap != pmap_current()) && (pmap != pmap_get_kernel_pmap())) { + return; + } + + if (pmap == pmap_get_kernel_pmap()) { + cpu_tlb_flush_all(); + } else { + cpu_tlb_flush(); + } +} + +static int +pmap_update_enter(struct pmap *pmap, int flush, + const struct pmap_update_enter_args *args) +{ + int error; + + error = pmap_enter_local(pmap, args->va, args->pa, args->prot, args->flags); + + if (error) { + return error; + } + + if (flush) { + pmap_flush_tlb(pmap, args->va, args->va + PAGE_SIZE); + } + + return 0; +} + +static void +pmap_update_remove(struct pmap *pmap, int flush, + const struct pmap_update_remove_args *args) +{ + pmap_remove_local(pmap, args->start, args->end); + + if (flush) { + pmap_flush_tlb(pmap, args->start, args->end); + } +} + +static void +pmap_update_protect(struct pmap *pmap, int flush, + const struct pmap_update_protect_args *args) +{ + pmap_protect_local(pmap, args->start, args->end, args->prot); + + if (flush) { + pmap_flush_tlb(pmap, args->start, args->end); + } +} + +static int +pmap_update_local(const struct pmap_update_oplist *oplist, + unsigned int nr_mappings) +{ + const struct pmap_update_op *op; + struct pmap_syncer *syncer; + int error, global_tlb_flush; + unsigned int i; + + syncer = cpu_local_ptr(pmap_syncer); + syscnt_inc(&syncer->sc_updates); + global_tlb_flush = (nr_mappings > PMAP_UPDATE_MAX_MAPPINGS); + error = 0; + + for (i = 0; i < oplist->nr_ops; i++) { + op = &oplist->ops[i]; + + if (!cpumap_test(&op->cpumap, cpu_id())) { + continue; + } + + switch (op->operation) { + case PMAP_UPDATE_OP_ENTER: + syscnt_inc(&syncer->sc_update_enters); + error = pmap_update_enter(oplist->pmap, !global_tlb_flush, + &op->enter_args); + break; + case PMAP_UPDATE_OP_REMOVE: + syscnt_inc(&syncer->sc_update_removes); + pmap_update_remove(oplist->pmap, !global_tlb_flush, + &op->remove_args); + break; + case PMAP_UPDATE_OP_PROTECT: + syscnt_inc(&syncer->sc_update_protects); + pmap_update_protect(oplist->pmap, !global_tlb_flush, + &op->protect_args); + break; + default: + assert(!"invalid update operation"); + } + + if (error) { + return error; + } + } + + if (global_tlb_flush) { + pmap_flush_tlb_all(oplist->pmap); + } + + return 0; +} +#endif + +int +pmap_update(struct pmap *pmap) +{ +#if 0 + struct pmap_update_oplist *oplist; + struct pmap_update_request_array *array; + struct pmap_update_request *request; + struct pmap_update_queue *queue; + struct pmap_syncer *syncer; + unsigned int nr_mappings; + int error, cpu; + + oplist = pmap_update_oplist_get(); + + if (pmap != oplist->pmap) { + /* Make sure pmap_update() is called before manipulating another pmap */ + assert(oplist->pmap == NULL); + return 0; + } + + assert(oplist->nr_ops != 0); + + if (!pmap_do_remote_updates) { + nr_mappings = pmap_update_oplist_count_mappings(oplist, cpu_id()); + error = pmap_update_local(oplist, nr_mappings); + goto out; + } + + error = 0; + + array = pmap_update_request_array_acquire(); + + cpumap_for_each(&oplist->cpumap, cpu) { + syncer = percpu_ptr(pmap_syncer, cpu); + queue = &syncer->queue; + request = &array->requests[cpu]; + request->sender = thread_self(); + request->oplist = oplist; + request->nr_mappings = pmap_update_oplist_count_mappings(oplist, cpu); + request->done = 0; + request->error = 0; + + spinlock_lock(&queue->lock); + list_insert_tail(&queue->requests, &request->node); + thread_wakeup(syncer->thread); + spinlock_unlock(&queue->lock); + } + + cpumap_for_each(&oplist->cpumap, cpu) { + request = &array->requests[cpu]; + + spinlock_lock(&request->lock); + + while (!request->done) { + thread_sleep(&request->lock, request, "pmaprq"); + } + + if (!error && request->error) { + error = request->error; + } + + spinlock_unlock(&request->lock); + } + + pmap_update_request_array_release(array); + +out: + cpumap_zero(&oplist->cpumap); + oplist->pmap = NULL; + oplist->nr_ops = 0; + return error; +#else + (void)pmap; + return ERROR_AGAIN; +#endif +} + +#if 0 +static void +pmap_sync(void *arg) +{ + struct pmap_update_queue *queue; + struct pmap_update_request *request; + struct pmap_syncer *self; + int error; + + self = arg; + queue = &self->queue; + + for (;;) { + spinlock_lock(&queue->lock); + + while (list_empty(&queue->requests)) { + thread_sleep(&queue->lock, queue, "pmapq"); + } + + request = list_first_entry(&queue->requests, + struct pmap_update_request, node); + list_remove(&request->node); + + spinlock_unlock(&queue->lock); + + error = pmap_update_local(request->oplist, request->nr_mappings); + + spinlock_lock(&request->lock); + request->done = 1; + request->error = error; + thread_wakeup(request->sender); + spinlock_unlock(&request->lock); + } +} +#endif + +void +pmap_load(struct pmap *pmap) +{ +#if 0 + struct pmap_cpu_table *cpu_table; + + assert(!cpu_intr_enabled()); + assert(!thread_preempt_enabled()); + + if (pmap_current() == pmap) { + return; + } + + /* TODO Lazy TLB invalidation */ + + cpu_local_assign(pmap_current_ptr, pmap); + + /* TODO Implement per-CPU page tables for non-kernel pmaps */ + cpu_table = pmap->cpu_tables[cpu_id()]; + + cpu_set_cr3(cpu_table->root_ptp_pa); +#else + (void)pmap; +#endif +} diff --git a/arch/arm/machine/pmap.h b/arch/arm/machine/pmap.h new file mode 100644 index 00000000..3f55ab91 --- /dev/null +++ b/arch/arm/machine/pmap.h @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * TODO Comment. + */ + +#ifndef _ARM_PMAP_H +#define _ARM_PMAP_H + +#include <kern/macros.h> + +/* + * Virtual memory layout. + */ + +/* + * User space boundaries. + */ +#define PMAP_START_ADDRESS DECL_CONST(0, UL) +#define PMAP_END_ADDRESS DECL_CONST(0xc0000000, UL) + +/* + * Kernel space boundaries. + */ +#define PMAP_START_KERNEL_ADDRESS PMAP_END_ADDRESS +#define PMAP_END_KERNEL_ADDRESS DECL_CONST(0xfffff000, UL) + +/* + * Direct physical mapping boundaries. + */ +#define PMAP_START_DIRECTMAP_ADDRESS PMAP_START_KERNEL_ADDRESS +#define PMAP_END_DIRECTMAP_ADDRESS DECL_CONST(0xf8000000, UL) + +/* + * Kernel mapping offset. + */ +#define PMAP_KERNEL_OFFSET PMAP_START_DIRECTMAP_ADDRESS + +/* + * Kernel virtual space boundaries. + * + * In addition to the direct physical mapping, the kernel has its own virtual + * memory space. + */ +#define PMAP_START_KMEM_ADDRESS PMAP_END_DIRECTMAP_ADDRESS +#define PMAP_END_KMEM_ADDRESS PMAP_END_KERNEL_ADDRESS + +#ifndef __ASSEMBLER__ + +#include <stdbool.h> +#include <stdint.h> + +#include <kern/cpumap.h> +#include <kern/init.h> +#include <kern/list.h> +#include <kern/mutex.h> +#include <kern/thread.h> +#include <machine/cpu.h> +#include <machine/types.h> + +/* + * Page table entry types. + */ +#define PMAP_PTE_TYPE_FAULT 0x00000000 +#define PMAP_PTE_TYPE_COARSE 0x00000001 +#define PMAP_PTE_TYPE_SMALL 0x00000002 +#define PMAP_PTE_TYPE_SECTION 0x00000002 +#define PMAP_PTE_TYPE_MASK 0x00000003 + +/* + * Mapping creation flags. + */ +#define PMAP_PEF_GLOBAL 0x1 /* Create a mapping on all processors */ + +typedef phys_addr_t pmap_pte_t; + +/* + * Physical address map. + */ +struct pmap; + +static __always_inline bool +pmap_pte_valid(pmap_pte_t pte) +{ + return (pte & PMAP_PTE_TYPE_MASK) != PMAP_PTE_TYPE_FAULT; +} + +static inline struct pmap * +pmap_get_kernel_pmap(void) +{ + extern struct pmap pmap_kernel_pmap; + + return &pmap_kernel_pmap; +} + +unsigned int pmap_boot_get_last_level(size_t page_size); + +/* + * Early initialization of the MMU. + * + * This function is called before paging is enabled by the boot module. It + * maps the kernel at physical and virtual addresses, after which all kernel + * functions and data can be accessed. + */ +pmap_pte_t * pmap_setup_paging(void); + +/* + * This function is called by the AP bootstrap code before paging is enabled. + */ +pmap_pte_t * pmap_ap_setup_paging(void); + +/* + * Initialize the pmap module on APs. + */ +void pmap_ap_setup(void); + +/* + * Set up the pmap module for multiprocessor operations. + * + * This function copies the current page tables so that each processor has + * its own set of page tables. As a result, it must be called right before + * starting APs to make sure all processors have the same mappings. + * + * This function must be called before starting the scheduler whatever the + * number of processors. + */ +void pmap_mp_setup(void); + +/* + * Build/clean up pmap thread-local data for the given thread. + */ +int pmap_thread_build(struct thread *thread); +void pmap_thread_cleanup(struct thread *thread); + +/* + * Extract a mapping from the kernel map. + * + * This function walks the page tables to retrieve the physical address + * mapped at the given virtual address. + */ +int pmap_kextract(uintptr_t va, phys_addr_t *pap); + +/* + * Create a pmap for a user task. + */ +int pmap_create(struct pmap **pmapp); + +/* + * Create a mapping on a physical map. + * + * If protection is VM_PROT_NONE, this function behaves as if it were + * VM_PROT_READ. There must not be an existing valid mapping for the given + * virtual address. + * + * If the mapping is local, it is the responsibility of the caller to take + * care of migration. + * + * This function may trigger an implicit update. + */ +int pmap_enter(struct pmap *pmap, uintptr_t va, phys_addr_t pa, + int prot, int flags); + +/* + * Remove a mapping from a physical map. + * + * The caller may use this function on non-existent mappings. + * + * This function may trigger an implicit update. + */ +int pmap_remove(struct pmap *pmap, uintptr_t va, + const struct cpumap *cpumap); + +/* + * Set the protection of a mapping in a physical map. + * + * This function may trigger an implicit update. + */ +int pmap_protect(struct pmap *pmap, uintptr_t va, int prot, + const struct cpumap *cpumap); + +/* + * Force application of pending modifications on a physical map. + * + * The functions that may defer physical map modifications are : + * - pmap_enter + * - pmap_remove + * - pmap_protect + * + * On return, if successful, all operations previously performed by the + * calling thread are guaranteed to be applied on their respective + * processors. Note that this function doesn't guarantee that modifications + * performed by different threads are applied. + * + * If an error occurs, then some or all of the pending modifications + * could not be applied. This function lacks the knowledge to handle + * such cases. As a result, the caller is responsible for the complete + * set of affected mappings and must take appropriate actions to restore + * physical mappings consistency. Note that multiple errors may occur + * when calling this function. The caller shouldn't rely on the specific + * error value, and should consider the whole operation to have failed. + * + * Also note that the only operation that may fail is mapping creation. + * Therefore, if the caller only queues removals or protection changes + * between two calls to this function, it is guaranteed to succeed. + */ +int pmap_update(struct pmap *pmap); + +/* + * Load the given pmap on the current processor. + * + * This function must be called with interrupts and preemption disabled. + */ +void pmap_load(struct pmap *pmap); + +/* + * Return the pmap currently loaded on the processor. + * + * Since threads may borrow pmaps, this can be different than the pmap + * of the caller. + */ +#if 0 +static inline struct pmap * +pmap_current(void) +{ + extern struct pmap *pmap_current_ptr; + return cpu_local_read(pmap_current_ptr); +} +#endif + +/* + * This init operation provides : + * - kernel pmap operations + */ +INIT_OP_DECLARE(pmap_bootstrap); + +/* + * This init operation provides : + * - user pmap creation + * - module fully initialized + */ +INIT_OP_DECLARE(pmap_setup); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ARM_PMAP_H */ diff --git a/arch/arm/machine/pmem.h b/arch/arm/machine/pmem.h new file mode 100644 index 00000000..9a072401 --- /dev/null +++ b/arch/arm/machine/pmem.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Physical memory layout. + * + * This file is a top header in the inclusion hierarchy, and shouldn't include + * other headers that may cause circular dependencies. + * + * XXX Specific to the Qemu virtual machine. + */ + +#ifndef _ARM_PMEM_H +#define _ARM_PMEM_H + +#include <kern/macros.h> + +/* + * Zone boundaries. + */ + +#define PMEM_RAM_START DECL_CONST(0x40000000, UL) +#define PMEM_DMA_LIMIT DECL_CONST(0x44000000, UL) +#define PMEM_DMA32_LIMIT PMEM_DMA_LIMIT +#define PMEM_DIRECTMAP_LIMIT PMEM_DMA_LIMIT +#define PMEM_HIGHMEM_LIMIT PMEM_DMA_LIMIT + +#define PMEM_MAX_ZONES 1 + +/* + * Zone vm_page indexes. + */ + +#define PMEM_ZONE_DMA 0 +#define PMEM_ZONE_DMA32 PMEM_ZONE_DMA +#define PMEM_ZONE_DIRECTMAP PMEM_ZONE_DMA +#define PMEM_ZONE_HIGHMEM PMEM_ZONE_DMA + +#endif /* _ARM_PMEM_H */ diff --git a/arch/arm/machine/strace.c b/arch/arm/machine/strace.c new file mode 100644 index 00000000..9c47a73a --- /dev/null +++ b/arch/arm/machine/strace.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <kern/init.h> + +static int __init +strace_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(strace_setup); diff --git a/arch/arm/machine/strace.h b/arch/arm/machine/strace.h new file mode 100644 index 00000000..362f7b05 --- /dev/null +++ b/arch/arm/machine/strace.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Stack tracing. + */ + +#ifndef _ARM_STRACE_H +#define _ARM_STRACE_H + +static inline void +strace_dump(void) +{ +} + +/* + * This init operation provides : + * - module fully initialized + */ +INIT_OP_DECLARE(strace_setup); + +#endif /* _ARM_STRACE_H */ diff --git a/arch/arm/machine/string.h b/arch/arm/machine/string.h new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/arch/arm/machine/string.h diff --git a/arch/arm/machine/tcb.c b/arch/arm/machine/tcb.c new file mode 100644 index 00000000..dea2a1db --- /dev/null +++ b/arch/arm/machine/tcb.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <kern/error.h> +#include <kern/init.h> +#include <machine/cpu.h> +#include <machine/tcb.h> + +int +tcb_build(struct tcb *tcb, void *stack, void (*fn)(void *), void *arg) +{ + (void)tcb; + (void)stack; + (void)fn; + (void)arg; + return ERROR_AGAIN; +} + +void +tcb_cleanup(struct tcb *tcb) +{ + (void)tcb; +} + +static int __init +tcb_setup(void) +{ + cpu_halt(); + return 0; +} + +INIT_OP_DEFINE(tcb_setup); diff --git a/arch/arm/machine/tcb.h b/arch/arm/machine/tcb.h new file mode 100644 index 00000000..fc0ae372 --- /dev/null +++ b/arch/arm/machine/tcb.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Thread control block. + */ + +#ifndef _ARM_TCB_H +#define _ARM_TCB_H + +#include <stdnoreturn.h> + +#include <kern/init.h> +#include <machine/page.h> + +/* + * Thread stack size. + */ +#define TCB_STACK_SIZE PAGE_SIZE + +struct tcb { +}; + +int tcb_build(struct tcb *tcb, void *stack, void (*fn)(void *), void *arg); + +void tcb_cleanup(struct tcb *tcb); + +static inline struct tcb * +tcb_current(void) +{ + return NULL; +} + +static inline void +tcb_set_current(struct tcb *tcb) +{ + (void)tcb; +} + +noreturn void tcb_load(struct tcb *tcb); + +static inline void +tcb_switch(struct tcb *prev, struct tcb *next) +{ + (void)prev; + (void)next; +} + +/* + * This init operation provides : + * - current TCB handling + */ +INIT_OP_DECLARE(tcb_setup); + +#endif /* _ARM_TCB_H */ diff --git a/arch/arm/machine/tcb_asm.S b/arch/arm/machine/tcb_asm.S new file mode 100644 index 00000000..f58f429e --- /dev/null +++ b/arch/arm/machine/tcb_asm.S @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <machine/asm.h> + +ASM_FUNC(tcb_load): +ASM_END(tcb_load) diff --git a/arch/arm/machine/trap.h b/arch/arm/machine/trap.h new file mode 100644 index 00000000..d29fa41b --- /dev/null +++ b/arch/arm/machine/trap.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _ARM_TRAP_H +#define _ARM_TRAP_H + +#define TRAP_INTR_TABLE_SIZE 256 + +#endif /* _ARM_TRAP_H */ diff --git a/arch/arm/machine/types.h b/arch/arm/machine/types.h new file mode 100644 index 00000000..e2d72d76 --- /dev/null +++ b/arch/arm/machine/types.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _ARM_TYPES_H +#define _ARM_TYPES_H + +typedef unsigned long phys_addr_t; + +#endif /* _ARM_TYPES_H */ diff --git a/arch/arm/x15.lds.S b/arch/arm/x15.lds.S new file mode 100644 index 00000000..525c6475 --- /dev/null +++ b/arch/arm/x15.lds.S @@ -0,0 +1,112 @@ +/* OUTPUT_FORMAT("elf32-little") */ +/* OUTPUT_ARCH(arm) */ + +ENTRY(_start) + +#include <kern/init.h> +#include <machine/boot.h> +#include <machine/cpu.h> +#include <machine/page.h> +#include <machine/pmem.h> + +PHDRS +{ + /* Flags are actually similar to classic Unix permissions */ + load PT_LOAD FLAGS(7); + boot PT_LOAD FLAGS(7); + init PT_LOAD FLAGS(7); + percpu PT_LOAD FLAGS(6); + text PT_LOAD FLAGS(5); + rodata PT_LOAD FLAGS(4); + data PT_LOAD FLAGS(6); +} + +SECTIONS +{ + . = 0; + + .boot.load : { + *(BOOT_LOAD_SECTION) + } : load + + . += PMEM_RAM_START; + _boot = .; + + .boot ALIGN(PAGE_SIZE) : AT(BOOT_RTOL(ADDR(.boot))) { + *(BOOT_TEXT_SECTION) + *(BOOT_DATA_SECTION) + } : boot + + . = ALIGN(PAGE_SIZE); + _boot_end = .; + + . += (PMAP_START_KERNEL_ADDRESS - PMEM_RAM_START); + _init = .; + + .init ALIGN(PAGE_SIZE) : AT(BOOT_VTOL(ADDR(.init))) { + *(.init.text) + *(.init.data) + + . = ALIGN(INIT_OP_ALIGN); + _init_ops = .; + *(.init.ops) + _init_ops_end = .; + + } : init + + . = ALIGN(PAGE_SIZE); + _init_end = .; + _percpu = .; + + .percpu 0 : AT(BOOT_VTOL(_percpu)) { + *(.percpu) + } : percpu + + . = _percpu + SIZEOF(.percpu); + . = ALIGN(PAGE_SIZE); + _percpu_end = .; + _text = .; + + .text ALIGN(PAGE_SIZE) : AT(BOOT_VTOL(ADDR(.text))) { + *(.text) + } : text + + . = ALIGN(PAGE_SIZE); + _rodata = .; + + .rodata ALIGN(PAGE_SIZE) : AT(BOOT_VTOL(ADDR(.rodata))) { + *(.rodata) + } : rodata + + .notes ALIGN(CPU_DATA_ALIGN) : AT(BOOT_VTOL(ADDR(.notes))) { + *(.note.*) + } : rodata + + . = ALIGN(PAGE_SIZE); + _data = .; + + .data ALIGN(PAGE_SIZE) : AT(BOOT_VTOL(ADDR(.data))) { + . = ALIGN(CPU_L1_SIZE); + *(.data.read_mostly) + . = ALIGN(CPU_L1_SIZE); + *(.data) + } : data + + . = ALIGN(PAGE_SIZE); + _bss = .; + + .bss ALIGN(CPU_DATA_ALIGN) : AT(BOOT_VTOL(ADDR(.bss))) { + *(.bss) + } : data + + . = ALIGN(PAGE_SIZE); + _end = .; + + /* + * XXX A global offset section is created because of linking with libgcc. + * Is it safe to discard it ? + */ + /DISCARD/ : { + *(.eh_frame) + } +} diff --git a/arch/x86/machine/asm.h b/arch/x86/machine/asm.h index 204d6fed..754c8f2f 100644 --- a/arch/x86/machine/asm.h +++ b/arch/x86/machine/asm.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012 Richard Braun. + * Copyright (c) 2011-2012 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,8 +15,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef _X86_ASM_H -#define _X86_ASM_H +#ifndef _ARM_ASM_H +#define _ARM_ASM_H #ifndef __ASSEMBLER__ #warning "asm.h included from a C file" @@ -39,4 +39,4 @@ x: #define ASM_END(x) \ .size x, . - x -#endif /* _X86_ASM_H */ +#endif /* _ARM_ASM_H */ diff --git a/arch/x86/machine/boot.c b/arch/x86/machine/boot.c index afc41d70..9b05d60d 100644 --- a/arch/x86/machine/boot.c +++ b/arch/x86/machine/boot.c @@ -120,84 +120,6 @@ static char boot_panic_meminfo_msg[] __bootdata static char boot_panic_cmdline_msg[] __bootdata = "boot: command line too long"; -void * __boot -boot_memcpy(void *dest, const void *src, size_t n) -{ - const char *src_ptr; - char *dest_ptr; - size_t i; - - dest_ptr = dest; - src_ptr = src; - - for (i = 0; i < n; i++) { - *dest_ptr = *src_ptr; - dest_ptr++; - src_ptr++; - } - - return dest; -} - -void * __boot -boot_memmove(void *dest, const void *src, size_t n) -{ - const char *src_ptr; - char *dest_ptr; - size_t i; - - if (dest <= src) { - dest_ptr = dest; - src_ptr = src; - - for (i = 0; i < n; i++) { - *dest_ptr = *src_ptr; - dest_ptr++; - src_ptr++; - } - } else { - dest_ptr = dest + n - 1; - src_ptr = src + n - 1; - - for (i = 0; i < n; i++) { - *dest_ptr = *src_ptr; - dest_ptr--; - src_ptr--; - } - } - - return dest; -} - -void * __boot -boot_memset(void *s, int c, size_t n) -{ - char *buffer; - size_t i; - - buffer = s; - - for (i = 0; i < n; i++) { - buffer[i] = c; - } - - return s; -} - -size_t __boot -boot_strlen(const char *s) -{ - const char *start; - - start = s; - - while (*s != '\0') { - s++; - } - - return (s - start); -} - void __boot boot_panic(const char *msg) { diff --git a/kern/Makefile b/kern/Makefile index 0aa96fc3..dfd1cbd8 100644 --- a/kern/Makefile +++ b/kern/Makefile @@ -1,6 +1,7 @@ x15_SOURCES-y += \ kern/arg.c \ kern/bitmap.c \ + kern/bootmem.c \ kern/cbuf.c \ kern/clock.c \ kern/condition.c \ diff --git a/kern/bootmem.c b/kern/bootmem.c new file mode 100644 index 00000000..0bf92e19 --- /dev/null +++ b/kern/bootmem.c @@ -0,0 +1,787 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * The purpose of this module is to provide all memory-related services + * required during bootstrap, before paging is enabled. + * + * In order to meet the requirements of the various supported page table + * formats, where some page tables may be smaller than a page while others + * may be bigger, but always aligned on the page table size, this module + * implements a buddy memory allocator similar to the vm_page module. + */ + +#include <assert.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <kern/bootmem.h> +#include <kern/error.h> +#include <kern/macros.h> +#include <machine/boot.h> +#include <machine/page.h> +#include <machine/pmem.h> +#include <machine/types.h> +#include <vm/vm_kmem.h> +#include <vm/vm_page.h> + +#define BOOTMEM_MAX_RESERVED_RANGES 64 + +/* + * Contiguous block of physical memory. + * + * These are semantically the same as those used by the VM system, and are + * actually loaded into the VM system when it's enabled. + * + * The boundaries of a zone must be page-aligned and must not overlap. + */ +struct bootmem_zone { + phys_addr_t start; + phys_addr_t end; + bool registered; + bool direct_mapped; +}; + +static struct bootmem_zone bootmem_zones[PMEM_MAX_ZONES] __bootdata; + +/* + * Physical memory range descriptor. + * + * Such ranges are used to describe memory containing data that must be + * preserved during bootstrap. If temporary, a range is released to the + * VM system when it's enabled. + * + * The boundary addresses must not be fixed up (e.g. page-aligned), since + * ranges may overlap the same pages. + */ +struct bootmem_range { + phys_addr_t start; + phys_addr_t end; + bool temporary; +}; + +/* + * Sorted array of reserved range descriptors. + */ +static struct bootmem_range bootmem_reserved_ranges[BOOTMEM_MAX_RESERVED_RANGES] + __bootdata; +static unsigned int bootmem_nr_reserved_ranges __bootdata; + +#if BOOT_MEM_BLOCK_BITS > PAGE_BITS +#error "block size too large" +#endif + +/* + * Basic block size. + * + * A block descriptor of order 0 describes a block of this size, also aligned + * to this value. + */ +#define BOOTMEM_BLOCK_SIZE (1 << BOOT_MEM_BLOCK_BITS) + +/* + * Special order value for blocks that aren't in a free list. Such blocks are + * either allocated, or part of a free block of pages but not the head page. + */ +#define BOOTMEM_ORDER_UNLISTED ((unsigned short)-1) + +/* + * Descriptor for a block in the buddy allocator heap. + * + * The physical address member doesn't have phys_addr_t type because the + * heap is used to return directly accessible blocks, and has uintptr_t + * type because that's the proper type for integers that may safely be + * converted to pointers. + * + * The size of a block is BOOTMEM_BLOCK_SIZE * 2^block->order. + */ +struct bootmem_block { + uintptr_t phys_addr; + struct bootmem_block *next; + struct bootmem_block **pprev; + unsigned short order; + bool allocated; +}; + +struct bootmem_free_list { + struct bootmem_block *blocks; +}; + +struct bootmem_heap { + uintptr_t start; + uintptr_t end; + struct bootmem_block *blocks; + struct bootmem_block *blocks_end; + struct bootmem_free_list free_lists[BOOT_MEM_NR_FREE_LISTS]; +}; + +static struct bootmem_heap bootmem_heap __bootdata; + +static char bootmem_panic_msg_zone_overlapping[] __bootdata + = "bootmem: zone overlapping"; +static char bootmem_panic_msg_invalid_zone_index_msg[] __bootdata + = "bootmem: invalid zone index"; +static char bootmem_panic_msg_zone_already_registered[] __bootdata + = "bootmem: zone already registered"; +static char bootmem_panic_msg_invalid_reserved_range[] __bootdata + = "bootmem: invalid reserved range"; +static char bootmem_panic_msg_too_many_reserved_ranges[] __bootdata + = "bootmem: too many reserved ranges"; +static char bootmem_panic_msg_setup[] __bootdata + = "bootmem: unable to set up the early memory allocator"; +static char bootmem_panic_msg_nomem[] __bootdata + = "bootmem: unable to allocate memory"; +static char bootmem_panic_msg_invalid_argument[] __bootdata + = "bootmem: invalid argument"; + +void * __boot +bootmem_memcpy(void *dest, const void *src, size_t n) +{ + const char *src_ptr; + char *dest_ptr; + + dest_ptr = dest; + src_ptr = src; + + for (size_t i = 0; i < n; i++) { + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; + } + + return dest; +} + +void * __boot +bootmem_memmove(void *dest, const void *src, size_t n) +{ + const char *src_ptr; + char *dest_ptr; + + if (dest <= src) { + dest_ptr = dest; + src_ptr = src; + + for (size_t i = 0; i < n; i++) { + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; + } + } else { + dest_ptr = dest + n - 1; + src_ptr = src + n - 1; + + for (size_t i = 0; i < n; i++) { + *dest_ptr = *src_ptr; + dest_ptr--; + src_ptr--; + } + } + + return dest; +} + +void * __boot +bootmem_memset(void *s, int c, size_t n) +{ + char *buffer; + + buffer = s; + + for (size_t i = 0; i < n; i++) { + buffer[i] = c; + } + + return s; +} + +size_t __boot +bootmem_strlen(const char *s) +{ + const char *start; + + start = s; + + while (*s != '\0') { + s++; + } + + return (s - start); +} + +static bool __boot +bootmem_overlaps(phys_addr_t start1, phys_addr_t end1, + phys_addr_t start2, phys_addr_t end2) +{ + return ((end2 > start1) && (start2 < end1)); +} + +static bool __boot +bootmem_included(phys_addr_t start1, phys_addr_t end1, + phys_addr_t start2, phys_addr_t end2) +{ + return ((start2 >= start1) && (end2 <= end1)); +} + +static void __boot +bootmem_zone_init(struct bootmem_zone *zone, phys_addr_t start, + phys_addr_t end, bool direct_mapped) +{ + zone->start = start; + zone->end = end; + zone->registered = true; + zone->direct_mapped = direct_mapped; +} + +static phys_addr_t __boot +bootmem_zone_end(const struct bootmem_zone *zone) +{ + return zone->end; +} + +static phys_addr_t __boot +bootmem_zone_size(const struct bootmem_zone *zone) +{ + return zone->end - zone->start; +} + +static bool __boot +bootmem_zone_registered(const struct bootmem_zone *zone) +{ + return zone->registered; +} + +static bool __boot +bootmem_zone_overlaps(const struct bootmem_zone *zone, + phys_addr_t start, phys_addr_t end) +{ + return bootmem_overlaps(zone->start, zone->end, start, end); +} + +static struct bootmem_zone * __boot +bootmem_get_zone(unsigned int index) +{ + assert(index < ARRAY_SIZE(bootmem_zones)); + return &bootmem_zones[index]; +} + +void __boot +bootmem_register_zone(unsigned int zone_index, bool direct_mapped, + phys_addr_t start, phys_addr_t end) +{ + struct bootmem_zone *zone, *tmp; + + for (size_t i = 0; i < ARRAY_SIZE(bootmem_zones); i++) { + tmp = bootmem_get_zone(i); + + if (!bootmem_zone_registered(tmp)) { + continue; + } + + if (bootmem_zone_overlaps(tmp, start, end)) { + boot_panic(bootmem_panic_msg_zone_overlapping); + } + } + + zone = bootmem_get_zone(zone_index); + + if (zone == NULL) { + boot_panic(bootmem_panic_msg_invalid_zone_index_msg); + } + + if (bootmem_zone_registered(zone)) { + boot_panic(bootmem_panic_msg_zone_already_registered); + } + + bootmem_zone_init(zone, start, end, direct_mapped); +} + +static void __boot +bootmem_range_init(struct bootmem_range *range, phys_addr_t start, + phys_addr_t end, bool temporary) +{ + range->start = start; + range->end = end; + range->temporary = temporary; +} + +static phys_addr_t __boot +bootmem_range_start(const struct bootmem_range *range) +{ + return range->start; +} + +static bool __boot +bootmem_range_temporary(const struct bootmem_range *range) +{ + return range->temporary; +} + +static void __boot +bootmem_range_clear_temporary(struct bootmem_range *range) +{ + range->temporary = false; +} + +static bool __boot +bootmem_range_overlaps(const struct bootmem_range *range, + phys_addr_t start, phys_addr_t end) +{ + return bootmem_overlaps(range->start, range->end, start, end); +} + +static bool __boot +bootmem_range_included(const struct bootmem_range *range, + phys_addr_t start, phys_addr_t end) +{ + return bootmem_included(range->start, range->end, start, end); +} + +static int __boot +bootmem_range_clip_region(const struct bootmem_range *range, + phys_addr_t *region_start, phys_addr_t *region_end) +{ + phys_addr_t range_start, range_end; + + range_start = vm_page_trunc(range->start); + range_end = vm_page_round(range->end); + + if (range_end < range->end) { + boot_panic(bootmem_panic_msg_invalid_reserved_range); + } + + if ((range_end <= *region_start) || (range_start >= *region_end)) { + return 0; + } + + if (range_start > *region_start) { + *region_end = range_start; + } else { + if (range_end >= *region_end) { + return ERROR_NOMEM; + } + + *region_start = range_end; + } + + return 0; +} + +static struct bootmem_range * __boot +bootmem_get_reserved_range(unsigned int index) +{ + assert(index < ARRAY_SIZE(bootmem_reserved_ranges)); + return &bootmem_reserved_ranges[index]; +} + +static void __boot +bootmem_shift_ranges_up(struct bootmem_range *range) +{ + struct bootmem_range *end; + size_t size; + + end = bootmem_reserved_ranges + ARRAY_SIZE(bootmem_reserved_ranges); + size = (end - range - 1) * sizeof(*range); + bootmem_memmove(range + 1, range, size); +} + +void __boot +bootmem_reserve_range(phys_addr_t start, phys_addr_t end, bool temporary) +{ + struct bootmem_range *range; + + if (start >= end) { + boot_panic(bootmem_panic_msg_invalid_reserved_range); + } + + if (bootmem_nr_reserved_ranges >= ARRAY_SIZE(bootmem_reserved_ranges)) { + boot_panic(bootmem_panic_msg_too_many_reserved_ranges); + } + + range = NULL; + + for (unsigned int i = 0; i < bootmem_nr_reserved_ranges; i++) { + range = bootmem_get_reserved_range(i); + + if (bootmem_range_overlaps(range, start, end)) { + /* + * If the range overlaps, check whether it's part of another + * range. For example, this applies to debugging symbols directly + * taken from the kernel image. + */ + if (bootmem_range_included(range, start, end)) { + /* + * If it's completely included, make sure that a permanent + * range remains permanent. + * + * XXX This means that if one big range is first registered + * as temporary, and a smaller range inside of it is + * registered as permanent, the bigger range becomes + * permanent. It's not easy nor useful in practice to do + * better than that. + */ + if (bootmem_range_temporary(range) != temporary) { + bootmem_range_clear_temporary(range); + } + + return; + } + + boot_panic(bootmem_panic_msg_invalid_reserved_range); + } + + if (end <= bootmem_range_start(range)) { + break; + } + } + + if (range == NULL) { + range = bootmem_reserved_ranges; + } + + bootmem_shift_ranges_up(range); + bootmem_range_init(range, start, end, temporary); + bootmem_nr_reserved_ranges++; +} + +static uintptr_t __boot +bootmem_block_round(uintptr_t size) +{ + return P2ROUND(size, BOOTMEM_BLOCK_SIZE); +} + +static uintptr_t __boot +bootmem_byte2block(uintptr_t byte) +{ + return byte >> BOOT_MEM_BLOCK_BITS; +} + +static uintptr_t __boot +bootmem_block2byte(uintptr_t block) +{ + return block << BOOT_MEM_BLOCK_BITS; +} + +static uintptr_t __boot +bootmem_compute_blocks(uintptr_t start, uintptr_t end) +{ + return bootmem_byte2block(end - start); +} + +static uintptr_t __boot +bootmem_compute_table_size(uintptr_t nr_blocks) +{ + return bootmem_block_round(nr_blocks * sizeof(struct bootmem_block)); +} + +static void __boot +bootmem_block_init(struct bootmem_block *block, uintptr_t pa) +{ + block->phys_addr = pa; + block->order = BOOTMEM_ORDER_UNLISTED; + block->allocated = true; +} + +static void __boot +bootmem_free_list_init(struct bootmem_free_list *list) +{ + list->blocks = NULL; +} + +static bool __boot +bootmem_free_list_empty(const struct bootmem_free_list *list) +{ + return list->blocks == NULL; +} + +static void __boot +bootmem_free_list_insert(struct bootmem_free_list *list, + struct bootmem_block *block) +{ + struct bootmem_block *blocks; + + blocks = list->blocks; + block->next = blocks; + block->pprev = &list->blocks; + + if (blocks != NULL) { + blocks->pprev = &block->next; + } + + list->blocks = block; +} + +static void __boot +bootmem_free_list_remove(struct bootmem_block *block) +{ + if (block->next != NULL) { + block->next->pprev = block->pprev; + } + + *block->pprev = block->next; +} + +static struct bootmem_block * __boot +bootmem_free_list_pop(struct bootmem_free_list *list) +{ + struct bootmem_block *block; + + block = list->blocks; + bootmem_free_list_remove(block); + return block; +} + +static struct bootmem_free_list * __boot +bootmem_heap_get_free_list(struct bootmem_heap *heap, unsigned int index) +{ + assert(index < ARRAY_SIZE(heap->free_lists)); + return &heap->free_lists[index]; +} + +static struct bootmem_block * __boot +bootmem_heap_get_block(struct bootmem_heap *heap, uintptr_t pa) +{ + return &heap->blocks[bootmem_byte2block(pa - heap->start)]; +} + +static void __boot +bootmem_heap_free(struct bootmem_heap *heap, struct bootmem_block *block, + unsigned short order) +{ + struct bootmem_block *buddy; + uintptr_t pa, buddy_pa; + + assert(block >= heap->blocks); + assert(block < heap->blocks_end); + assert(block->order == BOOTMEM_ORDER_UNLISTED); + assert(order < BOOTMEM_ORDER_UNLISTED); + assert(block->allocated); + + block->allocated = false; + pa = block->phys_addr; + + while (order < (BOOT_MEM_NR_FREE_LISTS - 1)) { + buddy_pa = pa ^ bootmem_block2byte(1 << order); + + if ((buddy_pa < heap->start) || (buddy_pa >= heap->end)) { + break; + } + + buddy = &heap->blocks[bootmem_byte2block(buddy_pa - heap->start)]; + + if (buddy->order != order) { + break; + } + + bootmem_free_list_remove(buddy); + buddy->order = BOOTMEM_ORDER_UNLISTED; + order++; + pa &= -bootmem_block2byte(1 << order); + block = &heap->blocks[bootmem_byte2block(pa - heap->start)]; + } + + bootmem_free_list_insert(&heap->free_lists[order], block); + block->order = order; +} + +static struct bootmem_block * __boot +bootmem_heap_alloc(struct bootmem_heap *heap, unsigned short order) +{ + struct bootmem_free_list *free_list; + struct bootmem_block *block, *buddy; + unsigned int i; + + assert(order < BOOT_MEM_NR_FREE_LISTS); + + for (i = order; i < BOOT_MEM_NR_FREE_LISTS; i++) { + free_list = &heap->free_lists[i]; + + if (!bootmem_free_list_empty(free_list)) { + break; + } + } + + if (i == BOOT_MEM_NR_FREE_LISTS) { + return NULL; + } + + block = bootmem_free_list_pop(free_list); + block->order = BOOTMEM_ORDER_UNLISTED; + + while (i > order) { + i--; + buddy = &block[1 << i]; + bootmem_free_list_insert(bootmem_heap_get_free_list(heap, i), buddy); + buddy->order = i; + } + + return block; +} + +static void __boot +bootmem_heap_init(struct bootmem_heap *heap, uintptr_t start, uintptr_t end) +{ + uintptr_t heap_blocks, table_size, table_blocks; + + bootmem_reserve_range(start, end, false); + + heap->start = start; + heap->end = end; + heap_blocks = bootmem_compute_blocks(start, end); + table_size = bootmem_compute_table_size(heap_blocks); + assert((end - table_size) > start); + heap->blocks = (struct bootmem_block *)(end - table_size); + heap->blocks_end = &heap->blocks[heap_blocks]; + + for (size_t i = 0; i < ARRAY_SIZE(heap->free_lists); i++) { + bootmem_free_list_init(&heap->free_lists[i]); + } + + for (phys_addr_t pa = start; pa < end; pa += BOOTMEM_BLOCK_SIZE) { + bootmem_block_init(bootmem_heap_get_block(heap, pa), pa); + } + + table_blocks = bootmem_byte2block(table_size); + heap_blocks -= table_blocks; + + for (size_t i = 0; i < heap_blocks; i++) { + bootmem_heap_free(heap, &heap->blocks[i], 0); + } +} + +static struct bootmem_heap * __boot +bootmem_get_heap(void) +{ + return &bootmem_heap; +} + +/* + * Find available memory. + * + * The search starts at the given start address, up to the given end address. + * If a range is found, it is stored through the region_startp and region_endp + * pointers. + * + * The range boundaries are page-aligned on return. + */ +static int __boot +bootmem_find_avail(phys_addr_t start, phys_addr_t end, + phys_addr_t *region_start, phys_addr_t *region_end) +{ + phys_addr_t orig_start; + int error; + + assert(start <= end); + + orig_start = start; + start = vm_page_round(start); + end = vm_page_trunc(end); + + if ((start < orig_start) || (start >= end)) { + return ERROR_INVAL; + } + + *region_start = start; + *region_end = end; + + for (unsigned int i = 0; i < bootmem_nr_reserved_ranges; i++) { + error = bootmem_range_clip_region(bootmem_get_reserved_range(i), + region_start, region_end); + + if (error) { + return error; + } + } + + return 0; +} + +void __boot +bootmem_setup(void) +{ + phys_addr_t heap_start, heap_end, max_heap_start, max_heap_end; + phys_addr_t start, end; + int error; + + bootmem_reserve_range((uintptr_t)&_boot, BOOT_VTOP((uintptr_t)&_end), false); + + /* + * Find some memory for the heap. Look for the largest unused area in + * upper memory, carefully avoiding all boot data. + */ + end = bootmem_directmap_end(); + + max_heap_start = 0; + max_heap_end = 0; + start = PMEM_RAM_START; + + for (;;) { + error = bootmem_find_avail(start, end, &heap_start, &heap_end); + + if (error) { + break; + } + + if ((heap_end - heap_start) > (max_heap_end - max_heap_start)) { + max_heap_start = heap_start; + max_heap_end = heap_end; + } + + start = heap_end; + } + + if (max_heap_start >= max_heap_end) { + boot_panic(bootmem_panic_msg_setup); + } + + assert(max_heap_start == (uintptr_t)max_heap_start); + assert(max_heap_end == (uintptr_t)max_heap_end); + bootmem_heap_init(bootmem_get_heap(), max_heap_start, max_heap_end); +} + +static unsigned short __boot +bootmem_alloc_order(size_t size) +{ + return iorder2(bootmem_byte2block(bootmem_block_round(size))); +} + +void * __boot +bootmem_alloc(size_t size) +{ + struct bootmem_block *block; + + block = bootmem_heap_alloc(bootmem_get_heap(), bootmem_alloc_order(size)); + + if (block == NULL) { + boot_panic(bootmem_panic_msg_nomem); + } + + return (void *)block->phys_addr; +} + +size_t __boot +bootmem_directmap_end(void) +{ + if (bootmem_zone_size(bootmem_get_zone(PMEM_ZONE_DIRECTMAP)) != 0) { + return bootmem_zone_end(bootmem_get_zone(PMEM_ZONE_DIRECTMAP)); + } else if (bootmem_zone_size(bootmem_get_zone(PMEM_ZONE_DMA32)) != 0) { + return bootmem_zone_end(bootmem_get_zone(PMEM_ZONE_DMA32)); + } else { + return bootmem_zone_end(bootmem_get_zone(PMEM_ZONE_DMA)); + } +} diff --git a/kern/bootmem.h b/kern/bootmem.h new file mode 100644 index 00000000..69a62115 --- /dev/null +++ b/kern/bootmem.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * This module provides memory-related services at bootstrap, before + * paging is enabled. In particular : + * - zone registration + * - memory allocation (mainly for page tables) + * - boot data reservation + */ + +#ifndef _KERN_BOOTMEM_H +#define _KERN_BOOTMEM_H + +#include <stdbool.h> +#include <stddef.h> + +#include <kern/init.h> +#include <machine/types.h> + +/* + * Helper functions available before paging is enabled. + * + * Any memory passed to these must also be accessible without paging. + */ +void * bootmem_memcpy(void *dest, const void *src, size_t n); +void * bootmem_memmove(void *dest, const void *src, size_t n); +void * bootmem_memset(void *s, int c, size_t n); +size_t bootmem_strlen(const char *s); + +/* + * Register a physical memory zone. + * + * Zones are expected to be sorted in ascending order of addresses and + * not overlap. They are later loaded to the VM system. Set direct_mapped + * to true if the zone is part of the direct mapping of physical memory. + * + * This function is called before paging is enabled. + */ +void bootmem_register_zone(unsigned int zone_index, bool direct_mapped, + phys_addr_t start, phys_addr_t end); + +/* + * Report reserved addresses to the bootmem module. + * + * The kernel is automatically reserved. + * + * Once all reserved ranges have been registered, the user can initialize + * the module. + * + * If the range is marked temporary, it will be unregistered once + * the boot data have been saved/consumed so that their backing + * pages are loaded into the VM system. + * + * This function is called before paging is enabled. + */ +void bootmem_reserve_range(phys_addr_t start, phys_addr_t end, bool temporary); + +/* + * Initialize the early page allocator. + * + * This function builds a heap based on the registered zones while carefully + * avoiding reserved data. + * + * This function is called before paging is enabled. + */ +void bootmem_setup(void); + +/* + * Allocate memory. + * + * The size of the returned region is a power-of-two, and its address is + * aligned on that size. The region is guaranteed to be part of the direct + * physical mapping when paging is enabled. + * + * This function is called before paging is enabled. + */ +void * bootmem_alloc(size_t size); + +/* + * Return the end address of the direct physical mapping. + */ +size_t bootmem_directmap_end(void); + +#endif /* _KERN_BOOTMEM_H */ diff --git a/kern/log2.h b/kern/log2.h index ed12b441..a4e57bf4 100644 --- a/kern/log2.h +++ b/kern/log2.h @@ -16,6 +16,9 @@ * * * Integer base 2 logarithm operations. + * + * The functions provided by this module are always inlined so they may + * safely be used before paging is enabled. */ #ifndef _KERN_LOG2_H @@ -24,14 +27,16 @@ #include <assert.h> #include <limits.h> -static inline unsigned int +#include <kern/macros.h> + +static __always_inline unsigned int ilog2(unsigned long x) { assert(x != 0); return LONG_BIT - __builtin_clzl(x) - 1; } -static inline unsigned int +static __always_inline unsigned int iorder2(unsigned long size) { assert(size != 0); diff --git a/kern/thread.c b/kern/thread.c index 09e15aa0..f5b0727e 100644 --- a/kern/thread.c +++ b/kern/thread.c @@ -2641,6 +2641,8 @@ thread_schedule_intr(void) assert(thread_check_intr_context()); + /* TODO Explain */ + runq = thread_runq_local(); syscnt_inc(&runq->sc_schedule_intrs); } diff --git a/tools/qemu_arm.sh b/tools/qemu_arm.sh new file mode 100755 index 00000000..9a69d0f1 --- /dev/null +++ b/tools/qemu_arm.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# XXX These parameters are currently hardcoded in the board configuration. +# XXX The script assumes an x86 host with an arm-none-eabi toolchain. + +QEMU_EXE=qemu-system-arm + +NR_CPUS=4 +RAM=64 + +X15=$PWD/x15 +TMPDIR=$(mktemp -d) +BIN=$TMPDIR/x15.bin +IMG=$TMPDIR/flash.img + +arm-none-eabi-objcopy -O binary x15 $BIN +dd if=/dev/zero of=$IMG bs=1M seek=64 count=0 +dd if=$BIN of=$IMG conv=notrunc + +$QEMU_EXE \ + -M virt-2.8 \ + -ctrl-grab \ + -gdb tcp::1234 \ + -m $RAM \ + -smp $NR_CPUS \ + -monitor stdio \ + -drive file=$IMG,if=pflash,format=raw + +rm -rf $TMPDIR diff --git a/vm/Makefile b/vm/Makefile index a42fe244..9cbf9d41 100644 --- a/vm/Makefile +++ b/vm/Makefile @@ -2,4 +2,5 @@ x15_SOURCES-y += \ vm/vm_kmem.c \ vm/vm_map.c \ vm/vm_object.c \ - vm/vm_page.c + vm/vm_page.c \ + vm/vm_ptable.c diff --git a/vm/vm_kmem.h b/vm/vm_kmem.h index 5ae4a169..96bbb541 100644 --- a/vm/vm_kmem.h +++ b/vm/vm_kmem.h @@ -38,6 +38,7 @@ extern char _text; extern char _rodata; extern char _data; +extern char _bss; extern char _end; /* diff --git a/vm/vm_ptable.c b/vm/vm_ptable.c new file mode 100644 index 00000000..70c6a505 --- /dev/null +++ b/vm/vm_ptable.c @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2010-2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * TODO Define "page table". + * TODO Review locking. + */ + +#include <assert.h> +#include <stdalign.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <kern/bootmem.h> +#include <kern/cpumap.h> +#include <kern/error.h> +#include <kern/init.h> +#include <kern/kmem.h> +#include <kern/list.h> +#include <kern/log.h> +#include <kern/macros.h> +#include <kern/mutex.h> +#include <kern/panic.h> +#include <kern/percpu.h> +#include <kern/spinlock.h> +#include <kern/syscnt.h> +#include <kern/thread.h> +#include <machine/boot.h> +#include <machine/cpu.h> +#include <machine/page.h> +#include <machine/pmap.h> +#include <machine/tcb.h> +#include <machine/trap.h> +#include <machine/types.h> +#include <vm/vm_kmem.h> +#include <vm/vm_page.h> +#include <vm/vm_ptable.h> +#include <vm/vm_prot.h> + +static const struct vm_ptable_tlp *vm_ptable_boot_tlps __bootdata; +static size_t vm_ptable_boot_nr_levels __bootdata; + +static struct vm_ptable_cpu_pt vm_ptable_boot_cpu_pt __bootdata; + +/* + * Structures related to inter-processor page table updates. + */ + +#define VM_PTABLE_UPDATE_OP_ENTER 1 +#define VM_PTABLE_UPDATE_OP_REMOVE 2 +#define VM_PTABLE_UPDATE_OP_PROTECT 3 + +struct vm_ptable_update_enter_args { + uintptr_t va; + phys_addr_t pa; + int prot; + int flags; +}; + +struct vm_ptable_update_remove_args { + uintptr_t start; + uintptr_t end; +}; + +struct vm_ptable_update_protect_args { + uintptr_t start; + uintptr_t end; + int prot; +}; + +struct vm_ptable_update_op { + struct cpumap cpumap; + unsigned int operation; + + union { + struct vm_ptable_update_enter_args enter_args; + struct vm_ptable_update_remove_args remove_args; + struct vm_ptable_update_protect_args protect_args; + }; +}; + +/* + * Maximum number of operations that can be batched before an implicit + * update. + */ +#define VM_PTABLE_UPDATE_MAX_OPS 32 + +/* + * List of update operations. + * + * A list of update operations is a container of operations that are pending + * for a pmap. Updating can be implicit, e.g. when a list has reached its + * maximum size, or explicit, when vm_ptable_update() is called. Operation lists + * are thread-local objects. + * + * The cpumap is the union of all processors affected by at least one + * operation. + */ +struct vm_ptable_update_oplist { + alignas(CPU_L1_SIZE) struct cpumap cpumap; + struct pmap *pmap; + unsigned int nr_ops; + struct vm_ptable_update_op ops[VM_PTABLE_UPDATE_MAX_OPS]; +}; + +/* + * Statically allocated data for the main booter thread. + */ +static struct vm_ptable_update_oplist vm_ptable_booter_oplist __initdata; + +/* + * Each regular thread gets an operation list from this cache. + */ +static struct kmem_cache vm_ptable_update_oplist_cache; + +/* + * Queue holding update requests from remote processors. + */ +struct vm_ptable_update_queue { + struct spinlock lock; + struct list requests; +}; + +/* + * Syncer thread. + * + * There is one such thread per processor. They are the recipients of + * update requests, providing thread context for the mapping operations + * they perform. + */ +struct vm_ptable_syncer { + alignas(CPU_L1_SIZE) struct thread *thread; + struct vm_ptable_update_queue queue; + struct syscnt sc_updates; + struct syscnt sc_update_enters; + struct syscnt sc_update_removes; + struct syscnt sc_update_protects; +}; + +#if 0 +static void vm_ptable_sync(void *arg); +#endif + +static struct vm_ptable_syncer vm_ptable_syncer __percpu; + +/* + * Maximum number of mappings for which individual TLB invalidations can be + * performed. Global TLB flushes are done beyond this value. + */ +#define VM_PTABLE_UPDATE_MAX_MAPPINGS 64 + +/* + * Per processor request, queued on a remote processor. + * + * The number of mappings is used to determine whether it's best to flush + * individual TLB entries or globally flush the TLB. + */ +struct vm_ptable_update_request { + alignas(CPU_L1_SIZE) struct list node; + struct spinlock lock; + struct thread *sender; + const struct vm_ptable_update_oplist *oplist; + unsigned int nr_mappings; + int done; + int error; +}; + +/* + * Per processor array of requests. + * + * When an operation list is to be applied, the thread triggering the update + * acquires the processor-local array of requests and uses it to queue requests + * on remote processors. + */ +struct vm_ptable_update_request_array { + struct vm_ptable_update_request requests[CONFIG_MAX_CPUS]; + struct mutex lock; +}; + +static struct vm_ptable_update_request_array vm_ptable_update_request_array + __percpu; + +static int vm_ptable_do_remote_updates __read_mostly; + +static char vm_ptable_panic_inval_msg[] __bootdata + = "vm_ptable: invalid physical address"; + +void __boot +vm_ptable_bootstrap(const struct vm_ptable_tlp *tlps, size_t nr_levels) +{ + assert(tlps); + assert(nr_levels != 0); + + vm_ptable_boot_tlps = tlps; + vm_ptable_boot_nr_levels = nr_levels; +} + +static const struct vm_ptable_tlp * __boot +vm_ptable_boot_get_tlp(unsigned int level) +{ + assert(level < vm_ptable_boot_nr_levels); + return &vm_ptable_boot_tlps[level]; +} + +static __always_inline unsigned long +vm_ptable_tlp_pte_index(const struct vm_ptable_tlp *tlp, uintptr_t va) +{ + return ((va >> tlp->skip) & ((1UL << tlp->bits) - 1)); +} + +static __always_inline phys_addr_t +vm_ptable_tlp_pa_mask(const struct vm_ptable_tlp *tlp) +{ + phys_addr_t size; + + if (tlp == vm_ptable_boot_tlps) { + return ~PAGE_MASK; + } else { + size = ((phys_addr_t)1 << tlp[-1].bits) * sizeof(pmap_pte_t); + return ~(size - 1); + } +} + +static __always_inline bool +vm_ptable_pa_aligned(phys_addr_t pa) +{ + phys_addr_t mask; + + mask = vm_ptable_tlp_pa_mask(vm_ptable_boot_get_tlp(0)); + return pa == (pa & mask); +} + +void __boot +vm_ptable_boot_build(struct vm_ptable *ptable) +{ + const struct vm_ptable_tlp *tlp; + struct vm_ptable_cpu_pt *pt; + void *ptr; + + tlp = vm_ptable_boot_get_tlp(vm_ptable_boot_nr_levels - 1); + pt = &vm_ptable_boot_cpu_pt; + ptr = bootmem_alloc(tlp->ptes_per_pt * sizeof(pmap_pte_t)); + pt->root = (void *)BOOT_PTOV((uintptr_t)ptr); + ptable->cpu_pts[0] = pt; + + for (size_t i = 1; i < ARRAY_SIZE(ptable->cpu_pts); i++) { + ptable->cpu_pts[i] = NULL; + } +} + +void __boot +vm_ptable_boot_enter(struct vm_ptable *ptable, uintptr_t va, + phys_addr_t pa, size_t page_size) +{ + const struct vm_ptable_tlp *tlp; + unsigned int level, last_level; + pmap_pte_t *pt, *next_pt, *pte; + phys_addr_t mask; + + if (!vm_ptable_pa_aligned(pa)) { + boot_panic(vm_ptable_panic_inval_msg); + } + + last_level = pmap_boot_get_last_level(page_size); + pt = (void *)BOOT_VTOP((uintptr_t)ptable->cpu_pts[0]->root); + + for (level = vm_ptable_boot_nr_levels - 1; level != last_level; level--) { + tlp = vm_ptable_boot_get_tlp(level); + pte = &pt[vm_ptable_tlp_pte_index(tlp, va)]; + + if (pmap_pte_valid(*pte)) { + mask = vm_ptable_tlp_pa_mask(tlp); + next_pt = (void *)(uintptr_t)(*pte & mask); + } else { + next_pt = bootmem_alloc(tlp[-1].ptes_per_pt * sizeof(pmap_pte_t)); + *pte = tlp->make_pte_fn((uintptr_t)next_pt, VM_PROT_ALL); + } + + pt = next_pt; + } + + tlp = vm_ptable_boot_get_tlp(last_level); + pte = &pt[vm_ptable_tlp_pte_index(tlp, va)]; + *pte = tlp->make_ll_pte_fn(pa, VM_PROT_ALL); +} + +pmap_pte_t * __boot +vm_ptable_boot_root(const struct vm_ptable *ptable) +{ + return (void *)BOOT_VTOP((uintptr_t)ptable->cpu_pts[0]->root); +} diff --git a/vm/vm_ptable.h b/vm/vm_ptable.h new file mode 100644 index 00000000..6967595c --- /dev/null +++ b/vm/vm_ptable.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * TODO Comment. + */ + +#ifndef _VM_VM_PTABLE_H +#define _VM_VM_PTABLE_H + +#include <stddef.h> +#include <stdint.h> + +#include <machine/pmap.h> +#include <machine/types.h> + +struct vm_ptable_cpu_pt { + pmap_pte_t *root; +}; + +typedef pmap_pte_t (*vm_ptable_make_pte_fn)(phys_addr_t pa, int prot); + +/* + * Translation level properties. + */ +struct vm_ptable_tlp { + unsigned int skip; + unsigned int bits; + unsigned int ptes_per_pt; + vm_ptable_make_pte_fn make_pte_fn; + vm_ptable_make_pte_fn make_ll_pte_fn; +}; + +struct vm_ptable { + struct vm_ptable_cpu_pt *cpu_pts[CONFIG_MAX_CPUS]; +}; + +void vm_ptable_bootstrap(const struct vm_ptable_tlp *tlps, size_t nr_levels); + +void vm_ptable_boot_build(struct vm_ptable *ptable); + +void vm_ptable_boot_enter(struct vm_ptable *ptable, uintptr_t va, + phys_addr_t pa, size_t page_size); + +pmap_pte_t * vm_ptable_boot_root(const struct vm_ptable *ptable); + +#endif /* _VM_VM_PTABLE_H */ |