diff options
154 files changed, 8190 insertions, 2891 deletions
diff --git a/.gdbinit b/.gdbinit new file mode 100644 index 00000000..f823e0a2 --- /dev/null +++ b/.gdbinit @@ -0,0 +1,3 @@ +define x15_connect_qemu + target remote localhost:1234 +end @@ -249,6 +249,10 @@ X15 Options likely to measurably reduce overall performance. It should only be enabled if you know what you're doing. +`--enable-shell' + Enable the diagnostics shell. Note that some shell commands may have + a real-time unsafe behaviour. + `--enable-thread-stack-guard' Enable the use of guard pages around kernel thread stacks to catch overflows. Note that this feature wastes precious kernel virtual diff --git a/Makefile.am b/Makefile.am index 77303136..50baae96 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,7 +32,6 @@ AM_CFLAGS += \ -fsigned-char \ -fno-common -# TODO Add stack protector support AM_CFLAGS += -fno-stack-protector AM_CFLAGS += -nostdlib diff --git a/Makefrag.am b/Makefrag.am index 2ed9f771..a99f822e 100644 --- a/Makefrag.am +++ b/Makefrag.am @@ -4,13 +4,14 @@ include doc/Makefrag.am EXTRA_DIST += tools/qemu.sh x15_SOURCES += \ + include/assert.h \ + include/limits.h \ include/stdio.h \ include/string.h x15_SOURCES += \ kern/arg.c \ kern/arg.h \ - kern/assert.h \ kern/atomic.h \ kern/bitmap.c \ kern/bitmap.h \ @@ -28,7 +29,11 @@ x15_SOURCES += \ kern/error.c \ kern/error.h \ kern/hash.h \ + kern/fmt.c \ + kern/fmt.h \ + kern/init.c \ kern/init.h \ + kern/init_i.h \ kern/intr.c \ kern/intr.h \ kern/kernel.c \ @@ -36,14 +41,16 @@ x15_SOURCES += \ kern/kmem.c \ kern/kmem.h \ kern/kmem_i.h \ - kern/limits.h \ kern/list.h \ kern/list_types.h \ kern/llsync.c \ kern/llsync.h \ kern/llsync_i.h \ + kern/log.c \ + kern/log.h \ kern/log2.h \ kern/macros.h \ + kern/mutex.c \ kern/mutex.h \ kern/mutex_types.h \ kern/mutex/mutex_adaptive_i.h \ @@ -54,7 +61,6 @@ x15_SOURCES += \ kern/mutex/mutex_plain_types.h \ kern/panic.c \ kern/panic.h \ - kern/param.h \ kern/percpu.c \ kern/percpu.h \ kern/plist.c \ @@ -75,16 +81,14 @@ x15_SOURCES += \ kern/semaphore.c \ kern/semaphore.h \ kern/semaphore_i.h \ - kern/shell.c \ - kern/shell.h \ + kern/shutdown.c \ + kern/shutdown.h \ kern/sleepq.c \ kern/sleepq.h \ kern/spinlock.c \ kern/spinlock.h \ kern/spinlock_i.h \ kern/spinlock_types.h \ - kern/sprintf.c \ - kern/sprintf.h \ kern/sref.c \ kern/sref.h \ kern/sref_i.h \ @@ -115,6 +119,12 @@ if !MUTEX_PI endif endif +if X15_SHELL +x15_SOURCES += \ + kern/shell.c \ + kern/shell.h +endif X15_SHELL + x15_SOURCES += \ vm/vm_adv.h \ vm/vm_inherit.h \ @@ -122,11 +132,12 @@ x15_SOURCES += \ vm/vm_kmem.h \ vm/vm_map.c \ vm/vm_map.h \ + vm/vm_object.c \ + vm/vm_object.h \ + vm/vm_object_types.h \ vm/vm_page.c \ vm/vm_page.h \ - vm/vm_prot.h \ - vm/vm_setup.c \ - vm/vm_setup.h + vm/vm_prot.h x15_SOURCES += \ test/test.h @@ -159,10 +170,6 @@ if TEST_VM_PAGE_FILL x15_SOURCES += test/test_vm_page_fill.c endif TEST_VM_PAGE_FILL -if TEST_X86_DOUBLE_FAULT -x15_SOURCES += test/test_x86_double_fault.c -endif TEST_X86_DOUBLE_FAULT - if TEST_XCALL x15_SOURCES += test/test_xcall.c endif TEST_XCALL @@ -1,5 +1,5 @@ X15 is an open source microkernel. Its purpose is to provide a foundation -for a Hurd-like operating system or an embedded kernel application. +for a Hurd-like operating system as well as real-time kernel applications. See https://www.sceen.net/x15/. diff --git a/arch/x86/Makefrag.am b/arch/x86/Makefrag.am index 0971530f..cf536987 100644 --- a/arch/x86/Makefrag.am +++ b/arch/x86/Makefrag.am @@ -24,8 +24,8 @@ x15_LDFLAGS += -m64 endif AMD64 x15_SOURCES += \ - arch/x86/machine/acpimp.c \ - arch/x86/machine/acpimp.h \ + arch/x86/machine/acpi.c \ + arch/x86/machine/acpi.h \ arch/x86/machine/atcons.c \ arch/x86/machine/atcons.h \ arch/x86/machine/atkbd.c \ @@ -50,16 +50,20 @@ x15_SOURCES += \ arch/x86/machine/lapic.c \ arch/x86/machine/lapic.h \ arch/x86/machine/multiboot.h \ - arch/x86/machine/param.h \ + arch/x86/machine/page.h \ arch/x86/machine/pic.c \ arch/x86/machine/pic.h \ arch/x86/machine/pit.c \ arch/x86/machine/pit.h \ arch/x86/machine/pmap.c \ arch/x86/machine/pmap.h \ + arch/x86/machine/pmem.h \ + arch/x86/machine/ssp.c \ + arch/x86/machine/ssp.h \ arch/x86/machine/strace.c \ arch/x86/machine/strace.h \ arch/x86/machine/string.c \ + arch/x86/machine/string.h \ arch/x86/machine/tcb_asm.S \ arch/x86/machine/tcb.c \ arch/x86/machine/tcb.h \ diff --git a/arch/x86/configfrag.ac b/arch/x86/configfrag.ac index 0ce93e0a..1d37a0e7 100644 --- a/arch/x86/configfrag.ac +++ b/arch/x86/configfrag.ac @@ -1,6 +1,5 @@ m4_define([x86_ENABLE_PAE], - [AC_DEFINE([X15_X86_PAE], [1], [use PAE page translation]) - AC_MSG_NOTICE([physical address extension enabled])]) + [AC_DEFINE([X15_X86_PAE], [1], [use PAE page translation])]) m4_define([x86_SELECT_I386], [machine=i386 @@ -27,7 +26,9 @@ m4_define([x86_SELECT], AC_ARG_ENABLE([pae], [AS_HELP_STRING([--enable-pae], - [enable physical address extension (i386 only)])]) + [enable physical address extension (i386 only)])], + [], + [enable_pae=no]) AS_IF([test x"$opt_i386$opt_amd64" = xyesyes], [AC_MSG_ERROR([select only one of i386 or amd64])], @@ -48,7 +49,9 @@ m4_define([x86_SELECT], -mno-avx]) AC_DEFINE_UNQUOTED([X15_X86_MACHINE], [$machine], [machine]) - AC_MSG_NOTICE([machine type: $arch ($machine)])]) + AC_MSG_NOTICE([machine type: $arch ($machine)]) + AS_IF([test x"$machine" = xi386], + [AC_MSG_NOTICE([physical address extension: $enable_pae])])]) AS_CASE(["$host_cpu"], [i?86|x86_64], [x86_SELECT]) diff --git a/arch/x86/machine/acpi.c b/arch/x86/machine/acpi.c new file mode 100644 index 00000000..77202267 --- /dev/null +++ b/arch/x86/machine/acpi.c @@ -0,0 +1,710 @@ +/* + * Copyright (c) 2012-2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <kern/init.h> +#include <kern/intr.h> +#include <kern/kmem.h> +#include <kern/log.h> +#include <kern/macros.h> +#include <kern/panic.h> +#include <kern/percpu.h> +#include <kern/shutdown.h> +#include <machine/acpi.h> +#include <machine/biosmem.h> +#include <machine/cpu.h> +#include <machine/io.h> +#include <machine/ioapic.h> +#include <machine/lapic.h> +#include <machine/pic.h> +#include <machine/types.h> +#include <vm/vm_kmem.h> + +/* + * Priority of the shutdown operations. + * + * Higher than all other methods. + */ +#define ACPI_SHUTDOWN_PRIORITY 10 + +#define ACPI_GAS_ASID_SYSIO 1 + +struct acpi_gas { + uint8_t asid; + uint8_t reg_width; + uint8_t reg_offset; + uint8_t access_size; + uint64_t addr; +} __packed; + +#define ACPI_RSDP_ALIGN 16 + +#define ACPI_RSDP_SIG "RSD PTR " + +struct acpi_rsdp { + uint8_t signature[8]; + uint8_t checksum; + uint8_t oem_id[6]; + uint8_t reserved; + uint32_t rsdt_address; +} __packed; + +#define ACPI_SIG_SIZE 5 + +struct acpi_sdth { + uint8_t signature[4]; + uint32_t length; + uint8_t revision; + uint8_t checksum; + uint8_t oem_id[6]; + uint8_t oem_table_id[8]; + uint32_t oem_revision; + uint8_t creator_id[4]; + uint32_t creator_revision; +} __packed; + +struct acpi_rsdt { + struct acpi_sdth header; + uint32_t entries[0]; +} __packed; + +#define ACPI_MADT_ENTRY_LAPIC 0 +#define ACPI_MADT_ENTRY_IOAPIC 1 +#define ACPI_MADT_ENTRY_ISO 2 + +struct acpi_madt_entry_hdr { + uint8_t type; + uint8_t length; +} __packed; + +#define ACPI_MADT_LAPIC_ENABLED 0x1 + +struct acpi_madt_entry_lapic { + struct acpi_madt_entry_hdr header; + uint8_t processor_id; + uint8_t apic_id; + uint32_t flags; +} __packed; + +struct acpi_madt_entry_ioapic { + struct acpi_madt_entry_hdr header; + uint8_t id; + uint8_t _reserved; + uint32_t addr; + uint32_t base; +} __packed; + +#define ACPI_MADT_ISO_POL_DEFAULT 0x00 +#define ACPI_MADT_ISO_POL_HIGH 0x01 +#define ACPI_MADT_ISO_POL_LOW 0x03 +#define ACPI_MADT_ISO_POL_MASK 0x03 +#define ACPI_MADT_ISO_TRIG_DEFAULT 0x00 +#define ACPI_MADT_ISO_TRIG_EDGE 0x04 +#define ACPI_MADT_ISO_TRIG_LEVEL 0x0c +#define ACPI_MADT_ISO_TRIG_MASK 0x0c + +struct acpi_madt_entry_iso { + struct acpi_madt_entry_hdr header; + uint8_t bus; + uint8_t source; + uint32_t gsi; + int16_t flags; +} __packed; + +union acpi_madt_entry { + uint8_t type; + struct acpi_madt_entry_hdr header; + struct acpi_madt_entry_lapic lapic; + struct acpi_madt_entry_ioapic ioapic; + struct acpi_madt_entry_iso iso; +} __packed; + +#define ACPI_MADT_PC_COMPAT 0x1 + +struct acpi_madt { + struct acpi_sdth header; + uint32_t lapic_addr; + uint32_t flags; + union acpi_madt_entry entries[0]; +} __packed; + +struct acpi_madt_iter { + const union acpi_madt_entry *entry; + const union acpi_madt_entry *end; +}; + +#define acpi_madt_foreach(madt, iter) \ +for (acpi_madt_iter_init(iter, madt); \ + acpi_madt_iter_valid(iter); \ + acpi_madt_iter_next(iter)) + +#define ACPI_FADT_FL_RESET_REG_SUP 0x400 + +struct acpi_fadt { + union { + struct acpi_sdth header; + char unused[112]; + }; + + uint32_t flags; + struct acpi_gas reset_reg; + uint8_t reset_value; +} __packed; + +struct acpi_table_addr { + const char *sig; + struct acpi_sdth *table; +}; + +static struct acpi_table_addr acpi_table_addrs[] __initdata = { + { "RSDT", NULL }, + { "APIC", NULL }, + { "FACP", NULL }, +}; + +static struct acpi_gas acpi_reset_reg; +static uint8_t acpi_reset_value; + +static void __init +acpi_table_sig(const struct acpi_sdth *table, char sig[ACPI_SIG_SIZE]) +{ + memcpy(sig, table->signature, sizeof(table->signature)); + sig[4] = '\0'; +} + +static int __init +acpi_table_required(const struct acpi_sdth *table) +{ + char sig[ACPI_SIG_SIZE]; + size_t i; + + acpi_table_sig(table, sig); + + for (i = 0; i < ARRAY_SIZE(acpi_table_addrs); i++) { + if (strcmp(sig, acpi_table_addrs[i].sig) == 0) { + return 1; + } + } + + return 0; +} + +static void __init +acpi_register_table(struct acpi_sdth *table) +{ + char sig[ACPI_SIG_SIZE]; + size_t i; + + acpi_table_sig(table, sig); + + for (i = 0; i < ARRAY_SIZE(acpi_table_addrs); i++) { + if (strcmp(sig, acpi_table_addrs[i].sig) == 0) { + if (acpi_table_addrs[i].table != NULL) { + log_warning("acpi: table %s ignored: already registered", sig); + return; + } + + acpi_table_addrs[i].table = table; + return; + } + } + + log_warning("acpi: table '%s' ignored: unknown table", sig); +} + +static struct acpi_sdth * __init +acpi_lookup_table(const char *sig) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(acpi_table_addrs); i++) { + if (strcmp(sig, acpi_table_addrs[i].sig) == 0) { + return acpi_table_addrs[i].table; + } + } + + return NULL; +} + +static int __init +acpi_check_tables(void) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(acpi_table_addrs); i++) { + if (acpi_table_addrs[i].table == NULL) { + log_err("acpi: table %s missing", acpi_table_addrs[i].sig); + return -1; + } + } + + return 0; +} + +static void __init +acpi_free_tables(void) +{ + struct acpi_sdth *table; + size_t i; + + for (i = 0; i < ARRAY_SIZE(acpi_table_addrs); i++) { + table = acpi_table_addrs[i].table; + + if (table != NULL) { + kmem_free(table, table->length); + } + } +} + +static unsigned int __init +acpi_checksum(const void *ptr, size_t size) +{ + const uint8_t *bytes; + uint8_t checksum; + size_t i; + + bytes = ptr; + checksum = 0; + + for (i = 0; i < size; i++) { + checksum += bytes[i]; + } + + return checksum; +} + +static int __init +acpi_check_rsdp(const struct acpi_rsdp *rsdp) +{ + unsigned int checksum; + + if (memcmp(rsdp->signature, ACPI_RSDP_SIG, sizeof(rsdp->signature)) != 0) { + return -1; + } + + checksum = acpi_checksum(rsdp, sizeof(*rsdp)); + + if (checksum != 0) { + return -1; + } + + return 0; +} + +static int __init +acpi_get_rsdp(phys_addr_t start, size_t size, struct acpi_rsdp *rsdp) +{ + const struct acpi_rsdp *src; + uintptr_t addr, end, map_addr; + size_t map_size; + int error; + + assert(size > 0); + assert(P2ALIGNED(size, ACPI_RSDP_ALIGN)); + + if (!P2ALIGNED(start, ACPI_RSDP_ALIGN)) { + return -1; + } + + addr = (uintptr_t)vm_kmem_map_pa(start, size, &map_addr, &map_size); + + if (addr == 0) { + panic("acpi: unable to map bios memory in kernel map"); + } + + for (end = addr + size; addr < end; addr += ACPI_RSDP_ALIGN) { + src = (const struct acpi_rsdp *)addr; + error = acpi_check_rsdp(src); + + if (!error) { + break; + } + } + + if (!(addr < end)) { + error = -1; + goto out; + } + + memcpy(rsdp, src, sizeof(*rsdp)); + error = 0; + +out: + vm_kmem_unmap_pa(map_addr, map_size); + return error; +} + +static int __init +acpi_find_rsdp(struct acpi_rsdp *rsdp) +{ + const uint16_t *ptr; + uintptr_t base, map_addr; + size_t map_size; + int error; + + ptr = vm_kmem_map_pa(BIOSMEM_EBDA_PTR, sizeof(*ptr), &map_addr, &map_size); + + if (ptr == NULL) { + panic("acpi: unable to map ebda pointer in kernel map"); + } + + base = *((const volatile uint16_t *)ptr); + vm_kmem_unmap_pa(map_addr, map_size); + + if (base != 0) { + base <<= 4; + error = acpi_get_rsdp(base, 1024, rsdp); + + if (!error) { + return 0; + } + } + + error = acpi_get_rsdp(BIOSMEM_EXT_ROM, BIOSMEM_END - BIOSMEM_EXT_ROM, + rsdp); + + if (!error) { + return 0; + } + + log_debug("acpi: unable to find root system description pointer"); + return -1; +} + +static void __init +acpi_info(void) +{ + const struct acpi_sdth *rsdt; + + rsdt = acpi_lookup_table("RSDT"); + assert(rsdt != NULL); + log_debug("acpi: revision: %u, oem: %.*s", rsdt->revision, + (int)sizeof(rsdt->oem_id), rsdt->oem_id); +} + +static struct acpi_sdth * __init +acpi_copy_table(uint32_t addr) +{ + const struct acpi_sdth *table; + struct acpi_sdth *copy; + uintptr_t map_addr; + size_t size, map_size; + unsigned int checksum; + + table = vm_kmem_map_pa(addr, sizeof(*table), &map_addr, &map_size); + + if (table == NULL) { + panic("acpi: unable to map acpi data in kernel map"); + } + + if (!acpi_table_required(table)) { + copy = NULL; + goto out; + } + + size = ((const volatile struct acpi_sdth *)table)->length; + vm_kmem_unmap_pa(map_addr, map_size); + + table = vm_kmem_map_pa(addr, size, &map_addr, &map_size); + + if (table == NULL) { + panic("acpi: unable to map acpi data in kernel map"); + } + + checksum = acpi_checksum(table, size); + + if (checksum != 0) { + char sig[ACPI_SIG_SIZE]; + + acpi_table_sig(table, sig); + log_err("acpi: table %s: invalid checksum", sig); + copy = NULL; + goto out; + } + + copy = kmem_alloc(size); + + if (copy == NULL) { + panic("acpi: unable to allocate memory for acpi data copy"); + } + + memcpy(copy, table, size); + +out: + vm_kmem_unmap_pa(map_addr, map_size); + return copy; +} + +static int __init +acpi_copy_tables(const struct acpi_rsdp *rsdp) +{ + struct acpi_rsdt *rsdt; + struct acpi_sdth *table; + uint32_t *addr, *end; + int error; + + table = acpi_copy_table(rsdp->rsdt_address); + + if (table == NULL) { + return -1; + } + + acpi_register_table(table); + + rsdt = structof(table, struct acpi_rsdt, header); + end = (void *)rsdt + rsdt->header.length; + + for (addr = rsdt->entries; addr < end; addr++) { + table = acpi_copy_table(*addr); + + if (table == NULL) { + continue; + } + + acpi_register_table(table); + } + + error = acpi_check_tables(); + + if (error) { + goto error; + } + + return 0; + +error: + acpi_free_tables(); + return -1; +} + +static void __init +acpi_madt_iter_init(struct acpi_madt_iter *iter, + const struct acpi_madt *madt) +{ + iter->entry = madt->entries; + iter->end = (void *)madt + madt->header.length; +} + +static int __init +acpi_madt_iter_valid(const struct acpi_madt_iter *iter) +{ + return iter->entry < iter->end; +} + +static void __init +acpi_madt_iter_next(struct acpi_madt_iter *iter) +{ + iter->entry = (void *)iter->entry + iter->entry->header.length; +} + +static void __init +acpi_load_lapic(const struct acpi_madt_entry_lapic *lapic, int *is_bsp) +{ + if (!(lapic->flags & ACPI_MADT_LAPIC_ENABLED)) { + return; + } + + cpu_mp_register_lapic(lapic->apic_id, *is_bsp); + *is_bsp = 0; +} + +static void __init +acpi_load_ioapic(const struct acpi_madt_entry_ioapic *ioapic) +{ + ioapic_register(ioapic->id, ioapic->addr, ioapic->base); +} + +static void __init +acpi_load_iso(const struct acpi_madt_entry_iso *iso) +{ + bool active_high, edge_triggered; + + if (iso->bus != 0) { + log_err("acpi: invalid interrupt source override bus"); + return; + } + + switch (iso->flags & ACPI_MADT_ISO_POL_MASK) { + case ACPI_MADT_ISO_POL_DEFAULT: + case ACPI_MADT_ISO_POL_HIGH: + active_high = true; + break; + case ACPI_MADT_ISO_POL_LOW: + active_high = false; + break; + default: + log_err("acpi: invalid polarity"); + return; + } + + switch (iso->flags & ACPI_MADT_ISO_TRIG_MASK) { + case ACPI_MADT_ISO_TRIG_DEFAULT: + case ACPI_MADT_ISO_TRIG_EDGE: + edge_triggered = true; + break; + case ACPI_MADT_ISO_TRIG_LEVEL: + edge_triggered = false; + break; + default: + log_err("acpi: invalid trigger mode"); + return; + } + + ioapic_override(iso->source, iso->gsi, active_high, edge_triggered); +} + +static void __init +acpi_load_madt(void) +{ + const struct acpi_sdth *table; + const struct acpi_madt *madt; + struct acpi_madt_iter iter; + int is_bsp; + + table = acpi_lookup_table("APIC"); + assert(table != NULL); + madt = structof(table, struct acpi_madt, header); + + lapic_setup(madt->lapic_addr); + is_bsp = 1; + + acpi_madt_foreach(madt, &iter) { + switch (iter.entry->type) { + case ACPI_MADT_ENTRY_LAPIC: + acpi_load_lapic(&iter.entry->lapic, &is_bsp); + break; + case ACPI_MADT_ENTRY_IOAPIC: + acpi_load_ioapic(&iter.entry->ioapic); + break; + case ACPI_MADT_ENTRY_ISO: + acpi_load_iso(&iter.entry->iso); + break; + } + } + + if (madt->flags & ACPI_MADT_PC_COMPAT) { + pic_setup_disabled(); + } +} + +static void +acpi_shutdown_reset_sysio(uint64_t addr) +{ + if (addr > UINT16_MAX) { + log_warning("acpi: invalid sysio address"); + return; + } + + io_write_byte((uint16_t)addr, acpi_reset_value); +} + +static void +acpi_shutdown_reset(void) +{ + if ((acpi_reset_reg.reg_width != 8) || (acpi_reset_reg.reg_offset != 0)) { + log_warning("acpi: invalid reset register"); + return; + } + + switch (acpi_reset_reg.asid) { + case ACPI_GAS_ASID_SYSIO: + acpi_shutdown_reset_sysio(acpi_reset_reg.addr); + break; + default: + log_warning("acpi: unsupported reset register type"); + } +} + +static struct shutdown_ops acpi_shutdown_ops = { + .reset = acpi_shutdown_reset, +}; + +static void __init +acpi_load_fadt(void) +{ + const struct acpi_sdth *table; + const struct acpi_fadt *fadt; + + table = acpi_lookup_table("FACP"); + + if (table == NULL) { + log_debug("acpi: unable to find FADT table"); + return; + } + + fadt = structof(table, struct acpi_fadt, header); + + if (!(fadt->flags & ACPI_FADT_FL_RESET_REG_SUP)) { + log_debug("acpi: reset register not supported"); + return; + } + + acpi_reset_reg = fadt->reset_reg; + acpi_reset_value = fadt->reset_value; + shutdown_register(&acpi_shutdown_ops, ACPI_SHUTDOWN_PRIORITY); +} + +static int __init +acpi_setup(void) +{ + struct acpi_rsdp rsdp; + int error; + + error = acpi_find_rsdp(&rsdp); + + if (error) { + goto error; + } + + error = acpi_copy_tables(&rsdp); + + if (error) { + goto error; + } + + acpi_info(); + acpi_load_madt(); + acpi_load_fadt(); + acpi_free_tables(); + + return 0; + +error: + /* + * For the sake of simplicity, it has been decided to ignore legacy + * specifications such as the multiprocessor specification, and use + * ACPI only. If ACPI is unavailable, consider the APIC system to + * be missing and fall back to using the legacy XT-PIC. + */ + pic_setup(); + return 0; +} + +INIT_OP_DEFINE(acpi_setup, + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(intr_bootstrap, true), + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(shutdown_bootstrap, true), + INIT_OP_DEP(trap_setup, true), + INIT_OP_DEP(vm_kmem_setup, true)); diff --git a/arch/x86/machine/acpimp.h b/arch/x86/machine/acpi.h index e66a87ae..1f65fd6f 100644 --- a/arch/x86/machine/acpimp.h +++ b/arch/x86/machine/acpi.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Richard Braun. + * Copyright (c) 2012-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,15 +19,17 @@ * specification v1.0. */ -#ifndef _X86_ACPIMP_H -#define _X86_ACPIMP_H +#ifndef _X86_ACPI_H +#define _X86_ACPI_H + +#include <kern/init.h> /* - * Load multiprocessor information. - * - * Return 0 if successful (an error usually means hardware doesn't support - * ACPI). + * This init operation provides : + * - Multiprocessor probing + * - registration of I/O APIC interrupt controllers + * - registration of ACPI shutdown operations */ -int acpimp_setup(void); +INIT_OP_DECLARE(acpi_setup); -#endif /* _X86_ACPIMP_H */ +#endif /* _X86_ACPI_H */ diff --git a/arch/x86/machine/acpimp.c b/arch/x86/machine/acpimp.c deleted file mode 100644 index bba2b8ca..00000000 --- a/arch/x86/machine/acpimp.c +++ /dev/null @@ -1,538 +0,0 @@ -/* - * Copyright (c) 2012 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> -#include <string.h> - -#include <kern/assert.h> -#include <kern/init.h> -#include <kern/kmem.h> -#include <kern/macros.h> -#include <kern/panic.h> -#include <machine/acpimp.h> -#include <machine/biosmem.h> -#include <machine/cpu.h> -#include <machine/io.h> -#include <machine/ioapic.h> -#include <machine/lapic.h> -#include <machine/types.h> -#include <vm/vm_kmem.h> - -/* - * Alignment of the RSDP. - */ -#define ACPIMP_RSDP_ALIGN 16 - -/* - * Signature of the root system description pointer. - */ -#define ACPIMP_RSDP_SIG "RSD PTR " - -struct acpimp_rsdp { - uint8_t signature[8]; - uint8_t checksum; - uint8_t oem_id[6]; - uint8_t reserved; - uint32_t rsdt_address; -} __packed; - -/* - * Size of a buffer which can store a table signature as a string. - */ -#define ACPIMP_SIG_SIZE 5 - -struct acpimp_sdth { - uint8_t signature[4]; - uint32_t length; - uint8_t revision; - uint8_t checksum; - uint8_t oem_id[6]; - uint8_t oem_table_id[8]; - uint32_t oem_revision; - uint8_t creator_id[4]; - uint32_t creator_revision; -} __packed; - -struct acpimp_rsdt { - struct acpimp_sdth header; - uint32_t entries[0]; -} __packed; - -/* - * MADT entry type codes. - */ -#define ACPIMP_MADT_ENTRY_LAPIC 0 -#define ACPIMP_MADT_ENTRY_IOAPIC 1 - -struct acpimp_madt_entry_hdr { - uint8_t type; - uint8_t length; -} __packed; - -#define ACPIMP_MADT_LAPIC_ENABLED 0x1 - -struct acpimp_madt_entry_lapic { - struct acpimp_madt_entry_hdr header; - uint8_t processor_id; - uint8_t apic_id; - uint32_t flags; -} __packed; - -struct acpimp_madt_entry_ioapic { - struct acpimp_madt_entry_hdr header; - uint8_t id; - uint8_t _reserved; - uint32_t addr; - uint32_t base; -} __packed; - -union acpimp_madt_entry { - uint8_t type; - struct acpimp_madt_entry_hdr header; - struct acpimp_madt_entry_lapic lapic; - struct acpimp_madt_entry_ioapic ioapic; -} __packed; - -struct acpimp_madt { - struct acpimp_sdth header; - uint32_t lapic_addr; - uint32_t flags; - union acpimp_madt_entry entries[0]; -} __packed; - -struct acpimp_madt_iter { - const union acpimp_madt_entry *entry; - const union acpimp_madt_entry *end; -}; - -#define acpimp_madt_foreach(madt, iter) \ -for (acpimp_madt_iter_init(iter, madt); \ - acpimp_madt_iter_valid(iter); \ - acpimp_madt_iter_next(iter)) - -struct acpimp_table_addr { - const char *sig; - struct acpimp_sdth *table; -}; - -static struct acpimp_table_addr acpimp_table_addrs[] __initdata = { - { "RSDT", NULL }, - { "APIC", NULL } -}; - -static void __init -acpimp_table_sig(const struct acpimp_sdth *table, char sig[ACPIMP_SIG_SIZE]) -{ - memcpy(sig, table->signature, sizeof(table->signature)); - sig[4] = '\0'; -} - -static int __init -acpimp_table_required(const struct acpimp_sdth *table) -{ - char sig[ACPIMP_SIG_SIZE]; - size_t i; - - acpimp_table_sig(table, sig); - - for (i = 0; i < ARRAY_SIZE(acpimp_table_addrs); i++) - if (strcmp(sig, acpimp_table_addrs[i].sig) == 0) { - return 1; - } - - return 0; -} - -static void __init -acpimp_register_table(struct acpimp_sdth *table) -{ - char sig[ACPIMP_SIG_SIZE]; - size_t i; - - acpimp_table_sig(table, sig); - - for (i = 0; i < ARRAY_SIZE(acpimp_table_addrs); i++) - if (strcmp(sig, acpimp_table_addrs[i].sig) == 0) { - if (acpimp_table_addrs[i].table != NULL) { - printf("acpimp: warning: table %s ignored:" - " already registered\n", sig); - return; - } - - acpimp_table_addrs[i].table = table; - return; - } - - printf("acpimp: warning: table '%s' ignored: unknown table\n", sig); -} - -static struct acpimp_sdth * __init -acpimp_lookup_table(const char *sig) -{ - size_t i; - - for (i = 0; i < ARRAY_SIZE(acpimp_table_addrs); i++) - if (strcmp(sig, acpimp_table_addrs[i].sig) == 0) { - return acpimp_table_addrs[i].table; - } - - return NULL; -} - -static int __init -acpimp_check_tables(void) -{ - size_t i; - - for (i = 0; i < ARRAY_SIZE(acpimp_table_addrs); i++) - if (acpimp_table_addrs[i].table == NULL) { - printf("acpimp: error: table %s missing\n", - acpimp_table_addrs[i].sig); - return -1; - } - - return 0; -} - -static void __init -acpimp_free_tables(void) -{ - struct acpimp_sdth *table; - size_t i; - - for (i = 0; i < ARRAY_SIZE(acpimp_table_addrs); i++) { - table = acpimp_table_addrs[i].table; - - if (table != NULL) { - kmem_free(table, table->length); - } - } -} - -static unsigned int __init -acpimp_checksum(const void *ptr, size_t size) -{ - const uint8_t *bytes; - uint8_t checksum; - size_t i; - - bytes = ptr; - checksum = 0; - - for (i = 0; i < size; i++) { - checksum += bytes[i]; - } - - return checksum; -} - -static int __init -acpimp_check_rsdp(const struct acpimp_rsdp *rsdp) -{ - unsigned int checksum; - - if (memcmp(rsdp->signature, ACPIMP_RSDP_SIG, sizeof(rsdp->signature)) != 0) { - return -1; - } - - checksum = acpimp_checksum(rsdp, sizeof(*rsdp)); - - if (checksum != 0) { - return -1; - } - - return 0; -} - -static int __init -acpimp_get_rsdp(phys_addr_t start, size_t size, struct acpimp_rsdp *rsdp) -{ - const struct acpimp_rsdp *src; - uintptr_t addr, end, map_addr; - size_t map_size; - int error; - - assert(size > 0); - assert(P2ALIGNED(size, ACPIMP_RSDP_ALIGN)); - - if (!P2ALIGNED(start, ACPIMP_RSDP_ALIGN)) { - return -1; - } - - addr = (uintptr_t)vm_kmem_map_pa(start, size, &map_addr, &map_size); - - if (addr == 0) { - panic("acpimp: unable to map bios memory in kernel map"); - } - - for (end = addr + size; addr < end; addr += ACPIMP_RSDP_ALIGN) { - src = (const struct acpimp_rsdp *)addr; - error = acpimp_check_rsdp(src); - - if (!error) { - break; - } - } - - if (!(addr < end)) { - error = -1; - goto out; - } - - memcpy(rsdp, src, sizeof(*rsdp)); - error = 0; - -out: - vm_kmem_unmap_pa(map_addr, map_size); - return error; -} - -static int __init -acpimp_find_rsdp(struct acpimp_rsdp *rsdp) -{ - const uint16_t *ptr; - uintptr_t base, map_addr; - size_t map_size; - int error; - - ptr = vm_kmem_map_pa(BIOSMEM_EBDA_PTR, sizeof(*ptr), &map_addr, &map_size); - - if (ptr == NULL) { - panic("acpimp: unable to map ebda pointer in kernel map"); - } - - base = *((const volatile uint16_t *)ptr); - vm_kmem_unmap_pa(map_addr, map_size); - - if (base != 0) { - base <<= 4; - error = acpimp_get_rsdp(base, 1024, rsdp); - - if (!error) { - return 0; - } - } - - error = acpimp_get_rsdp(BIOSMEM_EXT_ROM, BIOSMEM_END - BIOSMEM_EXT_ROM, - rsdp); - - if (!error) { - return 0; - } - - printf("acpimp: unable to find root system description pointer\n"); - return -1; -} - -static void __init -acpimp_info(void) -{ - const struct acpimp_sdth *rsdt; - - rsdt = acpimp_lookup_table("RSDT"); - assert(rsdt != NULL); - printf("acpimp: revision: %u, oem: %.*s\n", rsdt->revision, - (int)sizeof(rsdt->oem_id), rsdt->oem_id); -} - -static struct acpimp_sdth * __init -acpimp_copy_table(uint32_t addr) -{ - const struct acpimp_sdth *table; - struct acpimp_sdth *copy; - uintptr_t map_addr; - size_t size, map_size; - unsigned int checksum; - - table = vm_kmem_map_pa(addr, sizeof(*table), &map_addr, &map_size); - - if (table == NULL) { - panic("acpimp: unable to map acpi data in kernel map"); - } - - if (!acpimp_table_required(table)) { - copy = NULL; - goto out; - } - - size = ((const volatile struct acpimp_sdth *)table)->length; - vm_kmem_unmap_pa(map_addr, map_size); - - table = vm_kmem_map_pa(addr, size, &map_addr, &map_size); - - if (table == NULL) { - panic("acpimp: unable to map acpi data in kernel map"); - } - - checksum = acpimp_checksum(table, size); - - if (checksum != 0) { - char sig[ACPIMP_SIG_SIZE]; - - acpimp_table_sig(table, sig); - printf("acpimp: table %s: invalid checksum\n", sig); - copy = NULL; - goto out; - } - - copy = kmem_alloc(size); - - if (copy == NULL) { - panic("acpimp: unable to allocate memory for acpi data copy"); - } - - memcpy(copy, table, size); - -out: - vm_kmem_unmap_pa(map_addr, map_size); - return copy; -} - -static int __init -acpimp_copy_tables(const struct acpimp_rsdp *rsdp) -{ - struct acpimp_rsdt *rsdt; - struct acpimp_sdth *table; - uint32_t *addr, *end; - int error; - - table = acpimp_copy_table(rsdp->rsdt_address); - - if (table == NULL) { - return -1; - } - - acpimp_register_table(table); - - rsdt = structof(table, struct acpimp_rsdt, header); - end = (void *)rsdt + rsdt->header.length; - - for (addr = rsdt->entries; addr < end; addr++) { - table = acpimp_copy_table(*addr); - - if (table == NULL) { - continue; - } - - acpimp_register_table(table); - } - - error = acpimp_check_tables(); - - if (error) { - goto error; - } - - return 0; - -error: - acpimp_free_tables(); - return -1; -} - -static void __init -acpimp_madt_iter_init(struct acpimp_madt_iter *iter, - const struct acpimp_madt *madt) -{ - iter->entry = madt->entries; - iter->end = (void *)madt + madt->header.length; -} - -static int __init -acpimp_madt_iter_valid(const struct acpimp_madt_iter *iter) -{ - return iter->entry < iter->end; -} - -static void __init -acpimp_madt_iter_next(struct acpimp_madt_iter *iter) -{ - iter->entry = (void *)iter->entry + iter->entry->header.length; -} - -static void __init -acpimp_load_lapic(const struct acpimp_madt_entry_lapic *lapic, int *is_bsp) -{ - if (!(lapic->flags & ACPIMP_MADT_LAPIC_ENABLED)) { - return; - } - - cpu_mp_register_lapic(lapic->apic_id, *is_bsp); - *is_bsp = 0; -} - -static void __init -acpimp_load_ioapic(const struct acpimp_madt_entry_ioapic *ioapic) -{ - ioapic_register(ioapic->id, ioapic->addr, ioapic->base); -} - -static void __init -acpimp_load_madt(void) -{ - const struct acpimp_sdth *table; - const struct acpimp_madt *madt; - struct acpimp_madt_iter iter; - int is_bsp; - - table = acpimp_lookup_table("APIC"); - assert(table != NULL); - madt = structof(table, struct acpimp_madt, header); - lapic_setup(madt->lapic_addr); - is_bsp = 1; - - /* - * TODO Handle PCAT_COMPAT flag - * TODO Handle interrupt overrides - */ - - acpimp_madt_foreach(madt, &iter) { - switch (iter.entry->type) { - case ACPIMP_MADT_ENTRY_LAPIC: - acpimp_load_lapic(&iter.entry->lapic, &is_bsp); - break; - case ACPIMP_MADT_ENTRY_IOAPIC: - acpimp_load_ioapic(&iter.entry->ioapic); - } - } -} - -int __init -acpimp_setup(void) -{ - struct acpimp_rsdp rsdp; - int error; - - error = acpimp_find_rsdp(&rsdp); - - if (error) { - return error; - } - - error = acpimp_copy_tables(&rsdp); - - if (error) { - return error; - } - - acpimp_info(); - acpimp_load_madt(); - acpimp_free_tables(); - return 0; -} diff --git a/arch/x86/machine/asm.h b/arch/x86/machine/asm.h index 6118bb8a..204d6fed 100644 --- a/arch/x86/machine/asm.h +++ b/arch/x86/machine/asm.h @@ -22,16 +22,16 @@ #warning "asm.h included from a C file" #endif /* __ASSEMBLER__ */ -#include <machine/param.h> +#include <machine/cpu.h> #define ASM_ENTRY(x) \ -.align TEXT_ALIGN; \ +.align CPU_TEXT_ALIGN; \ .global x; \ .type x, STT_FUNC; \ x: #define ASM_DATA(x) \ -.align DATA_ALIGN; \ +.align CPU_DATA_ALIGN; \ .global x; \ .type x, STT_OBJECT; \ x: diff --git a/arch/x86/machine/atcons.c b/arch/x86/machine/atcons.c index c4c48069..0c35266f 100644 --- a/arch/x86/machine/atcons.c +++ b/arch/x86/machine/atcons.c @@ -15,42 +15,168 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <stdbool.h> +#include <string.h> + #include <kern/console.h> #include <kern/init.h> #include <machine/atcons.h> #include <machine/atkbd.h> #include <machine/cga.h> +#define ATCONS_ESC_SEQ_MAX_SIZE 8 + static struct console atcons_console; +typedef void (*atcons_esc_seq_fn_t)(void); + +struct atcons_esc_seq { + const char *str; + atcons_esc_seq_fn_t fn; +}; + +static bool atcons_escape; +static char atcons_esc_seq[ATCONS_ESC_SEQ_MAX_SIZE]; +static unsigned int atcons_esc_seq_index; + +static void +atcons_process_left(void) +{ + cga_cursor_left(); +} + +static void +atcons_process_right(void) +{ + cga_cursor_right(); +} + +static const struct atcons_esc_seq atcons_esc_seqs[] = { + { "[1D", atcons_process_left }, + { "[1C", atcons_process_right }, +}; + +static void +atcons_reset_esc_seq(void) +{ + atcons_escape = false; +} + +static int +atcons_esc_seq_lookup(const struct atcons_esc_seq **seqp) +{ + const struct atcons_esc_seq *seq; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(atcons_esc_seqs); i++) { + seq = &atcons_esc_seqs[i]; + + if (strncmp(seq->str, atcons_esc_seq, atcons_esc_seq_index) == 0) { + if (strlen(seq->str) == atcons_esc_seq_index) { + *seqp = seq; + return 0; + } else { + return ERROR_AGAIN; + } + } + } + + return ERROR_SRCH; +} + +static void +atcons_process_esc_seq(char c) +{ + const struct atcons_esc_seq *seq; + int error; + + if (atcons_esc_seq_index >= sizeof(atcons_esc_seq)) { + atcons_reset_esc_seq(); + return; + } + + atcons_esc_seq[atcons_esc_seq_index] = c; + atcons_esc_seq_index++; + + error = atcons_esc_seq_lookup(&seq); + + if (error) { + if (error != ERROR_AGAIN) { + atcons_reset_esc_seq(); + } + } else { + seq->fn(); + atcons_reset_esc_seq(); + } +} + static void atcons_putc(struct console *console, char c) { (void)console; - cga_putc(c); + + if (c == '\e') { + atcons_escape = true; + atcons_esc_seq_index = 0; + } else if (atcons_escape) { + atcons_process_esc_seq(c); + } else { + cga_putc(c); + } } static const struct console_ops atcons_ops = { .putc = atcons_putc, }; -void __init +static int __init atcons_bootstrap(void) { - cga_setup(); - console_init(&atcons_console, "atcons", &atcons_ops); console_register(&atcons_console); + return 0; } -void __init +INIT_OP_DEFINE(atcons_bootstrap, + INIT_OP_DEP(cga_setup, true), + INIT_OP_DEP(console_bootstrap, true)); + +static int __init atcons_setup(void) { - atkbd_setup(); + return 0; +} + +INIT_OP_DEFINE(atcons_setup, + INIT_OP_DEP(atcons_bootstrap, true), + INIT_OP_DEP(atkbd_setup, true)); + +void +atcons_intr(const char *s) +{ + console_intr(&atcons_console, s); +} + +void +atcons_left(void) +{ + atcons_intr("\e[D"); +} + +void +atcons_bottom(void) +{ + atcons_intr("\e[B"); +} + +void +atcons_right(void) +{ + atcons_intr("\e[C"); } void -atcons_intr(char c) +atcons_up(void) { - console_intr(&atcons_console, c); + atcons_intr("\e[A"); } diff --git a/arch/x86/machine/atcons.h b/arch/x86/machine/atcons.h index 05b870fc..0d4893a6 100644 --- a/arch/x86/machine/atcons.h +++ b/arch/x86/machine/atcons.h @@ -21,24 +21,35 @@ #ifndef _X86_ATCONS_H #define _X86_ATCONS_H +#include <kern/init.h> + /* - * Early initialization of the atcons module. + * Console interrupt handler. + * + * This function is called by the AT keyboard interrupt handler + * to handle machine-independent console management. */ -void atcons_bootstrap(void); +void atcons_intr(const char *s); /* - * Initialize the atcons module. - * - * This function enables keyboard interrupt handling. + * Direction control processing functions. */ -void atcons_setup(void); +void atcons_left(void); +void atcons_bottom(void); +void atcons_right(void); +void atcons_up(void); /* - * Console interrupt handler. - * - * This function is called by the AT keyboard interrupt handler - * to handle machine-independent console management. + * This init operation provides : + * - CGA output through the console module + */ +INIT_OP_DECLARE(atcons_bootstrap); + +/* + * This init operation provides : + * - AT keyboard input through the console module + * - module fully initialized */ -void atcons_intr(char c); +INIT_OP_DECLARE(atcons_setup); #endif /* _X86_ATCONS_H */ diff --git a/arch/x86/machine/atkbd.c b/arch/x86/machine/atkbd.c index ad66e21a..59f8d681 100644 --- a/arch/x86/machine/atkbd.c +++ b/arch/x86/machine/atkbd.c @@ -24,24 +24,37 @@ #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> +#include <kern/console.h> #include <kern/error.h> #include <kern/init.h> #include <kern/intr.h> +#include <kern/log.h> #include <kern/macros.h> #include <machine/atkbd.h> #include <machine/atcons.h> #include <machine/io.h> +/* + * Intel 8042 I/O ports. + */ #define ATKBD_PORT_DATA 0x60 #define ATKBD_PORT_STATUS 0x64 #define ATKBD_PORT_CMD 0x64 +/* + * Status register bits. + */ #define ATKBD_STATUS_OUT_FULL 0x01 #define ATKBD_STATUS_IN_FULL 0x02 +#define ATKBD_STATUS_TIMEOUT_ERROR 0x40 #define ATKBD_STATUS_PARITY_ERROR 0x80 +#define ATKBD_STATUS_ERROR (ATKBD_STATUS_PARITY_ERROR \ + | ATKBD_STATUS_TIMEOUT_ERROR) +/* + * Controller commands. + */ #define ATKBD_CMD_RDCONF 0x20 #define ATKBD_CMD_WRCONF 0x60 #define ATKBD_CMD_DIS2 0xa7 @@ -49,12 +62,26 @@ #define ATKBD_CMD_DIS1 0xad #define ATKBD_CMD_EN1 0xae +/* + * Controller configuration register bits. + */ #define ATKBD_CONF_ENINT1 0x01 #define ATKBD_CONF_ENINT2 0x02 #define ATKBD_CONF_ENTRANS 0x40 +/* + * Controller interrupt. + * + * This driver only supports keyboard input and doesn't use interrupt 12. + */ #define ATKBD_INTR1 1 +/* + * Key identifiers. + * + * Keys are obtained from static sparse tables, where undefined entries are + * initialized to 0. As a result, make that value an invalid key. + */ enum atkbd_key_id { ATKBD_KEY_INVALID = 0, ATKBD_KEY_1 = 1, @@ -150,6 +177,12 @@ enum atkbd_key_id { ATKBD_KEY_KP_DEL, }; +static const char atkbd_scroll_up[] = { CONSOLE_SCROLL_UP, '\0' }; +static const char atkbd_scroll_down[] = { CONSOLE_SCROLL_DOWN, '\0' }; + +/* + * Key modifiers. + */ #define ATKBD_KM_SHIFT 0x01 /* Shift / caps lock modifier applies */ #define ATKBD_KM_KP 0x02 /* Num lock modifier applies */ #define ATKBD_KM_CTL 0x04 /* Unmodified key is a control character */ @@ -159,6 +192,14 @@ struct atkbd_key { enum atkbd_key_id id; }; +/* + * Regular key table. + * + * Keys are indexed by their "scan code set 2" code. Undefined entries are + * initialized to 0, which is a known invalid identifier. Modifiers are used + * to select a different key->characters table, or key-specific processing + * in the case of control keys. + */ static const struct atkbd_key atkbd_keys[] = { [0x16] = { ATKBD_KM_SHIFT, ATKBD_KEY_1 }, [0x1e] = { ATKBD_KM_SHIFT, ATKBD_KEY_2 }, @@ -237,6 +278,12 @@ static const struct atkbd_key atkbd_keys[] = { [0x71] = { ATKBD_KM_KP, ATKBD_KEY_KP_DEL }, }; +/* + * Key table for e0 codes. + * + * This table is similar to the regular key table, except it is used if an + * e0 code was read in the input byte sequence. + */ static const struct atkbd_key atkbd_e0_keys[] = { [0x11] = { ATKBD_KM_CTL, ATKBD_KEY_ALTGR }, [0x14] = { ATKBD_KM_CTL, ATKBD_KEY_RCTRL }, @@ -245,18 +292,24 @@ static const struct atkbd_key atkbd_e0_keys[] = { [0x71] = { 0, ATKBD_KEY_DELETE }, [0x6c] = { 0, ATKBD_KEY_HOME }, [0x69] = { 0, ATKBD_KEY_END }, - [0x7d] = { 0, ATKBD_KEY_PGUP }, - [0x7a] = { 0, ATKBD_KEY_PGDOWN }, + [0x7d] = { ATKBD_KM_SHIFT, ATKBD_KEY_PGUP }, + [0x7a] = { ATKBD_KM_SHIFT, ATKBD_KEY_PGDOWN }, - [0x6b] = { 0, ATKBD_KEY_LEFT }, - [0x72] = { 0, ATKBD_KEY_BOTTOM }, - [0x74] = { 0, ATKBD_KEY_RIGHT }, - [0x75] = { 0, ATKBD_KEY_UP }, + [0x6b] = { ATKBD_KM_CTL, ATKBD_KEY_LEFT }, + [0x72] = { ATKBD_KM_CTL, ATKBD_KEY_BOTTOM }, + [0x74] = { ATKBD_KM_CTL, ATKBD_KEY_RIGHT }, + [0x75] = { ATKBD_KM_CTL, ATKBD_KEY_UP }, [0x4a] = { 0, ATKBD_KEY_KP_SLASH }, [0x5a] = { 0, ATKBD_KEY_KP_ENTER }, }; +/* + * Key to characters table. + * + * This table is used for keys with no modifiers or if none of the modifiers + * apply at input time. + */ static const char *atkbd_chars[] = { [ATKBD_KEY_1] = "1", [ATKBD_KEY_2] = "2", @@ -314,21 +367,23 @@ static const char *atkbd_chars[] = { [ATKBD_KEY_SPACE] = " ", [ATKBD_KEY_DELETE] = "\e[3~", - - [ATKBD_KEY_LEFT] = "\e[D", - [ATKBD_KEY_BOTTOM] = "\e[B", - [ATKBD_KEY_RIGHT] = "\e[C", - [ATKBD_KEY_UP] = "\e[A", + [ATKBD_KEY_HOME] = "\e[H", + [ATKBD_KEY_END] = "\e[F", [ATKBD_KEY_KP_SLASH] = "/", [ATKBD_KEY_KP_STAR] = "*", [ATKBD_KEY_KP_MINUS] = "-", + [ATKBD_KEY_KP_HOME] = "\e[H", [ATKBD_KEY_KP_PLUS] = "+", [ATKBD_KEY_KP_5] = "5", + [ATKBD_KEY_KP_END] = "\e[F", [ATKBD_KEY_KP_ENTER] = "\n", [ATKBD_KEY_KP_DEL] = "\e[3~", }; +/* + * Key to characters table when the shift modifier applies. + */ static const char *atkbd_shift_chars[] = { [ATKBD_KEY_1] = "!", [ATKBD_KEY_2] = "@", @@ -379,8 +434,14 @@ static const char *atkbd_shift_chars[] = { [ATKBD_KEY_COMMA] = "<", [ATKBD_KEY_DOT] = ">", [ATKBD_KEY_SLASH] = "?", + + [ATKBD_KEY_PGUP] = atkbd_scroll_up, + [ATKBD_KEY_PGDOWN] = atkbd_scroll_down, }; +/* + * Key to keypad characters table when the numlock modifier applies. + */ static const char *atkbd_kp_chars[] = { [ATKBD_KEY_KP_HOME] = "7", [ATKBD_KEY_KP_UP] = "8", @@ -394,13 +455,16 @@ static const char *atkbd_kp_chars[] = { [ATKBD_KEY_KP_DEL] = ".", }; -#define ATKBD_KF_E0 0x01 -#define ATKBD_KF_F0 0x02 -#define ATKBD_KF_LSHIFT 0x04 -#define ATKBD_KF_RSHIFT 0x08 -#define ATKBD_KF_NUMLOCK 0x10 -#define ATKBD_KF_CAPSLOCK 0x20 -#define ATKBD_KF_SCROLLLOCK 0x40 +/* + * Global flags. + */ +#define ATKBD_KF_E0 0x01 /* Current sequence includes an 0xe0 byte */ +#define ATKBD_KF_F0 0x02 /* Current sequence includes an 0xf0 byte, + which usually indicates a key release */ +#define ATKBD_KF_LSHIFT 0x04 /* Left shift key is being pressed */ +#define ATKBD_KF_RSHIFT 0x08 /* Right shift key is being pressed */ +#define ATKBD_KF_NUMLOCK 0x10 /* Numlock key has been toggled on */ +#define ATKBD_KF_CAPSLOCK 0x20 /* Caps lock key has been toggled on */ #define ATKBD_KF_SHIFT (ATKBD_KF_CAPSLOCK \ | ATKBD_KF_RSHIFT \ @@ -424,10 +488,33 @@ atkbd_write_data(uint8_t data) io_write_byte(ATKBD_PORT_DATA, data); } -static uint8_t -atkbd_read_status(void) +static int +atkbd_read_status(bool check_out) { - return io_read_byte(ATKBD_PORT_STATUS); + uint8_t status; + + status = io_read_byte(ATKBD_PORT_STATUS); + + /* + * XXX This case is rare enough that it can be used to identify the lack + * of hardware. + */ + if (status == 0xff) { + log_info("atkbd: no keyboard controller"); + return ERROR_NODEV; + } else if (status & ATKBD_STATUS_PARITY_ERROR) { + log_err("atkbd: parity error"); + return ERROR_IO; + } else if (status & ATKBD_STATUS_TIMEOUT_ERROR) { + log_err("atkbd: timeout error"); + return ERROR_TIMEDOUT; + } + + if (check_out) { + return (status & ATKBD_STATUS_OUT_FULL) ? 0 : ERROR_AGAIN; + } else { + return (status & ATKBD_STATUS_IN_FULL) ? ERROR_AGAIN : 0; + } } static void @@ -439,78 +526,84 @@ atkbd_write_cmd(uint8_t cmd) static int atkbd_out_wait(void) { - uint8_t status; + int error; for (;;) { - status = atkbd_read_status(); + error = atkbd_read_status(true); - if (status & ATKBD_STATUS_OUT_FULL) { + if (error != ERROR_AGAIN) { break; } } - if (status & ATKBD_STATUS_PARITY_ERROR) { - printf("atkbd: parity error\n"); - return ERROR_IO; - } - - return 0; + return error; } -static void +static int atkbd_in_wait(void) { - uint8_t status; + int error; for (;;) { - status = atkbd_read_status(); + error = atkbd_read_status(false); - if (!(status & ATKBD_STATUS_IN_FULL)) { + if (error != ERROR_AGAIN) { break; } } + + return error; } static int atkbd_read(uint8_t *datap, bool wait) { - uint8_t status; int error; if (wait) { error = atkbd_out_wait(); - - if (error) { - return error; - } } else { - status = atkbd_read_status(); + error = atkbd_read_status(true); + } - if (!(status & ATKBD_STATUS_OUT_FULL)) { - return ERROR_AGAIN; - } + if (error) { + return error; } *datap = atkbd_read_data(); return 0; } -static void +static int atkbd_write(uint8_t data) { - atkbd_in_wait(); + int error; + + error = atkbd_in_wait(); + + if (error) { + return error; + } + atkbd_write_data(data); + return 0; } -static void +static int __init atkbd_flush(void) { - uint8_t status; + uint8_t byte; int error; do { - error = atkbd_read(&status, false); + error = atkbd_read(&byte, false); } while (!error); + + if (error == ERROR_AGAIN) { + error = 0; + } + + return error; } static int __init @@ -521,9 +614,6 @@ atkbd_disable(void) atkbd_write_cmd(ATKBD_CMD_DIS1); atkbd_write_cmd(ATKBD_CMD_DIS2); - - atkbd_flush(); - atkbd_write_cmd(ATKBD_CMD_RDCONF); error = atkbd_read(&byte, true); @@ -533,9 +623,7 @@ atkbd_disable(void) byte &= ~(ATKBD_CONF_ENTRANS | ATKBD_CONF_ENINT2 | ATKBD_CONF_ENINT1); atkbd_write_cmd(ATKBD_CMD_WRCONF); - atkbd_write(byte); - - return 0; + return atkbd_write(byte); } static int __init @@ -556,11 +644,13 @@ atkbd_enable(void) byte &= ~(ATKBD_CONF_ENTRANS | ATKBD_CONF_ENINT2); byte |= ATKBD_CONF_ENINT1; atkbd_write_cmd(ATKBD_CMD_WRCONF); - atkbd_write(byte); + error = atkbd_write(byte); - atkbd_flush(); + if (error) { + return error; + } - return 0; + return atkbd_flush(); } static void @@ -592,10 +682,7 @@ atkbd_key_process_chars(const struct atkbd_key *key, s = chars[key->id]; if (s != NULL) { - while (*s != '\0') { - atcons_intr(*s); - s++; - } + atcons_intr(s); } } @@ -643,6 +730,31 @@ atkbd_key_process_ctl(const struct atkbd_key *key) if (!(atkbd_flags & ATKBD_KF_F0)) { atkbd_toggle_capslock(); } + break; + case ATKBD_KEY_LEFT: + if (!(atkbd_flags & ATKBD_KF_F0)) { + atcons_left(); + } + + break; + case ATKBD_KEY_BOTTOM: + if (!(atkbd_flags & ATKBD_KF_F0)) { + atcons_bottom(); + } + + break; + case ATKBD_KEY_RIGHT: + if (!(atkbd_flags & ATKBD_KF_F0)) { + atcons_right(); + } + + break; + case ATKBD_KEY_UP: + if (!(atkbd_flags & ATKBD_KF_F0)) { + atcons_up(); + } + + break; default: break; } @@ -652,7 +764,7 @@ static void atkbd_key_process(const struct atkbd_key *key) { if (key->id == ATKBD_KEY_INVALID) { - return; + goto out; } if ((key->modifiers & ATKBD_KM_SHIFT) && (atkbd_flags & ATKBD_KF_SHIFT)) { @@ -666,6 +778,7 @@ atkbd_key_process(const struct atkbd_key *key) atkbd_key_process_chars(key, atkbd_chars, ARRAY_SIZE(atkbd_chars)); } +out: atkbd_flags &= ~ATKBD_KF_F0; } @@ -677,11 +790,13 @@ atkbd_process_e0_code(uint8_t code) return; } - if (code >= ARRAY_SIZE(atkbd_keys)) { - return; + if (code >= ARRAY_SIZE(atkbd_e0_keys)) { + goto out; } atkbd_key_process(&atkbd_e0_keys[code]); + +out: atkbd_flags &= ~ATKBD_KF_E0; } @@ -728,27 +843,39 @@ atkbd_intr(void *arg) return 0; } -void __init +static int __init atkbd_setup(void) { int error; + error = atkbd_flush(); + + if (error) { + return 0; + } + error = atkbd_disable(); if (error) { - return; + return 0; } error = intr_register(ATKBD_INTR1, atkbd_intr, NULL); if (error) { - printf("atkbd: error: unable to register interrupt handler\n"); - return; + log_err("atkbd: unable to register interrupt handler"); + return 0; } error = atkbd_enable(); if (error) { - return; + return 0; } + + return 0; } + +INIT_OP_DEFINE(atkbd_setup, + INIT_OP_DEP(atcons_bootstrap, true), + INIT_OP_DEP(intr_setup, true)); diff --git a/arch/x86/machine/atkbd.h b/arch/x86/machine/atkbd.h index 0bed7ada..9efeefbf 100644 --- a/arch/x86/machine/atkbd.h +++ b/arch/x86/machine/atkbd.h @@ -21,9 +21,12 @@ #ifndef _X86_ATKBD_H #define _X86_ATKBD_H +#include <kern/init.h> + /* - * Initialize the atkbd module. + * This init operation provides : + * - module fully initialized */ -void atkbd_setup(void); +INIT_OP_DECLARE(atkbd_setup); #endif /* _X86_ATKBD_H */ diff --git a/arch/x86/machine/biosmem.c b/arch/x86/machine/biosmem.c index 65329a06..fc03614c 100644 --- a/arch/x86/machine/biosmem.c +++ b/arch/x86/machine/biosmem.c @@ -15,27 +15,26 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <machine/biosmem.h> #include <machine/boot.h> #include <machine/cpu.h> #include <machine/multiboot.h> +#include <machine/page.h> +#include <machine/pmap.h> +#include <machine/pmem.h> #include <machine/types.h> -#include <vm/vm_kmem.h> #include <vm/vm_page.h> -#define DEBUG 0 - #define BIOSMEM_MAX_BOOT_DATA 64 /* @@ -110,7 +109,7 @@ struct biosmem_zone { /* * Physical zone boundaries. */ -static struct biosmem_zone biosmem_zones[VM_PAGE_MAX_ZONES] __bootdata; +static struct biosmem_zone biosmem_zones[PMEM_MAX_ZONES] __bootdata; /* * Boundaries of the simple bootstrap heap. @@ -228,17 +227,15 @@ biosmem_unregister_boot_data(phys_addr_t start, phys_addr_t end) return; } -#if DEBUG - printf("biosmem: unregister boot data: %llx:%llx\n", - (unsigned long long)biosmem_boot_data_array[i].start, - (unsigned long long)biosmem_boot_data_array[i].end); -#endif /* DEBUG */ + log_debug("biosmem: unregister boot data: %llx:%llx", + (unsigned long long)biosmem_boot_data_array[i].start, + (unsigned long long)biosmem_boot_data_array[i].end); biosmem_nr_boot_data--; - boot_memmove(&biosmem_boot_data_array[i], - &biosmem_boot_data_array[i + 1], - (biosmem_nr_boot_data - i) * sizeof(*biosmem_boot_data_array)); + memmove(&biosmem_boot_data_array[i], + &biosmem_boot_data_array[i + 1], + (biosmem_nr_boot_data - i) * sizeof(*biosmem_boot_data_array)); } static void __boot @@ -610,8 +607,8 @@ biosmem_setup_allocator(const struct multiboot_raw_info *mbi) end = vm_page_trunc((mbi->mem_upper + 1024) << 10); #ifndef __LP64__ - if (end > VM_PAGE_DIRECTMAP_LIMIT) { - end = VM_PAGE_DIRECTMAP_LIMIT; + if (end > PMEM_DIRECTMAP_LIMIT) { + end = PMEM_DIRECTMAP_LIMIT; } #endif /* __LP64__ */ @@ -665,46 +662,46 @@ biosmem_bootstrap(const struct multiboot_raw_info *mbi) biosmem_map_adjust(); phys_start = BIOSMEM_BASE; - phys_end = VM_PAGE_DMA_LIMIT; + phys_end = PMEM_DMA_LIMIT; error = biosmem_map_find_avail(&phys_start, &phys_end); if (error) { boot_panic(biosmem_panic_nozone_msg); } - biosmem_set_zone(VM_PAGE_ZONE_DMA, phys_start, phys_end); + biosmem_set_zone(PMEM_ZONE_DMA, phys_start, phys_end); - phys_start = VM_PAGE_DMA_LIMIT; -#ifdef VM_PAGE_DMA32_LIMIT - phys_end = VM_PAGE_DMA32_LIMIT; + phys_start = PMEM_DMA_LIMIT; +#ifdef PMEM_DMA32_LIMIT + phys_end = PMEM_DMA32_LIMIT; error = biosmem_map_find_avail(&phys_start, &phys_end); if (error) { goto out; } - biosmem_set_zone(VM_PAGE_ZONE_DMA32, phys_start, phys_end); + biosmem_set_zone(PMEM_ZONE_DMA32, phys_start, phys_end); - phys_start = VM_PAGE_DMA32_LIMIT; -#endif /* VM_PAGE_DMA32_LIMIT */ - phys_end = VM_PAGE_DIRECTMAP_LIMIT; + phys_start = PMEM_DMA32_LIMIT; +#endif /* PMEM_DMA32_LIMIT */ + phys_end = PMEM_DIRECTMAP_LIMIT; error = biosmem_map_find_avail(&phys_start, &phys_end); if (error) { goto out; } - biosmem_set_zone(VM_PAGE_ZONE_DIRECTMAP, phys_start, phys_end); + biosmem_set_zone(PMEM_ZONE_DIRECTMAP, phys_start, phys_end); - phys_start = VM_PAGE_DIRECTMAP_LIMIT; - phys_end = VM_PAGE_HIGHMEM_LIMIT; + phys_start = PMEM_DIRECTMAP_LIMIT; + phys_end = PMEM_HIGHMEM_LIMIT; error = biosmem_map_find_avail(&phys_start, &phys_end); if (error) { goto out; } - biosmem_set_zone(VM_PAGE_ZONE_HIGHMEM, phys_start, phys_end); + biosmem_set_zone(PMEM_ZONE_HIGHMEM, phys_start, phys_end); out: biosmem_setup_allocator(mbi); @@ -754,17 +751,15 @@ biosmem_get_bda(void) phys_addr_t __boot biosmem_directmap_end(void) { - if (biosmem_zone_size(VM_PAGE_ZONE_DIRECTMAP) != 0) { - return biosmem_zone_end(VM_PAGE_ZONE_DIRECTMAP); - } else if (biosmem_zone_size(VM_PAGE_ZONE_DMA32) != 0) { - return biosmem_zone_end(VM_PAGE_ZONE_DMA32); + if (biosmem_zone_size(PMEM_ZONE_DIRECTMAP) != 0) { + return biosmem_zone_end(PMEM_ZONE_DIRECTMAP); + } else if (biosmem_zone_size(PMEM_ZONE_DMA32) != 0) { + return biosmem_zone_end(PMEM_ZONE_DMA32); } else { - return biosmem_zone_end(VM_PAGE_ZONE_DMA); + return biosmem_zone_end(PMEM_ZONE_DMA); } } -#if DEBUG - static const char * __init biosmem_type_desc(unsigned int type) { @@ -789,24 +784,21 @@ biosmem_map_show(void) { const struct biosmem_map_entry *entry, *end; - printf("biosmem: physical memory map:\n"); + log_debug("biosmem: physical memory map:"); for (entry = biosmem_map, end = entry + biosmem_map_size; entry < end; entry++) - printf("biosmem: %018llx:%018llx, %s\n", entry->base_addr, - entry->base_addr + entry->length, - biosmem_type_desc(entry->type)); - - printf("biosmem: heap: %llx:%llx\n", - (unsigned long long)biosmem_heap_start, - (unsigned long long)biosmem_heap_end); + log_debug("biosmem: %018llx:%018llx, %s", + (unsigned long long)entry->base_addr, + (unsigned long long)(entry->base_addr + entry->length), + biosmem_type_desc(entry->type)); + + log_debug("biosmem: heap: %llx:%llx", + (unsigned long long)biosmem_heap_start, + (unsigned long long)biosmem_heap_end); } -#else /* DEBUG */ -#define biosmem_map_show() -#endif /* DEBUG */ - static void __init biosmem_load_zone(struct biosmem_zone *zone, uint64_t max_phys_end) { @@ -819,13 +811,14 @@ biosmem_load_zone(struct biosmem_zone *zone, uint64_t max_phys_end) if (phys_end > max_phys_end) { if (max_phys_end <= phys_start) { - printf("biosmem: warning: zone %s physically unreachable, " - "not loaded\n", vm_page_zone_name(zone_index)); + log_warning("biosmem: zone %s physically unreachable, " + "not loaded", vm_page_zone_name(zone_index)); return; } - printf("biosmem: warning: zone %s truncated to %#llx\n", - vm_page_zone_name(zone_index), (unsigned long long)max_phys_end); + log_warning("biosmem: warning: zone %s truncated to %#llx", + vm_page_zone_name(zone_index), + (unsigned long long)max_phys_end); phys_end = max_phys_end; } @@ -853,7 +846,7 @@ biosmem_load_zone(struct biosmem_zone *zone, uint64_t max_phys_end) } } -void __init +static int __init biosmem_setup(void) { uint64_t max_phys_end; @@ -876,8 +869,14 @@ biosmem_setup(void) zone = &biosmem_zones[i]; biosmem_load_zone(zone, max_phys_end); } + + return 0; } +INIT_OP_DEFINE(biosmem_setup, + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(log_setup, true)); + static void __init biosmem_unregister_temporary_boot_data(void) { @@ -901,11 +900,9 @@ biosmem_free_usable_range(phys_addr_t start, phys_addr_t end) { struct vm_page *page; -#if DEBUG - printf("biosmem: release to vm_page: %llx:%llx (%lluk)\n", - (unsigned long long)start, (unsigned long long)end, - (unsigned long long)((end - start) >> 10)); -#endif + log_debug("biosmem: release to vm_page: %llx:%llx (%lluk)", + (unsigned long long)start, (unsigned long long)end, + (unsigned long long)((end - start) >> 10)); while (start < end) { page = vm_page_lookup(start); @@ -933,7 +930,7 @@ biosmem_free_usable_entry(phys_addr_t start, phys_addr_t end) } } -void __init +static int __init biosmem_free_usable(void) { struct biosmem_map_entry *entry; @@ -951,14 +948,14 @@ biosmem_free_usable(void) start = vm_page_round(entry->base_addr); - if (start >= VM_PAGE_HIGHMEM_LIMIT) { + if (start >= PMEM_HIGHMEM_LIMIT) { break; } end = vm_page_trunc(entry->base_addr + entry->length); - if (end > VM_PAGE_HIGHMEM_LIMIT) { - end = VM_PAGE_HIGHMEM_LIMIT; + if (end > PMEM_HIGHMEM_LIMIT) { + end = PMEM_HIGHMEM_LIMIT; } if (start < BIOSMEM_BASE) { @@ -971,4 +968,12 @@ biosmem_free_usable(void) biosmem_free_usable_entry(start, end); } + + return 0; } + +INIT_OP_DEFINE(biosmem_free_usable, + INIT_OP_DEP(boot_save_data, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(vm_page_setup, true)); diff --git a/arch/x86/machine/biosmem.h b/arch/x86/machine/biosmem.h index d877af4c..2a701dec 100644 --- a/arch/x86/machine/biosmem.h +++ b/arch/x86/machine/biosmem.h @@ -20,6 +20,7 @@ #include <stdbool.h> +#include <kern/init.h> #include <machine/multiboot.h> #include <machine/types.h> @@ -53,9 +54,9 @@ * Once all boot data have been registered, the user can set up the * early page allocator. * - * If the range is marked temporary, it will be unregistered when - * biosmem_free_usable() is called, so that pages that used to store - * these boot data may be released to the VM system. + * If the range is marked temporary, it will be unregistered once + * the boot data have been saved/consumed so that their backing + * pages are loaded into the VM system. * * This function is called before paging is enabled. */ @@ -102,17 +103,9 @@ const void * biosmem_get_bda(void); phys_addr_t biosmem_directmap_end(void); /* - * Set up physical memory based on the information obtained during bootstrap - * and load it in the VM system. + * This init operation provides : + * - heaps of physical memory are loaded by the VM system */ -void biosmem_setup(void); - -/* - * Free all usable memory. - * - * This function releases all pages that aren't used by boot data and have - * not already been loaded into the VM system. - */ -void biosmem_free_usable(void); +INIT_OP_DECLARE(biosmem_setup); #endif /* _X86_BIOSMEM_H */ diff --git a/arch/x86/machine/boot.c b/arch/x86/machine/boot.c index b7a9b0ee..25e92ff6 100644 --- a/arch/x86/machine/boot.c +++ b/arch/x86/machine/boot.c @@ -42,27 +42,20 @@ * to "enabling paging" do not refer to this initial identity mapping. */ -#include <stdbool.h> +#include <stdalign.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> #include <string.h> #include <kern/arg.h> -#include <kern/console.h> #include <kern/init.h> -#include <kern/intr.h> #include <kern/kmem.h> #include <kern/kernel.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> -#include <kern/percpu.h> -#include <kern/sleepq.h> -#include <kern/sref.h> -#include <kern/syscnt.h> #include <kern/thread.h> -#include <kern/turnstile.h> +#include <machine/acpi.h> #include <machine/atcons.h> #include <machine/biosmem.h> #include <machine/boot.h> @@ -70,27 +63,45 @@ #include <machine/cpu.h> #include <machine/elf.h> #include <machine/multiboot.h> -#include <machine/pic.h> -#include <machine/pit.h> +#include <machine/page.h> #include <machine/pmap.h> #include <machine/strace.h> -#include <machine/trap.h> #include <machine/uart.h> #include <vm/vm_kmem.h> -#include <vm/vm_setup.h> -char boot_stack[STACK_SIZE] __aligned(DATA_ALIGN) __bootdata; -char boot_ap_stack[STACK_SIZE] __aligned(DATA_ALIGN) __bootdata; +alignas(CPU_DATA_ALIGN) char boot_stack[BOOT_STACK_SIZE] __bootdata; +alignas(CPU_DATA_ALIGN) char boot_ap_stack[BOOT_STACK_SIZE] __bootdata; unsigned int boot_ap_id __bootdata; #ifdef __LP64__ -pmap_pte_t boot_pml4[PMAP_L3_PTES_PER_PT] __aligned(PAGE_SIZE) __bootdata; -pmap_pte_t boot_pdpt[PMAP_L2_PTES_PER_PT] __aligned(PAGE_SIZE) __bootdata; -pmap_pte_t boot_pdir[4 * PMAP_L1_PTES_PER_PT] __aligned(PAGE_SIZE) __bootdata; +alignas(PAGE_SIZE) pmap_pte_t boot_pml4[PMAP_L3_PTES_PER_PT] __bootdata; +alignas(PAGE_SIZE) pmap_pte_t boot_pdpt[PMAP_L2_PTES_PER_PT] __bootdata; +alignas(PAGE_SIZE) pmap_pte_t boot_pdir[4 * PMAP_L1_PTES_PER_PT] __bootdata; char boot_panic_long_mode_msg[] __bootdata = "boot: processor doesn't support long mode"; #endif /* __LP64__ */ +struct cpu_tls_seg boot_tls_seg __bootdata = { + .ssp_guard_word = SSP_GUARD_WORD, +}; + +/* + * This TLS segment descriptor is copied early at boot time into the + * temporary boot GDT. However, it is incomplete. Assembly code + * completes it before writing it into the boot GDT before calling + * any C function, since those may require the TLS segment ready if + * stack protection is enabled. + */ +struct cpu_seg_desc boot_tls_seg_desc __bootdata = { + .low = (sizeof(boot_tls_seg) & 0xffff), + + .high = CPU_DESC_DB + | ((sizeof(boot_tls_seg) >> 16) & 0xf) + | CPU_DESC_PRESENT + | CPU_DESC_S + | CPU_DESC_TYPE_DATA, +}; + /* * Copies of the multiboot data passed by the boot loader. */ @@ -118,7 +129,9 @@ boot_memcpy(void *dest, const void *src, size_t n) src_ptr = src; for (i = 0; i < n; i++) { - *dest_ptr++ = *src_ptr++; + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; } return dest; @@ -136,14 +149,18 @@ boot_memmove(void *dest, const void *src, size_t n) src_ptr = src; for (i = 0; i < n; i++) { - *dest_ptr++ = *src_ptr++; + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; } } else { dest_ptr = dest + n - 1; src_ptr = src + n - 1; for (i = 0; i < n; i++) { - *dest_ptr-- = *src_ptr--; + *dest_ptr = *src_ptr; + dest_ptr--; + src_ptr--; } } @@ -168,15 +185,15 @@ boot_memset(void *s, int c, size_t n) size_t __boot boot_strlen(const char *s) { - size_t i; + const char *start; - i = 0; + start = s; - while (*s++ != '\0') { - i++; + while (*s != '\0') { + s++; } - return i; + return (s - start); } void __boot @@ -191,17 +208,22 @@ boot_panic(const char *msg) s = boot_panic_intro_msg; while ((ptr < end) && (*s != '\0')) { - *ptr++ = (BOOT_CGACOLOR << 8) | *s++; + *ptr = (BOOT_CGACOLOR << 8) | *s; + ptr++; + s++; } s = msg; while ((ptr < end) && (*s != '\0')) { - *ptr++ = (BOOT_CGACOLOR << 8) | *s++; + *ptr = (BOOT_CGACOLOR << 8) | *s; + ptr++; + s++; } while (ptr < end) { - *ptr++ = (BOOT_CGACOLOR << 8) | ' '; + *ptr = (BOOT_CGACOLOR << 8) | ' '; + ptr++; } cpu_halt(); @@ -326,14 +348,14 @@ boot_setup_paging(struct multiboot_raw_info *mbi, unsigned long eax) return pmap_setup_paging(); } -static void __init -boot_show_version(void) +void __init +boot_log_info(void) { - printf(KERNEL_NAME "/" QUOTE(X15_X86_MACHINE) " " KERNEL_VERSION + log_info(KERNEL_NAME "/" QUOTE(X15_X86_MACHINE) " " KERNEL_VERSION #ifdef X15_X86_PAE - " PAE" + " PAE" #endif /* X15_X86_PAE */ - "\n"); + ); } static void * __init @@ -448,46 +470,24 @@ boot_save_mods(void) * the boot loader. Once the boot data are managed as kernel buffers, their * backing pages can be freed. */ -static void __init +static int __init boot_save_data(void) { boot_mbi.flags = boot_raw_mbi.flags; boot_save_mods(); - strace_setup(&boot_raw_mbi); + return 0; } +INIT_OP_DEFINE(boot_save_data, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(vm_kmem_setup, true)); + void __init boot_main(void) { - arg_setup(boot_tmp_cmdline); - sleepq_bootstrap(); - turnstile_bootstrap(); - syscnt_setup(); - percpu_bootstrap(); - trap_setup(); - pit_setup_free_running(); - cpu_setup(); - thread_bootstrap(); - console_setup(); - atcons_bootstrap(); - uart_bootstrap(); - printf_setup(); - boot_show_version(); - arg_info(); - uart_info(); - pmap_bootstrap(); - sref_bootstrap(); - cpu_check(cpu_current()); - cpu_info(cpu_current()); - biosmem_setup(); - vm_setup(); - boot_save_data(); - biosmem_free_usable(); - intr_setup(); - cpu_mp_probe(); - pic_setup(); - atcons_setup(); - uart_setup(); + arg_set_cmdline(boot_tmp_cmdline); + strace_set_mbi(&boot_raw_mbi); kernel_main(); /* Never reached */ @@ -497,9 +497,61 @@ void __init boot_ap_main(void) { cpu_ap_setup(); - thread_ap_bootstrap(); - pmap_ap_bootstrap(); + thread_ap_setup(); + pmap_ap_setup(); kernel_ap_main(); /* Never reached */ } + +/* + * Init operation aliases. + */ + +static int __init +boot_bootstrap_console(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_bootstrap_console, + INIT_OP_DEP(atcons_bootstrap, true), + INIT_OP_DEP(uart_bootstrap, true)); + +static int __init +boot_setup_console(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_setup_console, + INIT_OP_DEP(atcons_setup, true), + INIT_OP_DEP(uart_setup, true)); + +static int __init +boot_load_vm_page_zones(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_load_vm_page_zones, + INIT_OP_DEP(biosmem_setup, true)); + +static int __init +boot_setup_intr(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_setup_intr, + INIT_OP_DEP(acpi_setup, true)); + +static int __init +boot_setup_shutdown(void) +{ + return 0; +} + +INIT_OP_DEFINE(boot_setup_shutdown, + INIT_OP_DEP(acpi_setup, true), + INIT_OP_DEP(cpu_setup_shutdown, true)); diff --git a/arch/x86/machine/boot.h b/arch/x86/machine/boot.h index bd5eb2da..c85c2648 100644 --- a/arch/x86/machine/boot.h +++ b/arch/x86/machine/boot.h @@ -18,8 +18,16 @@ #ifndef _X86_BOOT_H #define _X86_BOOT_H +#include <stdnoreturn.h> + #include <kern/macros.h> -#include <machine/param.h> +#include <machine/page.h> +#include <machine/pmap.h> + +/* + * Size of the stack used when booting a processor. + */ +#define BOOT_STACK_SIZE PAGE_SIZE /* * Macros used by the very early panic functions. @@ -31,7 +39,7 @@ /* * The kernel is physically loaded at BOOT_OFFSET by the boot loader. It * is divided in two parts: the .boot section which uses physical addresses - * and the main kernel code and data at VM_KERNEL_OFFSET. + * and the main kernel code and data at PMAP_KERNEL_OFFSET. * * See the linker script for more information. */ @@ -40,7 +48,7 @@ /* * Virtual to physical address translation macro. */ -#define BOOT_VTOP(addr) ((addr) - VM_KERNEL_OFFSET) +#define BOOT_VTOP(addr) ((addr) - PMAP_KERNEL_OFFSET) /* * Address where the MP trampoline code is copied and run at. @@ -52,6 +60,7 @@ #ifndef __ASSEMBLER__ +#include <kern/init.h> #include <machine/multiboot.h> #include <machine/pmap.h> @@ -69,9 +78,6 @@ extern char _boot; extern char _boot_end; -extern char boot_stack[STACK_SIZE]; -extern char boot_ap_stack[STACK_SIZE]; - /* * This variable contains the CPU ID of an AP during early initialization. */ @@ -96,7 +102,7 @@ void * boot_memcpy(void *dest, const void *src, size_t n); void * boot_memmove(void *dest, const void *src, size_t n); void * boot_memset(void *s, int c, size_t n); size_t boot_strlen(const char *s); -void __noreturn boot_panic(const char *s); +noreturn void boot_panic(const char *s); /* * This function is called by the bootstrap code before paging is enabled. @@ -117,6 +123,47 @@ void boot_main(void); */ void boot_ap_main(void); +/* + * Log kernel version and other architecture-specific information. + */ +void boot_log_info(void); + +/* + * This init operation provides : + * - boot data are saved + */ +INIT_OP_DECLARE(boot_save_data); + +/* + * This init operation provides : + * - all console devices are bootstrapped + */ +INIT_OP_DECLARE(boot_bootstrap_console); + +/* + * This init operation provides : + * - all console devices are fully initialized + */ +INIT_OP_DECLARE(boot_setup_console); + +/* + * This init operation provides : + * - physical memory has been loaded to the VM system + */ +INIT_OP_DECLARE(boot_load_vm_page_zones); + +/* + * This init operation provides : + * - all interrupt controllers have been registered + */ +INIT_OP_DECLARE(boot_setup_intr); + +/* + * This init operation provides : + * - all shutdown operations have been registered + */ +INIT_OP_DECLARE(boot_setup_shutdown); + #endif /* __ASSEMBLER__ */ #endif /* _X86_BOOT_H */ diff --git a/arch/x86/machine/boot_asm.S b/arch/x86/machine/boot_asm.S index 3015b1fd..0aaf342e 100644 --- a/arch/x86/machine/boot_asm.S +++ b/arch/x86/machine/boot_asm.S @@ -19,10 +19,19 @@ #include <machine/cpu.h> #include <machine/boot.h> #include <machine/multiboot.h> -#include <machine/param.h> +#include <machine/page.h> #include <machine/pmap.h> /* + * Boot GDT segment selectors. + */ +#define BOOT_GDT_SEL_NULL 0 +#define BOOT_GDT_SEL_CODE 8 +#define BOOT_GDT_SEL_DATA 16 +#define BOOT_GDT_SEL_CODE64 24 +#define BOOT_GDT_SEL_TLS 24 + +/* * Convert a physical address in the .boot section to its real address in * the MP trampoline code. */ @@ -48,27 +57,28 @@ ASM_ENTRY(_start) lgdt boot_gdtr /* Keep %eax and %ebx */ - movl $0x10, %ecx + movl $BOOT_GDT_SEL_DATA, %ecx movl %ecx, %ds movl %ecx, %es movl %ecx, %ss xorl %ecx, %ecx movl %ecx, %fs movl %ecx, %gs - ljmp $8, $1f + ljmp $BOOT_GDT_SEL_CODE, $1f 1: - movl $(boot_stack + STACK_SIZE), %esp + movl $(boot_stack + BOOT_STACK_SIZE), %esp #ifdef __LP64__ call boot_check_long_mode + call boot_setup_tls call boot_setup_long_mode /* * At this point, the processor runs in long mode, but still uses the * compatibility mode code segment. Switch to 64-bit mode with a far return. */ - pushl $0x18 + pushl $BOOT_GDT_SEL_CODE64 pushl $1f lret @@ -77,6 +87,10 @@ ASM_ENTRY(_start) movl %ebx, %edi movl %eax, %esi #else /* __LP64__ */ + call boot_build_tls_seg_desc + movl $BOOT_GDT_SEL_TLS, %ecx + movl %ecx, %gs + pushl %eax pushl %ebx #endif /* __LP64__ */ @@ -85,16 +99,16 @@ ASM_ENTRY(_start) #ifdef __LP64__ movq %rax, %cr3 - movq $(boot_stack + STACK_SIZE), %rsp + movq $(boot_stack + BOOT_STACK_SIZE), %rsp #else /* __LP64__ */ movl %eax, %cr3 movl %cr0, %eax orl $CPU_CR0_PG, %eax movl %eax, %cr0 - ljmp $8, $1f + ljmp $BOOT_GDT_SEL_CODE, $1f 1: - movl $(boot_stack + STACK_SIZE), %esp + movl $(boot_stack + BOOT_STACK_SIZE), %esp #endif /* __LP64__ */ xorl %ebp, %ebp @@ -205,12 +219,68 @@ ASM_ENTRY(boot_setup_long_mode) movl %cr0, %eax orl $CPU_CR0_PG, %eax movl %eax, %cr0 - ljmp $8, $1f + ljmp $BOOT_GDT_SEL_CODE, $1f 1: movl %edi, %eax ret ASM_END(boot_setup_long_mode) + +ASM_ENTRY(boot_setup_tls) + pushl %eax + movl $boot_tls_seg, %eax + movl $0, %edx + movl $CPU_MSR_GSBASE, %ecx + wrmsr + popl %eax + ret +ASM_END(boot_setup_tls) + +#else /* __LP64__ */ + +/* + * Complete the temporary boot TLS segment descriptor and copy it into + * the boot GDT. + * + * The %eax and %ebx registers must be preserved. + */ +ASM_ENTRY(boot_build_tls_seg_desc) + pushl %eax + + /* Load boot_tls_seg_desc address */ + movl $boot_tls_seg_desc, %ecx + + /* Add the base 15:0 bits to boot_tls_seg_desc.low */ + movl $boot_tls_seg, %eax + shll $16, %eax + orl %eax, (%ecx) + + /* Add the base 23:16 bits to boot_tls_seg_desc.high */ + movl $boot_tls_seg, %eax + shrl $16, %eax + andl $0xff, %eax + orl %eax, 4(%ecx) + + /* Add the base 31:24 bits to boot_tls_seg_desc.high */ + movl $boot_tls_seg, %eax + andl $0xff000000, %eax + orl %eax, 4(%ecx) + + /* Load boot_gdt[BOOT_GDT_SEL_TLS] address */ + movl $boot_gdt + BOOT_GDT_SEL_TLS, %edx + + /* Set boot_gdt[BOOT_GDT_SEL_TLS].low */ + movl (%ecx), %eax + movl %eax, (%edx) + + /* Set boot_gdt[BOOT_GDT_SEL_TLS].high */ + movl 4(%ecx), %eax + movl %eax, 4(%edx) + + popl %eax + ret +ASM_END(boot_build_tls_seg_desc) + #endif /* __LP64__ */ /* @@ -223,21 +293,23 @@ ASM_ENTRY(boot_ap_start32) * is enabled. */ lgdt boot_gdtr - movl $0x10, %eax + movl $BOOT_GDT_SEL_DATA, %eax movl %eax, %ds movl %eax, %es movl %eax, %ss xorl %eax, %eax movl %eax, %fs + movl $BOOT_GDT_SEL_TLS, %eax movl %eax, %gs - ljmp $8, $1f + ljmp $BOOT_GDT_SEL_CODE, $1f 1: - movl $(boot_ap_stack + STACK_SIZE), %esp + movl $(boot_ap_stack + BOOT_STACK_SIZE), %esp #ifdef __LP64__ + call boot_setup_tls call boot_setup_long_mode - pushl $0x18 + pushl $BOOT_GDT_SEL_CODE64 pushl $1f lret @@ -254,7 +326,7 @@ ASM_ENTRY(boot_ap_start32) movl %cr0, %eax orl $CPU_CR0_PG, %eax movl %eax, %cr0 - ljmp $8, $1f + ljmp $BOOT_GDT_SEL_CODE, $1f 1: #endif /* __LP64__ */ @@ -263,10 +335,10 @@ ASM_ENTRY(boot_ap_start32) #ifdef __LP64__ movq %rax, %rsp - addq $STACK_SIZE, %rsp + addq $BOOT_STACK_SIZE, %rsp #else /* __LP64__ */ movl %eax, %esp - addl $STACK_SIZE, %esp + addl $BOOT_STACK_SIZE, %esp #endif /* __LP64__ */ xorl %ebp, %ebp @@ -296,19 +368,19 @@ ASM_ENTRY(boot_mp_trampoline) movl %cr0, %eax orl $CPU_CR0_PE, %eax movl %eax, %cr0 - ljmp $8, $BOOT_MP_ADDR_PTOT(1f) + ljmp $BOOT_GDT_SEL_CODE, $BOOT_MP_ADDR_PTOT(1f) .align 4 1: .code32 - movl $0x10, %eax + movl $BOOT_GDT_SEL_DATA, %eax movl %eax, %ds movl %eax, %es movl %eax, %ss xorl %eax, %eax movl %eax, %fs movl %eax, %gs - ljmp $8, $boot_ap_start32 + ljmp $BOOT_GDT_SEL_CODE, $boot_ap_start32 ASM_END(boot_mp_trampoline) ASM_DATA(boot_ap_gdtr) @@ -317,12 +389,13 @@ ASM_DATA(boot_ap_gdtr) ASM_END(boot_ap_gdtr) ASM_DATA(boot_gdt) - .quad 0x0000000000000000 /* Null selector */ - .quad 0x00cf9a000000ffff /* Code segment selector */ - .quad 0x00cf92000000ffff /* Data segment selector */ - + .quad 0x0000000000000000 /* Null descriptor */ + .quad 0x00cf9a000000ffff /* Code segment descriptor */ + .quad 0x00cf92000000ffff /* Data segment descriptor */ #ifdef __LP64__ .quad 0x00209a0000000000 /* 64-bit code segment selector */ +#else /* __LP64__ */ + .quad 0x0 /* TLS segment descriptor, filled at boot time */ #endif /* __LP64__ */ ASM_END(boot_gdt) boot_gdt_end: diff --git a/arch/x86/machine/cga.c b/arch/x86/machine/cga.c index 1ec80354..b519ba1f 100644 --- a/arch/x86/machine/cga.c +++ b/arch/x86/machine/cga.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2014 Richard Braun. + * Copyright (c) 2010-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,12 +15,16 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdbool.h> #include <stdint.h> #include <string.h> +#include <kern/console.h> +#include <kern/error.h> #include <kern/init.h> +#include <kern/cbuf.h> #include <kern/macros.h> -#include <kern/param.h> #include <machine/io.h> #include <machine/cga.h> #include <vm/vm_page.h> @@ -34,52 +38,90 @@ /* * Text mode mapped memory and size. */ -#define CGA_MEMORY 0xb8000 -#define CGA_MEMORY_SIZE (CGA_COLUMNS * CGA_LINES * 2) +#define CGA_MEMORY 0xb8000 +#define CGA_MEMORY_LINE_SIZE (CGA_COLUMNS * 2) +#define CGA_MEMORY_SIZE (CGA_MEMORY_LINE_SIZE * CGA_LINES) /* * I/O ports. */ -#define CGA_MISC_OUTPUT_REGISTER_READ 0x3cc -#define CGA_MISC_OUTPUT_REGISTER_WRITE 0x3c2 -#define CGA_CRTC_ADDRESS_REGISTER 0x3d4 -#define CGA_CRTC_DATA_REGISTER 0x3d5 +#define CGA_PORT_MISC_OUT_READ 0x3cc +#define CGA_PORT_MISC_OUT_WRITE 0x3c2 +#define CGA_PORT_CRTC_ADDR 0x3d4 +#define CGA_PORT_CRTC_DATA 0x3d5 + +/* + * Miscellaneous output register bits. + */ +#define CGA_MISC_OUT_IOAS 0x1 /* * CRTC registers. */ -#define CGA_CRTC_CURSOR_LOCATION_HIGH_REGISTER 0xe -#define CGA_CRTC_CURSOR_LOCATION_LOW_REGISTER 0xf +#define CGA_CRTC_CURSOR_START_REG 0xa +#define CGA_CRTC_CURSOR_LOC_HIGH_REG 0xe +#define CGA_CRTC_CURSOR_LOC_LOW_REG 0xf /* - * Foreground screen color. + * Cursor start register bits. */ -#define CGA_FOREGROUND_COLOR 0x7 +#define CGA_CSR_DISABLED 0x10 /* - * Blank space 16 bits word. + * Foreground screen color. */ -#define CGA_BLANK ((CGA_FOREGROUND_COLOR << 8) | ' ') +#define CGA_FOREGROUND_COLOR 0x7 /* * Number of spaces to display for a tabulation. */ #define CGA_TABULATION_SPACES 8 -static uint8_t *cga_memory __read_mostly; -static uint16_t cga_cursor; +static void *cga_memory __read_mostly; + +#define CGA_BACK_BUFFER_SIZE (64 * 1024) + +#if CGA_BACK_BUFFER_SIZE < CGA_MEMORY_SIZE +#error "back buffer size must be at least as large as video memory" +#endif + +#define CGA_SCROLL_PAGE ((CGA_LINES / 2) * CGA_MEMORY_LINE_SIZE) + +/* + * Back buffer. + */ +struct cga_bbuf { + struct cbuf cbuf; + size_t view; + size_t cursor; + char buf[CGA_BACK_BUFFER_SIZE]; + bool cursor_enabled; +}; + +static struct cga_bbuf cga_bbuf; + +static uint16_t +cga_build_cell(char c) +{ + return (CGA_FOREGROUND_COLOR << 8) | c; +} + +static void +cga_write(size_t index, const void *ptr, size_t size) +{ + assert((index + size) <= CGA_MEMORY_SIZE); + memcpy(cga_memory + index, ptr, size); +} static uint16_t cga_get_cursor_position(void) { uint16_t tmp; - io_write_byte(CGA_CRTC_ADDRESS_REGISTER, - CGA_CRTC_CURSOR_LOCATION_HIGH_REGISTER); - tmp = io_read_byte(CGA_CRTC_DATA_REGISTER) << 8; - io_write_byte(CGA_CRTC_ADDRESS_REGISTER, - CGA_CRTC_CURSOR_LOCATION_LOW_REGISTER); - tmp |= io_read_byte(CGA_CRTC_DATA_REGISTER); + io_write_byte(CGA_PORT_CRTC_ADDR, CGA_CRTC_CURSOR_LOC_HIGH_REG); + tmp = io_read_byte(CGA_PORT_CRTC_DATA) << 8; + io_write_byte(CGA_PORT_CRTC_ADDR, CGA_CRTC_CURSOR_LOC_LOW_REG); + tmp |= io_read_byte(CGA_PORT_CRTC_DATA); return tmp; } @@ -87,97 +129,344 @@ cga_get_cursor_position(void) static void cga_set_cursor_position(uint16_t position) { - io_write_byte(CGA_CRTC_ADDRESS_REGISTER, - CGA_CRTC_CURSOR_LOCATION_HIGH_REGISTER); - io_write_byte(CGA_CRTC_DATA_REGISTER, position >> 8); - io_write_byte(CGA_CRTC_ADDRESS_REGISTER, - CGA_CRTC_CURSOR_LOCATION_LOW_REGISTER); - io_write_byte(CGA_CRTC_DATA_REGISTER, position & 0xff); + io_write_byte(CGA_PORT_CRTC_ADDR, CGA_CRTC_CURSOR_LOC_HIGH_REG); + io_write_byte(CGA_PORT_CRTC_DATA, position >> 8); + io_write_byte(CGA_PORT_CRTC_ADDR, CGA_CRTC_CURSOR_LOC_LOW_REG); + io_write_byte(CGA_PORT_CRTC_DATA, position & 0xff); } -static uint8_t -cga_get_cursor_column(void) +static void +cga_enable_cursor(void) { - return cga_cursor % CGA_COLUMNS; + uint8_t tmp; + + io_write_byte(CGA_PORT_CRTC_ADDR, CGA_CRTC_CURSOR_START_REG); + tmp = io_read_byte(CGA_PORT_CRTC_DATA); + io_write_byte(CGA_PORT_CRTC_DATA, tmp & ~CGA_CSR_DISABLED); } static void -cga_scroll_lines(void) +cga_disable_cursor(void) { - uint16_t *last_line; - int i; + uint8_t tmp; - memmove(cga_memory, (uint16_t *)cga_memory + CGA_COLUMNS, - CGA_MEMORY_SIZE - (CGA_COLUMNS * 2)); - last_line = (uint16_t *)cga_memory + (CGA_COLUMNS * (CGA_LINES - 1)); + io_write_byte(CGA_PORT_CRTC_ADDR, CGA_CRTC_CURSOR_START_REG); + tmp = io_read_byte(CGA_PORT_CRTC_DATA); + io_write_byte(CGA_PORT_CRTC_DATA, tmp | CGA_CSR_DISABLED); +} + +static void +cga_bbuf_init(struct cga_bbuf *bbuf, size_t cursor) +{ + cbuf_init(&bbuf->cbuf, bbuf->buf, sizeof(bbuf->buf)); + bbuf->view = cbuf_start(&bbuf->cbuf); + cbuf_write(&bbuf->cbuf, bbuf->view, cga_memory, CGA_MEMORY_SIZE); + bbuf->cursor = bbuf->view + cursor; + bbuf->cursor_enabled = true; +} + +static size_t +cga_bbuf_cursor_offset(const struct cga_bbuf *bbuf) +{ + return bbuf->cursor - bbuf->view; +} - for(i = 0; i < CGA_COLUMNS; i++) { - last_line[i] = CGA_BLANK; +static int +cga_bbuf_get_phys_cursor(const struct cga_bbuf *bbuf, uint16_t *cursorp) +{ + uint16_t cursor; + + cursor = cga_bbuf_cursor_offset(bbuf); + assert((cursor & 1) == 0); + + if (cursor >= CGA_MEMORY_SIZE) { + return ERROR_NODEV; } + + *cursorp = (cursor >> 1); + return 0; } -void -cga_putc(char c) +static void +cga_bbuf_update_phys_cursor(struct cga_bbuf *bbuf) { - if (c == '\r') { - return; - } else if (c == '\n') { - cga_cursor += CGA_COLUMNS - cga_get_cursor_column(); + bool cursor_enabled; + uint16_t cursor = 0; + int error; - if (cga_cursor >= (CGA_LINES * CGA_COLUMNS)) { - cga_scroll_lines(); - cga_cursor -= CGA_COLUMNS; - } + error = cga_bbuf_get_phys_cursor(bbuf, &cursor); + cursor_enabled = !error; - cga_set_cursor_position(cga_cursor); - } else if (c == '\b') { - if (cga_cursor > 0) { - cga_cursor--; - ((uint16_t *)cga_memory)[cga_cursor] = CGA_BLANK; - cga_set_cursor_position(cga_cursor); - } - } else if (c == '\t') { - int i; + if (cursor_enabled != bbuf->cursor_enabled) { + bbuf->cursor_enabled = cursor_enabled; - for(i = 0; i < CGA_TABULATION_SPACES; i++) { - cga_putc(' '); - } - } else { - if ((cga_cursor + 1) >= CGA_COLUMNS * CGA_LINES) { - cga_scroll_lines(); - cga_cursor -= CGA_COLUMNS; + if (cursor_enabled) { + cga_enable_cursor(); + } else { + cga_disable_cursor(); } + } - ((uint16_t *)cga_memory)[cga_cursor] = ((CGA_FOREGROUND_COLOR << 8) - | c); - cga_cursor++; - cga_set_cursor_position(cga_cursor); + if (cursor_enabled) { + cga_set_cursor_position(cursor); } } -void __init -cga_setup(void) +static void +cga_bbuf_redraw(struct cga_bbuf *bbuf) { - uint8_t misc_output_register; + size_t size; + int error; - cga_memory = (void *)vm_page_direct_va(CGA_MEMORY); + size = CGA_MEMORY_SIZE; + error = cbuf_read(&bbuf->cbuf, bbuf->view, cga_memory, &size); + assert(!error); + assert(size == CGA_MEMORY_SIZE); + cga_bbuf_update_phys_cursor(bbuf); +} + +static bool +cga_bbuf_view_needs_scrolling(const struct cga_bbuf *bbuf) +{ + size_t view_size; + + view_size = cbuf_end(&bbuf->cbuf) - bbuf->view; + + if (view_size > CGA_MEMORY_SIZE) { + return true; + } + + /* Consider the cursor as a valid cell */ + view_size = (bbuf->cursor + 1) - bbuf->view; + + if (view_size > CGA_MEMORY_SIZE) { + return true; + } + + return false; +} + +static void +cga_bbuf_scroll_once(struct cga_bbuf *bbuf) +{ + uint16_t spaces[CGA_COLUMNS]; + size_t i; + + for (i = 0; i < ARRAY_SIZE(spaces); i++) { + spaces[i] = cga_build_cell(' '); + } + + cbuf_write(&bbuf->cbuf, cbuf_end(&bbuf->cbuf), spaces, sizeof(spaces)); + bbuf->view += sizeof(spaces); + cga_bbuf_redraw(bbuf); +} + +static void +cga_bbuf_reset_view(struct cga_bbuf *bbuf) +{ + if ((bbuf->view + CGA_MEMORY_SIZE) == cbuf_end(&bbuf->cbuf)) { + return; + } + + bbuf->view = cbuf_end(&bbuf->cbuf) - CGA_MEMORY_SIZE; + cga_bbuf_redraw(bbuf); +} + +static void +cga_bbuf_push(struct cga_bbuf *bbuf, char c) +{ + size_t offset; + uint16_t cell; + + cga_bbuf_reset_view(bbuf); + + cell = cga_build_cell(c); + cbuf_write(&bbuf->cbuf, bbuf->cursor, &cell, sizeof(cell)); + offset = cga_bbuf_cursor_offset(bbuf); + bbuf->cursor += sizeof(cell); + + if (cga_bbuf_view_needs_scrolling(bbuf)) { + cga_bbuf_scroll_once(bbuf); + } else { + cga_write(offset, &cell, sizeof(cell)); + cga_bbuf_update_phys_cursor(bbuf); + } +} + +static void +cga_bbuf_newline(struct cga_bbuf *bbuf) +{ + uint16_t cursor = 0, spaces[CGA_COLUMNS]; + size_t i, nr_spaces, offset, size; + int error; + + cga_bbuf_reset_view(bbuf); + + error = cga_bbuf_get_phys_cursor(bbuf, &cursor); + assert(!error); + + nr_spaces = CGA_COLUMNS - (cursor % CGA_COLUMNS); + + for (i = 0; i < nr_spaces; i++) { + spaces[i] = cga_build_cell(' '); + } /* - * Check if the Input/Output Address Select bit is set. + * The cursor may not point at the end of the view, in which case + * any existing data must be preserved. */ - misc_output_register = io_read_byte(CGA_MISC_OUTPUT_REGISTER_READ); + size = sizeof(spaces); + cbuf_read(&bbuf->cbuf, bbuf->cursor, spaces, &size); + + cbuf_write(&bbuf->cbuf, bbuf->cursor, + spaces, nr_spaces * sizeof(spaces[0])); + offset = cga_bbuf_cursor_offset(bbuf); + bbuf->cursor += nr_spaces * sizeof(spaces[0]); + + if (cga_bbuf_view_needs_scrolling(bbuf)) { + cga_bbuf_scroll_once(bbuf); + } else { + cga_write(offset, spaces, nr_spaces * sizeof(spaces[0])); + cga_bbuf_update_phys_cursor(bbuf); + } + + cga_bbuf_update_phys_cursor(bbuf); +} + +static void +cga_bbuf_move_cursor(struct cga_bbuf *bbuf, bool forward) +{ + cga_bbuf_reset_view(bbuf); + + if ((!forward && (bbuf->cursor == bbuf->view)) + || (forward && (bbuf->cursor == cbuf_end(&bbuf->cbuf)))) { + return; + } + + if (forward) { + bbuf->cursor += sizeof(uint16_t); + } else { + bbuf->cursor -= sizeof(uint16_t); + } + + cga_bbuf_update_phys_cursor(bbuf); +} + +static void +cga_bbuf_move_cursor_left(struct cga_bbuf *bbuf) +{ + cga_bbuf_move_cursor(bbuf, false); +} + +static void +cga_bbuf_move_cursor_right(struct cga_bbuf *bbuf) +{ + cga_bbuf_move_cursor(bbuf, true); +} + +static void +cga_bbuf_backspace(struct cga_bbuf *bbuf) +{ + cga_bbuf_move_cursor_left(bbuf); +} + +static void +cga_bbuf_scroll_up(struct cga_bbuf *bbuf) +{ + size_t start, size; - if (!(misc_output_register & 0x1)) { - /* - * Set the I/O AS bit. - */ - misc_output_register |= 0x1; + bbuf->view -= CGA_SCROLL_PAGE; - /* - * Update the misc output register. - */ - io_write_byte(CGA_MISC_OUTPUT_REGISTER_WRITE, misc_output_register); + /* The back buffer size is a power-of-two, not a line multiple */ + size = cbuf_size(&bbuf->cbuf); + size -= size % CGA_MEMORY_LINE_SIZE; + start = cbuf_end(&bbuf->cbuf) - size; + + if ((bbuf->view - start) >= size) { + bbuf->view = start; } - cga_cursor = cga_get_cursor_position(); + cga_bbuf_redraw(bbuf); +} + +static void +cga_bbuf_scroll_down(struct cga_bbuf *bbuf) +{ + size_t end; + + bbuf->view += CGA_SCROLL_PAGE; + end = bbuf->view + CGA_MEMORY_SIZE; + + if (!cbuf_range_valid(&bbuf->cbuf, bbuf->view, end)) { + bbuf->view = cbuf_end(&bbuf->cbuf) - CGA_MEMORY_SIZE; + } + + cga_bbuf_redraw(bbuf); +} + +void +cga_putc(char c) +{ + unsigned int i; + + switch (c) { + case '\r': + return; + case '\n': + cga_bbuf_newline(&cga_bbuf); + break; + case '\b': + cga_bbuf_backspace(&cga_bbuf); + break; + case '\t': + for(i = 0; i < CGA_TABULATION_SPACES; i++) { + cga_putc(' '); + } + + break; + case CONSOLE_SCROLL_UP: + cga_bbuf_scroll_up(&cga_bbuf); + break; + case CONSOLE_SCROLL_DOWN: + cga_bbuf_scroll_down(&cga_bbuf); + break; + default: + cga_bbuf_push(&cga_bbuf, c); + } +} + +static void __init +cga_setup_misc_out(void) +{ + uint8_t reg; + + reg = io_read_byte(CGA_PORT_MISC_OUT_READ); + + if (!(reg & CGA_MISC_OUT_IOAS)) { + reg |= CGA_MISC_OUT_IOAS; + io_write_byte(CGA_PORT_MISC_OUT_WRITE, reg); + } +} + +static int __init +cga_setup(void) +{ + cga_memory = (void *)vm_page_direct_va(CGA_MEMORY); + cga_setup_misc_out(); + cga_bbuf_init(&cga_bbuf, cga_get_cursor_position()); + return 0; +} + +INIT_OP_DEFINE(cga_setup); + +void +cga_cursor_left(void) +{ + cga_bbuf_move_cursor_left(&cga_bbuf); +} + +void +cga_cursor_right(void) +{ + cga_bbuf_move_cursor_right(&cga_bbuf); } diff --git a/arch/x86/machine/cga.h b/arch/x86/machine/cga.h index f5a5d285..2708ec4d 100644 --- a/arch/x86/machine/cga.h +++ b/arch/x86/machine/cga.h @@ -21,14 +21,23 @@ #ifndef _X86_CGA_H #define _X86_CGA_H -/* - * Initialize the cga module. - */ -void cga_setup(void); +#include <kern/init.h> /* * Append a character to the CGA screen. */ void cga_putc(char c); +/* + * Cursor control functions. + */ +void cga_cursor_left(void); +void cga_cursor_right(void); + +/* + * This init operation provides : + * - module fully initialized + */ +INIT_OP_DECLARE(cga_setup); + #endif /* _X86_CGA_H */ diff --git a/arch/x86/machine/cpu.c b/arch/x86/machine/cpu.c index eaa43ae7..98d3680e 100644 --- a/arch/x86/machine/cpu.c +++ b/arch/x86/machine/cpu.c @@ -15,27 +15,31 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> +#include <kern/shutdown.h> #include <kern/thread.h> #include <kern/xcall.h> -#include <machine/acpimp.h> +#include <machine/acpi.h> #include <machine/biosmem.h> #include <machine/boot.h> #include <machine/cpu.h> #include <machine/io.h> #include <machine/lapic.h> +#include <machine/page.h> +#include <machine/pic.h> #include <machine/pit.h> #include <machine/pmap.h> +#include <machine/ssp.h> #include <machine/trap.h> #include <vm/vm_page.h> @@ -75,27 +79,11 @@ #define CPU_MP_CMOS_RESET_VECTOR 0x467 /* - * Gate/segment descriptor bits and masks. + * Priority of the shutdown operations. + * + * Last resort, lower than everything else. */ -#define CPU_DESC_TYPE_DATA 0x00000200 -#define CPU_DESC_TYPE_CODE 0x00000a00 -#define CPU_DESC_TYPE_TSS 0x00000900 -#define CPU_DESC_TYPE_GATE_INTR 0x00000e00 -#define CPU_DESC_TYPE_GATE_TASK 0x00000500 -#define CPU_DESC_S 0x00001000 -#define CPU_DESC_PRESENT 0x00008000 -#define CPU_DESC_LONG 0x00200000 -#define CPU_DESC_DB 0x00400000 -#define CPU_DESC_GRAN_4KB 0x00800000 - -#define CPU_DESC_GATE_OFFSET_LOW_MASK 0x0000ffff -#define CPU_DESC_GATE_OFFSET_HIGH_MASK 0xffff0000 -#define CPU_DESC_SEG_IST_MASK 0x00000007 -#define CPU_DESC_SEG_BASE_LOW_MASK 0x0000ffff -#define CPU_DESC_SEG_BASE_MID_MASK 0x00ff0000 -#define CPU_DESC_SEG_BASE_HIGH_MASK 0xff000000 -#define CPU_DESC_SEG_LIMIT_LOW_MASK 0x0000ffff -#define CPU_DESC_SEG_LIMIT_HIGH_MASK 0x000f0000 +#define CPU_SHUTDOWN_PRIORITY 0 /* * Gate descriptor. @@ -110,14 +98,6 @@ struct cpu_gate_desc { }; /* - * Code or data segment descriptor. - */ -struct cpu_seg_desc { - uint32_t low; - uint32_t high; -}; - -/* * LDT or TSS system segment descriptor. */ struct cpu_sysseg_desc { @@ -131,7 +111,7 @@ struct cpu_sysseg_desc { struct cpu_pseudo_desc { uint16_t limit; - unsigned long address; + uintptr_t address; } __packed; void *cpu_local_area __percpu; @@ -144,7 +124,7 @@ struct cpu cpu_desc __percpu; /* * Number of active processors. */ -unsigned int cpu_nr_active __read_mostly; +unsigned int cpu_nr_active __read_mostly = 1; /* * Processor frequency, assumed fixed and equal on all processors. @@ -152,9 +132,19 @@ unsigned int cpu_nr_active __read_mostly; static uint64_t cpu_freq __read_mostly; /* + * TLS segment, as expected by the compiler. + * + * TLS isn't actually used inside the kernel. The current purpose of this + * segment is to implement stack protection. + */ +static const struct cpu_tls_seg cpu_tls_seg = { + .ssp_guard_word = SSP_GUARD_WORD, +}; + +/* * Interrupt descriptor table. */ -static struct cpu_gate_desc cpu_idt[CPU_IDT_SIZE] __aligned(8) __read_mostly; +static alignas(8) struct cpu_gate_desc cpu_idt[CPU_IDT_SIZE] __read_mostly; /* * Double fault handler, and stack for the main processor. @@ -163,7 +153,7 @@ static struct cpu_gate_desc cpu_idt[CPU_IDT_SIZE] __aligned(8) __read_mostly; * memory. */ static unsigned long cpu_double_fault_handler; -static char cpu_double_fault_stack[STACK_SIZE] __aligned(DATA_ALIGN); +static alignas(CPU_DATA_ALIGN) char cpu_double_fault_stack[TRAP_STACK_SIZE]; void cpu_delay(unsigned long usecs) @@ -277,9 +267,10 @@ cpu_seg_set_tss(char *table, unsigned int selector, struct cpu_tss *tss) /* * Set the given GDT for the current processor. * - * On i386, the ds, es and ss segment registers are reloaded. In any case, - * the gs segment register is set to the null selector. The fs segment - * register, which points to the percpu area, must be set separately. + * On i386, the ds, es and ss segment registers are reloaded. + * + * The fs and gs segment registers, which point to the percpu and the TLS + * areas respectively, must be set separately. */ void cpu_load_gdt(struct cpu_pseudo_desc *gdtr); @@ -298,6 +289,19 @@ cpu_set_percpu_area(const struct cpu *cpu, void *area) percpu_var(cpu_local_area, cpu->id) = area; } +static inline void __init +cpu_set_tls_area(void) +{ +#ifdef __LP64__ + unsigned long va; + + va = (unsigned long)&cpu_tls_seg; + cpu_set_msr(CPU_MSR_GSBASE, (uint32_t)(va >> 32), (uint32_t)va); +#else /* __LP64__ */ + asm volatile("mov %0, %%gs" : : "r" (CPU_GDT_SEL_TLS)); +#endif /* __LP64__ */ +} + static void __init cpu_init_gdtr(struct cpu_pseudo_desc *gdtr, const struct cpu *cpu) { @@ -321,11 +325,13 @@ cpu_init_gdt(struct cpu *cpu) #ifndef __LP64__ cpu_seg_set_tss(cpu->gdt, CPU_GDT_SEL_DF_TSS, &cpu->double_fault_tss); cpu_seg_set_data(cpu->gdt, CPU_GDT_SEL_PERCPU, (unsigned long)pcpu_area); + cpu_seg_set_data(cpu->gdt, CPU_GDT_SEL_TLS, (unsigned long)&cpu_tls_seg); #endif /* __LP64__ */ cpu_init_gdtr(&gdtr, cpu); cpu_load_gdt(&gdtr); cpu_set_percpu_area(cpu, pcpu_area); + cpu_set_tls_area(); } static void __init @@ -345,7 +351,7 @@ cpu_init_tss(struct cpu *cpu) #ifdef __LP64__ assert(cpu->double_fault_stack != NULL); tss->ist[CPU_TSS_IST_DF] = (unsigned long)cpu->double_fault_stack - + STACK_SIZE; + + TRAP_STACK_SIZE; #endif /* __LP64__ */ asm volatile("ltr %w0" : : "q" (CPU_GDT_SEL_TSS)); @@ -365,7 +371,7 @@ cpu_init_double_fault_tss(struct cpu *cpu) tss->cr3 = cpu_get_cr3(); tss->eip = cpu_double_fault_handler; tss->eflags = CPU_EFL_ONE; - tss->ebp = (unsigned long)cpu->double_fault_stack + STACK_SIZE; + tss->ebp = (unsigned long)cpu->double_fault_stack + TRAP_STACK_SIZE; tss->esp = tss->ebp; tss->es = CPU_GDT_SEL_DATA; tss->cs = CPU_GDT_SEL_CODE; @@ -415,12 +421,12 @@ cpu_idt_set_double_fault(void (*isr)(void)) } static void -cpu_load_idt(void) +cpu_load_idt(const void *idt, size_t size) { - static volatile struct cpu_pseudo_desc idtr; + struct cpu_pseudo_desc idtr; - idtr.address = (unsigned long)cpu_idt; - idtr.limit = sizeof(cpu_idt) - 1; + idtr.address = (uintptr_t)idt; + idtr.limit = size - 1; asm volatile("lidt %0" : : "m" (idtr)); } @@ -445,7 +451,7 @@ cpu_init(struct cpu *cpu) #ifndef __LP64__ cpu_init_double_fault_tss(cpu); #endif /* __LP64__ */ - cpu_load_idt(); + cpu_load_idt(cpu_idt, sizeof(cpu_idt)); eax = 0; cpu_cpuid(&eax, &ebx, &ecx, &edx); @@ -542,6 +548,8 @@ cpu_measure_freq(void) { uint64_t start, end; + pit_setup_free_running(); + start = cpu_get_tsc(); pit_delay(CPU_FREQ_CAL_DELAY); end = cpu_get_tsc(); @@ -549,7 +557,7 @@ cpu_measure_freq(void) cpu_freq = (end - start) / (1000000 / CPU_FREQ_CAL_DELAY); } -void __init +static int __init cpu_setup(void) { struct cpu *cpu; @@ -558,18 +566,23 @@ cpu_setup(void) cpu_preinit(cpu, 0, CPU_INVALID_APIC_ID); cpu->double_fault_stack = cpu_double_fault_stack; /* XXX */ cpu_init(cpu); - cpu_nr_active = 1; cpu_measure_freq(); + + return 0; } +INIT_OP_DEFINE(cpu_setup, + INIT_OP_DEP(percpu_bootstrap, true), + INIT_OP_DEP(trap_setup, true)); + static void __init cpu_panic_on_missing_feature(const char *feature) { panic("cpu: %s feature missing", feature); } -void __init +static void __init cpu_check(const struct cpu *cpu) { if (!(cpu->features2 & CPU_FEATURE2_FPU)) { @@ -589,25 +602,36 @@ cpu_check(const struct cpu *cpu) #endif } +static int __init +cpu_check_bsp(void) +{ + cpu_check(cpu_current()); + return 0; +} + +INIT_OP_DEFINE(cpu_check_bsp, + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(panic_setup, true)); + void -cpu_info(const struct cpu *cpu) +cpu_log_info(const struct cpu *cpu) { - printf("cpu%u: %s, type %u, family %u, model %u, stepping %u\n", - cpu->id, cpu->vendor_id, cpu->type, cpu->family, cpu->model, - cpu->stepping); + log_info("cpu%u: %s, type %u, family %u, model %u, stepping %u", + cpu->id, cpu->vendor_id, cpu->type, cpu->family, cpu->model, + cpu->stepping); if (strlen(cpu->model_name) > 0) { - printf("cpu%u: %s\n", cpu->id, cpu->model_name); + log_info("cpu%u: %s", cpu->id, cpu->model_name); } if ((cpu->phys_addr_width != 0) && (cpu->virt_addr_width != 0)) { - printf("cpu%u: address widths: physical: %hu, virtual: %hu\n", - cpu->id, cpu->phys_addr_width, cpu->virt_addr_width); + log_info("cpu%u: address widths: physical: %hu, virtual: %hu", + cpu->id, cpu->phys_addr_width, cpu->virt_addr_width); } - printf("cpu%u: frequency: %llu.%02llu MHz\n", cpu->id, - (unsigned long long)cpu_freq / 1000000, - (unsigned long long)cpu_freq % 1000000); + log_info("cpu%u: frequency: %llu.%02llu MHz", cpu->id, + (unsigned long long)cpu_freq / 1000000, + (unsigned long long)cpu_freq % 1000000); } void __init @@ -638,26 +662,43 @@ cpu_mp_register_lapic(unsigned int apic_id, int is_bsp) cpu_nr_active++; } -void __init +static void +cpu_shutdown_reset(void) +{ + cpu_load_idt(NULL, 1); + trap_trigger_double_fault(); +} + +static struct shutdown_ops cpu_shutdown_ops = { + .reset = cpu_shutdown_reset, +}; + +static int __init cpu_mp_probe(void) { - int error; + log_info("cpu: %u processor(s) configured", cpu_count()); + return 0; +} - error = acpimp_setup(); +INIT_OP_DEFINE(cpu_mp_probe, + INIT_OP_DEP(acpi_setup, true), + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(log_setup, true)); - if (error) { - /* - * For the sake of simplicity, it has been decided to ignore legacy - * specifications such as the multiprocessor specification, and use - * ACPI only. If ACPI is unavailable, consider the APIC system to - * be missing and fall back to using the legaxy XT-PIC. - */ - lapic_setup_unused(); +static int __init +cpu_setup_shutdown(void) +{ + if (cpu_count() == 1) { + shutdown_register(&cpu_shutdown_ops, CPU_SHUTDOWN_PRIORITY); } - printf("cpu: %u processor(s) configured\n", cpu_count()); + return 0; } +INIT_OP_DEFINE(cpu_setup_shutdown, + INIT_OP_DEP(cpu_mp_probe, true), + INIT_OP_DEP(shutdown_bootstrap, true)); + void __init cpu_mp_setup(void) { @@ -689,18 +730,20 @@ cpu_mp_setup(void) io_write_byte(CPU_MP_CMOS_PORT_REG, CPU_MP_CMOS_REG_RESET); io_write_byte(CPU_MP_CMOS_PORT_DATA, CPU_MP_CMOS_DATA_RESET_WARM); + /* TODO Allocate stacks out of the slab allocator for sub-page sizes */ + for (i = 1; i < cpu_count(); i++) { cpu = percpu_ptr(cpu_desc, i); - page = vm_page_alloc(vm_page_order(STACK_SIZE), VM_PAGE_SEL_DIRECTMAP, - VM_PAGE_KERNEL); + page = vm_page_alloc(vm_page_order(BOOT_STACK_SIZE), + VM_PAGE_SEL_DIRECTMAP, VM_PAGE_KERNEL); if (page == NULL) { panic("cpu: unable to allocate boot stack for cpu%u", i); } cpu->boot_stack = vm_page_direct_ptr(page); - page = vm_page_alloc(vm_page_order(STACK_SIZE), VM_PAGE_SEL_DIRECTMAP, - VM_PAGE_KERNEL); + page = vm_page_alloc(vm_page_order(TRAP_STACK_SIZE), + VM_PAGE_SEL_DIRECTMAP, VM_PAGE_KERNEL); if (page == NULL) { panic("cpu: unable to allocate double fault stack for cpu%u", i); diff --git a/arch/x86/machine/cpu.h b/arch/x86/machine/cpu.h index b384e555..39f17f84 100644 --- a/arch/x86/machine/cpu.h +++ b/arch/x86/machine/cpu.h @@ -18,6 +18,31 @@ #ifndef _X86_CPU_H #define _X86_CPU_H +#include <limits.h> + +/* + * L1 cache line size. + * + * XXX Use this value until processor selection is available. + */ +#define CPU_L1_SIZE 64 + +/* + * Data alignment, normally the word size. + */ +#define CPU_DATA_ALIGN (LONG_BIT / 8) + +/* + * Function alignment. + * + * Aligning functions improves instruction fetching. + * + * Used for assembly functions only. + * + * XXX Use this value until processor selection is available. + */ +#define CPU_TEXT_ALIGN 16 + /* * Processor privilege levels. */ @@ -46,7 +71,7 @@ /* * EFLAGS register flags. */ -#define CPU_EFL_ONE 0x00000002 +#define CPU_EFL_ONE 0x00000002 /* Reserved, must be one */ #define CPU_EFL_IF 0x00000200 /* @@ -54,6 +79,7 @@ */ #define CPU_MSR_EFER 0xc0000080 #define CPU_MSR_FSBASE 0xc0000100 +#define CPU_MSR_GSBASE 0xc0000101 /* * EFER MSR flags. @@ -89,19 +115,55 @@ #else /* __LP64__ */ #define CPU_GDT_SEL_DF_TSS 32 #define CPU_GDT_SEL_PERCPU 40 -#define CPU_GDT_SIZE 48 +#define CPU_GDT_SEL_TLS 48 +#define CPU_GDT_SIZE 56 #endif /* __LP64__ */ #define CPU_IDT_SIZE 256 #ifndef __ASSEMBLER__ +#include <stdalign.h> #include <stdint.h> +#include <stdnoreturn.h> +#include <kern/init.h> #include <kern/macros.h> #include <kern/percpu.h> #include <machine/lapic.h> #include <machine/pit.h> +#include <machine/ssp.h> + +/* + * Gate/segment descriptor bits and masks. + */ +#define CPU_DESC_TYPE_DATA 0x00000200 +#define CPU_DESC_TYPE_CODE 0x00000a00 +#define CPU_DESC_TYPE_TSS 0x00000900 +#define CPU_DESC_TYPE_GATE_INTR 0x00000e00 +#define CPU_DESC_TYPE_GATE_TASK 0x00000500 +#define CPU_DESC_S 0x00001000 +#define CPU_DESC_PRESENT 0x00008000 +#define CPU_DESC_LONG 0x00200000 +#define CPU_DESC_DB 0x00400000 +#define CPU_DESC_GRAN_4KB 0x00800000 + +#define CPU_DESC_GATE_OFFSET_LOW_MASK 0x0000ffff +#define CPU_DESC_GATE_OFFSET_HIGH_MASK 0xffff0000 +#define CPU_DESC_SEG_IST_MASK 0x00000007 +#define CPU_DESC_SEG_BASE_LOW_MASK 0x0000ffff +#define CPU_DESC_SEG_BASE_MID_MASK 0x00ff0000 +#define CPU_DESC_SEG_BASE_HIGH_MASK 0xff000000 +#define CPU_DESC_SEG_LIMIT_LOW_MASK 0x0000ffff +#define CPU_DESC_SEG_LIMIT_HIGH_MASK 0x000f0000 + +/* + * Code or data segment descriptor. + */ +struct cpu_seg_desc { + uint32_t low; + uint32_t high; +}; /* * Forward declaration. @@ -179,7 +241,7 @@ struct cpu { unsigned int features4; unsigned short phys_addr_width; unsigned short virt_addr_width; - char gdt[CPU_GDT_SIZE] __aligned(8); + alignas(8) char gdt[CPU_GDT_SIZE]; struct cpu_tss tss; #ifndef __LP64__ struct cpu_tss double_fault_tss; @@ -189,6 +251,11 @@ struct cpu { void *double_fault_stack; }; +struct cpu_tls_seg { + uintptr_t unused[SSP_WORD_TLS_OFFSET]; + uintptr_t ssp_guard_word; +}; + /* * Macro to create functions that read/write control registers. */ @@ -334,7 +401,7 @@ cpu_idle(void) * * Implies a compiler barrier. */ -static __noreturn __always_inline void +noreturn static __always_inline void cpu_halt(void) { cpu_intr_disable(); @@ -548,24 +615,16 @@ void * cpu_get_boot_stack(void); /* * Install an interrupt handler in the IDT. + * + * These functions may be called before the cpu module is initialized. */ void cpu_idt_set_gate(unsigned int vector, void (*isr)(void)); void cpu_idt_set_double_fault(void (*isr)(void)); /* - * Set up the cpu module. - */ -void cpu_setup(void); - -/* - * Make sure the CPU has some required features. + * Log processor information. */ -void cpu_check(const struct cpu *cpu); - -/* - * Display processor information. - */ -void cpu_info(const struct cpu *cpu); +void cpu_log_info(const struct cpu *cpu); /* * Register the presence of a local APIC. @@ -573,13 +632,6 @@ void cpu_info(const struct cpu *cpu); void cpu_mp_register_lapic(unsigned int apic_id, int is_bsp); /* - * Probe application processors. - * - * On return, cpu_count() gives the actual number of managed processors. - */ -void cpu_mp_probe(void); - -/* * Start application processors. * * The x86 architecture uses per-CPU page tables, which are created as a @@ -630,6 +682,26 @@ cpu_send_thread_schedule(unsigned int cpu) */ void cpu_thread_schedule_intr(struct trap_frame *frame); +/* + * This init operation provides : + * - initialization of the BSP structure. + * - cpu_delay() + * - cpu_local_ptr() and cpu_local_var() + */ +INIT_OP_DECLARE(cpu_setup); + +/* + * This init operation provides : + * - cpu_count() + */ +INIT_OP_DECLARE(cpu_mp_probe); + +/* + * This init operation provides : + * - cpu shutdown operations registered + */ +INIT_OP_DECLARE(cpu_setup_shutdown); + #endif /* __ASSEMBLER__ */ #endif /* _X86_CPU_H */ diff --git a/arch/x86/machine/cpu_asm.S b/arch/x86/machine/cpu_asm.S index 9f8c0773..9d479913 100644 --- a/arch/x86/machine/cpu_asm.S +++ b/arch/x86/machine/cpu_asm.S @@ -34,9 +34,6 @@ ASM_ENTRY(cpu_load_gdt) movl %eax, %es movl %eax, %ss - movl $CPU_GDT_SEL_NULL, %eax - movl %eax, %gs - /* Alter the stack to reload the code segment using a far return */ #ifdef __LP64__ popq %rax diff --git a/arch/x86/machine/ioapic.c b/arch/x86/machine/ioapic.c index 66178b3a..82fbd6d5 100644 --- a/arch/x86/machine/ioapic.c +++ b/arch/x86/machine/ioapic.c @@ -15,20 +15,22 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stdbool.h> #include <stdint.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/init.h> #include <kern/intr.h> #include <kern/kmem.h> +#include <kern/log.h> +#include <kern/macros.h> #include <kern/panic.h> #include <kern/spinlock.h> #include <machine/cpu.h> #include <machine/ioapic.h> #include <machine/lapic.h> +#include <machine/pic.h> #include <machine/trap.h> #include <vm/vm_kmem.h> @@ -40,6 +42,10 @@ #define IOAPIC_VERSION_MAXREDIR_MASK 0x00ff0000 #define IOAPIC_VERSION_MAXREDIR_SHIFT 16 +#define IOAPIC_ENTLOW_FIXED_DEST 0x00000 +#define IOAPIC_ENTLOW_PHYS_DELIVERY 0x00000 +#define IOAPIC_ENTLOW_ACTIVE_LOW 0x02000 +#define IOAPIC_ENTLOW_LEVEL 0x08000 #define IOAPIC_ENTLOW_INTRMASK 0x10000 #define IOAPIC_MAX_ENTRIES 24 @@ -55,18 +61,73 @@ struct ioapic_map { uint32_t win; }; +/* + * Interrupt source override descriptor. + */ +struct ioapic_iso { + uint32_t gsi; + uint8_t source; + bool active_high; + bool edge_triggered; +}; + struct ioapic { struct spinlock lock; unsigned int id; unsigned int apic_id; unsigned int version; volatile struct ioapic_map *map; - unsigned int first_intr; - unsigned int last_intr; + unsigned int first_gsi; + unsigned int last_gsi; }; static unsigned int ioapic_nr_devs; +static struct ioapic_iso ioapic_isos[PIC_MAX_INTR + 1]; +static unsigned int ioapic_nr_isos; + +static void +ioapic_iso_init(struct ioapic_iso *iso, uint8_t source, uint32_t gsi, + bool active_high, bool edge_triggered) +{ + iso->source = source; + iso->gsi = gsi; + iso->active_high = active_high; + iso->edge_triggered = edge_triggered; +} + +static struct ioapic_iso * __init +ioapic_alloc_iso(void) +{ + struct ioapic_iso *iso; + + if (ioapic_nr_isos >= ARRAY_SIZE(ioapic_isos)) { + log_err("ioapic: too many interrupt overrides"); + return NULL; + } + + iso = &ioapic_isos[ioapic_nr_isos]; + ioapic_nr_isos++; + return iso; +} + +static struct ioapic_iso * +ioapic_lookup_iso(unsigned int intr) +{ + struct ioapic_iso *iso; + unsigned int i; + + for (i = 0; i < ioapic_nr_isos; i++) { + iso = &ioapic_isos[i]; + + if (intr == iso->source) { + return iso; + } + } + + return NULL; +} + static uint32_t ioapic_read(struct ioapic *ioapic, uint8_t reg) { @@ -102,10 +163,10 @@ ioapic_intr(struct trap_frame *frame) } static struct ioapic * __init -ioapic_create(unsigned int apic_id, uintptr_t addr, unsigned int intr_base) +ioapic_create(unsigned int apic_id, uintptr_t addr, unsigned int gsi_base) { struct ioapic *ioapic; - unsigned int i, nr_intrs; + unsigned int i, nr_gsis; uint32_t value; ioapic = kmem_alloc(sizeof(*ioapic)); @@ -117,7 +178,7 @@ ioapic_create(unsigned int apic_id, uintptr_t addr, unsigned int intr_base) spinlock_init(&ioapic->lock); ioapic->id = ioapic_nr_devs; ioapic->apic_id = apic_id; - ioapic->first_intr = intr_base; + ioapic->first_gsi = gsi_base; ioapic->map = vm_kmem_map_pa(addr, sizeof(*ioapic->map), NULL, NULL); @@ -128,51 +189,86 @@ ioapic_create(unsigned int apic_id, uintptr_t addr, unsigned int intr_base) value = ioapic_read(ioapic, IOAPIC_REG_VERSION); ioapic->version = (value & IOAPIC_VERSION_VERSION_MASK) >> IOAPIC_VERSION_VERSION_SHIFT; - nr_intrs = ((value & IOAPIC_VERSION_MAXREDIR_MASK) - >> IOAPIC_VERSION_MAXREDIR_SHIFT) + 1; - ioapic->last_intr = ioapic->first_intr + nr_intrs - 1; + nr_gsis = ((value & IOAPIC_VERSION_MAXREDIR_MASK) + >> IOAPIC_VERSION_MAXREDIR_SHIFT) + 1; + ioapic->last_gsi = ioapic->first_gsi + nr_gsis - 1; - if (ioapic->last_intr > (TRAP_INTR_LAST - TRAP_INTR_FIRST)) { + /* XXX This assumes that interrupts are mapped 1:1 to traps */ + if (ioapic->last_gsi > (TRAP_INTR_LAST - TRAP_INTR_FIRST)) { panic("ioapic: invalid interrupt range"); } - for (i = ioapic->first_intr; i < ioapic->last_intr; i++) { + for (i = ioapic->first_gsi; i < ioapic->last_gsi; i++) { trap_register(TRAP_INTR_FIRST + i, ioapic_intr); } - printf("ioapic%u: version:%#x intrs:%u-%u\n", ioapic->id, - ioapic->version, ioapic->first_intr, ioapic->last_intr); + log_info("ioapic%u: version:%#x gsis:%u-%u", ioapic->id, + ioapic->version, ioapic->first_gsi, ioapic->last_gsi); ioapic_nr_devs++; return ioapic; } static bool -ioapic_has_intr(const struct ioapic *ioapic, unsigned int intr) +ioapic_has_gsi(const struct ioapic *ioapic, unsigned int gsi) { - return ((intr >= ioapic->first_intr) && (intr <= ioapic->last_intr)); + return ((gsi >= ioapic->first_gsi) && (gsi <= ioapic->last_gsi)); } static unsigned int -ioapic_compute_id(const struct ioapic *ioapic, unsigned int intr) +ioapic_compute_id(const struct ioapic *ioapic, unsigned int gsi) +{ + assert(ioapic_has_gsi(ioapic, gsi)); + return gsi - ioapic->first_gsi; +} + +static void +ioapic_compute_entry(uint32_t *highp, uint32_t *lowp, + unsigned int apic_id, unsigned int intr, + bool active_high, bool edge_triggered) { - assert(ioapic_has_intr(ioapic, intr)); - return intr - ioapic->first_intr; + assert(apic_id < 16); + assert(intr < (TRAP_NR_VECTORS - TRAP_INTR_FIRST)); + + *highp = apic_id << 24; + *lowp = (!edge_triggered ? IOAPIC_ENTLOW_LEVEL : 0) + | (!active_high ? IOAPIC_ENTLOW_ACTIVE_LOW : 0) + | IOAPIC_ENTLOW_PHYS_DELIVERY + | IOAPIC_ENTLOW_FIXED_DEST + | (TRAP_INTR_FIRST + intr); } static void ioapic_enable(void *priv, unsigned int intr, unsigned int cpu) { + bool active_high, edge_triggered; + const struct ioapic_iso *iso; + uint32_t high, low, gsi; struct ioapic *ioapic; unsigned long flags; unsigned int id; + iso = ioapic_lookup_iso(intr); + + /* XXX These are defaults that should work with architectural devices */ + if (iso == NULL) { + active_high = true; + edge_triggered = true; + gsi = intr; + } else { + active_high = iso->active_high; + edge_triggered = iso->edge_triggered; + gsi = iso->gsi; + } + ioapic = priv; - id = ioapic_compute_id(ioapic, intr); + id = ioapic_compute_id(ioapic, gsi); + ioapic_compute_entry(&high, &low, cpu_apic_id(cpu), intr, + active_high, edge_triggered); spinlock_lock_intr_save(&ioapic->lock, &flags); - ioapic_write_entry_high(ioapic, id, cpu_apic_id(cpu) << 24); - ioapic_write_entry_low(ioapic, id, TRAP_INTR_FIRST + intr); + ioapic_write_entry_high(ioapic, id, high); + ioapic_write_entry_low(ioapic, id, low); spinlock_unlock_intr_restore(&ioapic->lock, flags); } @@ -207,12 +303,37 @@ static const struct intr_ops ioapic_ops = { }; void __init -ioapic_register(unsigned int apic_id, uintptr_t addr, unsigned int intr_base) +ioapic_setup(void) +{ + ioapic_nr_devs = 0; + ioapic_nr_isos = 0; +} + +void __init +ioapic_register(unsigned int apic_id, uintptr_t addr, unsigned int gsi_base) { struct ioapic *ioapic; - ioapic = ioapic_create(apic_id, addr, intr_base); - intr_register_ctl(&ioapic_ops, ioapic, - ioapic->first_intr, ioapic->last_intr); + ioapic = ioapic_create(apic_id, addr, gsi_base); + + /* + * XXX This assumes that any interrupt override source is included + * in the GSI range. + */ + intr_register_ctl(&ioapic_ops, ioapic, ioapic->first_gsi, ioapic->last_gsi); } +void __init +ioapic_override(uint8_t source, uint32_t gsi, + bool active_high, bool edge_triggered) +{ + struct ioapic_iso *iso; + + iso = ioapic_alloc_iso(); + + if (iso == NULL) { + return; + } + + ioapic_iso_init(iso, source, gsi, active_high, edge_triggered); +} diff --git a/arch/x86/machine/ioapic.h b/arch/x86/machine/ioapic.h index d508904e..b9a702d3 100644 --- a/arch/x86/machine/ioapic.h +++ b/arch/x86/machine/ioapic.h @@ -18,12 +18,24 @@ #ifndef _KERN_IOAPIC_H #define _KERN_IOAPIC_H +#include <stdbool.h> #include <stdint.h> /* + * Initialize the ioapic module. + */ +void ioapic_setup(void); + +/* * Register an I/O APIC controller. */ void ioapic_register(unsigned int apic_id, uintptr_t addr, unsigned int gsi_base); +/* + * Report an interrupt source override. + */ +void ioapic_override(uint8_t source, uint32_t gsi, + bool active_high, bool edge_triggered); + #endif /* _KERN_IOAPIC_H */ diff --git a/arch/x86/machine/lapic.c b/arch/x86/machine/lapic.c index 6bd1d723..a0eee852 100644 --- a/arch/x86/machine/lapic.c +++ b/arch/x86/machine/lapic.c @@ -15,16 +15,15 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/thread.h> #include <machine/cpu.h> #include <machine/lapic.h> @@ -184,9 +183,6 @@ static volatile struct lapic_map *lapic_map __read_mostly; */ static uint32_t lapic_bus_freq __read_mostly; -static bool lapic_initialized __initdata; -static bool lapic_is_unused __initdata; - static uint32_t lapic_read(const volatile struct lapic_register *r) { @@ -213,9 +209,9 @@ lapic_compute_freq(void) cpu_delay(LAPIC_TIMER_CAL_DELAY); c2 = lapic_read(&lapic_map->timer_ccr); lapic_bus_freq = (c1 - c2) * (1000000 / LAPIC_TIMER_CAL_DELAY); - printf("lapic: bus frequency: %u.%02u MHz\n", lapic_bus_freq / 1000000, - lapic_bus_freq % 1000000); - lapic_write(&lapic_map->timer_icr, lapic_bus_freq / HZ); + log_info("lapic: bus frequency: %u.%02u MHz", lapic_bus_freq / 1000000, + lapic_bus_freq % 1000000); + lapic_write(&lapic_map->timer_icr, lapic_bus_freq / THREAD_TICK_FREQ); lapic_write(&lapic_map->svr, 0); } @@ -242,21 +238,7 @@ lapic_setup_registers(void) lapic_write(&lapic_map->lvt_lint1, LAPIC_LVT_MASK_INTR); lapic_write(&lapic_map->lvt_error, TRAP_LAPIC_ERROR); lapic_write(&lapic_map->timer_dcr, LAPIC_TIMER_DCR_DIV1); - lapic_write(&lapic_map->timer_icr, lapic_bus_freq / HZ); -} - -bool __init -lapic_unused(void) -{ - assert(lapic_initialized); - return lapic_is_unused; -} - -void __init -lapic_setup_unused(void) -{ - lapic_initialized = true; - lapic_is_unused = true; + lapic_write(&lapic_map->timer_icr, lapic_bus_freq / THREAD_TICK_FREQ); } void __init @@ -278,8 +260,6 @@ lapic_setup(uint32_t map_addr) lapic_compute_freq(); lapic_setup_registers(); - - lapic_initialized = true; } void __init @@ -363,7 +343,7 @@ lapic_error_intr(struct trap_frame *frame) (void)frame; esr = lapic_read(&lapic_map->esr); - printf("lapic: error on cpu%u: esr:%08x\n", cpu_id(), esr); + log_err("lapic: error on cpu%u: esr:%08x", cpu_id(), esr); lapic_write(&lapic_map->esr, 0); lapic_eoi(); } @@ -372,7 +352,7 @@ void lapic_spurious_intr(struct trap_frame *frame) { (void)frame; - printf("lapic: warning: spurious interrupt\n"); + log_warning("lapic: spurious interrupt"); /* No EOI for this interrupt */ } diff --git a/arch/x86/machine/lapic.h b/arch/x86/machine/lapic.h index ff9c8937..ae7abfe1 100644 --- a/arch/x86/machine/lapic.h +++ b/arch/x86/machine/lapic.h @@ -29,17 +29,6 @@ void lapic_eoi(void); /* - * Report whether the local APIC is actually used or not. - */ -bool lapic_unused(void); - -/* - * Initialize the lapic module for the sole purpose of reporting that - * it's actually not used. - */ -void lapic_setup_unused(void); - -/* * Set up the lapic module. */ void lapic_setup(uint32_t map_addr); diff --git a/arch/x86/machine/page.h b/arch/x86/machine/page.h new file mode 100644 index 00000000..43ab237a --- /dev/null +++ b/arch/x86/machine/page.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * This file is a top header in the inclusion hierarchy, and shouldn't include + * other headers that may cause circular dependencies. + */ + +#ifndef _X86_PAGE_H +#define _X86_PAGE_H + +#define PAGE_SHIFT 12 +#define PAGE_SIZE (1 << PAGE_SHIFT) +#define PAGE_MASK (PAGE_SIZE - 1) + +#endif /* _X86_PAGE_H */ diff --git a/arch/x86/machine/param.h b/arch/x86/machine/param.h deleted file mode 100644 index 6ee11cce..00000000 --- a/arch/x86/machine/param.h +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2010-2014 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * - * This file is a top header in the inclusion hierarchy, and shouldn't include - * other headers that may cause circular dependencies. - */ - -#ifndef _X86_PARAM_H -#define _X86_PARAM_H - -#include <kern/macros.h> - -#ifndef __LITTLE_ENDIAN__ -#define __LITTLE_ENDIAN__ -#endif /* __LITTLE_ENDIAN__ */ - -/* - * L1 cache line size. - * - * XXX Use this value until processor selection is available. - */ -#define CPU_L1_SIZE 64 - -/* - * Code/data alignment. - */ -#define TEXT_ALIGN 16 - -#ifdef __LP64__ -#define DATA_ALIGN 8 -#else /* __LP64__ */ -#define DATA_ALIGN 4 -#endif /* __LP64__ */ - -/* - * Attributes for variables that are mostly read and seldom changed. - */ -#define __read_mostly __section(".data.read_mostly") - -/* - * Provide architecture-specific string functions. - */ -#define ARCH_STRING_MEMCPY -#define ARCH_STRING_MEMMOVE -#define ARCH_STRING_MEMSET -#define ARCH_STRING_MEMCMP -#define ARCH_STRING_STRLEN -#define ARCH_STRING_STRCPY -#define ARCH_STRING_STRCMP - -/* - * System timer frequency. - * - * The selected value of 200 translates to a period of 5ms, small enough to - * provide low latency, and is practical as both a dividend and divisor. - */ -#define HZ 200 - -/* - * 4 KiB pages. - */ -#define PAGE_SHIFT 12 -#define PAGE_SIZE (1 << PAGE_SHIFT) -#define PAGE_MASK (PAGE_SIZE - 1) - -/* - * Kernel stack size for threads and interrupt handlers. - */ -#define STACK_SIZE PAGE_SIZE - -/* - * Maximum number of available interrupts. - */ -#define INTR_TABLE_SIZE 256 - -/* - * Virtual memory properties. - */ - -/* - * User space boundaries. - */ -#define VM_MIN_ADDRESS DECL_CONST(0, UL) - -#ifdef __LP64__ -#define VM_MAX_ADDRESS DECL_CONST(0x800000000000, UL) -#else /* __LP64__ */ -#define VM_MAX_ADDRESS DECL_CONST(0xc0000000, UL) -#endif/* __LP64__ */ - -/* - * Kernel space boundaries. - */ -#ifdef __LP64__ -#define VM_MIN_KERNEL_ADDRESS DECL_CONST(0xffff800000000000, UL) -#define VM_MAX_KERNEL_ADDRESS DECL_CONST(0xfffffffffffff000, UL) -#else /* __LP64__ */ -#define VM_MIN_KERNEL_ADDRESS VM_MAX_ADDRESS -#define VM_MAX_KERNEL_ADDRESS DECL_CONST(0xfffff000, UL) -#endif /* __LP64__ */ - -/* - * Direct physical mapping boundaries. - */ -#ifdef __LP64__ -#define VM_MIN_DIRECTMAP_ADDRESS VM_MIN_KERNEL_ADDRESS -#define VM_MAX_DIRECTMAP_ADDRESS DECL_CONST(0xffffc00000000000, UL) -#else /* __LP64__ */ -#define VM_MIN_DIRECTMAP_ADDRESS VM_MAX_ADDRESS -#define VM_MAX_DIRECTMAP_ADDRESS DECL_CONST(0xf8000000, UL) -#endif /* __LP64__ */ - -/* - * Kernel mapping offset. - * - * On 32-bits systems, the kernel is linked at addresses included in the - * direct physical mapping, whereas on 64-bits systems, it is linked at - * -2 GiB because the "kernel" memory model is used when compiling (see - * the -mcmodel=kernel gcc option). - */ -#ifdef __LP64__ -#define VM_KERNEL_OFFSET DECL_CONST(0xffffffff80000000, UL) -#else /* __LP64__ */ -#define VM_KERNEL_OFFSET VM_MIN_DIRECTMAP_ADDRESS -#endif /* __LP64__ */ - -/* - * Kernel virtual space boundaries. - * - * In addition to the direct physical mapping, the kernel has its own virtual - * memory space. - */ -#define VM_MIN_KMEM_ADDRESS VM_MAX_DIRECTMAP_ADDRESS - -#ifdef __LP64__ -#define VM_MAX_KMEM_ADDRESS VM_KERNEL_OFFSET -#else /* __LP64__ */ -#define VM_MAX_KMEM_ADDRESS VM_MAX_KERNEL_ADDRESS -#endif /* __LP64__ */ - -/* - * Physical memory properties. - */ - -#define VM_PAGE_DMA_LIMIT DECL_CONST(0x1000000, UL) - -#ifdef __LP64__ -#define VM_PAGE_MAX_ZONES 4 -#define VM_PAGE_DMA32_LIMIT DECL_CONST(0x100000000, UL) -#define VM_PAGE_DIRECTMAP_LIMIT DECL_CONST(0x400000000000, UL) -#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, UL) -#else /* __LP64__ */ -#define VM_PAGE_DIRECTMAP_LIMIT DECL_CONST(0x38000000, ULL) -#ifdef X15_X86_PAE -#define VM_PAGE_MAX_ZONES 3 -#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, ULL) -#else /* X15_X86_PAE */ -#define VM_PAGE_MAX_ZONES 3 -#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0xfffff000, UL) -#endif /* X15_X86_PAE */ -#endif /* __LP64__ */ - -/* - * Physical zone indexes. - */ -#define VM_PAGE_ZONE_DMA 0 - -#ifdef __LP64__ -#define VM_PAGE_ZONE_DMA32 1 -#define VM_PAGE_ZONE_DIRECTMAP 2 -#define VM_PAGE_ZONE_HIGHMEM 3 -#else /* __LP64__ */ -#define VM_PAGE_ZONE_DMA32 1 /* Alias for the DIRECTMAP zone */ -#define VM_PAGE_ZONE_DIRECTMAP 1 -#define VM_PAGE_ZONE_HIGHMEM 2 -#endif /* __LP64__ */ - -#endif /* _X86_PARAM_H */ diff --git a/arch/x86/machine/pic.c b/arch/x86/machine/pic.c index 9084abce..8ef57538 100644 --- a/arch/x86/machine/pic.c +++ b/arch/x86/machine/pic.c @@ -15,9 +15,10 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdbool.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/init.h> #include <kern/intr.h> @@ -50,8 +51,6 @@ */ #define PIC_SLAVE_INTR 2 #define PIC_SPURIOUS_INTR 7 -#define PIC_NR_INTRS 8 -#define PIC_MAX_INTR ((PIC_NR_INTRS * 2) - 1) static unsigned int pic_nr_slave_intrs; @@ -198,8 +197,8 @@ pic_spurious_intr(void *arg) return 0; } -void __init -pic_setup(void) +static void __init +pic_setup_common(bool register_ctl) { int error; @@ -227,7 +226,7 @@ pic_setup(void) io_write_byte(PIC_MASTER_IMR, pic_master_mask); io_write_byte(PIC_SLAVE_IMR, pic_slave_mask); - if (lapic_unused()) { + if (register_ctl) { pic_register(); } @@ -241,3 +240,15 @@ pic_setup(void) &pic_slave_spurious_intr); error_check(error, __func__); } + +void __init +pic_setup(void) +{ + pic_setup_common(true); +} + +void __init +pic_setup_disabled(void) +{ + pic_setup_common(false); +} diff --git a/arch/x86/machine/pic.h b/arch/x86/machine/pic.h index 9da75b74..89833d00 100644 --- a/arch/x86/machine/pic.h +++ b/arch/x86/machine/pic.h @@ -18,11 +18,30 @@ #ifndef _X86_PIC_H #define _X86_PIC_H -#include <machine/trap.h> +/* + * Interrupts per PIC. + */ +#define PIC_NR_INTRS 8 + +/* + * Maximum global interrupt number. + */ +#define PIC_MAX_INTR ((PIC_NR_INTRS * 2) - 1) /* - * Set up the pic module. + * Initialize the pic module. */ void pic_setup(void); +/* + * Initialize the pic module in an APIC system. + * + * This function is called by the acpi module if ACPI reports the presence + * of legacy interrupt controllers. + * + * Since it doesn't register the legacy PIC as an interrupt controller, the + * acpi module must have registered I/O APICs before calling this function. + */ +void pic_setup_disabled(void); + #endif /* _X86_PIC_H */ diff --git a/arch/x86/machine/pit.c b/arch/x86/machine/pit.c index bf53ee11..c31c7e0b 100644 --- a/arch/x86/machine/pit.c +++ b/arch/x86/machine/pit.c @@ -15,7 +15,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <kern/assert.h> +#include <assert.h> + #include <kern/init.h> #include <machine/io.h> #include <machine/pit.h> diff --git a/arch/x86/machine/pmap.c b/arch/x86/machine/pmap.c index 1de9b093..7ab70bfd 100644 --- a/arch/x86/machine/pmap.c +++ b/arch/x86/machine/pmap.c @@ -18,20 +18,20 @@ * TODO Review locking. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> -#include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/cpumap.h> #include <kern/error.h> #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/syscnt.h> @@ -40,6 +40,7 @@ #include <machine/boot.h> #include <machine/cpu.h> #include <machine/lapic.h> +#include <machine/page.h> #include <machine/pmap.h> #include <machine/trap.h> #include <machine/types.h> @@ -113,8 +114,8 @@ struct pmap *pmap_current_ptr __percpu; /* * "Hidden" kernel root page tables for PAE mode. */ -static pmap_pte_t pmap_cpu_kpdpts[X15_MAX_CPUS][PMAP_L2_PTES_PER_PT] - __read_mostly __aligned(PMAP_PDPT_ALIGN); +static alignas(PMAP_PDPT_ALIGN) pmap_pte_t + pmap_cpu_kpdpts[X15_MAX_CPUS][PMAP_L2_PTES_PER_PT] __read_mostly; #endif /* X15_X86_PAE */ @@ -184,11 +185,11 @@ struct pmap_update_op { * operation. */ struct pmap_update_oplist { - struct cpumap cpumap; + alignas(CPU_L1_SIZE) struct cpumap cpumap; struct pmap *pmap; unsigned int nr_ops; struct pmap_update_op ops[PMAP_UPDATE_MAX_OPS]; -} __aligned(CPU_L1_SIZE); +}; static unsigned int pmap_oplist_tsd_key __read_mostly; @@ -219,13 +220,13 @@ struct pmap_update_queue { * they perform. */ struct pmap_syncer { - struct thread *thread; + alignas(CPU_L1_SIZE) struct thread *thread; struct pmap_update_queue queue; struct syscnt sc_updates; struct syscnt sc_update_enters; struct syscnt sc_update_removes; struct syscnt sc_update_protects; -} __aligned(CPU_L1_SIZE); +}; static void pmap_sync(void *arg); @@ -244,14 +245,14 @@ static struct pmap_syncer pmap_syncer __percpu; * individual TLB entries or globally flush the TLB. */ struct pmap_update_request { - struct list node; + alignas(CPU_L1_SIZE) struct list node; struct spinlock lock; struct thread *sender; const struct pmap_update_oplist *oplist; unsigned int nr_mappings; int done; int error; -} __aligned(CPU_L1_SIZE); +}; /* * Per processor array of requests. @@ -447,11 +448,12 @@ pmap_setup_paging(void) directmap_end = biosmem_directmap_end(); - if (directmap_end > (VM_MAX_DIRECTMAP_ADDRESS - VM_MIN_DIRECTMAP_ADDRESS)) { + if (directmap_end > (PMAP_MAX_DIRECTMAP_ADDRESS + - PMAP_MIN_DIRECTMAP_ADDRESS)) { boot_panic(pmap_panic_directmap_msg); } - va = VM_MIN_DIRECTMAP_ADDRESS; + va = PMAP_MIN_DIRECTMAP_ADDRESS; pa = 0; for (i = 0; i < directmap_end; i += pgsize) { @@ -464,7 +466,7 @@ pmap_setup_paging(void) /* * On 64-bits systems, the kernel isn't linked at addresses included * in the direct mapping, which requires the creation of an additional - * mapping for it. See param.h for more details. + * mapping for it. */ va = P2ALIGN((uintptr_t)&_init, pgsize); pa = BOOT_VTOP(va); @@ -509,14 +511,14 @@ pmap_ap_setup_paging(void) #define pmap_assert_range(pmap, start, end) \ MACRO_BEGIN \ assert((start) < (end)); \ - assert(((end) <= VM_MIN_DIRECTMAP_ADDRESS) \ - || ((start) >= VM_MAX_DIRECTMAP_ADDRESS)); \ + assert(((end) <= PMAP_MIN_DIRECTMAP_ADDRESS) \ + || ((start) >= PMAP_MAX_DIRECTMAP_ADDRESS)); \ \ if ((pmap) == kernel_pmap) { \ - assert(((start) >= VM_MIN_KMEM_ADDRESS) \ - && ((end) <= VM_MAX_KMEM_ADDRESS)); \ + assert(((start) >= PMAP_MIN_KMEM_ADDRESS) \ + && ((end) <= PMAP_MAX_KMEM_ADDRESS)); \ } else { \ - assert((end) <= VM_MAX_ADDRESS); \ + assert((end) <= PMAP_MAX_ADDRESS); \ } \ MACRO_END @@ -583,7 +585,7 @@ pmap_walk_vas(uintptr_t start, uintptr_t end, pmap_walk_fn_t walk_fn) assert(vm_page_aligned(start)); assert(start < end); #ifdef __LP64__ - assert((start < VM_MAX_ADDRESS) || (start >= VM_MIN_KERNEL_ADDRESS)); + assert((start < PMAP_MAX_ADDRESS) || (start >= PMAP_MIN_KERNEL_ADDRESS)); #endif /* __LP64__ */ va = start; @@ -592,8 +594,8 @@ pmap_walk_vas(uintptr_t start, uintptr_t end, pmap_walk_fn_t walk_fn) do { #ifdef __LP64__ /* Handle long mode canonical form */ - if (va == VM_MAX_ADDRESS) { - va = VM_MIN_KERNEL_ADDRESS; + if (va == PMAP_MAX_ADDRESS) { + va = PMAP_MIN_KERNEL_ADDRESS; } #endif /* __LP64__ */ @@ -641,7 +643,7 @@ pmap_setup_global_page(phys_addr_t ptp_pa, unsigned int index, static void __init pmap_setup_global_pages(void) { - pmap_walk_vas(VM_MIN_KERNEL_ADDRESS, VM_MAX_KERNEL_ADDRESS, + pmap_walk_vas(PMAP_MIN_KERNEL_ADDRESS, PMAP_MAX_KERNEL_ADDRESS, pmap_setup_global_page); pmap_pt_levels[0].mask |= PMAP_PTE_G; cpu_enable_global_pages(); @@ -826,7 +828,7 @@ pmap_syncer_init(struct pmap_syncer *syncer, unsigned int cpu) syscnt_register(&syncer->sc_update_protects, name); } -void __init +static int __init pmap_bootstrap(void) { struct pmap_cpu_table *cpu_table; @@ -862,20 +864,17 @@ pmap_bootstrap(void) if (cpu_has_global_pages()) { pmap_setup_global_pages(); } -} - -void __init -pmap_ap_bootstrap(void) -{ - cpu_local_assign(pmap_current_ptr, kernel_pmap); - if (cpu_has_global_pages()) { - cpu_enable_global_pages(); - } else { - cpu_tlb_flush(); - } + return 0; } +INIT_OP_DEFINE(pmap_bootstrap, + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_bootstrap, true)); + static void __init pmap_setup_set_ptp_type(phys_addr_t ptp_pa, unsigned int index, unsigned int level) @@ -900,11 +899,11 @@ pmap_setup_set_ptp_type(phys_addr_t ptp_pa, unsigned int index, static void __init pmap_setup_fix_ptps(void) { - pmap_walk_vas(VM_MIN_ADDRESS, VM_MAX_KERNEL_ADDRESS, + pmap_walk_vas(PMAP_MIN_ADDRESS, PMAP_MAX_KERNEL_ADDRESS, pmap_setup_set_ptp_type); } -void __init +static int __init pmap_setup(void) { pmap_setup_fix_ptps(); @@ -912,6 +911,25 @@ pmap_setup(void) kmem_cache_init(&pmap_update_oplist_cache, "pmap_update_oplist", sizeof(struct pmap_update_oplist), CPU_L1_SIZE, pmap_update_oplist_ctor, 0); + return 0; +} + +INIT_OP_DEFINE(pmap_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(pmap_bootstrap, true), + INIT_OP_DEP(vm_page_setup, true)); + +void __init +pmap_ap_setup(void) +{ + cpu_local_assign(pmap_current_ptr, kernel_pmap); + + if (cpu_has_global_pages()) { + cpu_enable_global_pages(); + } else { + cpu_tlb_flush(); + } } static void __init @@ -946,8 +964,8 @@ pmap_copy_cpu_table_recursive(const pmap_pte_t *sptp, unsigned int level, i++, va = P2END(va, 1UL << pt_level->skip)) { #ifdef __LP64__ /* Handle long mode canonical form */ - if (va == VM_MAX_ADDRESS) { - va = VM_MIN_KERNEL_ADDRESS; + if (va == PMAP_MAX_ADDRESS) { + va = PMAP_MIN_KERNEL_ADDRESS; } #endif /* __LP64__ */ @@ -1006,7 +1024,7 @@ pmap_copy_cpu_table(unsigned int cpu) dptp = vm_page_direct_ptr(page); #endif /* X15_X86_PAE */ - pmap_copy_cpu_table_recursive(sptp, level, dptp, VM_MIN_ADDRESS); + pmap_copy_cpu_table_recursive(sptp, level, dptp, PMAP_MIN_ADDRESS); } void __init @@ -1162,7 +1180,7 @@ pmap_enter_local(struct pmap *pmap, uintptr_t va, phys_addr_t pa, page = vm_page_alloc(0, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_PMAP); if (page == NULL) { - printf("pmap: warning: page table page allocation failure\n"); + log_warning("pmap: page table page allocation failure"); return ERROR_NOMEM; } diff --git a/arch/x86/machine/pmap.h b/arch/x86/machine/pmap.h index 9c76f5d8..114dcdac 100644 --- a/arch/x86/machine/pmap.h +++ b/arch/x86/machine/pmap.h @@ -16,6 +16,7 @@ * * * TODO Comment. + * TODO Rename MIN/MAX to START/END. */ #ifndef _X86_PMAP_H @@ -24,6 +25,71 @@ #include <kern/macros.h> /* + * Virtual memory layout. + */ + +/* + * User space boundaries. + */ +#define PMAP_MIN_ADDRESS DECL_CONST(0, UL) + +#ifdef __LP64__ +#define PMAP_MAX_ADDRESS DECL_CONST(0x800000000000, UL) +#else /* __LP64__ */ +#define PMAP_MAX_ADDRESS DECL_CONST(0xc0000000, UL) +#endif/* __LP64__ */ + +/* + * Kernel space boundaries. + */ +#ifdef __LP64__ +#define PMAP_MIN_KERNEL_ADDRESS DECL_CONST(0xffff800000000000, UL) +#define PMAP_MAX_KERNEL_ADDRESS DECL_CONST(0xfffffffffffff000, UL) +#else /* __LP64__ */ +#define PMAP_MIN_KERNEL_ADDRESS PMAP_MAX_ADDRESS +#define PMAP_MAX_KERNEL_ADDRESS DECL_CONST(0xfffff000, UL) +#endif /* __LP64__ */ + +/* + * Direct physical mapping boundaries. + */ +#ifdef __LP64__ +#define PMAP_MIN_DIRECTMAP_ADDRESS PMAP_MIN_KERNEL_ADDRESS +#define PMAP_MAX_DIRECTMAP_ADDRESS DECL_CONST(0xffffc00000000000, UL) +#else /* __LP64__ */ +#define PMAP_MIN_DIRECTMAP_ADDRESS PMAP_MAX_ADDRESS +#define PMAP_MAX_DIRECTMAP_ADDRESS DECL_CONST(0xf8000000, UL) +#endif /* __LP64__ */ + +/* + * Kernel mapping offset. + * + * On 32-bits systems, the kernel is linked at addresses included in the + * direct physical mapping, whereas on 64-bits systems, it is linked at + * -2 GiB because the "kernel" memory model is used when compiling (see + * the -mcmodel=kernel gcc option). + */ +#ifdef __LP64__ +#define PMAP_KERNEL_OFFSET DECL_CONST(0xffffffff80000000, UL) +#else /* __LP64__ */ +#define PMAP_KERNEL_OFFSET PMAP_MIN_DIRECTMAP_ADDRESS +#endif /* __LP64__ */ + +/* + * Kernel virtual space boundaries. + * + * In addition to the direct physical mapping, the kernel has its own virtual + * memory space. + */ +#define PMAP_MIN_KMEM_ADDRESS PMAP_MAX_DIRECTMAP_ADDRESS + +#ifdef __LP64__ +#define PMAP_MAX_KMEM_ADDRESS PMAP_KERNEL_OFFSET +#else /* __LP64__ */ +#define PMAP_MAX_KMEM_ADDRESS PMAP_MAX_KERNEL_ADDRESS +#endif /* __LP64__ */ + +/* * Page table entry flags. */ #define PMAP_PTE_P 0x00000001 @@ -99,11 +165,11 @@ #include <stdint.h> #include <kern/cpumap.h> +#include <kern/init.h> #include <kern/list.h> #include <kern/mutex.h> #include <kern/thread.h> #include <machine/cpu.h> -#include <machine/trap.h> #include <machine/types.h> /* @@ -138,22 +204,9 @@ pmap_pte_t * pmap_setup_paging(void); pmap_pte_t * pmap_ap_setup_paging(void); /* - * Early initialization of the pmap module. - */ -void pmap_bootstrap(void); - -/* - * Early initialization of the MMU on APs. - */ -void pmap_ap_bootstrap(void); - -/* - * Set up the pmap module. - * - * This function should only be called by the VM system, once kernel - * allocations can be performed safely. + * Initialize the pmap module on APs. */ -void pmap_setup(void); +void pmap_ap_setup(void); /* * Set up the pmap module for multiprocessor operations. @@ -265,6 +318,19 @@ pmap_current(void) return cpu_local_read(pmap_current_ptr); } +/* + * This init operation provides : + * - kernel pmap operations + */ +INIT_OP_DECLARE(pmap_bootstrap); + +/* + * This init operation provides : + * - user pmap creation + * - module fully initialized + */ +INIT_OP_DECLARE(pmap_setup); + #endif /* __ASSEMBLER__ */ #endif /* _X86_PMAP_H */ diff --git a/arch/x86/machine/pmem.h b/arch/x86/machine/pmem.h new file mode 100644 index 00000000..b6c6db0c --- /dev/null +++ b/arch/x86/machine/pmem.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Physical memory layout. + * + * This file is a top header in the inclusion hierarchy, and shouldn't include + * other headers that may cause circular dependencies. + */ + +#ifndef _X86_PMEM_H +#define _X86_PMEM_H + +#include <kern/macros.h> + +/* + * Zone boundaries. + */ + +#define PMEM_DMA_LIMIT DECL_CONST(0x1000000, UL) + +#ifdef __LP64__ +#define PMEM_MAX_ZONES 4 +#define PMEM_DMA32_LIMIT DECL_CONST(0x100000000, UL) +#define PMEM_DIRECTMAP_LIMIT DECL_CONST(0x400000000000, UL) +#define PMEM_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, UL) +#else /* __LP64__ */ +#define PMEM_DIRECTMAP_LIMIT DECL_CONST(0x38000000, ULL) +#ifdef X15_X86_PAE +#define PMEM_MAX_ZONES 3 +#define PMEM_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, ULL) +#else /* X15_X86_PAE */ +#define PMEM_MAX_ZONES 3 +#define PMEM_HIGHMEM_LIMIT DECL_CONST(0xfffff000, UL) +#endif /* X15_X86_PAE */ +#endif /* __LP64__ */ + +/* + * Zone vm_page indexes. + */ + +#define PMEM_ZONE_DMA 0 +#define PMEM_ZONE_DMA32 1 + +#ifdef __LP64__ +#define PMEM_ZONE_DIRECTMAP 2 +#define PMEM_ZONE_HIGHMEM 3 +#else /* __LP64__ */ +#define PMEM_ZONE_DMA32 1 +#define PMEM_ZONE_DIRECTMAP 1 /* Alias for the DMA32 zone */ +#define PMEM_ZONE_HIGHMEM 2 +#endif /* __LP64__ */ + +#endif /* _X86_PMEM_H */ diff --git a/vm/vm_setup.h b/arch/x86/machine/ssp.c index f52ddb24..77d5a2b5 100644 --- a/vm/vm_setup.h +++ b/arch/x86/machine/ssp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Richard Braun. + * Copyright (c) 2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,14 +15,16 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef _VM_VM_SETUP_H -#define _VM_VM_SETUP_H +#include <kern/macros.h> +#include <kern/panic.h> +#include <machine/ssp.h> -/* - * Set up the VM system. - * - * This function also initializes the kmem (kernel memory) allocator. - */ -void vm_setup(void); +void ssp_panic(void); + +void +ssp_panic(void) +{ + panic("ssp: stack corruption detected"); +} -#endif /* _VM_VM_SETUP_H */ +void __stack_chk_fail(void) __attribute__((alias("ssp_panic"))); diff --git a/kern/param.h b/arch/x86/machine/ssp.h index d0061728..f2783616 100644 --- a/kern/param.h +++ b/arch/x86/machine/ssp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Richard Braun. + * Copyright (c) 2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,9 +15,18 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef _KERN_PARAM_H -#define _KERN_PARAM_H +#ifndef _X86_SSP_H +#define _X86_SSP_H -#include <machine/param.h> +#ifdef __LP64__ +#define SSP_GUARD_WORD 0xdeadd00ddeadd00d +#else +#define SSP_GUARD_WORD 0xdeadd00d +#endif -#endif /* _KERN_PARAM_H */ +/* + * Offset, in words, of the SSP guard word. + */ +#define SSP_WORD_TLS_OFFSET 5 + +#endif /* _X86_SSP_H */ diff --git a/arch/x86/machine/strace.c b/arch/x86/machine/strace.c index 8aea4b05..cfb237e1 100644 --- a/arch/x86/machine/strace.c +++ b/arch/x86/machine/strace.c @@ -21,7 +21,8 @@ #include <kern/init.h> #include <kern/kmem.h> -#include <kern/param.h> +#include <kern/log.h> +#include <kern/macros.h> #include <machine/elf.h> #include <machine/multiboot.h> #include <machine/pmap.h> @@ -35,6 +36,8 @@ #define STRACE_ADDR_FORMAT "%#010lx" #endif /* __LP64__ */ +static const struct multiboot_raw_info *strace_mbi __initdata; + static struct elf_sym *strace_symtab __read_mostly; static struct elf_sym *strace_symtab_end __read_mostly; static char *strace_strtab __read_mostly; @@ -74,9 +77,9 @@ strace_show_one(unsigned int index, unsigned long ip) name = strace_lookup(ip, &offset, &size); if (name == NULL) { - printf("strace: #%u [" STRACE_ADDR_FORMAT "]\n", index, ip); + printf("#%u [" STRACE_ADDR_FORMAT "]\n", index, ip); } else { - printf("strace: #%u [" STRACE_ADDR_FORMAT "] %s+%#lx/%#lx\n", + printf("#%u [" STRACE_ADDR_FORMAT "] %s+%#lx/%#lx\n", index, ip, name, (unsigned long)offset, (unsigned long)size); } } @@ -89,7 +92,6 @@ strace_show(unsigned long ip, unsigned long bp) unsigned int i; int error; - printf("strace: stack trace:\n"); strace_show_one(0, ip); i = 1; @@ -118,8 +120,6 @@ strace_show(unsigned long ip, unsigned long bp) i++; frame = frame[0]; } - - printf("strace: end of trace\n"); } static void * __init @@ -133,14 +133,14 @@ strace_copy_section(const struct elf_shdr *shdr) src = vm_kmem_map_pa(shdr->addr, shdr->size, &map_addr, &map_size); if (src == NULL) { - printf("strace: unable to map section\n"); + log_err("strace: unable to map section"); goto error_map; } copy = kmem_alloc(shdr->size); if (copy == NULL) { - printf("strace: unable to allocate section copy\n"); + log_err("strace: unable to allocate section copy"); goto error_copy; } @@ -175,14 +175,23 @@ strace_lookup_section(const struct multiboot_raw_info *mbi, const void *table, } void __init -strace_setup(const struct multiboot_raw_info *mbi) +strace_set_mbi(const struct multiboot_raw_info *mbi) +{ + strace_mbi = mbi; +} + +static int __init +strace_setup(void) { const struct elf_shdr *shstrtab_hdr, *symtab_hdr, *strtab_hdr; + const struct multiboot_raw_info *mbi; uintptr_t map_addr, shstrtab_map_addr; size_t size, map_size, shstrtab_map_size; const char *shstrtab; const void *table; + mbi = strace_mbi; + if (!(mbi->flags & MULTIBOOT_LOADER_SHDR) || (mbi->shdr_num == 0)) { goto no_syms; } @@ -191,12 +200,12 @@ strace_setup(const struct multiboot_raw_info *mbi) table = vm_kmem_map_pa(mbi->shdr_addr, size, &map_addr, &map_size); if (table == NULL) { - printf("strace: unable to map section headers table\n"); + log_err("strace: unable to map section headers table"); goto no_syms; } if (mbi->shdr_strndx >= mbi->shdr_num) { - printf("strace: invalid section names index\n"); + log_err("strace: invalid section names index"); goto error_shstrndx; } @@ -205,21 +214,21 @@ strace_setup(const struct multiboot_raw_info *mbi) &shstrtab_map_addr, &shstrtab_map_size); if (shstrtab == NULL) { - printf("strace: unable to map section names\n"); + log_err("strace: unable to map section names"); goto error_shstrtab; } symtab_hdr = strace_lookup_section(mbi, table, shstrtab, ".symtab"); if (symtab_hdr == NULL) { - printf("strace: unable to find symbol table\n"); + log_err("strace: unable to find symbol table"); goto error_symtab_lookup; } strtab_hdr = strace_lookup_section(mbi, table, shstrtab, ".strtab"); if (strtab_hdr == NULL) { - printf("strace: unable to find symbol string table\n"); + log_err("strace: unable to find symbol string table"); goto error_strtab_lookup; } @@ -238,7 +247,7 @@ strace_setup(const struct multiboot_raw_info *mbi) vm_kmem_unmap_pa(shstrtab_map_addr, shstrtab_map_size); vm_kmem_unmap_pa(map_addr, map_size); - return; + return 0; error_strtab: kmem_free(strace_symtab, symtab_hdr->size); @@ -253,4 +262,12 @@ no_syms: strace_symtab = NULL; strace_symtab_end = NULL; strace_strtab = NULL; + return 0; } + +INIT_OP_DEFINE(strace_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(pmap_bootstrap, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(vm_kmem_setup, true)); diff --git a/arch/x86/machine/strace.h b/arch/x86/machine/strace.h index 66aa18b7..2f686a07 100644 --- a/arch/x86/machine/strace.h +++ b/arch/x86/machine/strace.h @@ -21,6 +21,7 @@ #ifndef _X86_STRACE_H #define _X86_STRACE_H +#include <kern/init.h> #include <kern/macros.h> #include <machine/multiboot.h> @@ -45,10 +46,17 @@ strace_dump(void) } /* - * Setup the stack tracing module. + * Pass the multiboot information structure. * - * If available, the symbol table is extracted from the boot data. + * If available, the symbol table is extracted from the boot data, when + * the strace module is initialized. */ -void strace_setup(const struct multiboot_raw_info *mbi); +void strace_set_mbi(const struct multiboot_raw_info *mbi); + +/* + * This init operation provides : + * - module fully initialized + */ +INIT_OP_DECLARE(strace_setup); #endif /* _X86_STRACE_H */ diff --git a/arch/x86/machine/string.c b/arch/x86/machine/string.c index 10d9ee0a..c44b62d5 100644 --- a/arch/x86/machine/string.c +++ b/arch/x86/machine/string.c @@ -13,14 +13,20 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Despite comparison and scan instructions not having side-effects, the + * memory clobber is used because the compiler cannot infer dependencies + * on the memory referenced by the pointers. */ #include <stddef.h> #include <string.h> -#include <kern/param.h> +#include <kern/macros.h> +#include <machine/string.h> -#ifdef ARCH_STRING_MEMCPY +#ifdef STRING_ARCH_MEMCPY void * memcpy(void *dest, const void *src, size_t n) { @@ -32,9 +38,9 @@ memcpy(void *dest, const void *src, size_t n) : : "memory"); return orig_dest; } -#endif /* ARCH_STRING_MEMCPY */ +#endif /* STRING_ARCH_MEMCPY */ -#ifdef ARCH_STRING_MEMMOVE +#ifdef STRING_ARCH_MEMMOVE void * memmove(void *dest, const void *src, size_t n) { @@ -56,9 +62,9 @@ memmove(void *dest, const void *src, size_t n) return orig_dest; } -#endif /* ARCH_STRING_MEMMOVE */ +#endif /* STRING_ARCH_MEMMOVE */ -#ifdef ARCH_STRING_MEMSET +#ifdef STRING_ARCH_MEMSET void * memset(void *s, int c, size_t n) { @@ -71,9 +77,9 @@ memset(void *s, int c, size_t n) : "memory"); return orig_s; } -#endif /* ARCH_STRING_MEMSET */ +#endif /* STRING_ARCH_MEMSET */ -#ifdef ARCH_STRING_MEMCMP +#ifdef STRING_ARCH_MEMCMP int memcmp(const void *s1, const void *s2, size_t n) { @@ -90,9 +96,9 @@ memcmp(const void *s1, const void *s2, size_t n) c2 = *(((const unsigned char *)s2) - 1); return (int)c1 - (int)c2; } -#endif /* ARCH_STRING_MEMCMP */ +#endif /* STRING_ARCH_MEMCMP */ -#ifdef ARCH_STRING_STRLEN +#ifdef STRING_ARCH_STRLEN size_t strlen(const char *s) { @@ -103,11 +109,11 @@ strlen(const char *s) : "+D" (s), "+c" (n) : "a" (0) : "memory"); - return ~n - 1; + return (size_t)-2 - n; } -#endif /* ARCH_STRING_STRLEN */ +#endif /* STRING_ARCH_STRLEN */ -#ifdef ARCH_STRING_STRCPY +#ifdef STRING_ARCH_STRCPY char * strcpy(char *dest, const char *src) { @@ -123,9 +129,9 @@ strcpy(char *dest, const char *src) : : "al", "memory"); return orig_dest; } -#endif /* ARCH_STRING_STRCPY */ +#endif /* STRING_ARCH_STRCPY */ -#ifdef ARCH_STRING_STRCMP +#ifdef STRING_ARCH_STRCMP int strcmp(const char *s1, const char *s2) { @@ -144,4 +150,51 @@ strcmp(const char *s1, const char *s2) c2 = *(((const unsigned char *)s2) - 1); return (int)c1 - (int)c2; } -#endif /* ARCH_STRING_STRCMP */ +#endif /* STRING_ARCH_STRCMP */ + +#ifdef STRING_ARCH_STRNCMP +int +strncmp(const char *s1, const char *s2, size_t n) +{ + unsigned char c1, c2; + + if (unlikely(n == 0)) { + return 0; + } + + asm volatile("1:\n" + "lodsb\n" + "scasb\n" + "jne 1f\n" + "testb %%al, %%al\n" + "jz 1f\n" + "dec %2\n" + "jnz 1b\n" + "1:\n" + : "+D" (s1), "+S" (s2), "+c" (n) + : : "al", "memory"); + c1 = *(((const unsigned char *)s1) - 1); + c2 = *(((const unsigned char *)s2) - 1); + return (int)c1 - (int)c2; +} +#endif /* STRING_ARCH_STRNCMP */ + +#ifdef STRING_ARCH_STRCHR +char * +strchr(const char *s, int c) +{ + asm volatile("1:\n" + "lodsb\n" + "cmpb %%al, %1\n" + "je 1f\n" + "testb %%al, %%al\n" + "jnz 1b\n" + "mov $1, %0\n" + "1:\n" + "dec %0\n" + : "+S" (s) + : "c" ((char)c) + : "al", "memory"); + return (char *)s; +} +#endif /* STRING_ARCH_STRCHR */ diff --git a/arch/x86/machine/string.h b/arch/x86/machine/string.h new file mode 100644 index 00000000..6111ba10 --- /dev/null +++ b/arch/x86/machine/string.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _X86_STRING_H +#define _X86_STRING_H + +/* + * Provide architecture-specific string functions. + */ +#define STRING_ARCH_MEMCPY +#define STRING_ARCH_MEMMOVE +#define STRING_ARCH_MEMSET +#define STRING_ARCH_MEMCMP +#define STRING_ARCH_STRLEN +#define STRING_ARCH_STRCPY +#define STRING_ARCH_STRCMP +#define STRING_ARCH_STRNCMP +#define STRING_ARCH_STRCHR + +#endif /* _X86_STRING_H */ diff --git a/arch/x86/machine/tcb.c b/arch/x86/machine/tcb.c index df1b222c..b54945c7 100644 --- a/arch/x86/machine/tcb.c +++ b/arch/x86/machine/tcb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2014 Richard Braun. + * Copyright (c) 2012-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,27 +15,67 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <stdnoreturn.h> + #include <kern/init.h> -#include <kern/macros.h> -#include <kern/param.h> #include <kern/thread.h> #include <machine/cpu.h> #include <machine/pmap.h> #include <machine/strace.h> #include <machine/tcb.h> -/* - * Low level functions. - */ -void __noreturn tcb_context_load(struct tcb *tcb); -void __noreturn tcb_start(void); +noreturn void tcb_context_load(struct tcb *tcb); +noreturn void tcb_start(void); +void tcb_context_restore(void); struct tcb *tcb_current_ptr __percpu; +static void +tcb_stack_push(struct tcb *tcb, uintptr_t word) +{ + uintptr_t *ptr; + + ptr = (uintptr_t *)tcb->sp; + ptr--; + *ptr = word; + tcb->sp = (uintptr_t)ptr; +} + +#ifdef __LP64__ + +static void +tcb_stack_forge(struct tcb *tcb, void (*fn)(void *), void *arg) +{ + tcb_stack_push(tcb, (uintptr_t)arg); + tcb_stack_push(tcb, (uintptr_t)fn); + tcb_stack_push(tcb, (uintptr_t)tcb_start); /* Return address */ + tcb_stack_push(tcb, CPU_EFL_ONE); /* RFLAGS */ + tcb_stack_push(tcb, 0); /* RBX */ + tcb_stack_push(tcb, 0); /* R12 */ + tcb_stack_push(tcb, 0); /* R13 */ + tcb_stack_push(tcb, 0); /* R14 */ + tcb_stack_push(tcb, 0); /* R15 */ +} + +#else /* __LP64__ */ + +static void +tcb_stack_forge(struct tcb *tcb, void (*fn)(void *), void *arg) +{ + tcb_stack_push(tcb, (uintptr_t)arg); + tcb_stack_push(tcb, (uintptr_t)fn); + tcb_stack_push(tcb, (uintptr_t)tcb_start); /* Return address */ + tcb_stack_push(tcb, CPU_EFL_ONE); /* EFLAGS */ + tcb_stack_push(tcb, 0); /* EBX */ + tcb_stack_push(tcb, 0); /* EDI */ + tcb_stack_push(tcb, 0); /* ESI */ +} + +#endif /* __LP64__ */ + int -tcb_init(struct tcb *tcb, void *stack, void (*fn)(void)) +tcb_init(struct tcb *tcb, void *stack, void (*fn)(void *), void *arg) { - void **ptr; int error; error = pmap_thread_init(thread_from_tcb(tcb)); @@ -45,12 +85,8 @@ tcb_init(struct tcb *tcb, void *stack, void (*fn)(void)) } tcb->bp = 0; - tcb->sp = (unsigned long)stack + STACK_SIZE - sizeof(unsigned long); - tcb->ip = (unsigned long)tcb_start; - - ptr = (void **)tcb->sp; - *ptr = fn; - + tcb->sp = (uintptr_t)stack + TCB_STACK_SIZE; + tcb_stack_forge(tcb, fn, arg); return 0; } @@ -66,5 +102,14 @@ tcb_load(struct tcb *tcb) void tcb_trace(const struct tcb *tcb) { - strace_show(tcb->ip, tcb->bp); + strace_show((uintptr_t)tcb_context_restore, tcb->bp); } + +static int __init +tcb_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(tcb_setup, + INIT_OP_DEP(cpu_setup, true)); diff --git a/arch/x86/machine/tcb.h b/arch/x86/machine/tcb.h index 5f455850..9107f7c4 100644 --- a/arch/x86/machine/tcb.h +++ b/arch/x86/machine/tcb.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2014 Richard Braun. + * Copyright (c) 2012-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,33 +21,36 @@ #ifndef _X86_TCB_H #define _X86_TCB_H -#include <kern/assert.h> -#include <kern/macros.h> +#include <assert.h> +#include <stdint.h> +#include <stdnoreturn.h> + +#include <kern/init.h> #include <machine/cpu.h> +#include <machine/page.h> + +/* + * Thread stack size. + */ +#define TCB_STACK_SIZE PAGE_SIZE /* * Thread control block. */ struct tcb { - unsigned long bp; - unsigned long sp; - unsigned long ip; + uintptr_t bp; + uintptr_t sp; }; /* * Initialize a TCB. * * Prepare the given stack for execution. The context is defined so that it - * will call fn() with interrupts disabled when loaded. + * will call thread_main(fn, arg) with interrupts disabled when loaded. * * In addition, initialize any thread-local machine-specific data. */ -int tcb_init(struct tcb *tcb, void *stack, void (*fn)(void)); - -/* - * Low level context switch function. - */ -void tcb_context_switch(struct tcb *prev, struct tcb *next); +int tcb_init(struct tcb *tcb, void *stack, void (*fn)(void *), void *arg); static inline struct tcb * tcb_current(void) @@ -68,7 +71,7 @@ tcb_set_current(struct tcb *tcb) * * Called with interrupts disabled. The caller context is lost. */ -void __noreturn tcb_load(struct tcb *tcb); +noreturn void tcb_load(struct tcb *tcb); /* * Context switch. @@ -78,6 +81,8 @@ void __noreturn tcb_load(struct tcb *tcb); static inline void tcb_switch(struct tcb *prev, struct tcb *next) { + void tcb_context_switch(struct tcb *prev, struct tcb *next); + assert(!cpu_intr_enabled()); tcb_set_current(next); @@ -87,8 +92,14 @@ tcb_switch(struct tcb *prev, struct tcb *next) /* * Dump the stack trace of a TCB. * - * The thread associated to the TCB should not be running. + * The thread associated to the TCB must not be running. */ void tcb_trace(const struct tcb *tcb); +/* + * This init operation provides : + * - current TCB handling + */ +INIT_OP_DECLARE(tcb_setup); + #endif /* _X86_TCB_H */ diff --git a/arch/x86/machine/tcb_asm.S b/arch/x86/machine/tcb_asm.S index 2089e939..a6e31da5 100644 --- a/arch/x86/machine/tcb_asm.S +++ b/arch/x86/machine/tcb_asm.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2013 Richard Braun. + * Copyright (c) 2012-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,24 +24,18 @@ #ifdef __LP64__ ASM_ENTRY(tcb_context_load) - movq (%rdi), %rbp - movq 8(%rdi), %rsp - movq 16(%rdi), %rax - pushq $CPU_EFL_ONE - popfq - jmp *%rax + movq (%rdi), %rbp /* load frame pointer from TCB */ + movq 8(%rdi), %rsp /* load stack pointer from TCB */ + jmp tcb_context_restore ASM_END(tcb_context_load) #else /* __LP64__ */ ASM_ENTRY(tcb_context_load) - movl 4(%esp), %eax - movl (%eax), %ebp - movl 4(%eax), %esp - movl 8(%eax), %ecx - pushl $CPU_EFL_ONE - popfl - jmp *%ecx + movl 4(%esp), %eax /* load TCB address */ + movl (%eax), %ebp /* load frame pointer from TCB */ + movl 4(%eax), %esp /* load stack pointer from TCB */ + jmp tcb_context_restore ASM_END(tcb_context_load) #endif /* __LP64__ */ @@ -51,30 +45,39 @@ ASM_END(tcb_context_load) #ifdef __LP64__ ASM_ENTRY(tcb_start) - popq %rax - call *%rax + popq %rdi /* load function */ + popq %rsi /* load argument */ + + /* + * Use the call instruction to start a clean stack trace. + * + * Note that, on amd64, the stack must be 16-byte before the call + * instruction, so that "$(rsp + 8) is a multiple is always a multiple + * of 16 when control is transferred to the function entry point", + * which is another reason to use call instead of a bare jump. + */ + call thread_main /* Never reached */ - nop + nop /* make the return address point to an instruction + inside the function to build a clean stack trace */ ASM_END(tcb_start) ASM_ENTRY(tcb_context_switch) - pushfq + pushfq /* store registers as required by ABI */ pushq %rbx pushq %r12 pushq %r13 pushq %r14 pushq %r15 - movq %rbp, (%rdi) - movq %rsp, 8(%rdi) - movq $1f, 16(%rdi) - movq (%rsi), %rbp - movq 8(%rsi), %rsp - movq 16(%rsi), %rax - jmp *%rax - -1: - popq %r15 + movq %rbp, (%rdi) /* store frame pointer into prev TCB */ + movq %rsp, 8(%rdi) /* store stack pointer into prev TCB */ + movq (%rsi), %rbp /* load frame pointer from next TCB */ + movq 8(%rsi), %rsp /* load stack pointer from next TCB */ + +.global tcb_context_restore +tcb_context_restore: + popq %r15 /* load registers as required by ABI */ popq %r14 popq %r13 popq %r12 @@ -86,29 +89,34 @@ ASM_END(tcb_context_switch) #else /* __LP64__ */ ASM_ENTRY(tcb_start) - popl %eax - call *%eax + call thread_main /* the stack already contains the expected arguments + in the expected order, use the call instruction to + start a clean stack trace */ /* Never reached */ - nop + nop /* make the return address point to an instruction + inside the function to build a clean stack trace */ ASM_END(tcb_start) ASM_ENTRY(tcb_context_switch) - movl 4(%esp), %eax - movl 8(%esp), %ecx - pushfl + movl 4(%esp), %eax /* load prev TCB address */ + movl 8(%esp), %ecx /* load next TCB address */ + pushfl /* store registers as required by ABI */ pushl %ebx pushl %edi pushl %esi - movl %ebp, (%eax) - movl %esp, 4(%eax) - movl $1f, 8(%eax) - movl (%ecx), %ebp - movl 4(%ecx), %esp - movl 8(%ecx), %edx - jmp *%edx - -1: + movl %ebp, (%eax) /* store frame pointer into prev TCB */ + movl %esp, 4(%eax) /* store stack pointer into prev TCB */ + movl (%ecx), %ebp /* load frame pointer from next TCB */ + movl 4(%ecx), %esp /* load stack pointer from next TCB */ + +/* + * This code is run on context restoration. The frame and stack pointers + * have already been loaded to their correct values. Load registers which + * were stored on the stack when the context was saved and return. + */ +.global tcb_context_restore +tcb_context_restore: popl %esi popl %edi popl %ebx diff --git a/arch/x86/machine/trap.c b/arch/x86/machine/trap.c index 3de4fd26..878f20c9 100644 --- a/arch/x86/machine/trap.c +++ b/arch/x86/machine/trap.c @@ -19,14 +19,14 @@ * additional configuration and resources to be properly handled. */ +#include <assert.h> +#include <stdalign.h> #include <stdint.h> #include <stdio.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/init.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/spinlock.h> #include <kern/thread.h> #include <machine/cpu.h> @@ -37,7 +37,7 @@ #include <machine/trap.h> struct trap_cpu_data { - unsigned char intr_stack[STACK_SIZE] __aligned(DATA_ALIGN); + alignas(CPU_DATA_ALIGN) unsigned char intr_stack[TRAP_STACK_SIZE]; }; static struct trap_cpu_data trap_cpu_data __percpu; @@ -171,7 +171,7 @@ trap_default(struct trap_frame *frame) cpu_halt(); } -void __init +static int __init trap_setup(void) { size_t i; @@ -213,8 +213,12 @@ trap_setup(void) trap_install(TRAP_LAPIC_TIMER, TRAP_HF_INTR, lapic_timer_intr); trap_install(TRAP_LAPIC_ERROR, TRAP_HF_INTR, lapic_error_intr); trap_install(TRAP_LAPIC_SPURIOUS, TRAP_HF_INTR, lapic_spurious_intr); + + return 0; } +INIT_OP_DEFINE(trap_setup); + void trap_main(struct trap_frame *frame) { diff --git a/arch/x86/machine/trap.h b/arch/x86/machine/trap.h index 7bda9cf3..699f19e7 100644 --- a/arch/x86/machine/trap.h +++ b/arch/x86/machine/trap.h @@ -16,11 +16,16 @@ * * * Trap (interrupt and exception) handling. + * + * This file is a top header in the inclusion hierarchy, and shouldn't include + * other headers that may cause circular dependencies. */ #ifndef _X86_TRAP_H #define _X86_TRAP_H +#include <machine/page.h> + /* * Architecture defined traps. */ @@ -61,16 +66,18 @@ #define TRAP_LAPIC_ERROR 254 #define TRAP_LAPIC_SPURIOUS 255 -/* - * Vector identifying an unhandled trap. - */ -#define TRAP_DEFAULT 256 +#define TRAP_NR_VECTORS 256 + +#define TRAP_INTR_TABLE_SIZE 256 + +#define TRAP_STACK_SIZE PAGE_SIZE #ifndef __ASSEMBLER__ #include <stdint.h> #include <stdio.h> +#include <kern/init.h> #include <kern/macros.h> #ifdef __LP64__ @@ -131,18 +138,13 @@ struct trap_frame { typedef void (*trap_handler_fn_t)(struct trap_frame *); static inline void -trap_test_double_fault(void) +trap_trigger_double_fault(void) { printf("trap: double fault test\n"); asm volatile("movl $0xdead, %esp; push $0"); } /* - * Set up the trap module. - */ -void trap_setup(void); - -/* * Unified trap entry point. */ void trap_main(struct trap_frame *frame); @@ -171,6 +173,13 @@ void trap_stack_show(struct trap_frame *frame); */ void * trap_get_interrupt_stack(const struct trap_frame *frame); +/* + * This init operation provides : + * - initialization of all IDT entries and trap handlers + * - double fault exception support + */ +INIT_OP_DECLARE(trap_setup); + #endif /* __ASSEMBLER__ */ #endif /* _X86_TRAP_H */ diff --git a/arch/x86/machine/trap_asm.S b/arch/x86/machine/trap_asm.S index a60bbf0f..2f683bd7 100644 --- a/arch/x86/machine/trap_asm.S +++ b/arch/x86/machine/trap_asm.S @@ -81,7 +81,7 @@ ASM_ENTRY(trap_common) call trap_get_interrupt_stack testq %rax, %rax /* switch stack ? */ jz 1f - movq %rax, %rsp /* switch to interrupt stack TODO conditional move ? */ + movq %rax, %rsp /* switch to interrupt stack ? */ 1: xorq %rbp, %rbp /* block stack tracing */ movq %rbx, %rdi diff --git a/arch/x86/machine/uart.c b/arch/x86/machine/uart.c index 82b6c83b..10a32886 100644 --- a/arch/x86/machine/uart.c +++ b/arch/x86/machine/uart.c @@ -13,19 +13,17 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * - * TODO Make serial line parameters configurable. */ +#include <assert.h> #include <stdint.h> -#include <stdio.h> -#include <kern/assert.h> +#include <kern/arg.h> #include <kern/console.h> #include <kern/error.h> #include <kern/init.h> #include <kern/intr.h> +#include <kern/log.h> #include <kern/macros.h> #include <machine/biosmem.h> #include <machine/io.h> @@ -53,6 +51,8 @@ #define UART_LCR_8BITS 0x03 #define UART_LCR_1S 0x00 #define UART_LCR_NP 0x00 +#define UART_LCR_OP 0x08 +#define UART_LCR_EP 0x18 #define UART_LCR_BEN 0x40 #define UART_LCR_DLAB 0x80 @@ -65,6 +65,19 @@ #define UART_MAX_DEVS 4 +#define UART_SPEED_MAX 115200 +#define UART_SPEED_DEFAULT UART_SPEED_MAX + +enum { + UART_PARITY_NONE, + UART_PARITY_ODD, + UART_PARITY_EVEN, +}; + +#define UART_PARITY_DEFAULT UART_PARITY_NONE + +#define UART_DATA_BITS_DEFAULT 8 + struct uart { struct console console; uint16_t port; @@ -132,6 +145,9 @@ static void uart_recv_intr(struct uart *uart) { uint8_t byte; + char tmp[2]; + + tmp[1] = '\0'; for (;;) { byte = uart_read(uart, UART_REG_LSR); @@ -141,7 +157,8 @@ uart_recv_intr(struct uart *uart) } byte = uart_read(uart, UART_REG_DAT); - console_intr(&uart->console, (char)byte); + tmp[0] = (char)byte; + console_intr(&uart->console, tmp); } } @@ -176,8 +193,8 @@ uart_enable_intr(struct uart *uart) error = intr_register(uart->intr, uart_intr, uart); if (error) { - printf("uart%zu: error: unable to register interrupt %u\n", - uart_get_id(uart), uart->intr); + log_err("uart%zu: unable to register interrupt %u", + uart_get_id(uart), uart->intr); return; } @@ -239,32 +256,174 @@ static const struct console_ops uart_console_ops = { }; static void __init +uart_init_default(unsigned int *speed, unsigned int *parity, + unsigned int *data_bits) +{ + *speed = UART_SPEED_DEFAULT; + *parity = UART_PARITY_DEFAULT; + *data_bits = UART_DATA_BITS_DEFAULT; +} + +static int __init +uart_init_check_speed(unsigned int speed) +{ + if (speed > UART_SPEED_MAX) { + return ERROR_INVAL; + } + + return 0; +} + +static int __init +uart_init_convert_parity_char(char c, unsigned int *parity) +{ + switch (c) { + case 'n': + *parity = UART_PARITY_NONE; + break; + case 'o': + *parity = UART_PARITY_ODD; + break; + case 'e': + *parity = UART_PARITY_EVEN; + break; + default: + return ERROR_INVAL; + } + + return 0; +} + +static int __init +uart_init_check_data_bits(unsigned int data_bits) +{ + switch (data_bits) { + case 5 ... 8: + break; + default: + return ERROR_INVAL; + } + + return 0; +} + +static void __init +uart_init_args(const struct uart *uart, const char *arg_str, + unsigned int *speed, unsigned int *parity, + unsigned int *data_bits) +{ + char parity_char; + int ret, error; + + ret = sscanf(arg_str, "%u%c%1u", speed, &parity_char, data_bits); + + if (ret < 1) { + goto set_defaults; + } + + error = uart_init_check_speed(*speed); + + if (error) { + goto set_defaults; + } else if (ret < 2) { + return; + } + + error = uart_init_convert_parity_char(parity_char, parity); + + if (error) { + goto set_defaults; + } else if (ret < 3) { + return; + } + + error = uart_init_check_data_bits(*data_bits); + + if (error) { + goto set_defaults; + } + + return; + +set_defaults: + log_warning("uart%zu: invalid serial configuration, using defaults", + uart_get_id(uart)); + uart_init_default(speed, parity, data_bits); +} + +static void __init uart_init(struct uart *uart, uint16_t port, uint16_t intr) { + unsigned int speed, parity, data_bits; + const char *arg_str; char name[CONSOLE_NAME_SIZE]; + uint16_t divisor; uint8_t byte; + snprintf(name, sizeof(name), "uart%zu", uart_get_id(uart)); + arg_str = arg_value(name); + + uart_init_default(&speed, &parity, &data_bits); + + if (arg_str != NULL) { + uart_init_args(uart, arg_str, &speed, &parity, &data_bits); + } + + log_debug("uart%zu: speed:%u parity:%u data_bits:%u", + uart_get_id(uart), speed, parity, data_bits); + uart->port = port; uart->intr = intr; uart_write(uart, UART_REG_IER, 0); + divisor = UART_SPEED_MAX / speed; + uart_set(uart, UART_REG_LCR, UART_LCR_DLAB); - uart_write(uart, UART_REG_DLH, 0); - uart_write(uart, UART_REG_DLL, 1); + uart_write(uart, UART_REG_DLH, divisor >> 8); + uart_write(uart, UART_REG_DLL, divisor & 0xff); uart_clear(uart, UART_REG_LCR, UART_LCR_DLAB); uart_write(uart, UART_REG_MCR, UART_MCR_AUX2 | UART_MCR_RTS | UART_MCR_DTR); - byte = UART_LCR_8BITS | UART_LCR_NP | UART_LCR_1S; + byte = UART_LCR_1S; + + switch (parity) { + case UART_PARITY_NONE: + byte |= UART_LCR_NP; + break; + case UART_PARITY_ODD: + byte |= UART_LCR_OP; + break; + case UART_PARITY_EVEN: + byte |= UART_LCR_EP; + break; + } + + byte |= (data_bits - 5); uart_write(uart, UART_REG_LCR, byte); - snprintf(name, sizeof(name), "uart%zu", uart_get_id(uart)); console_init(&uart->console, name, &uart_console_ops); console_register(&uart->console); } -void __init +static void __init +uart_log_info(void) +{ + const struct uart *uart; + size_t i; + + for (i = 0; i < ARRAY_SIZE(uart_devs); i++) { + uart = uart_get_dev(i); + + if (uart->port != 0) { + log_info("uart%zu: port:%#x irq:%u", i, (unsigned int)uart->port, + (unsigned int)uart->intr); + } + } +} + +static int __init uart_bootstrap(void) { const uint16_t *ptr; @@ -279,9 +438,17 @@ uart_bootstrap(void) uart_init(uart_get_dev(i), ptr[i], uart_intrs[i]); } + + uart_log_info(); + return 0; } -void __init +INIT_OP_DEFINE(uart_bootstrap, + INIT_OP_DEP(arg_setup, true), + INIT_OP_DEP(console_bootstrap, true), + INIT_OP_DEP(log_setup, true)); + +static int __init uart_setup(void) { struct uart *uart; @@ -296,20 +463,10 @@ uart_setup(void) uart_enable_intr(uart); } -} -void __init -uart_info(void) -{ - const struct uart *uart; - size_t i; - - for (i = 0; i < ARRAY_SIZE(uart_devs); i++) { - uart = uart_get_dev(i); - - if (uart->port != 0) { - printf("uart%zu: port:%#x irq:%u\n", i, (unsigned int)uart->port, - (unsigned int)uart->intr); - } - } + return 0; } + +INIT_OP_DEFINE(uart_setup, + INIT_OP_DEP(intr_setup, true), + INIT_OP_DEP(uart_bootstrap, true)); diff --git a/arch/x86/machine/uart.h b/arch/x86/machine/uart.h index 60b195c6..0054da86 100644 --- a/arch/x86/machine/uart.h +++ b/arch/x86/machine/uart.h @@ -21,24 +21,19 @@ #ifndef _X86_UART_H #define _X86_UART_H -/* - * Early initialization of the uart module. - * - * Devices may only be used to report diagnostics until initialization - * is completed. - */ -void uart_bootstrap(void); +#include <kern/init.h> /* - * Initialize the uart module. - * - * On return, devices may be used for both input and output, using interrupts. + * This init operation provides : + * - UART output through the console module */ -void uart_setup(void); +INIT_OP_DECLARE(uart_bootstrap); /* - * Display device information. + * This init operation provides : + * - UART input through the console module + * - module fully initialized */ -void uart_info(void); +INIT_OP_DECLARE(uart_setup); #endif /* _X86_UART_H */ diff --git a/arch/x86/x15.lds.S b/arch/x86/x15.lds.S index 3fc57cad..352ebf4a 100644 --- a/arch/x86/x15.lds.S +++ b/arch/x86/x15.lds.S @@ -9,7 +9,9 @@ OUTPUT_ARCH(i386) ENTRY(_start) #include <machine/boot.h> -#include <machine/param.h> +#include <machine/cpu.h> +#include <machine/page.h> +#include <machine/pmap.h> PHDRS { @@ -36,12 +38,18 @@ SECTIONS . = ALIGN(PAGE_SIZE); _boot_end = .; - . += VM_KERNEL_OFFSET; + . += PMAP_KERNEL_OFFSET; _init = .; .init ALIGN(PAGE_SIZE) : AT(BOOT_VTOP(ADDR(.init))) { *(.init.text) *(.init.data) + + . = ALIGN(64); + _init_ops = .; + *(.init.ops) + _init_ops_end = .; + } : init . = ALIGN(PAGE_SIZE); @@ -68,7 +76,7 @@ SECTIONS *(.rodata) } : rodata - .notes ALIGN(DATA_ALIGN) : AT(BOOT_VTOP(ADDR(.notes))) { + .notes ALIGN(CPU_DATA_ALIGN) : AT(BOOT_VTOP(ADDR(.notes))) { *(.note.*) } : rodata @@ -82,7 +90,7 @@ SECTIONS *(.data) } : data - .bss ALIGN(DATA_ALIGN) : AT(BOOT_VTOP(ADDR(.bss))) { + .bss ALIGN(CPU_DATA_ALIGN) : AT(BOOT_VTOP(ADDR(.bss))) { *(.bss) } : data diff --git a/configure.ac b/configure.ac index 520fa02c..26939597 100644 --- a/configure.ac +++ b/configure.ac @@ -55,11 +55,21 @@ AC_ARG_ENABLE([mutex-pi], [AS_HELP_STRING([--enable-mutex-pi], [enable priority inheritance for regular mutexes (note that priority inheritance is always - enabled for real-time mutexes)])]) + enabled for real-time mutexes)])], + [], + [enable_mutex_pi=no]) + +AC_ARG_ENABLE([shell], + [AS_HELP_STRING([--enable-shell], + [enable the diagnostics shell])], + [], + [enable_shell=no]) AC_ARG_ENABLE([thread-stack-guard], [AS_HELP_STRING([--enable-thread-stack-guard], - [enable kernel thread stack guard pages])]) + [enable kernel thread stack guard pages])], + [], + [enable_thread_stack_guard=no]) AC_DEFINE([__KERNEL__], [1], [kernel code]) AC_DEFINE_UNQUOTED([X15_ARCH], [$arch], [arch]) @@ -73,7 +83,6 @@ m4_define([ENABLE_TEST_MODULE], [sref_noref], [test_sref_noref=yes], [sref_weakref], [test_sref_weakref=yes], [vm_page_fill], [test_vm_page_fill=yes], - [x86_double_fault], [test_x86_double_fault=yes], [xcall], [test_xcall=yes], [AC_MSG_ERROR([invalid test module])]) AC_DEFINE([X15_RUN_TEST_MODULE], [1], @@ -95,8 +104,6 @@ AM_CONDITIONAL([TEST_SREF_WEAKREF], [test x"$test_sref_weakref" = xyes]) AM_CONDITIONAL([TEST_VM_PAGE_FILL], [test x"$test_vm_page_fill" = xyes]) -AM_CONDITIONAL([TEST_X86_DOUBLE_FAULT], - [test x"$test_x86_double_fault" = xyes]) AM_CONDITIONAL([TEST_XCALL], [test x"$test_xcall" = xyes]) @@ -126,9 +133,17 @@ AS_IF([test x"$enable_mutex_pi" = xyes], AM_CONDITIONAL([MUTEX_PI], [test x"$enable_mutex_pi" = xyes]) +AM_CONDITIONAL([X15_SHELL], + [test x"$enable_shell" = xyes]) +AS_IF([test x"$enable_shell" = xyes], + [AC_DEFINE_UNQUOTED([X15_SHELL], [], + [Enable the diagnostics shell])]) +AC_MSG_NOTICE([diagnostics shell: $enable_shell]) + AS_IF([test x"$enable_thread_stack_guard" = xyes], [AC_DEFINE_UNQUOTED([X15_THREAD_STACK_GUARD], [], [Enable the use of guard pages for thread stacks])]) +AC_MSG_NOTICE([thread stack guard pages: $enable_thread_stack_guard]) AH_BOTTOM([#include <kern/config.h>]) AC_CONFIG_HEADER([config.h]) diff --git a/doc/intro.9.txt b/doc/intro.9.txt index 57a5c582..e3eb4ee2 100644 --- a/doc/intro.9.txt +++ b/doc/intro.9.txt @@ -13,7 +13,7 @@ DESCRIPTION ----------- X15 is an open source microkernel. Its purpose is to provide a foundation -for a Hurd-like operating system or an embedded kernel application. +for a Hurd-like operating system as well as real-time kernel applications. Section 9 of the manual describes kernel interfaces, both internal and provided to application code hosted in kernel mode. @@ -111,6 +111,8 @@ other development tools : module:kern/bitmap:: Arbitrary-length bit array. +module:kern/cbuf:: + Circular character buffer. module:kern/error:: Common errors and error handling functions. module:kern/kmem:: @@ -132,8 +134,9 @@ module:kern/syscnt:: X15 doesn't provide a generic queue interface, because the requirements often vary too much. Similarly, it doesn't provide a hash table interface. -Instead, users can easily build specialized queues and hash tables on top -of the provided facilities. Hash functions may be provided in the future. +Instead, users can easily build specialized queues, hash tables and ring +buffers on top of the provided facilities. Hash functions may be provided +in the future. [[multiprocessor_support]] MULTIPROCESSOR SUPPORT @@ -216,8 +219,6 @@ module:arch/atomic:: Architecture-specific support for atomic instructions. module:arch/cpu:: Processor interface. -module:arch/param:: - Miscellaneous parameters. module:arch/pmap:: Physical mappings, the MMU driver. module:arch/strace:: diff --git a/doc/style.9.txt b/doc/style.9.txt index 101f3406..72095671 100644 --- a/doc/style.9.txt +++ b/doc/style.9.txt @@ -321,7 +321,7 @@ NAMES First of all, use underscores instead of camel case to separate words in compound names. Function and variable names are in lower case. Macro -names generally are in upper caser, except for macros which behave +names generally are in upper case, except for macros which behave strictly like functions and could be replaced with inline functions with no API change. @@ -387,12 +387,6 @@ kmem_cache_alloc(struct kmem_cache *cache) void kmem_info(void); -------------------------------------------------------------------------------- -There is currently one exception to the naming rules, which is the -module:arch/param module. The only API changes allowed on this module -are removals, in order to move certain symbols where they should be, -and renames using the param namespace, for symbols that actually belong -to the module. - FUNCTIONS --------- @@ -498,6 +492,13 @@ error_subobj: } -------------------------------------------------------------------------------- +Local variables +~~~~~~~~~~~~~~~ + +Local variables must be declared at the beginning of their scope. An empty +line separates them from the function body. A notable exception is iterators, +which may be declared inside loop statements. + Common functions and methods ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -612,19 +613,8 @@ the macro used to declare an array again in the iteration code. Boolean Coercion ~~~~~~~~~~~~~~~~ -Boolean coercion is forbidden by default, especially on pointers. Always -explicitely compare pointers to NULL and integers to 0. For example : - -[source,c] --------------------------------------------------------------------------------- -/* Read "if ptr is not invalid and if ptr and ref are equal" */ -if ((ptr != NULL) && (memcmp(ptr, ref, sizeof(*ptr)) == 0)) { - do_something(); -} --------------------------------------------------------------------------------- - -The main exception to this rule is error codes, which developers are -encouraged to use as boolean values. Here is an example : +Boolean coercion is forbidden by default. It is only allowed when +checking error codes and pointers. For example : [source,c] -------------------------------------------------------------------------------- @@ -641,8 +631,25 @@ if (error) { } -------------------------------------------------------------------------------- -The other exception is the historical use of the *int* type, rather than -the C99 *bool* type, for boolean values. +There are historical uses of the *int* type for boolean values, but all of +them should be converted to the C99 *bool* type. + +Side effects and conditions +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Conditions must never be made of expressions with side effects. This +completely removes the need to think about short-circuit evaluations. + +Increment/decrement operators +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Increment and decrement operators may never be used inside larger expressions. +Their usage as expression-3 of a for loop is considered as an entire expression +and doesn't violate this rule. The rationale is to avoid tricky errors related +to sequence points. Since such operators may not be used inside larger +expressions, there is no difference between prefix and postfix operators. +The latter are preferred because they look more object-oriented, somewhat +matching the object.method() syntax. SEE --- diff --git a/kern/assert.h b/include/assert.h index cc8b80ea..8fdc2fe2 100644 --- a/kern/assert.h +++ b/include/assert.h @@ -15,8 +15,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef _KERN_ASSERT_H -#define _KERN_ASSERT_H +#ifndef _ASSERT_H +#define _ASSERT_H #define static_assert _Static_assert @@ -40,4 +40,4 @@ MACRO_END #endif /* NDEBUG */ -#endif /* _KERN_ASSERT_H */ +#endif /* _ASSERT_H */ diff --git a/kern/limits.h b/include/limits.h index fa468537..2d47e6b8 100644 --- a/kern/limits.h +++ b/include/limits.h @@ -15,8 +15,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef _KERN_LIMITS_H -#define _KERN_LIMITS_H +#ifndef _LIMITS_H +#define _LIMITS_H #define CHAR_BIT 8 @@ -26,4 +26,4 @@ #define LONG_BIT 32 #endif /* __LP64__ */ -#endif /* _KERN_LIMITS_H */ +#endif /* _LIMITS_H */ diff --git a/include/stdio.h b/include/stdio.h index e1aa1a33..96c03316 100644 --- a/include/stdio.h +++ b/include/stdio.h @@ -18,8 +18,8 @@ #ifndef _STDIO_H #define _STDIO_H +#include <kern/fmt.h> #include <kern/printf.h> -#include <kern/sprintf.h> #ifndef EOF #define EOF (-1) @@ -31,4 +31,12 @@ char console_getchar(void); #define getchar console_getchar #define putchar console_putchar +#define sprintf fmt_sprintf +#define snprintf fmt_snprintf +#define vsprintf fmt_vsprintf +#define vsnprintf fmt_vsnprintf + +#define sscanf fmt_sscanf +#define vsscanf fmt_vsscanf + #endif /* _STDIO_H */ @@ -15,14 +15,14 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> #include <string.h> #include <kern/arg.h> -#include <kern/assert.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> @@ -35,22 +35,17 @@ static char arg_cmdline[ARG_CMDLINE_MAX_SIZE] __initdata; static const char *arg_cmdline_end __initdata; void __init -arg_setup(const char *cmdline) +arg_set_cmdline(const char *cmdline) { - size_t i, length; - - if (cmdline == NULL) { - arg_cmdline[0] = '\0'; - return; - } - - length = strlen(cmdline); + strlcpy(arg_cmdline, cmdline, sizeof(arg_cmdline)); +} - if ((length + 1) > ARRAY_SIZE(arg_cmdline)) { - panic("arg: command line too long"); - } +static int __init +arg_setup(void) +{ + size_t i, length; - memcpy(arg_cmdline, cmdline, length + 1); + length = strlen(arg_cmdline); for (i = 0; i < length; i++) { if (arg_cmdline[i] == ' ') { @@ -59,12 +54,23 @@ arg_setup(const char *cmdline) } arg_cmdline_end = arg_cmdline + length; + return 0; } +INIT_OP_DEFINE(arg_setup); + void __init -arg_info(void) +arg_log_info(void) { - printf("arg: %s\n", arg_cmdline); + char cmdline[sizeof(arg_cmdline)]; + size_t i; + + for (i = 0; &arg_cmdline[i] < arg_cmdline_end; i++) { + cmdline[i] = (arg_cmdline[i] == '\0') ? ' ' : arg_cmdline[i]; + } + + cmdline[i] = '\0'; + log_info("arg: %s", cmdline); } static const char * __init @@ -27,17 +27,21 @@ #include <stdbool.h> +#include <kern/init.h> + #define ARG_CMDLINE_MAX_SIZE 256 /* - * Initialize the arg module. + * Set the command line string. + * + * This function must be called before calling the kernel main entry point. */ -void arg_setup(const char *cmdline); +void arg_set_cmdline(const char *cmdline); /* - * Display command line information. + * Log command line information. */ -void arg_info(void); +void arg_log_info(void); /* * Return true if an argument with the given name is present in the @@ -54,4 +58,10 @@ bool arg_present(const char *name); */ const char * arg_value(const char *name); +/* + * This init operation provides : + * - command line arguments can be retrieved + */ +INIT_OP_DECLARE(arg_setup); + #endif /* _KERN_ARG_H */ diff --git a/kern/atomic.h b/kern/atomic.h index 63f0ac73..6c0105dd 100644 --- a/kern/atomic.h +++ b/kern/atomic.h @@ -135,7 +135,4 @@ MACRO_END #define atomic_fetch_sub_acq_rel(ptr, val) \ atomic_fetch_sub(ptr, val, ATOMIC_ACQ_REL) -#define atomic_or_acq_rel(ptr, val) atomic_or(ptr, val, ATOMIC_ACQ_REL) -#define atomic_and_acq_rel(ptr, val) atomic_and(ptr, val, ATOMIC_ACQ_REL) - #endif /* _KERN_ATOMIC_H */ diff --git a/kern/bitmap.c b/kern/bitmap.c index d270b9f5..97e497d6 100644 --- a/kern/bitmap.c +++ b/kern/bitmap.c @@ -15,11 +15,11 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <limits.h> #include <string.h> #include <kern/bitmap.h> #include <kern/bitmap_i.h> -#include <kern/limits.h> int bitmap_cmp(const unsigned long *a, const unsigned long *b, int nr_bits) @@ -78,7 +78,7 @@ bitmap_find_next_bit(const unsigned long *bm, int nr_bits, int bit, end = bm + BITMAP_LONGS(nr_bits); if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); nr_bits -= ((bm - start) * LONG_BIT); } diff --git a/kern/bitmap.h b/kern/bitmap.h index a10fb512..9a3d7a0b 100644 --- a/kern/bitmap.h +++ b/kern/bitmap.h @@ -24,11 +24,11 @@ #ifndef _KERN_BITMAP_H #define _KERN_BITMAP_H +#include <limits.h> #include <string.h> #include <kern/atomic.h> #include <kern/bitmap_i.h> -#include <kern/limits.h> #define BITMAP_DECLARE(name, nr_bits) unsigned long name[BITMAP_LONGS(nr_bits)] @@ -65,7 +65,7 @@ static inline void bitmap_set(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } *bm |= bitmap_mask(bit); @@ -75,17 +75,17 @@ static inline void bitmap_set_atomic(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } - atomic_or_acq_rel(bm, bitmap_mask(bit)); + atomic_or(bm, bitmap_mask(bit), ATOMIC_RELEASE); } static inline void bitmap_clear(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } *bm &= ~bitmap_mask(bit); @@ -95,22 +95,32 @@ static inline void bitmap_clear_atomic(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } - atomic_and_acq_rel(bm, ~bitmap_mask(bit)); + atomic_and(bm, ~bitmap_mask(bit), ATOMIC_RELEASE); } static inline int bitmap_test(const unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } return ((*bm & bitmap_mask(bit)) != 0); } +static inline int +bitmap_test_atomic(const unsigned long *bm, int bit) +{ + if (bit >= LONG_BIT) { + bitmap_lookup(&bm, &bit); + } + + return ((atomic_load(bm, ATOMIC_ACQUIRE) & bitmap_mask(bit)) != 0); +} + static inline void bitmap_and(unsigned long *a, const unsigned long *b, int nr_bits) { diff --git a/kern/bitmap_i.h b/kern/bitmap_i.h index 39a330c9..dc91a0ab 100644 --- a/kern/bitmap_i.h +++ b/kern/bitmap_i.h @@ -18,7 +18,8 @@ #ifndef _KERN_BITMAP_I_H #define _KERN_BITMAP_I_H -#include <kern/limits.h> +#include <limits.h> + #include <kern/macros.h> #define BITMAP_LONGS(nr_bits) DIV_CEIL(nr_bits, LONG_BIT) @@ -29,13 +30,13 @@ * * Implemented as a macro for const-correctness. */ -#define bitmap_lookup(bm, bit) \ +#define bitmap_lookup(bmp, bitp) \ MACRO_BEGIN \ int i; \ \ - i = BITMAP_LONGS((bit) + 1) - 1; \ - (bm) += i; \ - (bit) -= i * LONG_BIT; \ + i = BITMAP_LONGS(*(bitp) + 1) - 1; \ + *(bmp) += i; \ + *(bitp) -= i * LONG_BIT; \ MACRO_END static inline unsigned long diff --git a/kern/cbuf.c b/kern/cbuf.c index 17369afe..ea848c56 100644 --- a/kern/cbuf.c +++ b/kern/cbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Richard Braun. + * Copyright (c) 2015-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,16 +15,19 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <kern/assert.h> +#include <assert.h> +#include <stddef.h> +#include <string.h> + #include <kern/cbuf.h> #include <kern/error.h> #include <kern/macros.h> /* Negative close to 0 so that an overflow occurs early */ -#define CBUF_INIT_INDEX ((unsigned long)-500) +#define CBUF_INIT_INDEX ((size_t)-500) void -cbuf_init(struct cbuf *cbuf, char *buf, unsigned long capacity) +cbuf_init(struct cbuf *cbuf, char *buf, size_t capacity) { assert(ISP2(capacity)); @@ -34,24 +37,29 @@ cbuf_init(struct cbuf *cbuf, char *buf, unsigned long capacity) cbuf->end = cbuf->start; } -static unsigned long -cbuf_index(const struct cbuf *cbuf, unsigned long abs_index) +static size_t +cbuf_index(const struct cbuf *cbuf, size_t abs_index) { return abs_index & (cbuf->capacity - 1); } -void -cbuf_push(struct cbuf *cbuf, char byte) +static void +cbuf_update_start(struct cbuf *cbuf) { - cbuf->buf[cbuf_index(cbuf, cbuf->end)] = byte; - cbuf->end++; - /* Mind integer overflows */ if (cbuf_size(cbuf) > cbuf->capacity) { cbuf->start = cbuf->end - cbuf->capacity; } } +void +cbuf_push(struct cbuf *cbuf, char byte) +{ + cbuf->buf[cbuf_index(cbuf, cbuf->end)] = byte; + cbuf->end++; + cbuf_update_start(cbuf); +} + int cbuf_pop(struct cbuf *cbuf, char *bytep) { @@ -65,13 +73,76 @@ cbuf_pop(struct cbuf *cbuf, char *bytep) } int -cbuf_read(const struct cbuf *cbuf, unsigned long index, char *bytep) +cbuf_write(struct cbuf *cbuf, size_t index, const void *buf, size_t size) { - /* Mind integer overflows */ - if ((cbuf->end - index - 1) >= cbuf_size(cbuf)) { + char *start, *end, *buf_end; + size_t new_end, skip; + + if (!cbuf_range_valid(cbuf, index, cbuf->end)) { return ERROR_INVAL; } - *bytep = cbuf->buf[cbuf_index(cbuf, index)]; + new_end = index + size; + + if (!cbuf_range_valid(cbuf, cbuf->start, new_end)) { + cbuf->end = new_end; + + if (size > cbuf_capacity(cbuf)) { + skip = size - cbuf_capacity(cbuf); + buf += skip; + index += skip; + size = cbuf_capacity(cbuf); + } + } + + start = &cbuf->buf[cbuf_index(cbuf, index)]; + end = start + size; + buf_end = cbuf->buf + cbuf->capacity; + + if ((end <= cbuf->buf) || (end > buf_end)) { + skip = buf_end - start; + memcpy(start, buf, skip); + buf += skip; + start = cbuf->buf; + size -= skip; + } + + memcpy(start, buf, size); + cbuf_update_start(cbuf); + return 0; +} + +int +cbuf_read(const struct cbuf *cbuf, size_t index, void *buf, size_t *sizep) +{ + const char *start, *end, *buf_end; + size_t size; + + /* At least one byte must be available */ + if (!cbuf_range_valid(cbuf, index, index + 1)) { + return ERROR_INVAL; + } + + size = cbuf->end - index; + + if (*sizep > size) { + *sizep = size; + } + + start = &cbuf->buf[cbuf_index(cbuf, index)]; + end = start + *sizep; + buf_end = cbuf->buf + cbuf->capacity; + + if ((end > cbuf->buf) && (end <= buf_end)) { + size = *sizep; + } else { + size = buf_end - start; + memcpy(buf, start, size); + buf += size; + start = cbuf->buf; + size = *sizep - size; + } + + memcpy(buf, start, size); return 0; } diff --git a/kern/cbuf.h b/kern/cbuf.h index 3ea38418..4d69334d 100644 --- a/kern/cbuf.h +++ b/kern/cbuf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Richard Braun. + * Copyright (c) 2015-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,6 +21,9 @@ #ifndef _KERN_CBUF_H #define _KERN_CBUF_H +#include <stdbool.h> +#include <stddef.h> + /* * Circular buffer descriptor. * @@ -29,30 +32,30 @@ */ struct cbuf { char *buf; - unsigned long capacity; - unsigned long start; - unsigned long end; + size_t capacity; + size_t start; + size_t end; }; -static inline unsigned long +static inline size_t cbuf_capacity(const struct cbuf *cbuf) { return cbuf->capacity; } -static inline unsigned long +static inline size_t cbuf_start(const struct cbuf *cbuf) { return cbuf->start; } -static inline unsigned long +static inline size_t cbuf_end(const struct cbuf *cbuf) { return cbuf->end; } -static inline unsigned long +static inline size_t cbuf_size(const struct cbuf *cbuf) { return cbuf->end - cbuf->start; @@ -64,13 +67,21 @@ cbuf_clear(struct cbuf *cbuf) cbuf->start = cbuf->end; } +static inline bool +cbuf_range_valid(const struct cbuf *cbuf, size_t start, size_t end) +{ + return (((end - start) <= cbuf_size(cbuf)) + && ((start - cbuf->start) <= cbuf_size(cbuf)) + && ((cbuf->end - end) <= cbuf_size(cbuf))); +} + /* * Initialize a circular buffer. * * The descriptor is set to use the given buffer for storage. Capacity * must be a power-of-two. */ -void cbuf_init(struct cbuf *cbuf, char *buf, unsigned long capacity); +void cbuf_init(struct cbuf *cbuf, char *buf, size_t capacity); /* * Append a byte to a circular buffer. @@ -90,12 +101,25 @@ void cbuf_push(struct cbuf *cbuf, char byte); int cbuf_pop(struct cbuf *cbuf, char *bytep); /* - * Read a byte at a specific location. + * Write into a circular buffer at a specific location. + * + * If the given index is outside buffer boundaries, ERROR_INVAL is returned. + * Otherwise size bytes are copied into the circular buffer. If the range + * in the circular buffer goes beyond its end, the end index is updated as + * appropriate. If the buffer is full when extending its end, the oldest + * bytes are overwritten and the start index is updated accordingly. + */ +int cbuf_write(struct cbuf *cbuf, size_t index, const void *buf, size_t size); + +/* + * Read from a circular buffer at a specific location. * * If the given index is outside buffer boundaries, ERROR_INVAL is returned. - * Otherwise the byte is stored at the bytep address and 0 is returned. - * The buffer isn't changed by this operation. + * Otherwise at most *sizep bytes are copied into the given byte buffer, + * and *sizep is updated to the number of bytes actually copied. + * + * The circular buffer isn't changed by this operation. */ -int cbuf_read(const struct cbuf *cbuf, unsigned long index, char *bytep); +int cbuf_read(const struct cbuf *cbuf, size_t index, void *buf, size_t *sizep); #endif /* _KERN_CBUF_H */ diff --git a/kern/condition.c b/kern/condition.c index 4a837f4e..c8ea5f39 100644 --- a/kern/condition.c +++ b/kern/condition.c @@ -18,10 +18,10 @@ * Locking order : mutex -> sleep queue */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <kern/assert.h> #include <kern/condition.h> #include <kern/condition_types.h> #include <kern/mutex.h> diff --git a/kern/console.c b/kern/console.c index c82e7dc9..1f9a8a9c 100644 --- a/kern/console.c +++ b/kern/console.c @@ -15,20 +15,21 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> #include <string.h> #include <kern/arg.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/init.h> #include <kern/console.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/mutex.h> #include <kern/spinlock.h> #include <kern/thread.h> +#include <machine/boot.h> #include <machine/cpu.h> /* @@ -66,13 +67,28 @@ console_init(struct console *console, const char *name, strlcpy(console->name, name, sizeof(console->name)); } +static int +console_process_ctrl_char(struct console *console, char c) +{ + switch (c) { + case CONSOLE_SCROLL_UP: + case CONSOLE_SCROLL_DOWN: + break; + default: + return ERROR_INVAL; + } + + console->ops->putc(console, c); + return 0; +} + static void console_putc(struct console *console, char c) { unsigned long flags; spinlock_lock_intr_save(&console->lock, &flags); - console->ops->putc(console_dev, c); + console->ops->putc(console, c); spinlock_unlock_intr_restore(&console->lock, flags); } @@ -96,7 +112,11 @@ console_getc(struct console *console) error = cbuf_pop(&console->recvbuf, &c); if (!error) { - break; + error = console_process_ctrl_char(console, c); + + if (error) { + break; + } } thread_sleep(&console->lock, console, "consgetc"); @@ -110,13 +130,28 @@ out: return c; } -void __init -console_setup(void) +static int __init +console_bootstrap(void) { list_init(&console_devs); console_name = arg_value("console"); + return 0; +} + +INIT_OP_DEFINE(console_bootstrap, + INIT_OP_DEP(arg_setup, true), + INIT_OP_DEP(log_setup, true)); + +static int __init +console_setup(void) +{ + return 0; } +INIT_OP_DEFINE(console_setup, + INIT_OP_DEP(boot_setup_console, true), + INIT_OP_DEP(thread_setup, true)); + void __init console_register(struct console *console) { @@ -128,30 +163,35 @@ console_register(struct console *console) console_dev = console; } - printf("console: %s registered\n", console->name); + log_info("console: %s registered", console->name); if (console == console_dev) { - printf("console: %s selected as active console\n", console->name); + log_info("console: %s selected as active console", console->name); } } void -console_intr(struct console *console, char c) +console_intr(struct console *console, const char *s) { assert(!cpu_intr_enabled()); - spinlock_lock(&console->lock); - - if (cbuf_size(&console->recvbuf) == cbuf_capacity(&console->recvbuf)) { - goto out; + if (*s == '\0') { + return; } - cbuf_push(&console->recvbuf, c); + spinlock_lock(&console->lock); - if ((console->waiter != NULL) && (console->waiter != thread_self())) { - thread_wakeup(console->waiter); + while (*s != '\0') { + if (cbuf_size(&console->recvbuf) == cbuf_capacity(&console->recvbuf)) { + goto out; + } + + cbuf_push(&console->recvbuf, *s); + s++; } + thread_wakeup(console->waiter); + out: spinlock_unlock(&console->lock); } diff --git a/kern/console.h b/kern/console.h index b967c0dd..15789974 100644 --- a/kern/console.h +++ b/kern/console.h @@ -22,10 +22,14 @@ #define _KERN_CONSOLE_H #include <kern/cbuf.h> +#include <kern/init.h> #include <kern/list.h> #include <kern/spinlock.h> #include <kern/thread.h> +#define CONSOLE_SCROLL_UP 0x12 /* DC2 */ +#define CONSOLE_SCROLL_DOWN 0x14 /* DC4 */ + struct console; struct console_ops { @@ -59,11 +63,6 @@ void console_init(struct console *console, const char *name, const struct console_ops *ops); /* - * Initialize the console module. - */ -void console_setup(void); - -/* * Register a console device. * * The given console must be initialized before calling this function. @@ -81,7 +80,7 @@ void console_register(struct console *console); * * Interrupts must be disabled when calling this function. */ -void console_intr(struct console *console, char c); +void console_intr(struct console *console, const char *s); /* * Write/read a single character to all registered console devices. @@ -92,4 +91,17 @@ void console_intr(struct console *console, char c); void console_putchar(char c); char console_getchar(void); +/* + * This init operation provides : + * - registration of consoles + */ +INIT_OP_DECLARE(console_bootstrap); + +/* + * This init operation provides : + * - all consoles have been registered + * - module fully initialized + */ +INIT_OP_DECLARE(console_setup); + #endif /* _KERN_CONSOLE_H */ diff --git a/kern/cpumap.c b/kern/cpumap.c index 5582e681..d166b237 100644 --- a/kern/cpumap.c +++ b/kern/cpumap.c @@ -20,15 +20,16 @@ #include <kern/bitmap.h> #include <kern/cpumap.h> #include <kern/error.h> +#include <kern/init.h> #include <kern/kmem.h> -#include <kern/param.h> +#include <kern/macros.h> #include <machine/cpu.h> static struct cpumap cpumap_active_cpus __read_mostly = { { 1 } }; static struct kmem_cache cpumap_cache; -void +static int __init cpumap_setup(void) { unsigned int i, nr_cpus; @@ -40,8 +41,14 @@ cpumap_setup(void) for (i = 0; i < nr_cpus; i++) { cpumap_set(&cpumap_active_cpus, i); } + + return 0; } +INIT_OP_DEFINE(cpumap_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(cpu_mp_probe, true)); + const struct cpumap * cpumap_all(void) { diff --git a/kern/cpumap.h b/kern/cpumap.h index 1843a99a..fd07afc1 100644 --- a/kern/cpumap.h +++ b/kern/cpumap.h @@ -27,6 +27,7 @@ #define _KERN_CPUMAP_H #include <kern/bitmap.h> +#include <kern/init.h> struct cpumap { BITMAP_DECLARE(cpus, X15_MAX_CPUS); @@ -135,11 +136,6 @@ cpumap_find_first_zero(const struct cpumap *cpumap) bitmap_for_each_zero((cpumap)->cpus, X15_MAX_CPUS, index) /* - * Initialize the cpumap module. - */ -void cpumap_setup(void); - -/* * Return a cpumap representing all active processors. * * Until the cpumap module is initialized, the cpumap returned by this @@ -167,4 +163,11 @@ void cpumap_destroy(struct cpumap *cpumap); */ int cpumap_check(const struct cpumap *cpumap); +/* + * This init operation provides : + * - cpumap creation + * - module fully initialized + */ +INIT_OP_DECLARE(cpumap_setup); + #endif /* _KERN_CPUMAP_H */ diff --git a/kern/error.c b/kern/error.c index 02a26d0c..f5d43e48 100644 --- a/kern/error.c +++ b/kern/error.c @@ -15,6 +15,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <stddef.h> + #include <kern/error.h> #include <kern/panic.h> @@ -33,13 +35,17 @@ error_str(int error) case ERROR_BUSY: return "device or resource busy"; case ERROR_FAULT: - return "Bad address"; + return "bad address"; case ERROR_NODEV: - return "No such device"; + return "no such device"; case ERROR_EXIST: - return "Entry exists"; + return "entry exists"; case ERROR_IO: - return "Input/output error"; + return "input/output error"; + case ERROR_SRCH: + return "no such process"; + case ERROR_TIMEDOUT: + return "timeout error"; default: return "unknown error"; } diff --git a/kern/error.h b/kern/error.h index 96a26206..a77db973 100644 --- a/kern/error.h +++ b/kern/error.h @@ -18,14 +18,16 @@ #ifndef _KERN_ERROR_H #define _KERN_ERROR_H -#define ERROR_NOMEM 1 -#define ERROR_AGAIN 2 -#define ERROR_INVAL 3 -#define ERROR_BUSY 4 -#define ERROR_FAULT 5 -#define ERROR_NODEV 6 -#define ERROR_EXIST 7 -#define ERROR_IO 8 +#define ERROR_NOMEM 1 +#define ERROR_AGAIN 2 +#define ERROR_INVAL 3 +#define ERROR_BUSY 4 +#define ERROR_FAULT 5 +#define ERROR_NODEV 6 +#define ERROR_EXIST 7 +#define ERROR_IO 8 +#define ERROR_SRCH 9 +#define ERROR_TIMEDOUT 10 /* * Return a string describing the given error. diff --git a/kern/fmt.c b/kern/fmt.c new file mode 100644 index 00000000..d1067e5c --- /dev/null +++ b/kern/fmt.c @@ -0,0 +1,1467 @@ +/* + * Copyright (c) 2010-2017 Richard Braun. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * + * Upstream site with license notes : + * http://git.sceen.net/rbraun/librbraun.git/ + */ + +#include <assert.h> +#include <limits.h> +#include <stdbool.h> +#include <stdarg.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> + +#include <kern/error.h> +#include <kern/fmt.h> +#include <kern/macros.h> +#include <kern/types.h> + +/* + * Size for the temporary number buffer. The minimum base is 8 so 3 bits + * are consumed per digit. Add one to round up. The conversion algorithm + * doesn't use the null byte. + */ +#define FMT_MAX_NUM_SIZE (((sizeof(unsigned long long) * CHAR_BIT) / 3) + 1) + +/* + * Special size for fmt_vsnprintf(), used when the buffer size is unknown. + */ +#define FMT_NOLIMIT ((size_t)-1) + +/* + * Formatting flags. + * + * FMT_FORMAT_LOWER must be 0x20 as it is OR'd with fmt_digits, eg. + * '0': 0x30 | 0x20 => 0x30 ('0') + * 'A': 0x41 | 0x20 => 0x61 ('a') + */ +#define FMT_FORMAT_ALT_FORM 0x0001 /* "Alternate form" */ +#define FMT_FORMAT_ZERO_PAD 0x0002 /* Zero padding on the left */ +#define FMT_FORMAT_LEFT_JUSTIFY 0x0004 /* Align text on the left */ +#define FMT_FORMAT_BLANK 0x0008 /* Blank space before positive number */ +#define FMT_FORMAT_SIGN 0x0010 /* Always place a sign (either + or -) */ +#define FMT_FORMAT_LOWER 0x0020 /* To lowercase (for %x) */ +#define FMT_FORMAT_CONV_SIGNED 0x0040 /* Format specifies signed conversion */ +#define FMT_FORMAT_DISCARD 0x0080 /* Discard output (scanf) */ +#define FMT_FORMAT_CHECK_WIDTH 0x0100 /* Check field width (scanf) */ + +enum { + FMT_MODIFIER_NONE, + FMT_MODIFIER_CHAR, + FMT_MODIFIER_SHORT, + FMT_MODIFIER_LONG, + FMT_MODIFIER_LONGLONG, + FMT_MODIFIER_PTR, /* Used only for %p */ + FMT_MODIFIER_SIZE, + FMT_MODIFIER_PTRDIFF, +}; + +enum { + FMT_SPECIFIER_INVALID, + FMT_SPECIFIER_INT, + FMT_SPECIFIER_CHAR, + FMT_SPECIFIER_STR, + FMT_SPECIFIER_NRCHARS, + FMT_SPECIFIER_PERCENT, +}; + +/* + * Note that copies of the original va_list object are made, because va_arg() + * may not reliably be used by different callee functions, and despite the + * standard explicitely allowing pointers to va_list objects, it's apparently + * very difficult for implementations to provide and is best avoided. + */ + +struct fmt_sprintf_state { + const char *format; + va_list ap; + unsigned int flags; + int width; + int precision; + unsigned int modifier; + unsigned int specifier; + unsigned int base; + char *str; + char *start; + char *end; +}; + +struct fmt_sscanf_state { + const char *str; + const char *start; + const char *format; + va_list ap; + unsigned int flags; + int width; + int precision; + unsigned int modifier; + unsigned int specifier; + unsigned int base; + int nr_convs; +}; + +static const char fmt_digits[] = "0123456789ABCDEF"; + +static char +fmt_consume(const char **strp) +{ + char c; + + c = **strp; + (*strp)++; + return c; +} + +static void +fmt_restore(const char **strp) +{ + (*strp)--; +} + +static void +fmt_vsnprintf_produce(char **strp, char *end, char c) +{ + if (*strp < end) { + **strp = c; + } + + (*strp)++; +} + +static bool +fmt_isdigit(char c) +{ + return (c >= '0') && (c <= '9'); +} + +static bool +fmt_isxdigit(char c) +{ + return fmt_isdigit(c) + || ((c >= 'a') && (c <= 'f')) + || ((c >= 'A') && (c <= 'F')); +} + +static void +fmt_sprintf_state_init(struct fmt_sprintf_state *state, + char *str, size_t size, + const char *format, va_list ap) +{ + state->format = format; + va_copy(state->ap, ap); + state->str = str; + state->start = str; + + if (size == 0) { + state->end = NULL; + } else if (size == FMT_NOLIMIT) { + state->end = (char *)-1; + } else { + state->end = state->start + size - 1; + } +} + +static int +fmt_sprintf_state_finalize(struct fmt_sprintf_state *state) +{ + va_end(state->ap); + + if (state->str < state->end) { + *state->str = '\0'; + } else if (state->end != NULL) { + *state->end = '\0'; + } + + return state->str - state->start; +} + +static char +fmt_sprintf_state_consume_format(struct fmt_sprintf_state *state) +{ + return fmt_consume(&state->format); +} + +static void +fmt_sprintf_state_restore_format(struct fmt_sprintf_state *state) +{ + fmt_restore(&state->format); +} + +static void +fmt_sprintf_state_consume_flags(struct fmt_sprintf_state *state) +{ + bool found; + char c; + + found = true; + state->flags = 0; + + do { + c = fmt_sprintf_state_consume_format(state); + + switch (c) { + case '#': + state->flags |= FMT_FORMAT_ALT_FORM; + break; + case '0': + state->flags |= FMT_FORMAT_ZERO_PAD; + break; + case '-': + state->flags |= FMT_FORMAT_LEFT_JUSTIFY; + break; + case ' ': + state->flags |= FMT_FORMAT_BLANK; + break; + case '+': + state->flags |= FMT_FORMAT_SIGN; + break; + default: + found = false; + break; + } + } while (found); + + fmt_sprintf_state_restore_format(state); +} + +static void +fmt_sprintf_state_consume_width(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_sprintf_state_consume_format(state); + + if (fmt_isdigit(c)) { + state->width = 0; + + do { + state->width = state->width * 10 + (c - '0'); + c = fmt_sprintf_state_consume_format(state); + } while (fmt_isdigit(c)); + + fmt_sprintf_state_restore_format(state); + } else if (c == '*') { + state->width = va_arg(state->ap, int); + + if (state->width < 0) { + state->flags |= FMT_FORMAT_LEFT_JUSTIFY; + state->width = -state->width; + } + } else { + state->width = 0; + fmt_sprintf_state_restore_format(state); + } +} + +static void +fmt_sprintf_state_consume_precision(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_sprintf_state_consume_format(state); + + if (c == '.') { + c = fmt_sprintf_state_consume_format(state); + + if (fmt_isdigit(c)) { + state->precision = 0; + + do { + state->precision = state->precision * 10 + (c - '0'); + c = fmt_sprintf_state_consume_format(state); + } while (fmt_isdigit(c)); + + fmt_sprintf_state_restore_format(state); + } else if (c == '*') { + state->precision = va_arg(state->ap, int); + + if (state->precision < 0) { + state->precision = 0; + } + } else { + state->precision = 0; + fmt_sprintf_state_restore_format(state); + } + } else { + /* precision is >= 0 only if explicit */ + state->precision = -1; + fmt_sprintf_state_restore_format(state); + } +} + +static void +fmt_sprintf_state_consume_modifier(struct fmt_sprintf_state *state) +{ + char c, c2; + + c = fmt_sprintf_state_consume_format(state); + + switch (c) { + case 'h': + case 'l': + c2 = fmt_sprintf_state_consume_format(state); + + if (c == c2) { + state->modifier = (c == 'h') ? FMT_MODIFIER_CHAR + : FMT_MODIFIER_LONGLONG; + } else { + state->modifier = (c == 'h') ? FMT_MODIFIER_SHORT + : FMT_MODIFIER_LONG; + fmt_sprintf_state_restore_format(state); + } + + break; + case 'z': + state->modifier = FMT_MODIFIER_SIZE; + case 't': + state->modifier = FMT_MODIFIER_PTRDIFF; + break; + default: + state->modifier = FMT_MODIFIER_NONE; + fmt_sprintf_state_restore_format(state); + break; + } +} + +static void +fmt_sprintf_state_consume_specifier(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_sprintf_state_consume_format(state); + + switch (c) { + case 'd': + case 'i': + state->flags |= FMT_FORMAT_CONV_SIGNED; + case 'u': + state->base = 10; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'o': + state->base = 8; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'p': + state->flags |= FMT_FORMAT_ALT_FORM; + state->modifier = FMT_MODIFIER_PTR; + case 'x': + state->flags |= FMT_FORMAT_LOWER; + case 'X': + state->base = 16; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'c': + state->specifier = FMT_SPECIFIER_CHAR; + break; + case 's': + state->specifier = FMT_SPECIFIER_STR; + break; + case 'n': + state->specifier = FMT_SPECIFIER_NRCHARS; + break; + case '%': + state->specifier = FMT_SPECIFIER_PERCENT; + break; + default: + state->specifier = FMT_SPECIFIER_INVALID; + fmt_sprintf_state_restore_format(state); + break; + } +} + +static void +fmt_sprintf_state_produce_raw_char(struct fmt_sprintf_state *state, char c) +{ + fmt_vsnprintf_produce(&state->str, state->end, c); +} + +static int +fmt_sprintf_state_consume(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_consume(&state->format); + + if (c == '\0') { + return ERROR_IO; + } + + if (c != '%') { + fmt_sprintf_state_produce_raw_char(state, c); + return ERROR_AGAIN; + } + + fmt_sprintf_state_consume_flags(state); + fmt_sprintf_state_consume_width(state); + fmt_sprintf_state_consume_precision(state); + fmt_sprintf_state_consume_modifier(state); + fmt_sprintf_state_consume_specifier(state); + return 0; +} + +static void +fmt_sprintf_state_produce_int(struct fmt_sprintf_state *state) +{ + char c, sign, tmp[FMT_MAX_NUM_SIZE]; + unsigned int r, mask, shift; + unsigned long long n; + int i; + + switch (state->modifier) { + case FMT_MODIFIER_CHAR: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = (signed char)va_arg(state->ap, int); + } else { + n = (unsigned char)va_arg(state->ap, int); + } + + break; + case FMT_MODIFIER_SHORT: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = (short)va_arg(state->ap, int); + } else { + n = (unsigned short)va_arg(state->ap, int); + } + + break; + case FMT_MODIFIER_LONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, long); + } else { + n = va_arg(state->ap, unsigned long); + } + + break; + case FMT_MODIFIER_LONGLONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, long long); + } else { + n = va_arg(state->ap, unsigned long long); + } + + break; + case FMT_MODIFIER_PTR: + n = (uintptr_t)va_arg(state->ap, void *); + break; + case FMT_MODIFIER_SIZE: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, ssize_t); + } else { + n = va_arg(state->ap, size_t); + } + + break; + case FMT_MODIFIER_PTRDIFF: + n = va_arg(state->ap, ptrdiff_t); + break; + default: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, int); + } else { + n = va_arg(state->ap, unsigned int); + } + + break; + } + + if ((state->flags & FMT_FORMAT_LEFT_JUSTIFY) || (state->precision >= 0)) { + state->flags &= ~FMT_FORMAT_ZERO_PAD; + } + + sign = '\0'; + + if (state->flags & FMT_FORMAT_ALT_FORM) { + /* '0' for octal */ + state->width--; + + /* '0x' or '0X' for hexadecimal */ + if (state->base == 16) { + state->width--; + } + } else if (state->flags & FMT_FORMAT_CONV_SIGNED) { + if ((long long)n < 0) { + sign = '-'; + state->width--; + n = -(long long)n; + } else if (state->flags & FMT_FORMAT_SIGN) { + /* FMT_FORMAT_SIGN must precede FMT_FORMAT_BLANK. */ + sign = '+'; + state->width--; + } else if (state->flags & FMT_FORMAT_BLANK) { + sign = ' '; + state->width--; + } + } + + /* Conversion, in reverse order */ + + i = 0; + + if (n == 0) { + if (state->precision != 0) { + tmp[i] = '0'; + i++; + } + } else if (state->base == 10) { + /* + * Try to avoid 64 bits operations if the processor doesn't + * support them. Note that even when using modulus and + * division operators close to each other, the compiler may + * forge two functions calls to compute the quotient and the + * remainder, whereas processor instructions are generally + * correctly used once, giving both results at once, through + * plain or reciprocal division. + */ +#ifndef __LP64__ + if (state->modifier == FMT_MODIFIER_LONGLONG) { +#endif /* __LP64__ */ + do { + r = n % 10; + n /= 10; + tmp[i] = fmt_digits[r]; + i++; + } while (n != 0); +#ifndef __LP64__ + } else { + unsigned long m; + + m = (unsigned long)n; + + do { + r = m % 10; + m /= 10; + tmp[i] = fmt_digits[r]; + i++; + } while (m != 0); + } +#endif /* __LP64__ */ + } else { + mask = state->base - 1; + shift = (state->base == 8) ? 3 : 4; + + do { + r = n & mask; + n >>= shift; + tmp[i] = fmt_digits[r] | (state->flags & FMT_FORMAT_LOWER); + i++; + } while (n != 0); + } + + if (i > state->precision) { + state->precision = i; + } + + state->width -= state->precision; + + if (!(state->flags & (FMT_FORMAT_LEFT_JUSTIFY | FMT_FORMAT_ZERO_PAD))) { + while (state->width > 0) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } + + state->width--; + } + + if (state->flags & FMT_FORMAT_ALT_FORM) { + fmt_sprintf_state_produce_raw_char(state, '0'); + + if (state->base == 16) { + c = 'X' | (state->flags & FMT_FORMAT_LOWER); + fmt_sprintf_state_produce_raw_char(state, c); + } + } else if (sign != '\0') { + fmt_sprintf_state_produce_raw_char(state, sign); + } + + if (!(state->flags & FMT_FORMAT_LEFT_JUSTIFY)) { + c = (state->flags & FMT_FORMAT_ZERO_PAD) ? '0' : ' '; + + while (state->width > 0) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, c); + } + + state->width--; + } + + while (i < state->precision) { + state->precision--; + fmt_sprintf_state_produce_raw_char(state, '0'); + } + + state->precision--; + + while (i > 0) { + i--; + fmt_sprintf_state_produce_raw_char(state, tmp[i]); + } + + while (state->width > 0) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } + + state->width--; +} + +static void +fmt_sprintf_state_produce_char(struct fmt_sprintf_state *state) +{ + char c; + + c = va_arg(state->ap, int); + + if (!(state->flags & FMT_FORMAT_LEFT_JUSTIFY)) { + for (;;) { + state->width--; + + if (state->width <= 0) { + break; + } + + fmt_sprintf_state_produce_raw_char(state, ' '); + } + } + + fmt_sprintf_state_produce_raw_char(state, c); + + for (;;) { + state->width--; + + if (state->width <= 0) { + break; + } + + fmt_sprintf_state_produce_raw_char(state, ' '); + } +} + +static void +fmt_sprintf_state_produce_str(struct fmt_sprintf_state *state) +{ + int i, len; + char *s; + + s = va_arg(state->ap, char *); + + if (s == NULL) { + s = "(null)"; + } + + len = 0; + + for (len = 0; s[len] != '\0'; len++) { + if (len == state->precision) { + break; + } + } + + if (!(state->flags & FMT_FORMAT_LEFT_JUSTIFY)) { + while (len < state->width) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } + } + + for (i = 0; i < len; i++) { + fmt_sprintf_state_produce_raw_char(state, *s); + s++; + } + + while (len < state->width) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } +} + +static void +fmt_sprintf_state_produce_nrchars(struct fmt_sprintf_state *state) +{ + if (state->modifier == FMT_MODIFIER_CHAR) { + signed char *ptr = va_arg(state->ap, signed char *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_SHORT) { + short *ptr = va_arg(state->ap, short *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_LONG) { + long *ptr = va_arg(state->ap, long *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_LONGLONG) { + long long *ptr = va_arg(state->ap, long long *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_SIZE) { + ssize_t *ptr = va_arg(state->ap, ssize_t *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_PTRDIFF) { + ptrdiff_t *ptr = va_arg(state->ap, ptrdiff_t *); + *ptr = state->str - state->start; + } else { + int *ptr = va_arg(state->ap, int *); + *ptr = state->str - state->start; + } +} + +static void +fmt_sprintf_state_produce(struct fmt_sprintf_state *state) +{ + switch (state->specifier) { + case FMT_SPECIFIER_INT: + fmt_sprintf_state_produce_int(state); + break; + case FMT_SPECIFIER_CHAR: + fmt_sprintf_state_produce_char(state); + break; + case FMT_SPECIFIER_STR: + fmt_sprintf_state_produce_str(state); + break; + case FMT_SPECIFIER_NRCHARS: + fmt_sprintf_state_produce_nrchars(state); + break; + case FMT_SPECIFIER_PERCENT: + case FMT_SPECIFIER_INVALID: + fmt_sprintf_state_produce_raw_char(state, '%'); + break; + } +} + +int +fmt_sprintf(char *str, const char *format, ...) +{ + va_list ap; + int length; + + va_start(ap, format); + length = fmt_vsprintf(str, format, ap); + va_end(ap); + + return length; +} + +int +fmt_vsprintf(char *str, const char *format, va_list ap) +{ + return fmt_vsnprintf(str, FMT_NOLIMIT, format, ap); +} + +int +fmt_snprintf(char *str, size_t size, const char *format, ...) +{ + va_list ap; + int length; + + va_start(ap, format); + length = fmt_vsnprintf(str, size, format, ap); + va_end(ap); + + return length; +} + +int +fmt_vsnprintf(char *str, size_t size, const char *format, va_list ap) +{ + struct fmt_sprintf_state state; + int error; + + fmt_sprintf_state_init(&state, str, size, format, ap); + + for (;;) { + error = fmt_sprintf_state_consume(&state); + + if (error == ERROR_AGAIN) { + continue; + } else if (error) { + break; + } + + fmt_sprintf_state_produce(&state); + } + + return fmt_sprintf_state_finalize(&state); +} + +static char +fmt_atoi(char c) +{ + assert(fmt_isxdigit(c)); + + if (fmt_isdigit(c)) { + return c - '0'; + } else if (c >= 'a' && c <= 'f') { + return 10 + (c - 'a'); + } else { + return 10 + (c - 'A'); + } +} + +static bool +fmt_isspace(char c) +{ + if (c == ' ') { + return true; + } + + if ((c >= '\t') && (c <= '\f')) { + return true; + } + + return false; +} + +static void +fmt_skip(const char **strp) +{ + while (fmt_isspace(**strp)) { + (*strp)++; + } +} + +static void +fmt_sscanf_state_init(struct fmt_sscanf_state *state, const char *str, + const char *format, va_list ap) +{ + state->str = str; + state->start = str; + state->format = format; + state->flags = 0; + state->width = 0; + va_copy(state->ap, ap); + state->nr_convs = 0; +} + +static int +fmt_sscanf_state_finalize(struct fmt_sscanf_state *state) +{ + va_end(state->ap); + return state->nr_convs; +} + +static void +fmt_sscanf_state_report_conv(struct fmt_sscanf_state *state) +{ + if (state->nr_convs == EOF) { + state->nr_convs = 1; + return; + } + + state->nr_convs++; +} + +static void +fmt_sscanf_state_report_error(struct fmt_sscanf_state *state) +{ + if (state->nr_convs != 0) { + return; + } + + state->nr_convs = EOF; +} + +static void +fmt_sscanf_state_skip_space(struct fmt_sscanf_state *state) +{ + fmt_skip(&state->str); +} + +static char +fmt_sscanf_state_consume_string(struct fmt_sscanf_state *state) +{ + char c; + + c = fmt_consume(&state->str); + + if (state->flags & FMT_FORMAT_CHECK_WIDTH) { + if (state->width == 0) { + c = EOF; + } else { + state->width--; + } + } + + return c; +} + +static void +fmt_sscanf_state_restore_string(struct fmt_sscanf_state *state) +{ + fmt_restore(&state->str); +} + +static char +fmt_sscanf_state_consume_format(struct fmt_sscanf_state *state) +{ + return fmt_consume(&state->format); +} + +static void +fmt_sscanf_state_restore_format(struct fmt_sscanf_state *state) +{ + fmt_restore(&state->format); +} + +static void +fmt_sscanf_state_consume_flags(struct fmt_sscanf_state *state) +{ + bool found; + char c; + + found = true; + state->flags = 0; + + do { + c = fmt_sscanf_state_consume_format(state); + + switch (c) { + case '*': + state->flags |= FMT_FORMAT_DISCARD; + break; + default: + found = false; + break; + } + } while (found); + + fmt_sscanf_state_restore_format(state); +} + +static void +fmt_sscanf_state_consume_width(struct fmt_sscanf_state *state) +{ + char c; + + state->width = 0; + + for (;;) { + c = fmt_sscanf_state_consume_format(state); + + if (!fmt_isdigit(c)) { + break; + } + + state->width = state->width * 10 + (c - '0'); + } + + if (state->width != 0) { + state->flags |= FMT_FORMAT_CHECK_WIDTH; + } + + fmt_sscanf_state_restore_format(state); +} + +static void +fmt_sscanf_state_consume_modifier(struct fmt_sscanf_state *state) +{ + char c, c2; + + c = fmt_sscanf_state_consume_format(state); + + switch (c) { + case 'h': + case 'l': + c2 = fmt_sscanf_state_consume_format(state); + + if (c == c2) { + state->modifier = (c == 'h') ? FMT_MODIFIER_CHAR + : FMT_MODIFIER_LONGLONG; + } else { + state->modifier = (c == 'h') ? FMT_MODIFIER_SHORT + : FMT_MODIFIER_LONG; + fmt_sscanf_state_restore_format(state); + } + + break; + case 'z': + state->modifier = FMT_MODIFIER_SIZE; + break; + case 't': + state->modifier = FMT_MODIFIER_PTRDIFF; + break; + default: + state->modifier = FMT_MODIFIER_NONE; + fmt_sscanf_state_restore_format(state); + break; + } +} + +static void +fmt_sscanf_state_consume_specifier(struct fmt_sscanf_state *state) +{ + char c; + + c = fmt_sscanf_state_consume_format(state); + + switch (c) { + case 'i': + state->base = 0; + state->flags |= FMT_FORMAT_CONV_SIGNED; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'd': + state->flags |= FMT_FORMAT_CONV_SIGNED; + case 'u': + state->base = 10; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'o': + state->base = 8; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'p': + state->modifier = FMT_MODIFIER_PTR; + case 'x': + case 'X': + state->base = 16; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'c': + state->specifier = FMT_SPECIFIER_CHAR; + break; + case 's': + state->specifier = FMT_SPECIFIER_STR; + break; + case 'n': + state->specifier = FMT_SPECIFIER_NRCHARS; + break; + case '%': + state->specifier = FMT_SPECIFIER_PERCENT; + break; + default: + state->specifier = FMT_SPECIFIER_INVALID; + fmt_sscanf_state_restore_format(state); + break; + } +} + +static int +fmt_sscanf_state_discard_char(struct fmt_sscanf_state *state, char c) +{ + char c2; + + if (fmt_isspace(c)) { + fmt_sscanf_state_skip_space(state); + return 0; + } + + c2 = fmt_sscanf_state_consume_string(state); + + if (c != c2) { + if ((c2 == '\0') && (state->nr_convs == 0)) { + state->nr_convs = EOF; + } + + return ERROR_INVAL; + } + + return 0; +} + +static int +fmt_sscanf_state_consume(struct fmt_sscanf_state *state) +{ + int error; + char c; + + state->flags = 0; + + c = fmt_consume(&state->format); + + if (c == '\0') { + return ERROR_IO; + } + + if (c != '%') { + error = fmt_sscanf_state_discard_char(state, c); + + if (error) { + return error; + } + + return ERROR_AGAIN; + } + + fmt_sscanf_state_consume_flags(state); + fmt_sscanf_state_consume_width(state); + fmt_sscanf_state_consume_modifier(state); + fmt_sscanf_state_consume_specifier(state); + return 0; +} + +static int +fmt_sscanf_state_produce_int(struct fmt_sscanf_state *state) +{ + unsigned long long n, m, tmp; + char c, buf[FMT_MAX_NUM_SIZE]; + bool negative; + size_t i; + + negative = 0; + + fmt_sscanf_state_skip_space(state); + c = fmt_sscanf_state_consume_string(state); + + if (c == '-') { + negative = true; + c = fmt_sscanf_state_consume_string(state); + } + + if (c == '0') { + c = fmt_sscanf_state_consume_string(state); + + if ((c == 'x') || (c == 'X')) { + if (state->base == 0) { + state->base = 16; + } + + if (state->base == 16) { + c = fmt_sscanf_state_consume_string(state); + } else { + fmt_sscanf_state_restore_string(state); + c = '0'; + } + } else { + if (state->base == 0) { + state->base = 8; + } + + if (state->base != 8) { + fmt_sscanf_state_restore_string(state); + c = '0'; + } + } + } + + i = 0; + + while (c != '\0') { + if (state->base == 8) { + if (!((c >= '0') && (c <= '7'))) { + break; + } + } else if (state->base == 16) { + if (!fmt_isxdigit(c)) { + break; + } + } else { + if (!fmt_isdigit(c)) { + break; + } + } + + /* XXX Standard sscanf provides no way to cleanly handle overflows */ + if (i < (ARRAY_SIZE(buf) - 1)) { + buf[i] = c; + } else if (i == (ARRAY_SIZE(buf) - 1)) { + strcpy(buf, "1"); + negative = true; + } + + i++; + c = fmt_sscanf_state_consume_string(state); + } + + fmt_sscanf_state_restore_string(state); + + if (state->flags & FMT_FORMAT_DISCARD) { + return 0; + } + + if (i == 0) { + if (c == '\0') { + fmt_sscanf_state_report_error(state); + return ERROR_INVAL; + } + + buf[0] = '0'; + i = 1; + } + + if (i < ARRAY_SIZE(buf)) { + buf[i] = '\0'; + i--; + } else { + i = strlen(buf) - 1; + } + + n = 0; + +#ifndef __LP64__ + if (state->modifier == FMT_MODIFIER_LONGLONG) { +#endif /* __LP64__ */ + m = 1; + tmp = 0; + + while (&buf[i] >= buf) { + tmp += fmt_atoi(buf[i]) * m; + + if (tmp < n) { + n = 1; + negative = true; + break; + } + + n = tmp; + m *= state->base; + i--; + } +#ifndef __LP64__ + } else { + unsigned long _n, _m, _tmp; + + _n = 0; + _m = 1; + _tmp = 0; + + while (&buf[i] >= buf) { + _tmp += fmt_atoi(buf[i]) * _m; + + if (_tmp < _n) { + _n = 1; + negative = true; + break; + } + + _n = _tmp; + _m *= state->base; + i--; + } + + n = _n; + } +#endif /* __LP64__ */ + + if (negative) { + n = -n; + } + + switch (state->modifier) { + case FMT_MODIFIER_CHAR: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, char *) = n; + } else { + *va_arg(state->ap, unsigned char *) = n; + } + + break; + case FMT_MODIFIER_SHORT: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, short *) = n; + } else { + *va_arg(state->ap, unsigned short *) = n; + } + + break; + case FMT_MODIFIER_LONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, long *) = n; + } else { + *va_arg(state->ap, unsigned long *) = n; + } + + break; + case FMT_MODIFIER_LONGLONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, long long *) = n; + } else { + *va_arg(state->ap, unsigned long long *) = n; + } + + break; + case FMT_MODIFIER_PTR: + *va_arg(state->ap, uintptr_t *) = n; + break; + case FMT_MODIFIER_SIZE: + *va_arg(state->ap, size_t *) = n; + break; + case FMT_MODIFIER_PTRDIFF: + *va_arg(state->ap, ptrdiff_t *) = n; + break; + default: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, int *) = n; + } else { + *va_arg(state->ap, unsigned int *) = n; + } + } + + fmt_sscanf_state_report_conv(state); + return 0; +} + +static int +fmt_sscanf_state_produce_char(struct fmt_sscanf_state *state) +{ + char c, *dest; + int i, width; + + if (state->flags & FMT_FORMAT_DISCARD) { + dest = NULL; + } else { + dest = va_arg(state->ap, char *); + } + + if (state->flags & FMT_FORMAT_CHECK_WIDTH) { + width = state->width; + } else { + width = 1; + } + + for (i = 0; i < width; i++) { + c = fmt_sscanf_state_consume_string(state); + + if ((c == '\0') || (c == EOF)) { + break; + } + + if (dest != NULL) { + *dest = c; + dest++; + } + } + + if (i < width) { + fmt_sscanf_state_restore_string(state); + } + + if ((dest != NULL) && (i != 0)) { + fmt_sscanf_state_report_conv(state); + } + + return 0; +} + +static int +fmt_sscanf_state_produce_str(struct fmt_sscanf_state *state) +{ + const char *orig; + char c, *dest; + + orig = state->str; + + fmt_sscanf_state_skip_space(state); + + if (state->flags & FMT_FORMAT_DISCARD) { + dest = NULL; + } else { + dest = va_arg(state->ap, char *); + } + + for (;;) { + c = fmt_sscanf_state_consume_string(state); + + if ((c == '\0') || (c == ' ') || (c == EOF)) { + break; + } + + if (dest != NULL) { + *dest = c; + dest++; + } + } + + fmt_sscanf_state_restore_string(state); + + if (state->str == orig) { + fmt_sscanf_state_report_error(state); + return ERROR_INVAL; + } + + if (dest != NULL) { + *dest = '\0'; + fmt_sscanf_state_report_conv(state); + } + + return 0; +} + +static int +fmt_sscanf_state_produce_nrchars(struct fmt_sscanf_state *state) +{ + *va_arg(state->ap, int *) = state->str - state->start; + return 0; +} + +static int +fmt_sscanf_state_produce(struct fmt_sscanf_state *state) +{ + switch (state->specifier) { + case FMT_SPECIFIER_INT: + return fmt_sscanf_state_produce_int(state); + case FMT_SPECIFIER_CHAR: + return fmt_sscanf_state_produce_char(state); + case FMT_SPECIFIER_STR: + return fmt_sscanf_state_produce_str(state); + case FMT_SPECIFIER_NRCHARS: + return fmt_sscanf_state_produce_nrchars(state); + case FMT_SPECIFIER_PERCENT: + fmt_sscanf_state_skip_space(state); + return fmt_sscanf_state_discard_char(state, '%'); + default: + fmt_sscanf_state_report_error(state); + return ERROR_INVAL; + } +} + +int +fmt_sscanf(const char *str, const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = fmt_vsscanf(str, format, ap); + va_end(ap); + + return ret; +} + +int +fmt_vsscanf(const char *str, const char *format, va_list ap) +{ + struct fmt_sscanf_state state; + int error; + + fmt_sscanf_state_init(&state, str, format, ap); + + for (;;) { + error = fmt_sscanf_state_consume(&state); + + if (error == ERROR_AGAIN) { + continue; + } else if (error) { + break; + } + + error = fmt_sscanf_state_produce(&state); + + if (error) { + break; + } + } + + return fmt_sscanf_state_finalize(&state); +} diff --git a/kern/fmt.h b/kern/fmt.h new file mode 100644 index 00000000..a339ed46 --- /dev/null +++ b/kern/fmt.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2010-2017 Richard Braun. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * + * Formatted string functions. + * + * The functions provided by this module implement a subset of the standard + * sprintf- and sscanf-like functions. + * + * sprintf: + * - flags: # 0 - ' ' (space) + + * - field width is supported + * - precision is supported + * + * sscanf: + * - flags: * + * - field width is supported + * + * common: + * - modifiers: hh h l ll z t + * - specifiers: d i o u x X c s p n % + * + * + * Upstream site with license notes : + * http://git.sceen.net/rbraun/librbraun.git/ + */ + +#ifndef _FMT_H +#define _FMT_H + +#include <stdarg.h> +#include <stddef.h> + +int fmt_sprintf(char *str, const char *format, ...) + __attribute__((format(printf, 2, 3))); + +int fmt_vsprintf(char *str, const char *format, va_list ap) + __attribute__((format(printf, 2, 0))); + +int fmt_snprintf(char *str, size_t size, const char *format, ...) + __attribute__((format(printf, 3, 4))); + +int fmt_vsnprintf(char *str, size_t size, const char *format, va_list ap) + __attribute__((format(printf, 3, 0))); + +int fmt_sscanf(const char *str, const char *format, ...) + __attribute__((format(scanf, 2, 3))); + +int fmt_vsscanf(const char *str, const char *format, va_list ap) + __attribute__((format(scanf, 2, 0))); + +#endif /* _FMT_H */ diff --git a/kern/hash.h b/kern/hash.h index 108d4b46..db399960 100644 --- a/kern/hash.h +++ b/kern/hash.h @@ -39,11 +39,10 @@ #ifndef _KERN_HASH_H #define _KERN_HASH_H +#include <assert.h> #include <stdint.h> #include <string.h> -#include <kern/assert.h> - #ifdef __LP64__ #define HASH_ALLBITS 64 #define hash_long(n, bits) hash_int64(n, bits) diff --git a/kern/init.c b/kern/init.c new file mode 100644 index 00000000..1dc51144 --- /dev/null +++ b/kern/init.c @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * The main purpose of the init module is to provide a convenient interface + * to declare initialization operations and their dependency, infer an order + * of execution based on the resulting graph (which must be acyclic), and + * run the operations in that order. Topological sorting is achieved using + * Kahn's algorithm. + */ + +#include <assert.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <kern/error.h> +#include <kern/init.h> +#include <kern/list.h> +#include <kern/macros.h> +#include <machine/cpu.h> + +#define INIT_DEBUG 1 + +extern struct init_op _init_ops; +extern struct init_op _init_ops_end; + +static_assert(sizeof(struct init_op) == INIT_OP_ALIGN, "invalid init_op size"); + +/* + * List of initialization operations. + * + * This type is used for the output of the topological sort. + */ +struct init_ops_list { + struct list ops; + size_t size; +}; + +static void __init +init_ops_list_init(struct init_ops_list *list) +{ + list_init(&list->ops); + list->size = 0; +} + +static size_t __init +init_ops_list_size(const struct init_ops_list *list) +{ + return list->size; +} + +static void __init +init_ops_list_push(struct init_ops_list *list, struct init_op *op) +{ + list_insert_tail(&list->ops, &op->list_node); + list->size++; +} + +static struct init_op * __init +init_ops_list_pop(struct init_ops_list *list) +{ + struct init_op *op; + + if (list->size == 0) { + return NULL; + } + + op = list_last_entry(&list->ops, struct init_op, list_node); + list_remove(&op->list_node); + list->size--; + return op; +} + +/* + * Stack of initialization operations. + * + * This type is used internally by the topological sort algorithm. + */ +struct init_ops_stack { + struct list ops; +}; + +static void __init +init_ops_stack_init(struct init_ops_stack *stack) +{ + list_init(&stack->ops); +} + +static void __init +init_ops_stack_push(struct init_ops_stack *stack, struct init_op *op) +{ + list_insert_tail(&stack->ops, &op->stack_node); +} + +static struct init_op * __init +init_ops_stack_pop(struct init_ops_stack *stack) +{ + struct init_op *op; + + if (list_empty(&stack->ops)) { + return NULL; + } + + op = list_last_entry(&stack->ops, struct init_op, stack_node); + list_remove(&op->stack_node); + return op; +} + +static struct init_op_dep * __init +init_op_get_dep(const struct init_op *op, size_t index) +{ + assert(index < op->nr_deps); + return &op->deps[index]; +} + +static void __init +init_op_init(struct init_op *op) +{ + struct init_op_dep *dep; + + for (size_t i = 0; i < op->nr_deps; i++) { + dep = init_op_get_dep(op, i); + dep->op->nr_parents++; + assert(dep->op->nr_parents != 0); + } +} + +static bool __init +init_op_orphan(const struct init_op *op) +{ + return (op->nr_parents == 0); +} + +static bool __init +init_op_pending(const struct init_op *op) +{ + return op->state == INIT_OP_STATE_PENDING; +} + +static void __init +init_op_set_pending(struct init_op *op) +{ + assert(op->state == INIT_OP_STATE_UNLINKED); + op->state = INIT_OP_STATE_PENDING; +} + +static bool __init +init_op_complete(const struct init_op *op) +{ + return op->state == INIT_OP_STATE_COMPLETE; +} + +static void __init +init_op_set_complete(struct init_op *op) +{ + assert(init_op_pending(op)); + op->state = INIT_OP_STATE_COMPLETE; +} + +static bool __init +init_op_ready(const struct init_op *op) +{ + const struct init_op_dep *dep; + size_t i; + + for (i = 0; i < op->nr_deps; i++) { + dep = init_op_get_dep(op, i); + + if (!init_op_complete(dep->op) + || (dep->required && dep->op->error)) { + return false; + } + } + + return true; +} + +static void __init +init_op_run(struct init_op *op) +{ + if (init_op_ready(op)) { + op->error = op->fn(); + } + + init_op_set_complete(op); +} + +#if INIT_DEBUG + +#define INIT_DEBUG_LOG_BUFFER_SIZE 8192 + +struct init_debug_log { + char buffer[INIT_DEBUG_LOG_BUFFER_SIZE]; + size_t index; +}; + +/* + * Buffers used to store an easy-to-dump text representation of init operations. + * + * These buffers are meant to be retrieved through a debugging interface such + * as JTAG. + */ +struct init_debug_logs { + struct init_debug_log roots; /* graph roots */ + struct init_debug_log cycles; /* operations with dependency cycles */ + struct init_debug_log pending; /* operations successfully sorted */ + struct init_debug_log complete; /* executed operations */ +}; + +static struct init_debug_logs init_debug_logs; + +static void __init +init_debug_log_append(struct init_debug_log *log, const char *name) +{ + size_t size; + + if (log->index == sizeof(log->buffer)) { + return; + } + + size = sizeof(log->buffer) - log->index; + log->index += snprintf(log->buffer + log->index, size, "%s ", name); + + if (log->index >= sizeof(log->buffer)) { + log->index = sizeof(log->buffer); + } +} + +static void __init +init_debug_append_root(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.roots, op->name); +} + +static void __init +init_debug_append_cycle(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.cycles, op->name); +} + +static void __init +init_debug_append_pending(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.pending, op->name); +} + +static void __init +init_debug_append_complete(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.complete, op->name); +} + +static void __init +init_debug_scan_not_pending(void) +{ + const struct init_op *op; + + for (op = &_init_ops; op < &_init_ops_end; op++) { + if (!init_op_pending(op)) { + init_debug_append_cycle(op); + } + } +} + +#else /* INIT_DEBUG */ +#define init_debug_append_root(roots) +#define init_debug_append_pending(op) +#define init_debug_append_complete(op) +#define init_debug_scan_not_pending() +#endif /* INIT_DEBUG */ + +static void __init +init_add_pending_op(struct init_ops_list *pending_ops, struct init_op *op) +{ + assert(!init_op_pending(op)); + + init_op_set_pending(op); + init_ops_list_push(pending_ops, op); + init_debug_append_pending(op); +} + +static void __init +init_op_visit(struct init_op *op, struct init_ops_stack *stack) +{ + struct init_op_dep *dep; + + for (size_t i = 0; i < op->nr_deps; i++) { + dep = init_op_get_dep(op, i); + assert(dep->op->nr_parents != 0); + dep->op->nr_parents--; + + if (init_op_orphan(dep->op)) { + init_ops_stack_push(stack, dep->op); + } + } +} + +static void __init +init_check_ops_alignment(void) +{ + uintptr_t start, end; + + start = (uintptr_t)&_init_ops; + end = (uintptr_t)&_init_ops_end; + + if (((end - start) % INIT_OP_ALIGN) != 0) { + cpu_halt(); + } +} + +static void __init +init_bootstrap(void) +{ + struct init_op *op; + + init_check_ops_alignment(); + + for (op = &_init_ops; op < &_init_ops_end; op++) { + init_op_init(op); + } +} + +static void __init +init_scan_roots(struct init_ops_stack *stack) +{ + struct init_op *op; + + init_ops_stack_init(stack); + + for (op = &_init_ops; op < &_init_ops_end; op++) { + if (init_op_orphan(op)) { + init_ops_stack_push(stack, op); + init_debug_append_root(op); + } + } +} + +static void __init +init_scan_ops(struct init_ops_list *pending_ops) +{ + struct init_ops_stack stack; + struct init_op *op; + size_t nr_ops; + + init_scan_roots(&stack); + + for (;;) { + op = init_ops_stack_pop(&stack); + + if (op == NULL) { + break; + } + + init_add_pending_op(pending_ops, op); + init_op_visit(op, &stack); + } + + init_debug_scan_not_pending(); + + nr_ops = &_init_ops_end - &_init_ops; + + if (init_ops_list_size(pending_ops) != nr_ops) { + cpu_halt(); + } +} + +static void __init +init_run_ops(struct init_ops_list *pending_ops) +{ + struct init_op *op; + + for (;;) { + op = init_ops_list_pop(pending_ops); + + if (op == NULL) { + break; + } + + init_op_run(op); + init_debug_append_complete(op); + } +} + +void __init +init_setup(void) +{ + struct init_ops_list pending_ops; + + init_ops_list_init(&pending_ops); + + init_bootstrap(); + init_scan_ops(&pending_ops); + init_run_ops(&pending_ops); +} diff --git a/kern/init.h b/kern/init.h index 62fecf40..d15d9503 100644 --- a/kern/init.h +++ b/kern/init.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2012 Richard Braun. + * Copyright (c) 2010-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,6 +13,11 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Init sections and operations. + * + * TODO Make the x86 linker script use macros from this header. */ #ifndef _KERN_INIT_H @@ -25,8 +30,15 @@ #define INIT_SECTION .init.text #define INIT_DATA_SECTION .init.data +/* + * This section must only contain init operation structures, and must be + * located inside the .init section. + */ +#define INIT_OPS_SECTION .init.ops + #ifndef __ASSEMBLER__ +#include <kern/error.h> #include <kern/macros.h> #define __init __section(QUOTE(INIT_SECTION)) @@ -38,6 +50,60 @@ extern char _init; extern char _init_end; +/* + * Type for initialization operation functions. + */ +typedef int (*init_op_fn_t)(void); + +#include <kern/init_i.h> + +/* + * Forge an init operation declaration. + */ +#define INIT_OP_DECLARE(fn) extern struct init_op INIT_OP(fn) + +/* + * Foge an entry suitable as an init operation dependency. + * + * If a dependency isn't required, it's still used to determine run + * order, but its result is ignored, and the operation depending on it + * behaves as if that dependency succeeded. + */ +#define INIT_OP_DEP(fn, required) { &INIT_OP(fn), required } + +/* + * Init operation definition macro. + * + * This macro is used to define a structure named after the given function. + * Init operations are placed in a specific section which doesn't contain + * any other object type, making it a system-wide array of init operations. + * There is no need to actively register init operations; this module finds + * them all from their section. Dependencies are given as a variable-length + * argument list of entries built with the INIT_OP_DEP() macro. + */ +#define INIT_OP_DEFINE(_fn, ...) \ + static struct init_op_dep INIT_OP_DEPS(_fn)[] __initdata = { \ + __VA_ARGS__ \ + }; \ + \ + struct init_op INIT_OP(_fn) __initop = { \ + .name = QUOTE(_fn), \ + .fn = _fn, \ + .deps = INIT_OP_DEPS(_fn), \ + .error = ERROR_AGAIN, \ + .state = INIT_OP_STATE_UNLINKED, \ + .nr_deps = ARRAY_SIZE(INIT_OP_DEPS(_fn)), \ + .nr_parents = 0, \ + } + +/* + * Initialize the init module. + * + * Scan the section containing init operations, resolve all dependencies, + * and run operations in an appropriate order. + */ +void init_setup(void); + #endif /* __ASSEMBLER__ */ #endif /* _KERN_INIT_H */ diff --git a/kern/init_i.h b/kern/init_i.h new file mode 100644 index 00000000..be23073c --- /dev/null +++ b/kern/init_i.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _KERN_INIT_I_H +#define _KERN_INIT_I_H + +/* + * Alignment is important to make sure initialization operations are + * stored as a C array in the reserved init op section. + */ +#define INIT_OP_ALIGN 64 + +#include <stdalign.h> +#include <stdbool.h> +#include <stddef.h> + +#include <kern/list_types.h> +#include <kern/macros.h> + +#define __initop __section(QUOTE(INIT_OPS_SECTION)) + +#define INIT_OP_STATE_UNLINKED 0 +#define INIT_OP_STATE_PENDING 1 +#define INIT_OP_STATE_COMPLETE 2 + +struct init_op { + alignas(INIT_OP_ALIGN) struct list list_node; + struct list stack_node; + const char *name; + init_op_fn_t fn; + struct init_op_dep *deps; + int error; + unsigned char state; + unsigned char nr_deps; + unsigned char nr_parents; +}; + +struct init_op_dep { + struct init_op *op; + bool required; +}; + +#define __INIT_OP_DEPS(fn) fn ## _init_op_deps +#define INIT_OP_DEPS(fn) __INIT_OP_DEPS(fn) + +#define __INIT_OP(fn) fn ## _init_op +#define INIT_OP(fn) __INIT_OP(fn) + +#endif /* _KERN_INIT_I_H */ diff --git a/kern/intr.c b/kern/intr.c index fe69e3e9..8579b744 100644 --- a/kern/intr.c +++ b/kern/intr.c @@ -22,27 +22,29 @@ * Shared interrupts are supported. */ +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> #include <kern/atomic.h> #include <kern/kmem.h> #include <kern/init.h> #include <kern/intr.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/spinlock.h> #include <kern/thread.h> +#include <machine/boot.h> #include <machine/cpu.h> +#include <machine/trap.h> struct intr_handler { - struct list node; + alignas(CPU_L1_SIZE) struct list node; intr_handler_fn_t fn; void *arg; -} __aligned(CPU_L1_SIZE); +}; /* * Interrupt controller. @@ -67,16 +69,16 @@ struct intr_ctl { * span a cache line to avoid false sharing. */ struct intr_entry { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; struct intr_ctl *ctl; unsigned int cpu; struct list handlers; -} __aligned(CPU_L1_SIZE); +}; /* * Interrupt table. */ -static struct intr_entry intr_table[INTR_TABLE_SIZE]; +static struct intr_entry intr_table[TRAP_INTR_TABLE_SIZE]; /* * List of registered controllers. @@ -314,8 +316,8 @@ intr_get_entry(unsigned int intr) return &intr_table[intr]; } -void __init -intr_setup(void) +static int __init +intr_bootstrap(void) { unsigned int i; @@ -327,8 +329,27 @@ intr_setup(void) for (i = 0; i < ARRAY_SIZE(intr_table); i++) { intr_entry_init(intr_get_entry(i)); } + + return 0; } +INIT_OP_DEFINE(intr_bootstrap, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(thread_bootstrap, true)); + +static int __init +intr_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(intr_setup, + INIT_OP_DEP(boot_setup_intr, true), + INIT_OP_DEP(intr_bootstrap, true)); + static void __init intr_check_range(unsigned int first_intr, unsigned int last_intr) { @@ -388,7 +409,7 @@ intr_unregister(unsigned int intr, intr_handler_fn_t fn) handler = intr_entry_remove(intr_get_entry(intr), fn); if (handler == NULL) { - printf("intr: warning: attempting to unregister unknown handler\n"); + log_warning("intr: attempting to unregister unknown handler"); return; } @@ -410,7 +431,7 @@ intr_handle(unsigned int intr) spinlock_lock(&entry->lock); if (intr_entry_empty(entry)) { - printf("intr: spurious interrupt %u\n", intr); + log_warning("intr: spurious interrupt %u", intr); goto out; } diff --git a/kern/intr.h b/kern/intr.h index 1cdab5ef..ac3f8cf2 100644 --- a/kern/intr.h +++ b/kern/intr.h @@ -21,6 +21,8 @@ #ifndef _KERN_INTR_H #define _KERN_INTR_H +#include <kern/init.h> + /* * Type for interrupt handler functions. * @@ -42,11 +44,6 @@ struct intr_ops { }; /* - * Initialize the intr module. - */ -void intr_setup(void); - -/* * Register an interrupt controller. * * This function isn't thread-safe and can only be called during system @@ -66,4 +63,17 @@ void intr_unregister(unsigned int intr, intr_handler_fn_t fn); */ void intr_handle(unsigned int intr); +/* + * This init operation provides : + * - registration of interrupt controllers and handlers + */ +INIT_OP_DECLARE(intr_bootstrap); + +/* + * This init operation provides : + * - all interrupt controllers have been registered + * - module fully initialized + */ +INIT_OP_DECLARE(intr_setup); + #endif /* _KERN_INTR_H */ diff --git a/kern/kernel.c b/kern/kernel.c index 05cf8392..434297d3 100644 --- a/kern/kernel.c +++ b/kern/kernel.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2014 Richard Braun. + * Copyright (c) 2011-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,47 +15,19 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <kern/cpumap.h> #include <kern/init.h> #include <kern/kernel.h> -#include <kern/llsync.h> -#include <kern/percpu.h> -#include <kern/shell.h> -#include <kern/sleepq.h> -#include <kern/sref.h> -#include <kern/task.h> #include <kern/thread.h> -#include <kern/turnstile.h> -#include <kern/work.h> -#include <kern/xcall.h> #include <machine/cpu.h> #include <vm/vm_page.h> -#ifdef X15_RUN_TEST_MODULE -#include <test/test.h> -#endif /* X15_RUN_TEST_MODULE */ - void __init kernel_main(void) { assert(!cpu_intr_enabled()); - percpu_cleanup(); - cpumap_setup(); - xcall_setup(); - task_setup(); - sleepq_setup(); - turnstile_setup(); - thread_setup(); - work_setup(); - llsync_setup(); - sref_setup(); - shell_setup(); - vm_page_info(); - -#ifdef X15_RUN_TEST_MODULE - test_setup(); -#endif /* X15_RUN_TEST_MODULE */ + init_setup(); + vm_page_log_info(); /* * Enabling application processors is done late in the boot process for diff --git a/kern/kernel.h b/kern/kernel.h index 868066e6..22cae43b 100644 --- a/kern/kernel.h +++ b/kern/kernel.h @@ -18,7 +18,7 @@ #ifndef _KERN_KERNEL_H #define _KERN_KERNEL_H -#include <kern/macros.h> +#include <stdnoreturn.h> /* * Kernel properties. @@ -31,13 +31,13 @@ * * Interrupts must be disabled when calling this function. */ -void __noreturn kernel_main(void); +noreturn void kernel_main(void); /* * Entry point for APs. * * Interrupts must be disabled when calling this function. */ -void __noreturn kernel_ap_main(void); +noreturn void kernel_ap_main(void); #endif /* _KERN_KERNEL_H */ diff --git a/kern/kmem.c b/kern/kmem.c index 5cce5589..d98be0d6 100644 --- a/kern/kmem.c +++ b/kern/kmem.c @@ -41,15 +41,15 @@ * TODO Rework the CPU pool layer to use the SLQB algorithm by Nick Piggin. */ +#include <assert.h> +#include <limits.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> #include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/init.h> -#include <kern/limits.h> #include <kern/list.h> #include <kern/log2.h> #include <kern/kmem.h> @@ -57,9 +57,10 @@ #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> +#include <kern/shell.h> #include <kern/thread.h> #include <machine/cpu.h> +#include <machine/page.h> #include <machine/pmap.h> #include <vm/vm_kmem.h> #include <vm/vm_page.h> @@ -1092,41 +1093,28 @@ kmem_cache_info(struct kmem_cache *cache) { char flags_str[64]; - if (cache == NULL) { - mutex_lock(&kmem_cache_list_lock); - - list_for_each_entry(&kmem_cache_list, cache, node) { - kmem_cache_info(cache); - } - - mutex_unlock(&kmem_cache_list_lock); - - return; - } - snprintf(flags_str, sizeof(flags_str), "%s%s", (cache->flags & KMEM_CF_SLAB_EXTERNAL) ? " SLAB_EXTERNAL" : "", (cache->flags & KMEM_CF_VERIFY) ? " VERIFY" : ""); mutex_lock(&cache->lock); - printf("kmem: name: %s\n" - "kmem: flags: 0x%x%s\n" - "kmem: obj_size: %zu\n" - "kmem: align: %zu\n" - "kmem: buf_size: %zu\n" - "kmem: bufctl_dist: %zu\n" - "kmem: slab_size: %zu\n" - "kmem: color_max: %zu\n" + printf("kmem: flags: 0x%x%s\n" + "kmem: obj_size: %zu\n" + "kmem: align: %zu\n" + "kmem: buf_size: %zu\n" + "kmem: bufctl_dist: %zu\n" + "kmem: slab_size: %zu\n" + "kmem: color_max: %zu\n" "kmem: bufs_per_slab: %lu\n" - "kmem: nr_objs: %lu\n" - "kmem: nr_bufs: %lu\n" - "kmem: nr_slabs: %lu\n" + "kmem: nr_objs: %lu\n" + "kmem: nr_bufs: %lu\n" + "kmem: nr_slabs: %lu\n" "kmem: nr_free_slabs: %lu\n" - "kmem: buftag_dist: %zu\n" - "kmem: redzone_pad: %zu\n" - "kmem: cpu_pool_size: %d\n", cache->name, cache->flags, flags_str, - cache->obj_size, cache->align, cache->buf_size, cache->bufctl_dist, + "kmem: buftag_dist: %zu\n" + "kmem: redzone_pad: %zu\n" + "kmem: cpu_pool_size: %d\n", cache->flags, flags_str, cache->obj_size, + cache->align, cache->buf_size, cache->bufctl_dist, cache->slab_size, cache->color_max, cache->bufs_per_slab, cache->nr_objs, cache->nr_bufs, cache->nr_slabs, cache->nr_free_slabs, cache->buftag_dist, cache->redzone_pad, @@ -1135,8 +1123,71 @@ kmem_cache_info(struct kmem_cache *cache) mutex_unlock(&cache->lock); } -void __init -kmem_setup(void) +#ifdef X15_SHELL + +static struct kmem_cache * +kmem_lookup_cache(const char *name) +{ + struct kmem_cache *cache; + + mutex_lock(&kmem_cache_list_lock); + + list_for_each_entry(&kmem_cache_list, cache, node) { + if (strcmp(cache->name, name) == 0) { + goto out; + } + } + + cache = NULL; + +out: + mutex_unlock(&kmem_cache_list_lock); + + return cache; +} + +static void +kmem_shell_info(int argc, char **argv) +{ + struct kmem_cache *cache; + + if (argc < 2) { + kmem_info(); + } else { + cache = kmem_lookup_cache(argv[1]); + + if (cache == NULL) { + printf("kmem: info: invalid argument\n"); + return; + } + + kmem_cache_info(cache); + } +} + +static struct shell_cmd kmem_shell_cmds[] = { + SHELL_CMD_INITIALIZER("kmem_info", kmem_shell_info, + "kmem_info [<cache_name>]", + "display information about kernel memory and caches"), +}; + +static int __init +kmem_setup_shell(void) +{ + SHELL_REGISTER_CMDS(kmem_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(kmem_setup_shell, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(thread_setup, true)); + +#endif /* X15_SHELL */ + +static int __init +kmem_bootstrap(void) { struct kmem_cpu_pool_type *cpu_pool_type; char name[KMEM_NAME_SIZE]; @@ -1170,8 +1221,24 @@ kmem_setup(void) kmem_cache_init(&kmem_caches[i], name, size, 0, NULL, 0); size <<= 1; } + + return 0; +} + +INIT_OP_DEFINE(kmem_bootstrap, + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(vm_page_setup, true)); + +static int __init +kmem_setup(void) +{ + return 0; } +INIT_OP_DEFINE(kmem_setup, + INIT_OP_DEP(kmem_bootstrap, true), + INIT_OP_DEP(vm_kmem_setup, true)); + static inline size_t kmem_get_index(unsigned long size) { @@ -1282,8 +1349,11 @@ kmem_free(void *ptr, size_t size) void kmem_info(void) { + size_t mem_usage, mem_reclaimable, total, total_reclaimable; struct kmem_cache *cache; - size_t mem_usage, mem_reclaimable; + + total = 0; + total_reclaimable = 0; printf("kmem: cache obj slab bufs objs bufs " " total reclaimable\n" @@ -1297,6 +1367,8 @@ kmem_info(void) mem_usage = (cache->nr_slabs * cache->slab_size) >> 10; mem_reclaimable = (cache->nr_free_slabs * cache->slab_size) >> 10; + total += mem_usage; + total_reclaimable += mem_reclaimable; printf("kmem: %-19s %6zu %3zuk %4lu %6lu %6lu %7zuk %10zuk\n", cache->name, cache->obj_size, cache->slab_size >> 10, @@ -1307,4 +1379,6 @@ kmem_info(void) } mutex_unlock(&kmem_cache_list_lock); + + printf("total: %zuk reclaimable: %zuk\n", total, total_reclaimable); } diff --git a/kern/kmem.h b/kern/kmem.h index 8f6c773c..3296ddb4 100644 --- a/kern/kmem.h +++ b/kern/kmem.h @@ -23,6 +23,8 @@ #include <stddef.h> +#include <kern/init.h> + /* * Object cache. */ @@ -77,14 +79,6 @@ void kmem_cache_free(struct kmem_cache *cache, void *obj); void kmem_cache_info(struct kmem_cache *cache); /* - * Set up the kernel memory allocator module. - * - * This function should only be called by the VM system. Once it returns, - * caches can be initialized. - */ -void kmem_setup(void); - -/* * Allocate size bytes of uninitialized memory. */ void * kmem_alloc(size_t size); @@ -106,4 +100,16 @@ void kmem_free(void *ptr, size_t size); */ void kmem_info(void); +/* + * This init operation provides : + * - allocation from caches backed by the page allocator + */ +INIT_OP_DECLARE(kmem_bootstrap); + +/* + * This init operation provides : + * - allocation from all caches + */ +INIT_OP_DECLARE(kmem_setup); + #endif /* _KERN_KMEM_H */ diff --git a/kern/kmem_i.h b/kern/kmem_i.h index f3ad8228..beae6c45 100644 --- a/kern/kmem_i.h +++ b/kern/kmem_i.h @@ -18,11 +18,12 @@ #ifndef _KERN_KMEM_I_H #define _KERN_KMEM_I_H +#include <stdalign.h> #include <stddef.h> #include <kern/list.h> #include <kern/mutex.h> -#include <kern/param.h> +#include <machine/cpu.h> /* * Per-processor cache of pre-constructed objects. @@ -30,13 +31,13 @@ * The flags member is a read-only CPU-local copy of the parent cache flags. */ struct kmem_cpu_pool { - struct mutex lock; + alignas(CPU_L1_SIZE) struct mutex lock; int flags; int size; int transfer_size; int nr_objs; void **array; -} __aligned(CPU_L1_SIZE); +}; /* * When a cache is created, its CPU pool type is determined from the buffer @@ -70,17 +71,17 @@ union kmem_bufctl { * Redzone guard word. */ #ifdef __LP64__ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_REDZONE_WORD 0xfeedfacefeedfaceUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_REDZONE_WORD 0xcefaedfecefaedfeUL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #else /* __LP64__ */ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_REDZONE_WORD 0xfeedfaceUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_REDZONE_WORD 0xcefaedfeUL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #endif /* __LP64__ */ /* @@ -104,21 +105,21 @@ struct kmem_buftag { * Values the buftag state member can take. */ #ifdef __LP64__ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_BUFTAG_ALLOC 0xa110c8eda110c8edUL #define KMEM_BUFTAG_FREE 0xf4eeb10cf4eeb10cUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_BUFTAG_ALLOC 0xedc810a1edc810a1UL #define KMEM_BUFTAG_FREE 0x0cb1eef40cb1eef4UL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #else /* __LP64__ */ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_BUFTAG_ALLOC 0xa110c8edUL #define KMEM_BUFTAG_FREE 0xf4eeb10cUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_BUFTAG_ALLOC 0xedc810a1UL #define KMEM_BUFTAG_FREE 0x0cb1eef4UL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #endif /* __LP64__ */ /* @@ -127,13 +128,13 @@ struct kmem_buftag { * These values are unconditionally 64-bit wide since buffers are at least * 8-byte aligned. */ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_FREE_PATTERN 0xdeadbeefdeadbeefULL #define KMEM_UNINIT_PATTERN 0xbaddcafebaddcafeULL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_FREE_PATTERN 0xefbeaddeefbeaddeULL #define KMEM_UNINIT_PATTERN 0xfecaddbafecaddbaULL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ /* * Page-aligned collection of unconstructed buffers. diff --git a/kern/llsync.c b/kern/llsync.c index 1942576d..c432fc68 100644 --- a/kern/llsync.c +++ b/kern/llsync.c @@ -32,24 +32,24 @@ * TODO Gracefully handle large amounts of deferred works. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/condition.h> #include <kern/cpumap.h> #include <kern/init.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/llsync.h> #include <kern/llsync_i.h> #include <kern/macros.h> #include <kern/mutex.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/syscnt.h> #include <kern/work.h> +#include <kern/thread.h> #include <machine/cpu.h> /* @@ -82,7 +82,7 @@ llsync_ready(void) return llsync_is_ready; } -void __init +static int __init llsync_setup(void) { struct llsync_cpu_data *cpu_data; @@ -104,9 +104,18 @@ llsync_setup(void) work_queue_init(&cpu_data->queue0); } - llsync_is_ready = true; + return 0; } +INIT_OP_DEFINE(llsync_setup, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(work_setup, true)); + static void llsync_process_global_checkpoint(void) { @@ -122,7 +131,7 @@ llsync_process_global_checkpoint(void) /* TODO Handle hysteresis */ if (!llsync_data.no_warning && (nr_works >= LLSYNC_NR_PENDING_WORKS_WARN)) { llsync_data.no_warning = 1; - printf("llsync: warning: large number of pending works\n"); + log_warning("llsync: large number of pending works\n"); } if (llsync_data.nr_registered_cpus == 0) { @@ -182,6 +191,10 @@ llsync_register(void) unsigned long flags; unsigned int cpu; + if (!llsync_is_ready) { + llsync_is_ready = true; + } + cpu = cpu_id(); cpu_data = llsync_get_cpu_data(); diff --git a/kern/llsync.h b/kern/llsync.h index 6736d6f9..3852b0b9 100644 --- a/kern/llsync.h +++ b/kern/llsync.h @@ -123,11 +123,6 @@ llsync_read_exit(void) bool llsync_ready(void); /* - * Initialize the llsync module. - */ -void llsync_setup(void); - -/* * Manage registration of the current processor. * * The caller must not be allowed to migrate when calling these functions. diff --git a/kern/llsync_i.h b/kern/llsync_i.h index 6bc1bf1d..e043eb9d 100644 --- a/kern/llsync_i.h +++ b/kern/llsync_i.h @@ -18,10 +18,11 @@ #ifndef _KERN_LLSYNC_I_H #define _KERN_LLSYNC_I_H -#include <kern/assert.h> +#include <assert.h> +#include <stdalign.h> + #include <kern/cpumap.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/spinlock.h> #include <kern/syscnt.h> #include <kern/work.h> @@ -57,7 +58,7 @@ struct llsync_data { * - apply optimistic accesses to reduce contention */ struct { - volatile unsigned int value __aligned(CPU_L1_SIZE); + alignas(CPU_L1_SIZE) volatile unsigned int value; } gcid; }; diff --git a/kern/log.c b/kern/log.c new file mode 100644 index 00000000..3d6ee378 --- /dev/null +++ b/kern/log.c @@ -0,0 +1,672 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <limits.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdint.h> +#include <string.h> + +#include <kern/arg.h> +#include <kern/cbuf.h> +#include <kern/init.h> +#include <kern/log.h> +#include <kern/macros.h> +#include <kern/panic.h> +#include <kern/shell.h> +#include <kern/spinlock.h> +#include <kern/thread.h> +#include <machine/boot.h> +#include <machine/cpu.h> + +#define LOG_BUFFER_SIZE 16384 + +#if !ISP2(LOG_BUFFER_SIZE) +#error "log buffer size must be a power-of-two" +#endif + +#define LOG_MSG_SIZE 128 + +#define LOG_MARKER 0x1 + +static struct thread *log_thread; + +static struct cbuf log_cbuf; +static char log_buffer[LOG_BUFFER_SIZE]; + +/* + * This index is used by the log thread to report what it has consumed + * so that producers can detect overruns. + */ +static size_t log_index; + +static size_t log_nr_overruns; + +static unsigned int log_print_level; + +/* + * Global lock. + * + * Interrupts must be disabled when holding this lock. + */ +static struct spinlock log_lock; + +/* + * A record starts with a marker byte and ends with a null terminating byte. + * + * There must be no null byte inside the header, and the buffer is expected + * to be a standard null-terminated string. + */ +struct log_record { + uint8_t mark; + uint8_t level; + char buffer[LOG_MSG_SIZE]; +}; + +/* + * A consumer context allows faster, buffered reads of the circular buffer, + * by retaining bytes that weren't consumed in case a read spans multiple + * records. + */ +struct log_consume_ctx { + struct cbuf *cbuf; + size_t cbuf_index; + char buf[LOG_MSG_SIZE]; + size_t index; + size_t size; +}; + +static void +log_consume_ctx_init(struct log_consume_ctx *ctx, struct cbuf *cbuf) +{ + ctx->cbuf = cbuf; + ctx->cbuf_index = cbuf_start(cbuf); + ctx->index = 0; + ctx->size = 0; +} + +static size_t +log_consume_ctx_index(const struct log_consume_ctx *ctx) +{ + return ctx->cbuf_index; +} + +static void +log_consume_ctx_set_index(struct log_consume_ctx *ctx, size_t index) +{ + ctx->cbuf_index = index; +} + +static bool +log_consume_ctx_empty(const struct log_consume_ctx *ctx) +{ + return ctx->cbuf_index == cbuf_end(ctx->cbuf); +} + +static int +log_consume_ctx_pop(struct log_consume_ctx *ctx, char *byte) +{ + int error; + + if (ctx->index >= ctx->size) { + ctx->index = 0; + ctx->size = sizeof(ctx->buf); + error = cbuf_read(ctx->cbuf, ctx->cbuf_index, ctx->buf, &ctx->size); + + if (error) { + ctx->cbuf_index = cbuf_start(ctx->cbuf); + return error; + } + } + + ctx->cbuf_index++; + *byte = ctx->buf[ctx->index]; + ctx->index++; + return 0; +} + +static const char * +log_level2str(unsigned int level) +{ + switch (level) { + case LOG_EMERG: + return "emerg"; + case LOG_ALERT: + return "alert"; + case LOG_CRIT: + return "crit"; + case LOG_ERR: + return "error"; + case LOG_WARNING: + return "warning"; + case LOG_NOTICE: + return "notice"; + case LOG_INFO: + return "info"; + case LOG_DEBUG: + return "debug"; + default: + return NULL; + } +} + +static char +log_level2char(unsigned int level) +{ + assert(level < LOG_NR_LEVELS); + return '0' + level; +} + +static uint8_t +log_char2level(char c) +{ + uint8_t level; + + level = c - '0'; + assert(level < LOG_NR_LEVELS); + return level; +} + +static void +log_record_init_produce(struct log_record *record, unsigned int level) +{ + record->mark = LOG_MARKER; + record->level = log_level2char(level); +} + +static void +log_record_consume(struct log_record *record, char c, size_t *sizep) +{ + char *ptr; + + assert(*sizep < sizeof(*record)); + + ptr = (char *)record; + ptr[*sizep] = c; + (*sizep)++; +} + +static int +log_record_init_consume(struct log_record *record, struct log_consume_ctx *ctx) +{ + bool marker_found; + size_t size; + int error; + char c; + + marker_found = false; + size = 0; + + for (;;) { + if (log_consume_ctx_empty(ctx)) { + if (!marker_found) { + return ERROR_INVAL; + } + + break; + } + + error = log_consume_ctx_pop(ctx, &c); + + if (error) { + continue; + } + + if (!marker_found) { + if (c != LOG_MARKER) { + continue; + } + + marker_found = true; + log_record_consume(record, c, &size); + continue; + } else if (size == offsetof(struct log_record, level)) { + record->level = log_char2level(c); + size++; + continue; + } + + log_record_consume(record, c, &size); + + if (c == '\0') { + break; + } + } + + return 0; +} + +static void +log_record_print(const struct log_record *record, unsigned int level) +{ + if (record->level > level) { + return; + } + + if (record->level <= LOG_WARNING) { + printf("%7s %s\n", log_level2str(record->level), record->buffer); + } else { + printf("%s\n", record->buffer); + } +} + +static void +log_run(void *arg) +{ + unsigned long flags, nr_overruns; + struct log_consume_ctx ctx; + struct log_record record; + bool start_shell; + int error; + + (void)arg; + + nr_overruns = 0; + start_shell = true; + + spinlock_lock_intr_save(&log_lock, &flags); + + log_consume_ctx_init(&ctx, &log_cbuf); + + for (;;) { + while (log_consume_ctx_empty(&ctx)) { + /* + * Starting the shell after the log thread sleeps for the first + * time cleanly serializes log messages and shell prompt, making + * a clean ordered output. + */ + if (start_shell) { + spinlock_unlock_intr_restore(&log_lock, flags); + shell_start(); + start_shell = false; + spinlock_lock_intr_save(&log_lock, &flags); + } + + log_index = log_consume_ctx_index(&ctx); + + thread_sleep(&log_lock, &log_cbuf, "log_cbuf"); + + log_consume_ctx_set_index(&ctx, log_index); + } + + error = log_record_init_consume(&record, &ctx); + + /* Drain the log buffer before reporting overruns */ + if (log_consume_ctx_empty(&ctx)) { + nr_overruns = log_nr_overruns; + log_nr_overruns = 0; + } + + log_index = log_consume_ctx_index(&ctx); + + spinlock_unlock_intr_restore(&log_lock, flags); + + if (!error) { + log_record_print(&record, log_print_level); + } + + if (nr_overruns != 0) { + log_msg(LOG_ERR, "log: buffer overruns, %lu bytes dropped", + nr_overruns); + nr_overruns = 0; + } + + spinlock_lock_intr_save(&log_lock, &flags); + + log_consume_ctx_set_index(&ctx, log_index); + } +} + +#ifdef X15_SHELL + +static void +log_dump(unsigned int level) +{ + struct log_consume_ctx ctx; + struct log_record record; + unsigned long flags; + int error; + + spinlock_lock_intr_save(&log_lock, &flags); + + log_consume_ctx_init(&ctx, &log_cbuf); + + for (;;) { + error = log_record_init_consume(&record, &ctx); + + if (error) { + break; + } + + spinlock_unlock_intr_restore(&log_lock, flags); + + log_record_print(&record, level); + + spinlock_lock_intr_save(&log_lock, &flags); + } + + spinlock_unlock_intr_restore(&log_lock, flags); +} + +static void +log_shell_dump(int argc, char **argv) +{ + unsigned int level; + int ret; + + if (argc != 2) { + level = log_print_level; + } else { + ret = sscanf(argv[1], "%u", &level); + + if ((ret != 1) || (level >= LOG_NR_LEVELS)) { + printf("log: dump: invalid arguments\n"); + return; + } + } + + log_dump(level); +} + +static struct shell_cmd log_shell_cmds[] = { + SHELL_CMD_INITIALIZER2("log_dump", log_shell_dump, + "log_dump [<level>]", + "dump the log buffer", + "Only records of level less than or equal to the given level" + " are printed. Level may be one of :\n" + " 0: emergency\n" + " 1: alert\n" + " 2: critical\n" + " 3: error\n" + " 4: warning\n" + " 5: notice\n" + " 6: info\n" + " 7: debug"), +}; + +static int __init +log_setup_shell(void) +{ + SHELL_REGISTER_CMDS(log_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(log_setup_shell, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(shell_setup, true)); + +#endif /* X15_SHELL */ + +static int __init +log_setup(void) +{ + cbuf_init(&log_cbuf, log_buffer, sizeof(log_buffer)); + log_index = cbuf_start(&log_cbuf); + spinlock_init(&log_lock); + log_print_level = LOG_INFO; + + boot_log_info(); + arg_log_info(); + cpu_log_info(cpu_current()); + + return 0; +} + +INIT_OP_DEFINE(log_setup, + INIT_OP_DEP(arg_setup, true), + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(spinlock_setup, true)); + +static int __init +log_start(void) +{ + struct thread_attr attr; + int error; + + thread_attr_init(&attr, THREAD_KERNEL_PREFIX "log_run"); + thread_attr_set_detached(&attr); + error = thread_create(&log_thread, &attr, log_run, NULL); + + if (error) { + panic("log: unable to create thread"); + } + + return 0; +} + +INIT_OP_DEFINE(log_start, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(thread_setup, true)); + +static void +log_write(const void *s, size_t size) +{ + int error; + + error = cbuf_write(&log_cbuf, cbuf_end(&log_cbuf), s, size); + assert(!error); + + if (!cbuf_range_valid(&log_cbuf, log_index, log_index + 1)) { + log_nr_overruns += cbuf_start(&log_cbuf) - log_index; + log_index = cbuf_start(&log_cbuf); + } +} + +int +log_msg(unsigned int level, const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vmsg(level, format, ap); + va_end(ap); + + return ret; +} + +int +log_vmsg(unsigned int level, const char *format, va_list ap) +{ + struct log_record record; + unsigned long flags; + int nr_chars; + size_t size; + char *ptr; + + log_record_init_produce(&record, level); + nr_chars = vsnprintf(record.buffer, sizeof(record.buffer), format, ap); + + if ((unsigned int)nr_chars >= sizeof(record.buffer)) { + log_msg(LOG_ERR, "log: message too large"); + goto out; + } + + ptr = strchr(record.buffer, '\n'); + + if (ptr != NULL) { + *ptr = '\0'; + nr_chars = ptr - record.buffer; + } + + assert(nr_chars >= 0); + size = offsetof(struct log_record, buffer) + nr_chars + 1; + + spinlock_lock_intr_save(&log_lock, &flags); + log_write(&record, size); + thread_wakeup(log_thread); + spinlock_unlock_intr_restore(&log_lock, flags); + +out: + return nr_chars; +} + +int +log_emerg(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vemerg(format, ap); + va_end(ap); + + return ret; +} + +int +log_alert(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_valert(format, ap); + va_end(ap); + + return ret; +} + +int +log_crit(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vcrit(format, ap); + va_end(ap); + + return ret; +} + +int +log_err(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_verr(format, ap); + va_end(ap); + + return ret; +} + +int +log_warning(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vwarning(format, ap); + va_end(ap); + + return ret; +} + +int +log_notice(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vnotice(format, ap); + va_end(ap); + + return ret; +} + +int +log_info(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vinfo(format, ap); + va_end(ap); + + return ret; +} + +int +log_debug(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vdebug(format, ap); + va_end(ap); + + return ret; +} + +int +log_vemerg(const char *format, va_list ap) +{ + return log_vmsg(LOG_EMERG, format, ap); +} + +int +log_valert(const char *format, va_list ap) +{ + return log_vmsg(LOG_ALERT, format, ap); +} + +int +log_vcrit(const char *format, va_list ap) +{ + return log_vmsg(LOG_CRIT, format, ap); +} + +int +log_verr(const char *format, va_list ap) +{ + return log_vmsg(LOG_ERR, format, ap); +} + +int +log_vwarning(const char *format, va_list ap) +{ + return log_vmsg(LOG_WARNING, format, ap); +} + +int +log_vnotice(const char *format, va_list ap) +{ + return log_vmsg(LOG_NOTICE, format, ap); +} + +int +log_vinfo(const char *format, va_list ap) +{ + return log_vmsg(LOG_INFO, format, ap); +} + +int +log_vdebug(const char *format, va_list ap) +{ + return log_vmsg(LOG_DEBUG, format, ap); +} diff --git a/kern/log.h b/kern/log.h new file mode 100644 index 00000000..581a7840 --- /dev/null +++ b/kern/log.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * System logging. + */ + +#ifndef _KERN_LOG_H +#define _KERN_LOG_H + +#include <stdarg.h> + + +#include <kern/init.h> +enum { + LOG_EMERG, + LOG_ALERT, + LOG_CRIT, + LOG_ERR, + LOG_WARNING, + LOG_NOTICE, + LOG_INFO, + LOG_DEBUG, + LOG_NR_LEVELS, +}; + +/* + * Generate a message and send it to the log thread. + * + * The arguments and return value are similar to printf(), with + * these exceptions : + * - a level is associated to each log message + * - processing stops at the first terminating null byte or newline + * character, whichever occurs first + * + * This function may safely be called in interrupt context. + */ +int log_msg(unsigned int level, const char *format, ...) + __attribute__((format(printf, 2, 3))); + +int log_vmsg(unsigned int level, const char *format, va_list ap) + __attribute__((format(printf, 2, 0))); + +/* + * Convenience wrappers. + */ + +int log_emerg(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_alert(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_crit(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_err(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_warning(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_notice(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_info(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_debug(const char *format, ...) __attribute__((format(printf, 1, 2))); + +int log_vemerg(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_valert(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vcrit(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_verr(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vwarning(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vnotice(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vinfo(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vdebug(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); + +/* + * This init operation provides : + * - message logging + * + * The log thread isn't yet started and messages are merely stored in an + * internal buffer. + */ +INIT_OP_DECLARE(log_setup); + +#endif /* _KERN_LOG_H */ diff --git a/kern/log2.h b/kern/log2.h index 0a3768a7..ed12b441 100644 --- a/kern/log2.h +++ b/kern/log2.h @@ -21,8 +21,8 @@ #ifndef _KERN_LOG2_H #define _KERN_LOG2_H -#include <kern/assert.h> -#include <kern/limits.h> +#include <assert.h> +#include <limits.h> static inline unsigned int ilog2(unsigned long x) diff --git a/kern/macros.h b/kern/macros.h index a5b7b032..6b203e52 100644 --- a/kern/macros.h +++ b/kern/macros.h @@ -16,14 +16,22 @@ * * * Helper macros. + * + * This file is a top header in the inclusion hierarchy, and shouldn't include + * other headers that may cause circular dependencies. */ #ifndef _KERN_MACROS_H #define _KERN_MACROS_H -#ifndef __ASSEMBLER__ -#include <stddef.h> -#endif /* __ASSEMBLER__ */ +#if !defined(__GNUC__) || (__GNUC__ < 4) +#error "GCC 4+ required" +#endif + +/* + * Attributes for variables that are mostly read and seldom changed. + */ +#define __read_mostly __section(".data.read_mostly") #define MACRO_BEGIN ({ #define MACRO_END }) @@ -35,6 +43,16 @@ #define DECL_CONST(x, s) x #else /* __ASSEMBLER__ */ #define __DECL_CONST(x, s) x##s +void cga_putc(char c); + +static inline void +moo_print(const char *s) +{ + while (*s != '\0') { + cga_putc(*s); + s++; + } +} #define DECL_CONST(x, s) __DECL_CONST(x, s) #endif /* __ASSEMBLER__ */ @@ -48,32 +66,36 @@ #define P2ALIGNED(x, a) (((x) & ((a) - 1)) == 0) #define ISP2(x) P2ALIGNED(x, x) -#define P2ALIGN(x, a) ((x) & -(a)) -#define P2ROUND(x, a) (-(-(x) & -(a))) -#define P2END(x, a) (-(~(x) & -(a))) +#define P2ALIGN(x, a) ((x) & -(a)) /* decreases if not aligned */ +#define P2ROUND(x, a) (-(-(x) & -(a))) /* increases if not aligned */ +#define P2END(x, a) (-(~(x) & -(a))) /* always increases */ #define structof(ptr, type, member) \ ((type *)((char *)(ptr) - offsetof(type, member))) -#define alignof(x) __alignof__(x) - #define likely(expr) __builtin_expect(!!(expr), 1) #define unlikely(expr) __builtin_expect(!!(expr), 0) #define barrier() asm volatile("" : : : "memory") -#define __noreturn __attribute__((noreturn)) -#define __aligned(x) __attribute__((aligned(x))) +/* + * The following macros may be provided by the C environment. + */ + +#ifndef __noinline +#define __noinline __attribute__((noinline)) +#endif #ifndef __always_inline #define __always_inline inline __attribute__((always_inline)) -#endif /* __attribute__ */ +#endif +#ifndef __section #define __section(x) __attribute__((section(x))) -#define __packed __attribute__((packed)) -#define __alias(x) __attribute__((alias(x))) +#endif -#define __format_printf(fmt, args) \ - __attribute__((format(printf, fmt, args))) +#ifndef __packed +#define __packed __attribute__((packed)) +#endif #endif /* _KERN_MACROS_H */ diff --git a/test/test_x86_double_fault.c b/kern/mutex.c index cd12758a..62609768 100644 --- a/test/test_x86_double_fault.c +++ b/kern/mutex.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Richard Braun. + * Copyright (c) 2013-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,16 +13,16 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * - * Cause a double fault, see that it's correctly handled. */ -#include <test/test.h> -#include <machine/trap.h> +#include <kern/init.h> +#include <kern/thread.h> -void -test_setup(void) +static int __init +mutex_setup(void) { - trap_test_double_fault(); + return 0; } + +INIT_OP_DEFINE(mutex_setup, + INIT_OP_DEP(thread_setup_booter, true)); diff --git a/kern/mutex.h b/kern/mutex.h index 2936d82f..f192a70a 100644 --- a/kern/mutex.h +++ b/kern/mutex.h @@ -35,6 +35,7 @@ #include <kern/mutex/mutex_plain_i.h> #endif +#include <kern/init.h> #include <kern/mutex_types.h> #include <kern/thread.h> @@ -93,4 +94,12 @@ mutex_unlock(struct mutex *mutex) thread_wakeup_last_cond(); } +/* + * This init operation provides : + * - uncontended mutex locking + * + * Contended locking may only occur after starting the scheduler. + */ +INIT_OP_DECLARE(mutex_setup); + #endif /* _KERN_MUTEX_H */ diff --git a/kern/mutex/mutex_adaptive.c b/kern/mutex/mutex_adaptive.c index f4274ce1..ffc47169 100644 --- a/kern/mutex/mutex_adaptive.c +++ b/kern/mutex/mutex_adaptive.c @@ -35,7 +35,7 @@ mutex_adaptive_get_thread(uintptr_t owner) static void mutex_adaptive_set_contended(struct mutex *mutex) { - atomic_or_acq_rel(&mutex->owner, MUTEX_ADAPTIVE_CONTENDED); + atomic_or(&mutex->owner, MUTEX_ADAPTIVE_CONTENDED, ATOMIC_RELEASE); } static inline bool diff --git a/kern/mutex/mutex_adaptive_i.h b/kern/mutex/mutex_adaptive_i.h index 2dd06792..b9952ec6 100644 --- a/kern/mutex/mutex_adaptive_i.h +++ b/kern/mutex/mutex_adaptive_i.h @@ -23,9 +23,9 @@ " use <kern/mutex.h> instead" #endif +#include <assert.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/error.h> #include <kern/macros.h> diff --git a/kern/mutex/mutex_plain_i.h b/kern/mutex/mutex_plain_i.h index 9e41ff27..4f112b89 100644 --- a/kern/mutex/mutex_plain_i.h +++ b/kern/mutex/mutex_plain_i.h @@ -23,7 +23,8 @@ " use <kern/mutex.h> instead" #endif -#include <kern/assert.h> +#include <assert.h> + #include <kern/atomic.h> #include <kern/error.h> #include <kern/mutex_types.h> diff --git a/kern/panic.c b/kern/panic.c index 9e7d1d53..c610e3fa 100644 --- a/kern/panic.c +++ b/kern/panic.c @@ -19,6 +19,7 @@ #include <stdio.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/panic.h> #include <machine/cpu.h> #include <machine/strace.h> @@ -54,3 +55,14 @@ panic(const char *format, ...) * Never reached. */ } + +static int __init +panic_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(panic_setup, + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(strace_setup, true)); diff --git a/kern/panic.h b/kern/panic.h index 9d1d31e0..e6cbe8ba 100644 --- a/kern/panic.h +++ b/kern/panic.h @@ -18,11 +18,20 @@ #ifndef _KERN_PANIC_H #define _KERN_PANIC_H -#include <kern/macros.h> +#include <stdnoreturn.h> + +#include <kern/init.h> /* * Print the given message and halt the system immediately. */ -void __noreturn panic(const char *format, ...) __format_printf(1, 2); +noreturn void panic(const char *format, ...) + __attribute__((format(printf, 1, 2))); + +/* + * This init operation provides : + * - module fully initialized + */ +INIT_OP_DECLARE(panic_setup); #endif /* _KERN_PANIC_H */ diff --git a/kern/percpu.c b/kern/percpu.c index 5e4ff827..7621bb29 100644 --- a/kern/percpu.c +++ b/kern/percpu.c @@ -15,17 +15,16 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> #include <machine/cpu.h> #include <vm/vm_kmem.h> @@ -37,25 +36,28 @@ static void *percpu_area_content __initdata; static size_t percpu_area_size __initdata; static int percpu_skip_warning __initdata; -void __init +static int __init percpu_bootstrap(void) { percpu_areas[0] = &_percpu; + return 0; } -void __init +INIT_OP_DEFINE(percpu_bootstrap); + +static int __init percpu_setup(void) { struct vm_page *page; unsigned int order; percpu_area_size = &_percpu_end - &_percpu; - printf("percpu: max_cpus: %u, section size: %zuk\n", X15_MAX_CPUS, - percpu_area_size >> 10); + log_info("percpu: max_cpus: %u, section size: %zuk", X15_MAX_CPUS, + percpu_area_size >> 10); assert(vm_page_aligned(percpu_area_size)); if (percpu_area_size == 0) { - return; + return 0; } order = vm_page_order(percpu_area_size); @@ -67,8 +69,13 @@ percpu_setup(void) percpu_area_content = vm_page_direct_ptr(page); memcpy(percpu_area_content, &_percpu, percpu_area_size); + return 0; } +INIT_OP_DEFINE(percpu_setup, + INIT_OP_DEP(percpu_bootstrap, true), + INIT_OP_DEP(vm_page_setup, true)); + int __init percpu_add(unsigned int cpu) { @@ -77,8 +84,8 @@ percpu_add(unsigned int cpu) if (cpu >= ARRAY_SIZE(percpu_areas)) { if (!percpu_skip_warning) { - printf("percpu: ignoring processor beyond id %zu\n", - ARRAY_SIZE(percpu_areas) - 1); + log_warning("percpu: ignoring processor beyond id %zu", + ARRAY_SIZE(percpu_areas) - 1); percpu_skip_warning = 1; } @@ -86,7 +93,7 @@ percpu_add(unsigned int cpu) } if (percpu_areas[cpu] != NULL) { - printf("percpu: error: id %u ignored, already registered\n", cpu); + log_err("percpu: id %u ignored, already registered", cpu); return ERROR_INVAL; } @@ -98,7 +105,7 @@ percpu_add(unsigned int cpu) page = vm_page_alloc(order, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_KERNEL); if (page == NULL) { - printf("percpu: error: unable to allocate percpu area\n"); + log_err("percpu: unable to allocate percpu area"); return ERROR_NOMEM; } @@ -109,7 +116,7 @@ out: return 0; } -void +static int __init percpu_cleanup(void) { struct vm_page *page; @@ -118,4 +125,10 @@ percpu_cleanup(void) va = (uintptr_t)percpu_area_content; page = vm_page_lookup(vm_page_direct_pa(va)); vm_page_free(page, vm_page_order(percpu_area_size)); + return 0; } + +INIT_OP_DEFINE(percpu_cleanup, + INIT_OP_DEP(cpu_mp_probe, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(vm_page_setup, true)); diff --git a/kern/percpu.h b/kern/percpu.h index 59959518..879767ff 100644 --- a/kern/percpu.h +++ b/kern/percpu.h @@ -53,9 +53,11 @@ #ifndef _KERN_PERCPU_H #define _KERN_PERCPU_H +#include <assert.h> +#include <stddef.h> #include <stdint.h> -#include <kern/assert.h> +#include <kern/init.h> #include <kern/macros.h> #define PERCPU_SECTION .percpu @@ -94,25 +96,6 @@ percpu_area(unsigned int cpu) } /* - * Early initialization of the percpu module. - * - * This function registers the percpu section as the percpu area of the - * BSP. If a percpu variable is modified before calling percpu_setup(), - * the modification will be part of the percpu section and propagated to - * new percpu areas. - */ -void percpu_bootstrap(void); - -/* - * Complete initialization of the percpu module. - * - * The BSP keeps using the percpu section, but its content is copied to a - * dedicated block of memory used as a template for subsequently added - * processors. - */ -void percpu_setup(void); - -/* * Register a processor. * * This function creates a percpu area from kernel virtual memory for the @@ -122,8 +105,16 @@ void percpu_setup(void); int percpu_add(unsigned int cpu); /* - * Release init data allocated for setup. + * This init operation provides : + * - percpu section is registered as the BSP percpu area + */ +INIT_OP_DECLARE(percpu_bootstrap); + +/* + * This init operation provides : + * - percpu section is copied to a kernel buffer subsequently used as + * the template for future percpu areas */ -void percpu_cleanup(void); +INIT_OP_DECLARE(percpu_setup); #endif /* _KERN_PERCPU_H */ diff --git a/kern/printf.c b/kern/printf.c index f657fdfc..68cf4ba9 100644 --- a/kern/printf.c +++ b/kern/printf.c @@ -15,10 +15,11 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <stdio.h> - #include <kern/console.h> +#include <kern/fmt.h> +#include <kern/init.h> #include <kern/spinlock.h> +#include <machine/boot.h> #include <machine/cpu.h> /* @@ -51,7 +52,7 @@ vprintf(const char *format, va_list ap) spinlock_lock_intr_save(&printf_lock, &flags); - length = vsnprintf(printf_buffer, sizeof(printf_buffer), format, ap); + length = fmt_vsnprintf(printf_buffer, sizeof(printf_buffer), format, ap); for (ptr = printf_buffer; *ptr != '\0'; ptr++) { console_putchar(*ptr); @@ -62,8 +63,14 @@ vprintf(const char *format, va_list ap) return length; } -void +static int __init printf_setup(void) { spinlock_init(&printf_lock); + return 0; } + +INIT_OP_DEFINE(printf_setup, + INIT_OP_DEP(boot_bootstrap_console, true), + INIT_OP_DEP(console_bootstrap, true), + INIT_OP_DEP(spinlock_setup, true)); diff --git a/kern/printf.h b/kern/printf.h index 6643d664..8185c735 100644 --- a/kern/printf.h +++ b/kern/printf.h @@ -33,10 +33,18 @@ #include <stdarg.h> -#include <kern/macros.h> +#include <kern/init.h> -int printf(const char *format, ...) __format_printf(1, 2); -int vprintf(const char *format, va_list ap) __format_printf(1, 0); -void printf_setup(void); +int printf(const char *format, ...) + __attribute__((format(printf, 1, 2))); + +int vprintf(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); + +/* + * This init operation provides : + * - printf is usable + */ +INIT_OP_DECLARE(printf_setup); #endif /* _KERN_PRINTF_H */ diff --git a/kern/rbtree.c b/kern/rbtree.c index e2ea54ad..adce033a 100644 --- a/kern/rbtree.c +++ b/kern/rbtree.c @@ -15,10 +15,10 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/macros.h> #include <kern/rbtree.h> #include <kern/rbtree_i.h> diff --git a/kern/rbtree.h b/kern/rbtree.h index 38083427..4ae8353f 100644 --- a/kern/rbtree.h +++ b/kern/rbtree.h @@ -21,10 +21,10 @@ #ifndef _KERN_RBTREE_H #define _KERN_RBTREE_H +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/macros.h> /* diff --git a/kern/rbtree_i.h b/kern/rbtree_i.h index 99722977..944148e0 100644 --- a/kern/rbtree_i.h +++ b/kern/rbtree_i.h @@ -18,10 +18,10 @@ #ifndef _KERN_RBTREE_I_H #define _KERN_RBTREE_I_H +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/macros.h> /* diff --git a/kern/rdxtree.c b/kern/rdxtree.c index 788f3ba8..77005b64 100644 --- a/kern/rdxtree.c +++ b/kern/rdxtree.c @@ -15,15 +15,16 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <limits.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> #include <string.h> -#include <kern/assert.h> #include <kern/error.h> +#include <kern/init.h> #include <kern/kmem.h> -#include <kern/limits.h> #include <kern/llsync.h> #include <kern/macros.h> #include <kern/rdxtree.h> @@ -895,10 +896,14 @@ rdxtree_remove_all(struct rdxtree *tree) } } -void +static int __init rdxtree_setup(void) { kmem_cache_init(&rdxtree_node_cache, "rdxtree_node", sizeof(struct rdxtree_node), 0, - rdxtree_node_ctor, 0); + rdxtree_node_ctor, KMEM_CACHE_PAGE_ONLY); + return 0; } + +INIT_OP_DEFINE(rdxtree_setup, + INIT_OP_DEP(kmem_bootstrap, true)); diff --git a/kern/rdxtree.h b/kern/rdxtree.h index e3a2ba06..6ae1e6e7 100644 --- a/kern/rdxtree.h +++ b/kern/rdxtree.h @@ -29,7 +29,10 @@ #include <stddef.h> #include <stdint.h> -typedef uint32_t rdxtree_key_t; +#include <kern/init.h> +#include <kern/llsync.h> + +typedef uint64_t rdxtree_key_t; /* * Radix tree initialization flags. @@ -154,6 +157,12 @@ rdxtree_lookup_slot(const struct rdxtree *tree, rdxtree_key_t key) return rdxtree_lookup_common(tree, key, 1); } +static inline void * +rdxtree_load_slot(void **slot) +{ + return llsync_read_ptr(*slot); +} + /* * Replace a pointer in a tree. * @@ -192,8 +201,9 @@ rdxtree_iter_key(const struct rdxtree_iter *iter) void rdxtree_remove_all(struct rdxtree *tree); /* - * Initialize the rdxtree module. + * This init operation provides : + * - module fully initialized */ -void rdxtree_setup(void); +INIT_OP_DECLARE(rdxtree_setup); #endif /* _KERN_RDXTREE_H */ diff --git a/kern/rtmutex.c b/kern/rtmutex.c index 6909ce18..fcf5f358 100644 --- a/kern/rtmutex.c +++ b/kern/rtmutex.c @@ -15,10 +15,10 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/rtmutex.h> #include <kern/rtmutex_i.h> @@ -29,7 +29,7 @@ static void rtmutex_set_contended(struct rtmutex *rtmutex) { - atomic_or_acq_rel(&rtmutex->owner, RTMUTEX_CONTENDED); + atomic_or(&rtmutex->owner, RTMUTEX_CONTENDED, ATOMIC_RELEASE); } void @@ -64,8 +64,7 @@ rtmutex_lock_slow(struct rtmutex *rtmutex) turnstile_own(turnstile); if (turnstile_empty(turnstile)) { - /* TODO Review memory order */ - prev_owner = atomic_swap_acquire(&rtmutex->owner, owner); + prev_owner = atomic_swap(&rtmutex->owner, owner, ATOMIC_RELAXED); assert(prev_owner == (owner | bits)); } diff --git a/kern/rtmutex.h b/kern/rtmutex.h index f64274d7..ec79afa9 100644 --- a/kern/rtmutex.h +++ b/kern/rtmutex.h @@ -24,9 +24,9 @@ #ifndef _KERN_RTMUTEX_H #define _KERN_RTMUTEX_H +#include <assert.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/macros.h> #include <kern/rtmutex_i.h> diff --git a/kern/rtmutex_i.h b/kern/rtmutex_i.h index 2f2cc17f..984cfd16 100644 --- a/kern/rtmutex_i.h +++ b/kern/rtmutex_i.h @@ -18,10 +18,10 @@ #ifndef _KERN_RTMUTEX_I_H #define _KERN_RTMUTEX_I_H +#include <assert.h> #include <stdbool.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/rtmutex_types.h> #include <kern/thread.h> diff --git a/kern/semaphore.h b/kern/semaphore.h index d7219b4f..e08927ec 100644 --- a/kern/semaphore.h +++ b/kern/semaphore.h @@ -32,7 +32,8 @@ #ifndef _KERN_SEMAPHORE_H #define _KERN_SEMAPHORE_H -#include <kern/assert.h> +#include <assert.h> + #include <kern/atomic.h> #include <kern/error.h> #include <kern/macros.h> diff --git a/kern/semaphore_i.h b/kern/semaphore_i.h index 54985062..acd7cd48 100644 --- a/kern/semaphore_i.h +++ b/kern/semaphore_i.h @@ -18,7 +18,8 @@ #ifndef _KERN_SEMAPHORE_I_H #define _KERN_SEMAPHORE_I_H -#include <kern/assert.h> +#include <assert.h> + #include <kern/atomic.h> struct semaphore { diff --git a/kern/shell.c b/kern/shell.c index c4675bf4..a27c53f3 100644 --- a/kern/shell.c +++ b/kern/shell.c @@ -18,10 +18,10 @@ #include <stdio.h> #include <string.h> -#include <kern/console.h> #include <kern/error.h> -#include <kern/init.h> #include <kern/hash.h> +#include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/mutex.h> #include <kern/shell.h> @@ -224,6 +224,10 @@ shell_cmd_lookup(const char *name) } /* + * Look up the first command that matches a given string. + * + * The input string is defined by the given string pointer and size. + * * The global lock must be acquired before calling this function. */ static const struct shell_cmd * @@ -242,6 +246,21 @@ shell_cmd_match(const struct shell_cmd *cmd, const char *str, } /* + * Attempt command auto-completion. + * + * The given string is the beginning of a command, or the empty string. + * The sizep parameter initially points to the size of the given string. + * If the string matches any registered command, the cmdp pointer is + * updated to point to the first matching command in the sorted list of + * commands, and sizep is updated to the number of characters in the + * command name that are common in subsequent commands. The command + * pointer and the returned size can be used to print a list of commands + * eligible for completion. + * + * If there is a single match for the given string, return 0. If there + * are more than one match, return ERROR_AGAIN. If there is no match, + * return ERROR_INVAL. + * * The global lock must be acquired before calling this function. */ static int @@ -309,6 +328,10 @@ shell_cmd_complete(const char *str, unsigned long *sizep, } /* + * Print a list of commands eligible for completion, starting at the + * given command. Other eligible commands share the same prefix, as + * defined by the size argument. + * * The global lock must be acquired before calling this function. */ static void @@ -422,7 +445,7 @@ shell_cmd_add(struct shell_cmd *cmd) for (;;) { if (strcmp(cmd->name, tmp->name) == 0) { - printf("shell: %s: shell command name collision", cmd->name); + log_err("shell: %s: shell command name collision", cmd->name); return ERROR_EXIST; } @@ -644,7 +667,7 @@ shell_cmd_help(int argc, char *argv[]) shell_cmd_acquire(); for (cmd = shell_list; cmd != NULL; cmd = cmd->ls_next) { - printf("%13s %s\n", cmd->name, cmd->short_desc); + printf("%19s %s\n", cmd->name, cmd->short_desc); } shell_cmd_release(); @@ -668,7 +691,7 @@ shell_cmd_history(int argc, char *argv[]) static struct shell_cmd shell_default_cmds[] = { SHELL_CMD_INITIALIZER("help", shell_cmd_help, "help [command]", - "obtain help about shell commands"), + "display help about shell commands"), SHELL_CMD_INITIALIZER("history", shell_cmd_history, "history", "display history list"), @@ -811,6 +834,10 @@ shell_process_raw_char(char c) goto out; } + /* + * This assumes that the backspace character only moves the cursor + * without erasing characters. + */ printf("%s", shell_line_str(current_line) + shell_cursor - 1); remaining_chars = shell_line_size(current_line) - shell_cursor; @@ -1155,11 +1182,9 @@ shell_run(void *arg) } } -void __init +static int __init shell_setup(void) { - struct thread_attr attr; - struct thread *thread; unsigned long i; int error; @@ -1170,6 +1195,21 @@ shell_setup(void) error_check(error, "shell_cmd_register"); } + return 0; +} + +INIT_OP_DEFINE(shell_setup, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(printf_setup, true)); + +void __init +shell_start(void) +{ + struct thread_attr attr; + struct thread *thread; + int error; + thread_attr_init(&attr, THREAD_KERNEL_PREFIX "shell"); thread_attr_set_detached(&attr); error = thread_create(&thread, &attr, shell_run, NULL); diff --git a/kern/shell.h b/kern/shell.h index 5ff4920d..cf56cebf 100644 --- a/kern/shell.h +++ b/kern/shell.h @@ -21,6 +21,23 @@ #ifndef _KERN_SHELL_H #define _KERN_SHELL_H +#include <kern/init.h> +#include <kern/error.h> +#include <kern/macros.h> + +#ifdef X15_SHELL + +#define SHELL_REGISTER_CMDS(cmds) \ +MACRO_BEGIN \ + size_t ___i; \ + int ___error; \ + \ + for (___i = 0; ___i < ARRAY_SIZE(cmds); ___i++) { \ + ___error = shell_cmd_register(&(cmds)[___i]); \ + error_check(___error, __func__); \ + } \ +MACRO_END + typedef void (*shell_fn_t)(int argc, char *argv[]); struct shell_cmd { @@ -49,11 +66,9 @@ void shell_cmd_init(struct shell_cmd *cmd, const char *name, const char *short_desc, const char *long_desc); /* - * Initialize the shell module. - * - * On return, shell commands can be registered. + * Start the shell thread. */ -void shell_setup(void); +void shell_start(void); /* * Register a shell command. @@ -66,4 +81,17 @@ void shell_setup(void); */ int shell_cmd_register(struct shell_cmd *cmd); +#else /* X15_SHELL */ +#define SHELL_REGISTER_CMDS(cmds) +#define shell_setup() +#define shell_start() +#endif /* X15_SHELL */ + +/* + * This init operation provides : + * - commands can be registered + * - module fully initialized + */ +INIT_OP_DECLARE(shell_setup); + #endif /* _KERN_SHELL_H */ diff --git a/kern/shutdown.c b/kern/shutdown.c new file mode 100644 index 00000000..1c950415 --- /dev/null +++ b/kern/shutdown.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <stdio.h> + +#include <kern/init.h> +#include <kern/plist.h> +#include <kern/shell.h> +#include <kern/shutdown.h> +#include <machine/boot.h> +#include <machine/cpu.h> + +static struct plist shutdown_ops_list; + +#ifdef X15_SHELL + +static void +shutdown_shell_halt(int argc, char **argv) +{ + (void)argc; + (void)argv; + + shutdown_halt(); +} + +static void +shutdown_shell_reboot(int argc, char **argv) +{ + (void)argc; + (void)argv; + + shutdown_reboot(); +} + +static struct shell_cmd shutdown_shell_cmds[] = { + SHELL_CMD_INITIALIZER("shutdown_halt", shutdown_shell_halt, + "shutdown_halt", + "halt the system"), + SHELL_CMD_INITIALIZER("shutdown_reboot", shutdown_shell_reboot, + "shutdown_reboot", + "reboot the system"), +}; + +static int __init +shutdown_setup_shell(void) +{ + SHELL_REGISTER_CMDS(shutdown_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(shutdown_setup_shell, + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(shutdown_setup, true)); + +#endif /* X15_SHELL */ + +static int __init +shutdown_bootstrap(void) +{ + plist_init(&shutdown_ops_list); + return 0; +} + +INIT_OP_DEFINE(shutdown_bootstrap); + +static int __init +shutdown_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(shutdown_setup, + INIT_OP_DEP(boot_setup_shutdown, true), + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shutdown_bootstrap, true)); + +void __init +shutdown_register(struct shutdown_ops *ops, unsigned int priority) +{ + plist_node_init(&ops->node, priority); + plist_add(&shutdown_ops_list, &ops->node); +} + +static void +shutdown_halt_other_cpus(void) +{ + cpu_intr_disable(); + cpu_halt_broadcast(); +} + +void +shutdown_halt(void) +{ + shutdown_halt_other_cpus(); + printf("shutdown: system halted\n"); + cpu_halt(); +} + +void +shutdown_reboot(void) +{ + struct shutdown_ops *ops; + + if (plist_empty(&shutdown_ops_list)) { + printf("shutdown: no reset operation available, halting\n"); + shutdown_halt(); + } + + shutdown_halt_other_cpus(); + printf("shutdown: rebooting...\n"); + + plist_for_each_entry_reverse(&shutdown_ops_list, ops, node) { + ops->reset(); + } + + cpu_halt(); +} diff --git a/kern/shutdown.h b/kern/shutdown.h new file mode 100644 index 00000000..f61079b3 --- /dev/null +++ b/kern/shutdown.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _KERN_SHUTDOWN_H +#define _KERN_SHUTDOWN_H + +#include <stdnoreturn.h> + +#include <kern/init.h> +#include <kern/plist.h> + +struct shutdown_ops { + struct plist_node node; + void (*reset)(void); +}; + +void shutdown_register(struct shutdown_ops *ops, unsigned int priority); + +noreturn void shutdown_halt(void); +noreturn void shutdown_reboot(void); + +/* + * This init operation provides : + * - registration of shutdown operations + */ +INIT_OP_DECLARE(shutdown_bootstrap); + +/* + * This init operation provides : + * - all shutdown operations have been registered + * - module fully initialized + */ +INIT_OP_DECLARE(shutdown_setup); + +#endif /* _KERN_SHUTDOWN_H */ diff --git a/kern/sleepq.c b/kern/sleepq.c index 5af3d064..44ad9962 100644 --- a/kern/sleepq.c +++ b/kern/sleepq.c @@ -18,24 +18,25 @@ * TODO Analyse hash parameters. */ +#include <assert.h> +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/sleepq.h> #include <kern/spinlock.h> #include <kern/thread.h> +#include <machine/cpu.h> struct sleepq_bucket { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; struct list list; -} __aligned(CPU_L1_SIZE); +}; struct sleepq { struct sleepq_bucket *bucket; @@ -192,7 +193,7 @@ sleepq_ctor(void *ptr) sleepq->next_free = NULL; } -void __init +static int __init sleepq_bootstrap(void) { unsigned int i; @@ -204,15 +205,24 @@ sleepq_bootstrap(void) for (i = 0; i < ARRAY_SIZE(sleepq_cond_htable); i++) { sleepq_bucket_init(&sleepq_cond_htable[i]); } + + return 0; } -void __init +INIT_OP_DEFINE(sleepq_bootstrap); + +static int __init sleepq_setup(void) { kmem_cache_init(&sleepq_cache, "sleepq", sizeof(struct sleepq), CPU_L1_SIZE, sleepq_ctor, 0); + return 0; } +INIT_OP_DEFINE(sleepq_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(sleepq_bootstrap, true)); + struct sleepq * sleepq_create(void) { diff --git a/kern/sleepq.h b/kern/sleepq.h index f55d8c4b..651b3e7c 100644 --- a/kern/sleepq.h +++ b/kern/sleepq.h @@ -38,21 +38,9 @@ #include <stdbool.h> -struct sleepq; +#include <kern/init.h> -/* - * Early initialization of the sleepq module. - * - * This module is initialized by architecture-specific code. It should - * be one of the first modules to be initialized since it's used by - * synchronization objects that may be accessed very early. - */ -void sleepq_bootstrap(void); - -/* - * Initialize the sleepq module. - */ -void sleepq_setup(void); +struct sleepq; /* * Create/destroy a sleep queue. @@ -142,4 +130,17 @@ void sleepq_wait(struct sleepq *sleepq, const char *wchan); */ void sleepq_signal(struct sleepq *sleepq); +/* + * This init operation provides : + * - ? TODO Review + */ +INIT_OP_DECLARE(sleepq_bootstrap); + +/* + * This init operation provides : + * - sleepq creation + * - module fully initialized + */ +INIT_OP_DECLARE(sleepq_setup); + #endif /* _KERN_SLEEPQ_H */ diff --git a/kern/spinlock.c b/kern/spinlock.c index 3ec36b36..fcb7c7b6 100644 --- a/kern/spinlock.c +++ b/kern/spinlock.c @@ -51,13 +51,25 @@ * and fast paths operations fail. The lock operation must make sure * that the lock value is restored to SPINLOCK_LOCKED if there is no * more contention, an operation called downgrading. + * + * In order to enforce correct visibility of the critical section, + * acquire and release atomic operations are used when accessing a + * qnode lock. Similarly, acquire and release semantics are also used + * when accessing the lock word which stores the first and last QIDs, + * so that the memory operations on the qnodes referenced by those QIDs + * are correctly enforced. Accessing the next QID in a qnode however + * is performed with no memory order constraint, because the qnodes + * referenced by those QID are never accessed unless they are the first + * or the last, cases which do enforce ordering. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/error.h> +#include <kern/init.h> #include <kern/macros.h> #include <kern/percpu.h> #include <kern/spinlock.h> @@ -118,7 +130,7 @@ struct spinlock_qnode { #endif struct spinlock_cpu_data { - struct spinlock_qnode qnodes[SPINLOCK_NR_CTXS - 1] __aligned(CPU_L1_SIZE); + alignas(CPU_L1_SIZE) struct spinlock_qnode qnodes[SPINLOCK_NR_CTXS - 1]; }; static struct spinlock_cpu_data spinlock_cpu_data __percpu; @@ -204,7 +216,7 @@ spinlock_load_first_qid(const struct spinlock *lock) { unsigned int value; - value = atomic_load(&lock->value, ATOMIC_ACQUIRE); + value = atomic_load_acquire(&lock->value); return (value >> SPINLOCK_QID_MAX_BITS) & SPINLOCK_QID_MASK; } @@ -268,7 +280,7 @@ spinlock_lock_slow(struct spinlock *lock) } for (;;) { - locked = atomic_load(&qnode->locked, ATOMIC_ACQUIRE); + locked = atomic_load_acquire(&qnode->locked); if (!locked) { break; @@ -316,5 +328,14 @@ spinlock_unlock_slow(struct spinlock *lock) } next_qnode = spinlock_get_remote_qnode(next_qid); - atomic_store(&next_qnode->locked, false, ATOMIC_RELEASE); + atomic_store_release(&next_qnode->locked, false); +} + +static int __init +spinlock_setup(void) +{ + return 0; } + +INIT_OP_DEFINE(spinlock_setup, + INIT_OP_DEP(thread_setup_booter, true)); diff --git a/kern/spinlock.h b/kern/spinlock.h index 4dc4c83e..d4105da0 100644 --- a/kern/spinlock.h +++ b/kern/spinlock.h @@ -26,6 +26,7 @@ #ifndef _KERN_SPINLOCK_H #define _KERN_SPINLOCK_H +#include <kern/init.h> #include <kern/macros.h> #include <kern/spinlock_i.h> #include <kern/spinlock_types.h> @@ -162,4 +163,12 @@ spinlock_unlock_intr_restore(struct spinlock *lock, unsigned long flags) thread_preempt_enable(); } +/* + * This init operation provides : + * - uncontended spinlock locking + * + * Contended locking may only occur after starting APs. + */ +INIT_OP_DECLARE(spinlock_setup); + #endif /* _KERN_SPINLOCK_H */ diff --git a/kern/spinlock_i.h b/kern/spinlock_i.h index 45018033..c9dcdd10 100644 --- a/kern/spinlock_i.h +++ b/kern/spinlock_i.h @@ -18,10 +18,10 @@ #ifndef _KERN_SPINLOCK_I_H #define _KERN_SPINLOCK_I_H +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/error.h> #include <kern/macros.h> diff --git a/kern/sprintf.c b/kern/sprintf.c deleted file mode 100644 index a606d866..00000000 --- a/kern/sprintf.c +++ /dev/null @@ -1,577 +0,0 @@ -/* - * Copyright (c) 2010 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <stdarg.h> -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> - -#include <kern/limits.h> -#include <kern/types.h> - -/* - * Formatting flags. - * - * FORMAT_LOWER must be 0x20 as it is OR'd with digits, eg. - * '0': 0x30 | 0x20 => 0x30 ('0') - * 'A': 0x41 | 0x20 => 0x61 ('a') - */ -#define SPRINTF_FORMAT_ALT_FORM 0x01 -#define SPRINTF_FORMAT_ZERO_PAD 0x02 -#define SPRINTF_FORMAT_LEFT_JUSTIFY 0x04 -#define SPRINTF_FORMAT_BLANK 0x08 -#define SPRINTF_FORMAT_SIGN 0x10 -#define SPRINTF_FORMAT_LOWER 0x20 -#define SPRINTF_FORMAT_CONV_SIGNED 0x40 - -enum { - SPRINTF_MODIFIER_NONE, - SPRINTF_MODIFIER_CHAR, - SPRINTF_MODIFIER_SHORT, - SPRINTF_MODIFIER_LONG, - SPRINTF_MODIFIER_LONGLONG, - SPRINTF_MODIFIER_PTR, /* Used only for %p */ - SPRINTF_MODIFIER_SIZE, - SPRINTF_MODIFIER_PTRDIFF -}; - -enum { - SPRINTF_SPECIFIER_INVALID, - SPRINTF_SPECIFIER_INT, - SPRINTF_SPECIFIER_CHAR, - SPRINTF_SPECIFIER_STR, - SPRINTF_SPECIFIER_NRCHARS, - SPRINTF_SPECIFIER_PERCENT -}; - -/* - * Size for the temporary number buffer. The minimum base is 8 so 3 bits - * are consumed per digit. Add one to round up. The conversion algorithm - * doesn't use the null byte. - */ -#define SPRINTF_MAX_NUM_SIZE (((sizeof(uint64_t) * CHAR_BIT) / 3) + 1) - -/* - * Special size for vsnprintf(), used by sprintf()/vsprintf() when the - * buffer size is unknown. - */ -#define SPRINTF_NOLIMIT ((size_t)-1) - -static const char sprintf_digits[] = "0123456789ABCDEF"; - -static inline char * -sprintf_putchar(char *str, char *end, char c) -{ - if (str < end) { - *str = c; - } - - str++; - - return str; -} - -static inline int -sprintf_isdigit(char c) -{ - return (c >= '0') && (c <= '9'); -} - -int -sprintf(char *str, const char *format, ...) -{ - va_list ap; - int length; - - va_start(ap, format); - length = vsprintf(str, format, ap); - va_end(ap); - - return length; -} - -int -vsprintf(char *str, const char *format, va_list ap) -{ - return vsnprintf(str, SPRINTF_NOLIMIT, format, ap); -} - -int -snprintf(char *str, size_t size, const char *format, ...) -{ - va_list ap; - int length; - - va_start(ap, format); - length = vsnprintf(str, size, format, ap); - va_end(ap); - - return length; -} - -int -vsnprintf(char *str, size_t size, const char *format, va_list ap) -{ - unsigned long long n; - int i, len, found, flags, width, precision, modifier, specifier, shift; - unsigned char r, base, mask; - char c, *s, *start, *end, sign, tmp[SPRINTF_MAX_NUM_SIZE]; - - start = str; - - if (size == 0) { - end = NULL; - } else if (size == SPRINTF_NOLIMIT) { - end = (char *)-1; - } else { - end = start + size - 1; - } - - while ((c = *format) != '\0') { - if (c != '%') { - str = sprintf_putchar(str, end, c); - format++; - continue; - } - - /* Flags */ - - found = 1; - flags = 0; - - do { - format++; - c = *format; - - switch (c) { - case '#': - flags |= SPRINTF_FORMAT_ALT_FORM; - break; - case '0': - flags |= SPRINTF_FORMAT_ZERO_PAD; - break; - case '-': - flags |= SPRINTF_FORMAT_LEFT_JUSTIFY; - break; - case ' ': - flags |= SPRINTF_FORMAT_BLANK; - break; - case '+': - flags |= SPRINTF_FORMAT_SIGN; - break; - default: - found = 0; - break; - } - } while (found); - - /* Width */ - - if (sprintf_isdigit(c)) { - width = 0; - - while (sprintf_isdigit(c)) { - width = width * 10 + (c - '0'); - format++; - c = *format; - } - } else if (c == '*') { - width = va_arg(ap, int); - - if (width < 0) { - flags |= SPRINTF_FORMAT_LEFT_JUSTIFY; - width = -width; - } - - format++; - c = *format; - } else { - width = 0; - } - - /* Precision */ - - if (c == '.') { - format++; - c = *format; - - if (sprintf_isdigit(c)) { - precision = 0; - - while (sprintf_isdigit(c)) { - precision = precision * 10 + (c - '0'); - format++; - c = *format; - } - } else if (c == '*') { - precision = va_arg(ap, int); - - if (precision < 0) { - precision = 0; - } - - format++; - c = *format; - } else { - precision = 0; - } - } else { - /* precision is >= 0 only if explicit */ - precision = -1; - } - - /* Length modifier */ - - switch (c) { - case 'h': - case 'l': - format++; - - if (c == *format) { - modifier = (c == 'h') - ? SPRINTF_MODIFIER_CHAR - : SPRINTF_MODIFIER_LONGLONG; - goto skip_modifier; - } else { - modifier = (c == 'h') - ? SPRINTF_MODIFIER_SHORT - : SPRINTF_MODIFIER_LONG; - c = *format; - } - - break; - case 'z': - modifier = SPRINTF_MODIFIER_SIZE; - goto skip_modifier; - case 't': - modifier = SPRINTF_MODIFIER_PTRDIFF; -skip_modifier: - format++; - c = *format; - break; - default: - modifier = SPRINTF_MODIFIER_NONE; - break; - } - - /* Specifier */ - - switch (c) { - case 'd': - case 'i': - flags |= SPRINTF_FORMAT_CONV_SIGNED; - case 'u': - base = 10; - goto integer; - case 'o': - base = 8; - goto integer; - case 'p': - flags |= SPRINTF_FORMAT_ALT_FORM; - modifier = SPRINTF_MODIFIER_PTR; - case 'x': - flags |= SPRINTF_FORMAT_LOWER; - case 'X': - base = 16; -integer: - specifier = SPRINTF_SPECIFIER_INT; - break; - case 'c': - specifier = SPRINTF_SPECIFIER_CHAR; - break; - case 's': - specifier = SPRINTF_SPECIFIER_STR; - break; - case 'n': - specifier = SPRINTF_SPECIFIER_NRCHARS; - break; - case '%': - specifier = SPRINTF_SPECIFIER_PERCENT; - break; - default: - specifier = SPRINTF_SPECIFIER_INVALID; - break; - } - - /* Output */ - - switch (specifier) { - case SPRINTF_SPECIFIER_INT: - switch (modifier) { - case SPRINTF_MODIFIER_CHAR: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = (signed char)va_arg(ap, int); - } else { - n = (unsigned char)va_arg(ap, int); - } - break; - case SPRINTF_MODIFIER_SHORT: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = (short)va_arg(ap, int); - } else { - n = (unsigned short)va_arg(ap, int); - } - break; - case SPRINTF_MODIFIER_LONG: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, long); - } else { - n = va_arg(ap, unsigned long); - } - break; - case SPRINTF_MODIFIER_LONGLONG: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, long long); - } else { - n = va_arg(ap, unsigned long long); - } - break; - case SPRINTF_MODIFIER_PTR: - n = (uintptr_t)va_arg(ap, void *); - break; - case SPRINTF_MODIFIER_SIZE: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, ssize_t); - } else { - n = va_arg(ap, size_t); - } - break; - case SPRINTF_MODIFIER_PTRDIFF: - n = va_arg(ap, ptrdiff_t); - break; - default: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, int); - } else { - n = va_arg(ap, unsigned int); - } - break; - } - - if ((flags & SPRINTF_FORMAT_LEFT_JUSTIFY) || (precision >= 0)) { - flags &= ~SPRINTF_FORMAT_ZERO_PAD; - } - - sign = 0; - - if (flags & SPRINTF_FORMAT_ALT_FORM) { - /* '0' for octal */ - width--; - - /* '0x' or '0X' for hexadecimal */ - if (base == 16) { - width--; - } - } else if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - if ((long long)n < 0) { - sign = '-'; - width--; - n = -(long long)n; - } else if (flags & SPRINTF_FORMAT_SIGN) { - /* SPRINTF_FORMAT_SIGN must precede SPRINTF_FORMAT_BLANK. */ - sign = '+'; - width--; - } else if (flags & SPRINTF_FORMAT_BLANK) { - sign = ' '; - width--; - } - } - - /* Conversion, in reverse order */ - - i = 0; - - if (n == 0) { - if (precision != 0) { - tmp[i++] = '0'; - } - } else if (base == 10) { - /* - * Try to avoid 64 bits operations if the processor doesn't - * support them. Note that even when using modulus and - * division operators close to each other, the compiler will - * forge two calls to __udivdi3() and __umoddi3() instead of - * one to __udivmoddi3(), whereas processor instructions are - * generally correctly used once, giving both the remainder - * and the quotient, through plain or reciprocal division. - */ -#ifndef __LP64__ - if (modifier == SPRINTF_MODIFIER_LONGLONG) { -#endif /* __LP64__ */ - do { - r = n % 10; - n /= 10; - tmp[i++] = sprintf_digits[r]; - } while (n != 0); -#ifndef __LP64__ - } else { - unsigned long m; - - m = (unsigned long)n; - - do { - r = m % 10; - m /= 10; - tmp[i++] = sprintf_digits[r]; - } while (m != 0); - } -#endif /* __LP64__ */ - } else { - mask = base - 1; - shift = (base == 8) ? 3 : 4; - - do { - r = (unsigned char)n & mask; - n >>= shift; - tmp[i++] = sprintf_digits[r] - | (flags & SPRINTF_FORMAT_LOWER); - } while (n != 0); - } - - if (i > precision) { - precision = i; - } - - width -= precision; - - if (!(flags & (SPRINTF_FORMAT_LEFT_JUSTIFY - | SPRINTF_FORMAT_ZERO_PAD))) - while (width-- > 0) { - str = sprintf_putchar(str, end, ' '); - } - - if (flags & SPRINTF_FORMAT_ALT_FORM) { - str = sprintf_putchar(str, end, '0'); - - if (base == 16) - str = sprintf_putchar(str, end, - 'X' | (flags & SPRINTF_FORMAT_LOWER)); - } else if (sign) { - str = sprintf_putchar(str, end, sign); - } - - if (!(flags & SPRINTF_FORMAT_LEFT_JUSTIFY)) { - c = (flags & SPRINTF_FORMAT_ZERO_PAD) ? '0' : ' '; - - while (width-- > 0) { - str = sprintf_putchar(str, end, c); - } - } - - while (i < precision--) { - str = sprintf_putchar(str, end, '0'); - } - - while (i-- > 0) { - str = sprintf_putchar(str, end, tmp[i]); - } - - while (width-- > 0) { - str = sprintf_putchar(str, end, ' '); - } - - break; - case SPRINTF_SPECIFIER_CHAR: - c = (unsigned char)va_arg(ap, int); - - if (!(flags & SPRINTF_FORMAT_LEFT_JUSTIFY)) - while (--width > 0) { - str = sprintf_putchar(str, end, ' '); - } - - str = sprintf_putchar(str, end, c); - - while (--width > 0) { - str = sprintf_putchar(str, end, ' '); - } - - break; - case SPRINTF_SPECIFIER_STR: - s = va_arg(ap, char *); - - if (s == NULL) { - s = "(null)"; - } - - len = 0; - - for (len = 0; s[len] != '\0'; len++) - if (len == precision) { - break; - } - - if (!(flags & SPRINTF_FORMAT_LEFT_JUSTIFY)) - while (len < width--) { - str = sprintf_putchar(str, end, ' '); - } - - for (i = 0; i < len; i++) { - str = sprintf_putchar(str, end, *s); - s++; - } - - while (len < width--) { - str = sprintf_putchar(str, end, ' '); - } - - break; - case SPRINTF_SPECIFIER_NRCHARS: - if (modifier == SPRINTF_MODIFIER_CHAR) { - signed char *ptr = va_arg(ap, signed char *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_SHORT) { - short *ptr = va_arg(ap, short *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_LONG) { - long *ptr = va_arg(ap, long *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_LONGLONG) { - long long *ptr = va_arg(ap, long long *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_SIZE) { - ssize_t *ptr = va_arg(ap, ssize_t *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_PTRDIFF) { - ptrdiff_t *ptr = va_arg(ap, ptrdiff_t *); - *ptr = str - start; - } else { - int *ptr = va_arg(ap, int *); - *ptr = str - start; - } - - break; - case SPRINTF_SPECIFIER_PERCENT: - case SPRINTF_SPECIFIER_INVALID: - str = sprintf_putchar(str, end, '%'); - break; - default: - break; - } - - if (specifier != SPRINTF_SPECIFIER_INVALID) { - format++; - } - } - - if (str < end) { - *str = '\0'; - } else if (end != NULL) { - *end = '\0'; - } - - return str - start; -} diff --git a/kern/sprintf.h b/kern/sprintf.h deleted file mode 100644 index 1cb51e7d..00000000 --- a/kern/sprintf.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2010 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * - * Formatted string functions. - * - * The functions provided by this module implement a subset of the C99 - * sprintf() like functions, mostly centered around character, string, and - * integer conversions. - * - * The supported specifiers are: d i o u x X c s p n % - * The supported length modifiers are: hh h l ll z t - */ - -#ifndef _KERN_SPRINTF_H -#define _KERN_SPRINTF_H - -#ifndef _STDIO_H -#error "do not use <kern/sprintf.h> directly; include <stdio.h> instead" -#endif /* _STDIO_H */ - -#include <stdarg.h> - -#include <kern/macros.h> - -int sprintf(char *str, const char *format, ...) __format_printf(2, 3); -int vsprintf(char *str, const char *format, va_list ap) __format_printf(2, 0); - -int snprintf(char *str, size_t size, const char *format, ...) - __format_printf(3, 4); -int vsnprintf(char *str, size_t size, const char *format, va_list ap) - __format_printf(3, 0); - -#endif /* _KERN_SPRINTF_H */ diff --git a/kern/sref.c b/kern/sref.c index 3f399a30..9c49a425 100644 --- a/kern/sref.c +++ b/kern/sref.c @@ -41,19 +41,18 @@ * TODO Reconsider whether it's possible to bring back local review queues. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/condition.h> #include <kern/cpumap.h> #include <kern/error.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/sref.h> @@ -259,13 +258,13 @@ sref_weakref_init(struct sref_weakref *weakref, struct sref_counter *counter) static void sref_weakref_mark_dying(struct sref_weakref *weakref) { - atomic_or_acq_rel(&weakref->addr, SREF_WEAKREF_DYING); + atomic_or(&weakref->addr, SREF_WEAKREF_DYING, ATOMIC_RELEASE); } static void sref_weakref_clear_dying(struct sref_weakref *weakref) { - atomic_and_acq_rel(&weakref->addr, SREF_WEAKREF_MASK); + atomic_and(&weakref->addr, SREF_WEAKREF_MASK, ATOMIC_RELEASE); } static int @@ -273,8 +272,8 @@ sref_weakref_kill(struct sref_weakref *weakref) { uintptr_t addr, oldval; - addr = weakref->addr | SREF_WEAKREF_DYING; - oldval = atomic_cas_release(&weakref->addr, addr, (uintptr_t)NULL); + addr = atomic_load(&weakref->addr, ATOMIC_RELAXED) | SREF_WEAKREF_DYING; + oldval = atomic_cas(&weakref->addr, addr, (uintptr_t)NULL, ATOMIC_RELAXED); if (oldval != addr) { assert((oldval & SREF_WEAKREF_MASK) == (addr & SREF_WEAKREF_MASK)); @@ -289,8 +288,9 @@ sref_weakref_tryget(struct sref_weakref *weakref) { uintptr_t addr, oldval, newval; + /* TODO Review */ do { - addr = weakref->addr; + addr = atomic_load(&weakref->addr, ATOMIC_RELAXED); newval = addr & SREF_WEAKREF_MASK; oldval = atomic_cas_acquire(&weakref->addr, addr, newval); } while (oldval != addr); @@ -503,7 +503,7 @@ sref_end_epoch(struct sref_queue *queue) if (!sref_data.no_warning && (sref_review_queue_size() >= SREF_NR_COUNTERS_WARN)) { sref_data.no_warning = 1; - printf("sref: warning: large number of counters in review queue\n"); + log_warning("sref: large number of counters in review queue"); } if (sref_data.nr_registered_cpus == 1) { @@ -816,7 +816,7 @@ sref_manage(void *arg) /* Never reached */ } -void __init +static int __init sref_bootstrap(void) { spinlock_init(&sref_data.lock); @@ -828,8 +828,13 @@ sref_bootstrap(void) syscnt_register(&sref_data.sc_true_zeroes, "sref_true_zeroes"); sref_cache_init(sref_cache_get(), 0); + + return 0; } +INIT_OP_DEFINE(sref_bootstrap, + INIT_OP_DEP(syscnt_setup, true)); + static void __init sref_setup_manager(struct sref_cache *cache, unsigned int cpu) { @@ -861,7 +866,7 @@ sref_setup_manager(struct sref_cache *cache, unsigned int cpu) cache->manager = manager; } -void __init +static int __init sref_setup(void) { unsigned int i; @@ -873,8 +878,20 @@ sref_setup(void) for (i = 0; i < cpu_count(); i++) { sref_setup_manager(percpu_ptr(sref_cache, i), i); } + + return 0; } +INIT_OP_DEFINE(sref_setup, + INIT_OP_DEP(cpu_mp_probe, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(sref_bootstrap, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_setup, true)); + void sref_register(void) { @@ -976,8 +993,7 @@ sref_report_periodic_event(void) cache = sref_cache_get(); - if (!sref_cache_is_registered(cache) - || (cache->manager == thread_self())) { + if (!sref_cache_is_registered(cache)) { return; } diff --git a/kern/sref.h b/kern/sref.h index ac16b13d..5299b9d2 100644 --- a/kern/sref.h +++ b/kern/sref.h @@ -49,20 +49,6 @@ typedef void (*sref_noref_fn_t)(struct sref_counter *); #include <kern/sref_i.h> /* - * Early initialization of the sref module. - * - * This function depends on the availability of percpu variables. - */ -void sref_bootstrap(void); - -/* - * Initialize the sref module. - * - * This function mostly takes care of setting up periodic maintenance. - */ -void sref_setup(void); - -/* * Manage registration of the current processor. * * Registering tells the sref module that the current processor reports diff --git a/kern/string.c b/kern/string.c index 584030b9..6c9c8abc 100644 --- a/kern/string.c +++ b/kern/string.c @@ -21,9 +21,10 @@ #include <stddef.h> #include <string.h> -#include <kern/param.h> +#include <kern/macros.h> +#include <machine/string.h> -#ifndef ARCH_STRING_MEMCPY +#ifndef STRING_ARCH_MEMCPY void * memcpy(void *dest, const void *src, size_t n) { @@ -35,14 +36,16 @@ memcpy(void *dest, const void *src, size_t n) src_ptr = src; for (i = 0; i < n; i++) { - *dest_ptr++ = *src_ptr++; + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; } return dest; } -#endif /* ARCH_STRING_MEMCPY */ +#endif /* STRING_ARCH_MEMCPY */ -#ifndef ARCH_STRING_MEMMOVE +#ifndef STRING_ARCH_MEMMOVE void * memmove(void *dest, const void *src, size_t n) { @@ -55,22 +58,26 @@ memmove(void *dest, const void *src, size_t n) src_ptr = src; for (i = 0; i < n; i++) { - *dest_ptr++ = *src_ptr++; + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; } } else { dest_ptr = dest + n - 1; src_ptr = src + n - 1; for (i = 0; i < n; i++) { - *dest_ptr-- = *src_ptr--; + *dest_ptr = *src_ptr; + dest_ptr--; + src_ptr--; } } return dest; } -#endif /* ARCH_STRING_MEMMOVE */ +#endif /* STRING_ARCH_MEMMOVE */ -#ifndef ARCH_STRING_MEMSET +#ifndef STRING_ARCH_MEMSET void * memset(void *s, int c, size_t n) { @@ -85,9 +92,9 @@ memset(void *s, int c, size_t n) return s; } -#endif /* ARCH_STRING_MEMSET */ +#endif /* STRING_ARCH_MEMSET */ -#ifndef ARCH_STRING_MEMCMP +#ifndef STRING_ARCH_MEMCMP int memcmp(const void *s1, const void *s2, size_t n) { @@ -97,32 +104,33 @@ memcmp(const void *s1, const void *s2, size_t n) a1 = s1; a2 = s2; - for (i = 0; i < n; i++) + for (i = 0; i < n; i++) { if (a1[i] != a2[i]) { return (int)a1[i] - (int)a2[i]; } + } return 0; } -#endif /* ARCH_STRING_MEMCMP */ +#endif /* STRING_ARCH_MEMCMP */ -#ifndef ARCH_STRING_STRLEN +#ifndef STRING_ARCH_STRLEN size_t strlen(const char *s) { - size_t i; + const char *start; - i = 0; + start = s; - while (*s++ != '\0') { - i++; + while (*s != '\0') { + s++; } - return i; + return (s - start); } -#endif /* ARCH_STRING_STRLEN */ +#endif /* STRING_ARCH_STRLEN */ -#ifndef ARCH_STRING_STRCPY +#ifndef STRING_ARCH_STRCPY char * strcpy(char *dest, const char *src) { @@ -137,7 +145,7 @@ strcpy(char *dest, const char *src) return tmp; } -#endif /* ARCH_STRING_STRCPY */ +#endif /* STRING_ARCH_STRCPY */ size_t strlcpy(char *dest, const char *src, size_t n) @@ -158,7 +166,7 @@ out: return len; } -#ifndef ARCH_STRING_STRCMP +#ifndef STRING_ARCH_STRCMP int strcmp(const char *s1, const char *s2) { @@ -175,16 +183,17 @@ strcmp(const char *s1, const char *s2) return (int)c1 - (int)c2; } -#endif /* ARCH_STRING_STRCMP */ +#endif /* STRING_ARCH_STRCMP */ -#ifndef ARCH_STRING_STRNCMP +#ifndef STRING_ARCH_STRNCMP int strncmp(const char *s1, const char *s2, size_t n) { char c1, c2; - c1 = '\0'; - c2 = '\0'; + if (unlikely(n == 0)) { + return 0; + } while ((n != 0) && (c1 = *s1) == (c2 = *s2)) { if (c1 == '\0') { @@ -198,9 +207,9 @@ strncmp(const char *s1, const char *s2, size_t n) return (int)c1 - (int)c2; } -#endif /* ARCH_STRING_STRNCMP */ +#endif /* STRING_ARCH_STRNCMP */ -#ifndef ARCH_STRING_STRCHR +#ifndef STRING_ARCH_STRCHR char * strchr(const char *s, int c) { @@ -214,4 +223,4 @@ strchr(const char *s, int c) s++; } } -#endif /* ARCH_STRING_STRCHR */ +#endif /* STRING_ARCH_STRCHR */ diff --git a/kern/string.h b/kern/string.h index 35490ffa..7cd79a60 100644 --- a/kern/string.h +++ b/kern/string.h @@ -20,13 +20,13 @@ #include <stddef.h> -void * memcpy(void *dest, const void *src, size_t n); +void * memcpy(void * restrict dest, const void * restrict src, size_t n); void * memmove(void *dest, const void *src, size_t n); void * memset(void *s, int c, size_t n); int memcmp(const void *s1, const void *s2, size_t n); size_t strlen(const char *s); -char * strcpy(char *dest, const char *src); -size_t strlcpy(char *dest, const char *src, size_t n); +char * strcpy(char * restrict dest, const char *restrict src); +size_t strlcpy(char * restrict dest, const char * restrict src, size_t n); int strcmp(const char *s1, const char *s2); int strncmp(const char *s1, const char *s2, size_t n); char * strchr(const char *s, int c); diff --git a/kern/syscnt.c b/kern/syscnt.c index 5b479f36..7cceabac 100644 --- a/kern/syscnt.c +++ b/kern/syscnt.c @@ -22,6 +22,7 @@ #include <kern/init.h> #include <kern/list.h> #include <kern/mutex.h> +#include <kern/shell.h> #include <kern/spinlock.h> #include <kern/syscnt.h> @@ -31,13 +32,47 @@ static struct list syscnt_list; static struct mutex syscnt_lock; -void __init +#ifdef X15_SHELL + +static void +syscnt_shell_info(int argc, char **argv) +{ + char *prefix; + + prefix = (argc >= 2) ? argv[1] : NULL; + syscnt_info(prefix); +} + +static struct shell_cmd syscnt_shell_cmds[] = { + SHELL_CMD_INITIALIZER("syscnt_info", syscnt_shell_info, + "syscnt_info [<prefix>]", + "display information about system counters"), +}; + +static int __init +syscnt_setup_shell(void) +{ + SHELL_REGISTER_CMDS(syscnt_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(syscnt_setup_shell, + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(syscnt_setup, true)); + +#endif /* X15_SHELL */ + +static int __init syscnt_setup(void) { list_init(&syscnt_list); mutex_init(&syscnt_lock); + return 0; } +INIT_OP_DEFINE(syscnt_setup, + INIT_OP_DEP(mutex_setup, true)); + void __init syscnt_register(struct syscnt *syscnt, const char *name) { @@ -61,8 +96,6 @@ syscnt_info(const char *prefix) prefix_length = (prefix == NULL) ? 0 : strlen(prefix); - printf("syscnt: name value\n"); - mutex_lock(&syscnt_lock); list_for_each_entry(&syscnt_list, syscnt, node) { @@ -77,7 +110,7 @@ syscnt_info(const char *prefix) value = syscnt_read(syscnt); - printf("syscnt: %-30s %17llu\n", syscnt->name, + printf("syscnt: %40s %20llu\n", syscnt->name, (unsigned long long)value); } diff --git a/kern/syscnt.h b/kern/syscnt.h index a4bae3d7..816aa4b9 100644 --- a/kern/syscnt.h +++ b/kern/syscnt.h @@ -27,6 +27,7 @@ #include <stdint.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/macros.h> #include <kern/spinlock.h> @@ -43,14 +44,6 @@ struct syscnt; /* - * Initialize the syscnt module. - * - * This module is initialized by architecture-specific code. It is normally - * safe to call this function very early at boot time. - */ -void syscnt_setup(void); - -/* * Initialize and register the given counter. * * The counter is set to 0. @@ -118,4 +111,10 @@ syscnt_dec(struct syscnt *syscnt) */ void syscnt_info(const char *prefix); +/* + * This init operation provides : + * - registration of system counters + */ +INIT_OP_DECLARE(syscnt_setup); + #endif /* _KERN_SYSCNT_H */ diff --git a/kern/task.c b/kern/task.c index 7f1c53d2..ab82ad83 100644 --- a/kern/task.c +++ b/kern/task.c @@ -23,7 +23,8 @@ #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> -#include <kern/param.h> +#include <kern/macros.h> +#include <kern/shell.h> #include <kern/spinlock.h> #include <kern/task.h> #include <kern/thread.h> @@ -56,13 +57,64 @@ static struct spinlock task_list_lock; static void task_init(struct task *task, const char *name, struct vm_map *map) { + task->nr_refs = 1; spinlock_init(&task->lock); list_init(&task->threads); task->map = map; strlcpy(task->name, name, sizeof(task->name)); } -void __init +#ifdef X15_SHELL + +static void +task_shell_info(int argc, char *argv[]) +{ + struct task *task; + int error; + + if (argc == 1) { + task_info(NULL); + return; + } else { + task = task_lookup(argv[1]); + + if (task == NULL) { + error = ERROR_INVAL; + goto error; + } + + task_info(task); + task_unref(task); + } + + return; + +error: + printf("task: info: %s\n", error_str(error)); +} + +static struct shell_cmd task_shell_cmds[] = { + SHELL_CMD_INITIALIZER("task_info", task_shell_info, + "task_info [<task_name>]", + "display tasks and threads"), +}; + +static int __init +task_setup_shell(void) +{ + SHELL_REGISTER_CMDS(task_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(task_setup_shell, + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(task_setup, true), + INIT_OP_DEP(thread_setup, true)); + +#endif /* X15_SHELL */ + +static int __init task_setup(void) { kmem_cache_init(&task_cache, "task", sizeof(struct task), 0, NULL, 0); @@ -70,8 +122,14 @@ task_setup(void) spinlock_init(&task_list_lock); task_init(kernel_task, "x15", kernel_map); list_insert_head(&task_list, &kernel_task->node); + return 0; } +INIT_OP_DEFINE(task_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(vm_map_setup, true)); + int task_create(struct task **taskp, const char *name) { @@ -107,6 +165,31 @@ error_task: return error; } +struct task * +task_lookup(const char *name) +{ + struct task *task; + + spinlock_lock(&task_list_lock); + + list_for_each_entry(&task_list, task, node) { + spinlock_lock(&task->lock); + + if (strcmp(task->name, name) == 0) { + task_ref(task); + spinlock_unlock(&task->lock); + spinlock_unlock(&task_list_lock); + return task; + } + + spinlock_unlock(&task->lock); + } + + spinlock_unlock(&task_list_lock); + + return NULL; +} + void task_add_thread(struct task *task, struct thread *thread) { @@ -123,6 +206,26 @@ task_remove_thread(struct task *task, struct thread *thread) spinlock_unlock(&task->lock); } +struct thread * +task_lookup_thread(struct task *task, const char *name) +{ + struct thread *thread; + + spinlock_lock(&task->lock); + + list_for_each_entry(&task->threads, thread, task_node) { + if (strcmp(thread->name, name) == 0) { + thread_ref(thread); + spinlock_unlock(&task->lock); + return thread; + } + } + + spinlock_unlock(&task->lock); + + return NULL; +} + void task_info(struct task *task) { @@ -132,7 +235,7 @@ task_info(struct task *task) spinlock_lock(&task_list_lock); list_for_each_entry(&task_list, task, node) { - printf("task: %s\n", task->name); + task_info(task); } spinlock_unlock(&task_list_lock); @@ -149,6 +252,8 @@ task_info(struct task *task) * can be used to debug in the middle of most critical sections. * Threads are only destroyed after being removed from their task * so holding the task lock is enough to guarantee existence. + * + * TODO Handle tasks and threads names modifications. */ list_for_each_entry(&task->threads, thread, task_node) { printf(TASK_INFO_ADDR_FMT " %c %8s:" TASK_INFO_ADDR_FMT diff --git a/kern/task.h b/kern/task.h index 67599399..1711fbee 100644 --- a/kern/task.h +++ b/kern/task.h @@ -18,6 +18,8 @@ #ifndef _KERN_TASK_H #define _KERN_TASK_H +#include <kern/atomic.h> +#include <kern/init.h> #include <kern/list.h> #include <kern/spinlock.h> #include <kern/thread.h> @@ -32,6 +34,7 @@ * Task structure. */ struct task { + unsigned long nr_refs; struct spinlock lock; struct list node; struct list threads; @@ -44,10 +47,33 @@ struct task { */ extern struct task *kernel_task; -/* - * Initialize the task module. - */ -void task_setup(void); +static inline void +task_ref(struct task *task) +{ + unsigned long nr_refs; + + nr_refs = atomic_fetch_add(&task->nr_refs, 1, ATOMIC_RELAXED); + assert(nr_refs != (unsigned long)-1); +} + +static inline void +task_unref(struct task *task) +{ + unsigned long nr_refs; + + nr_refs = atomic_fetch_sub_acq_rel(&task->nr_refs, 1); + assert(nr_refs != 0); + + if (nr_refs == 1) { + /* TODO Task destruction */ + } +} + +static inline struct vm_map * +task_get_vm_map(const struct task *task) +{ + return task->map; +} /* * Create a task. @@ -55,6 +81,15 @@ void task_setup(void); int task_create(struct task **taskp, const char *name); /* + * Look up a task from its name. + * + * If a task is found, it gains a reference. Otherwise, NULL is returned. + * + * This function is meant for debugging only. + */ +struct task * task_lookup(const char *name); + +/* * Add a thread to a task. */ void task_add_thread(struct task *task, struct thread *thread); @@ -65,10 +100,26 @@ void task_add_thread(struct task *task, struct thread *thread); void task_remove_thread(struct task *task, struct thread *thread); /* + * Look up a thread in a task from its name. + * + * If a thread is found, it gains a reference, Otherwise, NULL is returned. + * + * This function is meant for debugging only. + */ +struct thread * task_lookup_thread(struct task *task, const char *name); + +/* * Display task information. * * If task is NULL, this function displays all tasks. */ void task_info(struct task *task); +/* + * This init operation provides : + * - task creation + * - module fully initialized + */ +INIT_OP_DECLARE(task_setup); + #endif /* _KERN_TASK_H */ diff --git a/kern/thread.c b/kern/thread.c index 7ce22fb7..8e5b2b52 100644 --- a/kern/thread.c +++ b/kern/thread.c @@ -54,7 +54,7 @@ * * A few terms are used by both papers with slightly different meanings. Here * are the definitions used in this implementation : - * - The time unit is the system timer period (1 / HZ) + * - The time unit is the system timer period (1 / tick frequency) * - Work is the amount of execution time units consumed * - Weight is the amount of execution time units allocated * - A round is the shortest period during which all threads in a run queue @@ -81,13 +81,15 @@ * weights in a smoother way than a raw scaling). */ +#include <assert.h> +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> #include <stdio.h> +#include <stdnoreturn.h> #include <string.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/condition.h> #include <kern/cpumap.h> @@ -99,8 +101,8 @@ #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> +#include <kern/shell.h> #include <kern/sleepq.h> #include <kern/spinlock.h> #include <kern/sref.h> @@ -110,6 +112,7 @@ #include <kern/turnstile.h> #include <kern/work.h> #include <machine/cpu.h> +#include <machine/page.h> #include <machine/pmap.h> #include <machine/tcb.h> #include <vm/vm_map.h> @@ -158,7 +161,7 @@ /* * Default time slice for real-time round-robin scheduling. */ -#define THREAD_DEFAULT_RR_TIME_SLICE (HZ / 10) +#define THREAD_DEFAULT_RR_TIME_SLICE (THREAD_TICK_FREQ / 10) /* * Maximum number of threads which can be pulled from a remote run queue @@ -169,7 +172,7 @@ /* * Delay (in ticks) between two balance attempts when a run queue is idle. */ -#define THREAD_IDLE_BALANCE_TICKS (HZ / 2) +#define THREAD_IDLE_BALANCE_TICKS (THREAD_TICK_FREQ / 2) /* * Run queue properties for real-time threads. @@ -189,7 +192,7 @@ struct thread_rt_runq { /* * Round slice base unit for fair-scheduling threads. */ -#define THREAD_FS_ROUND_SLICE_BASE (HZ / 10) +#define THREAD_FS_ROUND_SLICE_BASE (THREAD_TICK_FREQ / 10) /* * Group of threads sharing the same weight. @@ -226,7 +229,7 @@ struct thread_fs_runq { * return path) may violate the locking order. */ struct thread_runq { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; unsigned int cpu; unsigned int nr_threads; struct thread *current; @@ -257,7 +260,7 @@ struct thread_runq { struct syscnt sc_schedule_intrs; struct syscnt sc_tick_intrs; struct syscnt sc_boosts; -} __aligned(CPU_L1_SIZE); +}; /* * Operations of a scheduling class. @@ -321,7 +324,7 @@ static struct cpumap thread_idle_runqs; * There can be moderate bouncing on this word so give it its own cache line. */ static struct { - volatile unsigned long value __aligned(CPU_L1_SIZE); + alignas(CPU_L1_SIZE) volatile unsigned long value; } thread_fs_highest_round_struct; #define thread_fs_highest_round (thread_fs_highest_round_struct.value) @@ -1514,7 +1517,7 @@ thread_sched_idle_select_runq(struct thread *thread) panic("thread: idler threads cannot be awaken"); } -static void __noreturn +static noreturn void thread_sched_idle_panic(void) { panic("thread: only idle threads are allowed in the idle class"); @@ -1684,12 +1687,10 @@ thread_reset_real_priority(struct thread *thread) } static void __init -thread_bootstrap_common(unsigned int cpu) +thread_init_booter(unsigned int cpu) { struct thread *booter; - cpumap_set(&thread_active_runqs, cpu); - /* Initialize only what's needed during bootstrap */ booter = &thread_booters[cpu]; booter->nr_refs = 0; /* Make sure booters aren't destroyed */ @@ -1705,10 +1706,20 @@ thread_bootstrap_common(unsigned int cpu) booter->task = kernel_task; snprintf(booter->name, sizeof(booter->name), THREAD_KERNEL_PREFIX "thread_boot/%u", cpu); - thread_runq_init(percpu_ptr(thread_runq, cpu), cpu, booter); } -void __init +static int __init +thread_setup_booter(void) +{ + tcb_set_current(&thread_booters[0].tcb); + thread_init_booter(0); + return 0; +} + +INIT_OP_DEFINE(thread_setup_booter, + INIT_OP_DEP(tcb_setup, true)); + +static int __init thread_bootstrap(void) { cpumap_zero(&thread_active_runqs); @@ -1716,18 +1727,17 @@ thread_bootstrap(void) thread_fs_highest_round = THREAD_FS_INITIAL_ROUND; - tcb_set_current(&thread_booters[0].tcb); - thread_bootstrap_common(0); + cpumap_set(&thread_active_runqs, 0); + thread_runq_init(cpu_local_ptr(thread_runq), 0, &thread_booters[0]); + return 0; } -void __init -thread_ap_bootstrap(void) -{ - tcb_set_current(&thread_booters[cpu_id()].tcb); -} +INIT_OP_DEFINE(thread_bootstrap, + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_setup_booter, true)); -static void -thread_main(void) +void +thread_main(void (*fn)(void *), void *arg) { struct thread *thread; @@ -1741,7 +1751,7 @@ thread_main(void) cpu_intr_enable(); thread_preempt_enable(); - thread->fn(thread->arg); + fn(arg); thread_exit(); } @@ -1828,14 +1838,12 @@ thread_init(struct thread *thread, void *stack, thread->task = task; thread->stack = stack; strlcpy(thread->name, attr->name, sizeof(thread->name)); - thread->fn = fn; - thread->arg = arg; if (attr->flags & THREAD_ATTR_DETACHED) { thread->flags |= THREAD_DETACHED; } - error = tcb_init(&thread->tcb, stack, thread_main); + error = tcb_init(&thread->tcb, stack, fn, arg); if (error) { goto error_tcb; @@ -1894,7 +1902,7 @@ thread_alloc_stack(void) void *mem; int error; - stack_size = vm_page_round(STACK_SIZE); + stack_size = vm_page_round(TCB_STACK_SIZE); mem = vm_kmem_alloc((PAGE_SIZE * 2) + stack_size); if (mem == NULL) { @@ -1923,24 +1931,18 @@ thread_alloc_stack(void) pmap_remove(kernel_pmap, va + PAGE_SIZE + stack_size, cpumap_all()); pmap_update(kernel_pmap); - vm_page_free(first_page, 0); - vm_page_free(last_page, 0); - - return (char *)va + PAGE_SIZE; + return (void *)va + PAGE_SIZE; } static void thread_free_stack(void *stack) { size_t stack_size; - char *va; - - stack_size = vm_page_round(STACK_SIZE); - va = (char *)stack - PAGE_SIZE; + void *va; - vm_kmem_free_va(va, PAGE_SIZE); - vm_kmem_free(va + PAGE_SIZE, stack_size); - vm_kmem_free_va(va + PAGE_SIZE + stack_size, PAGE_SIZE); + stack_size = vm_page_round(TCB_STACK_SIZE); + va = (void *)stack - PAGE_SIZE; + vm_kmem_free(va, (PAGE_SIZE * 2) + stack_size); } #else /* X15_THREAD_STACK_GUARD */ @@ -2228,20 +2230,107 @@ thread_setup_runq(struct thread_runq *runq) thread_setup_idler(runq); } -void __init +#ifdef X15_SHELL + +/* + * This function is meant for debugging only. As a result, it uses a weak + * locking policy which allows tracing threads which state may mutate during + * tracing. + */ +static void +thread_shell_trace(int argc, char *argv[]) +{ + const char *task_name, *thread_name; + struct thread_runq *runq; + struct thread *thread; + unsigned long flags; + struct task *task; + int error; + + if (argc != 3) { + error = ERROR_INVAL; + goto error; + } + + task_name = argv[1]; + thread_name = argv[2]; + + task = task_lookup(task_name); + + if (task == NULL) { + error = ERROR_SRCH; + goto error; + } + + thread = task_lookup_thread(task, thread_name); + task_unref(task); + + if (thread == NULL) { + error = ERROR_SRCH; + goto error; + } + + runq = thread_lock_runq(thread, &flags); + + if (thread == runq->current) { + printf("thread: trace: thread is running\n"); + } else { + tcb_trace(&thread->tcb); + } + + thread_unlock_runq(runq, flags); + + thread_unref(thread); + return; + +error: + printf("thread: trace: %s\n", error_str(error)); +} + +static struct shell_cmd thread_shell_cmds[] = { + SHELL_CMD_INITIALIZER("thread_trace", thread_shell_trace, + "thread_trace <task_name> <thread_name>", + "display the stack trace of a given thread"), +}; + +static int __init +thread_setup_shell(void) +{ + SHELL_REGISTER_CMDS(thread_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(thread_setup_shell, + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(task_setup, true), + INIT_OP_DEP(thread_setup, true)); + +#endif /* X15_SHELL */ + +static void __init +thread_setup_common(unsigned int cpu) +{ + assert(cpu != 0); + cpumap_set(&thread_active_runqs, cpu); + thread_init_booter(cpu); + thread_runq_init(percpu_ptr(thread_runq, cpu), cpu, &thread_booters[cpu]); +} + +static int __init thread_setup(void) { int cpu; for (cpu = 1; (unsigned int)cpu < cpu_count(); cpu++) { - thread_bootstrap_common(cpu); + thread_setup_common(cpu); } kmem_cache_init(&thread_cache, "thread", sizeof(struct thread), CPU_L1_SIZE, NULL, 0); #ifndef X15_THREAD_STACK_GUARD - kmem_cache_init(&thread_stack_cache, "thread_stack", STACK_SIZE, - DATA_ALIGN, NULL, 0); + kmem_cache_init(&thread_stack_cache, "thread_stack", TCB_STACK_SIZE, + CPU_DATA_ALIGN, NULL, 0); #endif /* X15_THREAD_STACK_GUARD */ thread_setup_reaper(); @@ -2249,6 +2338,29 @@ thread_setup(void) cpumap_for_each(&thread_active_runqs, cpu) { thread_setup_runq(percpu_ptr(thread_runq, cpu)); } + + return 0; +} + +INIT_OP_DEFINE(thread_setup, + INIT_OP_DEP(cpumap_setup, true), + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(pmap_setup, true), + INIT_OP_DEP(sleepq_setup, true), + INIT_OP_DEP(task_setup, true), + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(turnstile_setup, true), +#ifdef X15_THREAD_STACK_GUARD + INIT_OP_DEP(vm_kmem_setup, true), + INIT_OP_DEP(vm_map_setup, true), + INIT_OP_DEP(vm_page_setup, true), +#endif + ); + +void __init +thread_ap_setup(void) +{ + tcb_set_current(&thread_booters[cpu_id()].tcb); } int @@ -2395,6 +2507,10 @@ thread_wakeup(struct thread *thread) struct thread_runq *runq; unsigned long flags; + if ((thread == NULL) || (thread == thread_self())) { + return; + } + /* * There is at most one reference on threads that were never dispatched, * in which case there is no need to lock anything. diff --git a/kern/thread.h b/kern/thread.h index 4d85c59d..43a98e87 100644 --- a/kern/thread.h +++ b/kern/thread.h @@ -33,20 +33,29 @@ #ifndef _KERN_THREAD_H #define _KERN_THREAD_H +#include <assert.h> #include <stdbool.h> #include <stddef.h> +#include <stdnoreturn.h> -#include <kern/assert.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/condition.h> #include <kern/cpumap.h> -#include <kern/macros.h> #include <kern/spinlock_types.h> #include <kern/turnstile_types.h> #include <machine/cpu.h> #include <machine/tcb.h> /* + * Scheduler tick frequency. + * + * The selected value of 200 translates to a period of 5ms, small enough to + * provide low latency, and is practical as both a dividend and divisor. + */ +#define THREAD_TICK_FREQ 200 + +/* * Thread structure. */ struct thread; @@ -162,18 +171,16 @@ thread_attr_set_priority(struct thread_attr *attr, unsigned short priority) } /* - * Early initialization of the thread module. + * Thread entry point. * - * These function make it possible to use migration and preemption control - * operations (and in turn, spin locks) during bootstrap. + * Loaded TCBs are expected to call this function with interrupts disabled. */ -void thread_bootstrap(void); -void thread_ap_bootstrap(void); +void thread_main(void (*fn)(void *), void *arg); /* - * Initialize the thread module. + * Initialization of the thread module on APs. */ -void thread_setup(void); +void thread_ap_setup(void); /* * Create a thread. @@ -188,7 +195,7 @@ int thread_create(struct thread **threadp, const struct thread_attr *attr, /* * Terminate the calling thread. */ -void __noreturn thread_exit(void); +noreturn void thread_exit(void); /* * Wait for the given thread to terminate and release its resources. @@ -219,7 +226,8 @@ void thread_sleep(struct spinlock *interlock, const void *wchan_addr, /* * Schedule a thread for execution on a processor. * - * No action is performed if the target thread is already in the running state. + * No action is performed if the target thread is NULL, the calling thread, + * or already in the running state. */ void thread_wakeup(struct thread *thread); @@ -228,7 +236,7 @@ void thread_wakeup(struct thread *thread); * * Interrupts must be disabled when calling this function. */ -void __noreturn thread_run_scheduler(void); +noreturn void thread_run_scheduler(void); /* * Make the calling thread release the processor. @@ -732,4 +740,24 @@ thread_get_specific(unsigned int key) */ bool thread_is_running(const struct thread *thread); +/* + * This init operation provides : + * - a dummy thread context for the BSP, allowing the use of thread_self() + */ +INIT_OP_DECLARE(thread_setup_booter); + +/* + * This init operation provides : + * - same as thread_setup_booter + * - BSP run queue initialization + */ +INIT_OP_DECLARE(thread_bootstrap); + +/* + * This init operation provides : + * - thread creation + * - module fully initialized + */ +INIT_OP_DECLARE(thread_setup); + #endif /* _KERN_THREAD_H */ diff --git a/kern/thread_i.h b/kern/thread_i.h index 2e1b88aa..3f350889 100644 --- a/kern/thread_i.h +++ b/kern/thread_i.h @@ -18,16 +18,16 @@ #ifndef _KERN_THREAD_I_H #define _KERN_THREAD_I_H +#include <stdalign.h> #include <stdbool.h> #include <kern/atomic.h> #include <kern/condition_types.h> #include <kern/cpumap.h> #include <kern/list_types.h> -#include <kern/macros.h> #include <kern/mutex_types.h> -#include <kern/param.h> #include <kern/turnstile_types.h> +#include <machine/cpu.h> #include <machine/tcb.h> /* @@ -97,7 +97,7 @@ struct thread_fs_data { * ( ) read-only */ struct thread { - struct tcb tcb; /* (r) */ + alignas(CPU_L1_SIZE) struct tcb tcb; /* (r) */ unsigned long nr_refs; /* (a) */ unsigned long flags; /* (a) */ @@ -178,11 +178,7 @@ struct thread { struct list task_node; /* (T) */ void *stack; /* (-) */ char name[THREAD_NAME_SIZE]; /* ( ) */ - - /* TODO Move out of the structure and make temporary */ - void (*fn)(void *); - void *arg; -} __aligned(CPU_L1_SIZE); +}; #define THREAD_ATTR_DETACHED 0x1 @@ -195,13 +191,13 @@ void thread_destroy(struct thread *thread); static inline void thread_set_flag(struct thread *thread, unsigned long flag) { - atomic_or_acq_rel(&thread->flags, flag); + atomic_or(&thread->flags, flag, ATOMIC_RELEASE); } static inline void thread_clear_flag(struct thread *thread, unsigned long flag) { - atomic_and_acq_rel(&thread->flags, ~flag); + atomic_and(&thread->flags, ~flag, ATOMIC_RELEASE); } static inline int diff --git a/kern/turnstile.c b/kern/turnstile.c index 7b27d841..e724c1e9 100644 --- a/kern/turnstile.c +++ b/kern/turnstile.c @@ -43,30 +43,31 @@ * TODO Analyse hash parameters. */ +#include <assert.h> +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/plist.h> #include <kern/spinlock.h> #include <kern/thread.h> #include <kern/turnstile.h> #include <kern/turnstile_types.h> +#include <machine/cpu.h> /* * Locking keys : * (b) bucket */ struct turnstile_bucket { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; struct list list; /* (b) */ -} __aligned(CPU_L1_SIZE); +}; /* * Adding/removing waiters to/from a turnstile are performed while @@ -499,7 +500,7 @@ turnstile_ctor(void *ptr) turnstile->owner = NULL; } -void __init +static int __init turnstile_bootstrap(void) { unsigned int i; @@ -507,15 +508,24 @@ turnstile_bootstrap(void) for (i = 0; i < ARRAY_SIZE(turnstile_htable); i++) { turnstile_bucket_init(&turnstile_htable[i]); } + + return 0; } -void __init +INIT_OP_DEFINE(turnstile_bootstrap); + +static int __init turnstile_setup(void) { kmem_cache_init(&turnstile_cache, "turnstile", sizeof(struct turnstile), CPU_L1_SIZE, turnstile_ctor, 0); + return 0; } +INIT_OP_DEFINE(turnstile_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(turnstile_bootstrap, true)); + struct turnstile * turnstile_create(void) { diff --git a/kern/turnstile.h b/kern/turnstile.h index 4bc8f74d..74512fa1 100644 --- a/kern/turnstile.h +++ b/kern/turnstile.h @@ -29,6 +29,7 @@ #include <stdbool.h> #include <stddef.h> +#include <kern/init.h> #include <kern/plist.h> #include <kern/spinlock.h> #include <kern/thread.h> @@ -101,20 +102,6 @@ turnstile_td_get_turnstile(const struct turnstile_td *td) void turnstile_td_propagate_priority(struct turnstile_td *td); /* - * Early initialization of the turnstile module. - * - * This module is initialized by architecture-specific code. It should - * be one of the first modules to be initialized since it's used by - * synchronization objects that may be accessed very early. - */ -void turnstile_bootstrap(void); - -/* - * Initialize the turnstile module. - */ -void turnstile_setup(void); - -/* * Create/destroy a turnstile. */ struct turnstile * turnstile_create(void); @@ -197,4 +184,17 @@ void turnstile_signal(struct turnstile *turnstile); void turnstile_own(struct turnstile *turnstile); void turnstile_disown(struct turnstile *turnstile); +/* + * This init operation provides : + * - ? TODO Review + */ +INIT_OP_DECLARE(turnstile_bootstrap); + +/* + * This init operation provides : + * - turnstile creation + * - module fully initialized + */ +INIT_OP_DECLARE(turnstile_setup); + #endif /* _KERN_TURNSTILE_H */ diff --git a/kern/work.c b/kern/work.c index 714af35c..9aa3f40e 100644 --- a/kern/work.c +++ b/kern/work.c @@ -15,18 +15,18 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/bitmap.h> #include <kern/error.h> #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/syscnt.h> @@ -85,7 +85,7 @@ struct work_thread { * only use one queue. */ struct work_pool { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; int flags; struct work_queue queue0; struct work_queue queue1; @@ -98,7 +98,7 @@ struct work_pool { struct list available_threads; struct list dead_threads; BITMAP_DECLARE(bitmap, WORK_MAX_THREADS); -} __aligned(CPU_L1_SIZE); +}; static int work_thread_create(struct work_pool *pool, unsigned int id); static void work_thread_destroy(struct work_thread *worker); @@ -259,7 +259,7 @@ work_pool_wakeup_manager(struct work_pool *pool) return; } - if ((pool->manager != NULL) && (pool->manager->thread != thread_self())) { + if (pool->manager != NULL) { thread_wakeup(pool->manager->thread); } } @@ -374,7 +374,7 @@ work_process(void *arg) if (error) { work_pool_free_id(pool, id); - printf("work: warning: unable to create worker thread\n"); + log_warning("work: unable to create worker thread"); } } } @@ -472,7 +472,7 @@ work_thread_destroy(struct work_thread *worker) kmem_cache_free(&work_thread_cache, worker); } -void __init +static int __init work_setup(void) { unsigned int i; @@ -490,11 +490,23 @@ work_setup(void) work_pool_init(&work_pool_highprio, WORK_INVALID_CPU, WORK_PF_GLOBAL | WORK_PF_HIGHPRIO); - printf("work: threads per pool (per-cpu/global): %u/%u, spare: %u\n", - percpu_var(work_pool_cpu_main.max_threads, 0), - work_pool_main.max_threads, WORK_THREADS_SPARE); + log_info("work: threads per pool (per-cpu/global): %u/%u, spare: %u", + percpu_var(work_pool_cpu_main.max_threads, 0), + work_pool_main.max_threads, WORK_THREADS_SPARE); + + return 0; } +INIT_OP_DEFINE(work_setup, + INIT_OP_DEP(cpu_mp_probe, true), + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_bootstrap, true)); + void work_schedule(struct work *work, int flags) { diff --git a/kern/work.h b/kern/work.h index 6e3876f0..a8df1f7e 100644 --- a/kern/work.h +++ b/kern/work.h @@ -26,6 +26,8 @@ #ifndef _KERN_WORK_H #define _KERN_WORK_H +#include <kern/init.h> + /* * Work scheduling flags. */ @@ -134,11 +136,6 @@ work_init(struct work *work, work_fn_t fn) } /* - * Initialize the work module. - */ -void work_setup(void); - -/* * Schedule work for deferred processing. * * This function may be called from interrupt context. @@ -156,4 +153,11 @@ void work_queue_schedule(struct work_queue *queue, int flags); */ void work_report_periodic_event(void); +/* + * This init operation provides : + * - works can be scheduled + * - module fully initialized + */ +INIT_OP_DECLARE(work_setup); + #endif /* _KERN_WORK_H */ diff --git a/kern/xcall.c b/kern/xcall.c index 364ad0be..cccb9373 100644 --- a/kern/xcall.c +++ b/kern/xcall.c @@ -15,12 +15,13 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> -#include <kern/assert.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/thread.h> @@ -28,9 +29,9 @@ #include <machine/cpu.h> struct xcall { - xcall_fn_t fn; + alignas(CPU_L1_SIZE) xcall_fn_t fn; void *arg; -} __aligned(CPU_L1_SIZE); +}; /* * Per-CPU data. @@ -49,11 +50,11 @@ struct xcall { * between multiple cross-calls. */ struct xcall_cpu_data { - struct xcall send_calls[X15_MAX_CPUS]; + alignas(CPU_L1_SIZE) struct xcall send_calls[X15_MAX_CPUS]; struct xcall *recv_call; struct spinlock lock; -} __aligned(CPU_L1_SIZE); +}; static struct xcall_cpu_data xcall_cpu_data __percpu; @@ -92,7 +93,8 @@ xcall_cpu_data_get_recv_call(const struct xcall_cpu_data *cpu_data) } static void -xcall_cpu_data_set_recv_call(struct xcall_cpu_data *cpu_data, struct xcall *call) +xcall_cpu_data_set_recv_call(struct xcall_cpu_data *cpu_data, + struct xcall *call) { atomic_store(&cpu_data->recv_call, call, ATOMIC_RELEASE); } @@ -103,7 +105,7 @@ xcall_cpu_data_clear_recv_call(struct xcall_cpu_data *cpu_data) xcall_cpu_data_set_recv_call(cpu_data, NULL); } -void +static int __init xcall_setup(void) { unsigned int i; @@ -111,8 +113,15 @@ xcall_setup(void) for (i = 0; i < cpu_count(); i++) { xcall_cpu_data_init(percpu_ptr(xcall_cpu_data, i)); } + + return 0; } +INIT_OP_DEFINE(xcall_setup, + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(spinlock_setup, true)); + void xcall_call(xcall_fn_t fn, void *arg, unsigned int cpu) { diff --git a/kern/xcall.h b/kern/xcall.h index 37c85867..27b6af25 100644 --- a/kern/xcall.h +++ b/kern/xcall.h @@ -30,11 +30,6 @@ typedef void (*xcall_fn_t)(void *arg); /* - * Initialize the xcall module. - */ -void xcall_setup(void); - -/* * Run the given cross-call function on a specific processor. * * The operation is completely synchronous, returning only when the function diff --git a/test/test_llsync_defer.c b/test/test_llsync_defer.c index 26747998..4b8afbc7 100644 --- a/test/test_llsync_defer.c +++ b/test/test_llsync_defer.c @@ -39,9 +39,9 @@ #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/thread.h> #include <kern/work.h> +#include <machine/page.h> #include <test/test.h> #include <vm/vm_kmem.h> @@ -167,10 +167,11 @@ test_read(void *arg) s = (const unsigned char *)pdsc->addr; if (s != NULL) { - for (j = 0; j < PAGE_SIZE; j++) + for (j = 0; j < PAGE_SIZE; j++) { if (s[j] != TEST_VALIDATION_BYTE) { panic("invalid content"); } + } if ((i % TEST_LOOPS_PER_PRINT) == 0) { printf("read "); diff --git a/test/test_pmap_update_mp.c b/test/test_pmap_update_mp.c index 5588ce83..c058323a 100644 --- a/test/test_pmap_update_mp.c +++ b/test/test_pmap_update_mp.c @@ -33,8 +33,8 @@ #include <kern/error.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/thread.h> +#include <machine/page.h> #include <test/test.h> #include <vm/vm_kmem.h> diff --git a/test/test_vm_page_fill.c b/test/test_vm_page_fill.c index fe351038..2b7f1130 100644 --- a/test/test_vm_page_fill.c +++ b/test/test_vm_page_fill.c @@ -30,6 +30,7 @@ #include <kern/error.h> #include <kern/list.h> #include <kern/thread.h> +#include <machine/page.h> #include <machine/pmap.h> #include <test/test.h> #include <vm/vm_kmem.h> diff --git a/vm/vm_kmem.c b/vm/vm_kmem.c index 8a7272cb..dc204f85 100644 --- a/vm/vm_kmem.c +++ b/vm/vm_kmem.c @@ -13,25 +13,23 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * - * TODO Rework so that pmap update errors can be handled. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/cpumap.h> #include <kern/init.h> -#include <kern/panic.h> -#include <kern/param.h> +#include <kern/macros.h> +#include <machine/page.h> #include <machine/pmap.h> #include <machine/types.h> #include <vm/vm_adv.h> #include <vm/vm_inherit.h> #include <vm/vm_kmem.h> #include <vm/vm_map.h> +#include <vm/vm_object.h> #include <vm/vm_page.h> #include <vm/vm_prot.h> @@ -41,6 +39,31 @@ static struct vm_map kernel_map_store; struct vm_map *kernel_map __read_mostly = &kernel_map_store; +static struct vm_object vm_kmem_kernel_object; + +static uint64_t +vm_kmem_offset(uintptr_t va) +{ + assert(va >= PMAP_MIN_KMEM_ADDRESS); + return va - PMAP_MIN_KMEM_ADDRESS; +} + +static int __init +vm_kmem_setup(void) +{ + uint64_t size; + + size = vm_kmem_offset(PMAP_MAX_KMEM_ADDRESS); + vm_object_init(&vm_kmem_kernel_object, size); + return 0; +} + +INIT_OP_DEFINE(vm_kmem_setup, + INIT_OP_DEP(pmap_bootstrap, true), + INIT_OP_DEP(vm_map_bootstrap, true), + INIT_OP_DEP(vm_object_setup, true), + INIT_OP_DEP(vm_page_setup, true)); + static int vm_kmem_alloc_check(size_t size) { @@ -97,6 +120,7 @@ vm_kmem_alloc(size_t size) { struct vm_page *page; uintptr_t va, start, end; + int error; size = vm_page_round(size); va = (uintptr_t)vm_kmem_alloc_va(size); @@ -109,30 +133,38 @@ vm_kmem_alloc(size_t size) page = vm_page_alloc(0, VM_PAGE_SEL_HIGHMEM, VM_PAGE_KERNEL); if (page == NULL) { - goto error_page; + goto error; } - pmap_enter(kernel_pmap, start, vm_page_to_pa(page), - VM_PROT_READ | VM_PROT_WRITE, PMAP_PEF_GLOBAL); - } + /* + * The page becomes managed by the object and is freed in case + * of failure. + */ + error = vm_object_insert(&vm_kmem_kernel_object, page, + vm_kmem_offset(start)); - pmap_update(kernel_pmap); - return (void *)va; + if (error) { + goto error; + } -error_page: - size = start - va; + error = pmap_enter(kernel_pmap, start, vm_page_to_pa(page), + VM_PROT_READ | VM_PROT_WRITE, PMAP_PEF_GLOBAL); - if (size != 0) { - pmap_update(kernel_pmap); - vm_kmem_free((void *)va, size); + if (error || (start - va == vm_page_ptoa(1000))) { + goto error; + } } - size = end - start; + error = pmap_update(kernel_pmap); - if (size != 0) { - vm_kmem_free_va((void *)start, size); + if (error) { + goto error; } + return (void *)va; + +error: + vm_kmem_free((void *)va, size); return NULL; } @@ -140,10 +172,7 @@ void vm_kmem_free(void *addr, size_t size) { const struct cpumap *cpumap; - struct vm_page *page; uintptr_t va, end; - phys_addr_t pa; - int error; va = (uintptr_t)addr; size = vm_page_round(size); @@ -151,16 +180,14 @@ vm_kmem_free(void *addr, size_t size) cpumap = cpumap_all(); while (va < end) { - error = pmap_kextract(va, &pa); - assert(!error); pmap_remove(kernel_pmap, va, cpumap); - page = vm_page_lookup(pa); - assert(page != NULL); - vm_page_free(page, 0); va += PAGE_SIZE; } pmap_update(kernel_pmap); + vm_object_remove(&vm_kmem_kernel_object, + vm_kmem_offset((uintptr_t)addr), + vm_kmem_offset(end)); vm_kmem_free_va(addr, size); } @@ -171,6 +198,7 @@ vm_kmem_map_pa(phys_addr_t pa, size_t size, uintptr_t offset, map_va; size_t map_size; phys_addr_t start; + int error; start = vm_page_trunc(pa); map_size = vm_page_round(pa + size) - start; @@ -180,11 +208,20 @@ vm_kmem_map_pa(phys_addr_t pa, size_t size, return NULL; } - for (offset = 0; offset < map_size; offset += PAGE_SIZE) - pmap_enter(kernel_pmap, map_va + offset, start + offset, - VM_PROT_READ | VM_PROT_WRITE, PMAP_PEF_GLOBAL); + for (offset = 0; offset < map_size; offset += PAGE_SIZE) { + error = pmap_enter(kernel_pmap, map_va + offset, start + offset, + VM_PROT_READ | VM_PROT_WRITE, PMAP_PEF_GLOBAL); - pmap_update(kernel_pmap); + if (error) { + goto error; + } + } + + error = pmap_update(kernel_pmap); + + if (error) { + goto error; + } if (map_vap != NULL) { *map_vap = map_va; @@ -195,6 +232,10 @@ vm_kmem_map_pa(phys_addr_t pa, size_t size, } return (void *)(map_va + (uintptr_t)(pa & PAGE_MASK)); + +error: + vm_kmem_unmap_pa(map_va, map_size); + return NULL; } void diff --git a/vm/vm_kmem.h b/vm/vm_kmem.h index c4b5295c..e10c2201 100644 --- a/vm/vm_kmem.h +++ b/vm/vm_kmem.h @@ -20,15 +20,17 @@ #include <stdint.h> +#include <kern/init.h> +#include <machine/pmap.h> #include <machine/types.h> /* * The kernel space is required not to start at address 0, which is used to * report allocation errors. */ -#if VM_MIN_KMEM_ADDRESS == 0 +#if PMAP_MIN_KMEM_ADDRESS == 0 #error "kernel space must not start at address 0" -#endif /* VM_MIN_KMEM_ADDRESS == 0 */ +#endif /* PMAP_MIN_KMEM_ADDRESS == 0 */ /* * Special kernel addresses. @@ -88,4 +90,10 @@ void * vm_kmem_map_pa(phys_addr_t pa, size_t size, */ void vm_kmem_unmap_pa(uintptr_t map_va, size_t map_size); +/* + * This init operation provides : + * - kernel virtual memory allocation + */ +INIT_OP_DECLARE(vm_kmem_setup); + #endif /* _VM_VM_KMEM_H */ diff --git a/vm/vm_map.c b/vm/vm_map.c index 657f4b59..01e0ed5d 100644 --- a/vm/vm_map.c +++ b/vm/vm_map.c @@ -19,11 +19,11 @@ * needed for kernel allocation. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> #include <stdio.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/init.h> #include <kern/kmem.h> @@ -31,8 +31,10 @@ #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/rbtree.h> +#include <kern/shell.h> +#include <kern/task.h> +#include <machine/page.h> #include <machine/pmap.h> #include <vm/vm_adv.h> #include <vm/vm_inherit.h> @@ -695,18 +697,81 @@ vm_map_init(struct vm_map *map, struct pmap *pmap, map->pmap = pmap; } -void __init -vm_map_setup(void) +#ifdef X15_SHELL + +static void +vm_map_shell_info(int argc, char **argv) +{ + const struct task *task; + + if (argc < 2) { + goto error; + } else { + task = task_lookup(argv[1]); + + if (task == NULL) { + goto error; + } + + vm_map_info(task_get_vm_map(task)); + } + + return; + +error: + printf("vm_map: info: invalid arguments\n"); +} + +static struct shell_cmd vm_map_shell_cmds[] = { + SHELL_CMD_INITIALIZER("vm_map_info", vm_map_shell_info, + "vm_map_info <task_name>", + "display information about a VM map"), +}; + +static int __init +vm_map_setup_shell(void) +{ + SHELL_REGISTER_CMDS(vm_map_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(vm_map_setup_shell, + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(task_setup, true), + INIT_OP_DEP(vm_map_setup, true)); + +#endif /* X15_SHELL */ + +static int __init +vm_map_bootstrap(void) { vm_map_init(kernel_map, kernel_pmap, - VM_MIN_KMEM_ADDRESS, VM_MAX_KMEM_ADDRESS); + PMAP_MIN_KMEM_ADDRESS, PMAP_MAX_KMEM_ADDRESS); kmem_cache_init(&vm_map_entry_cache, "vm_map_entry", sizeof(struct vm_map_entry), 0, NULL, KMEM_CACHE_PAGE_ONLY); + return 0; +} + +INIT_OP_DEFINE(vm_map_bootstrap, + INIT_OP_DEP(kmem_bootstrap, true), + INIT_OP_DEP(thread_bootstrap, true)); + +static int __init +vm_map_setup(void) +{ kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), - 0, NULL, 0); + 0, NULL, KMEM_CACHE_PAGE_ONLY); + return 0; } +INIT_OP_DEFINE(vm_map_setup, + INIT_OP_DEP(pmap_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(vm_map_bootstrap, true)); + int vm_map_create(struct vm_map **mapp) { @@ -727,7 +792,7 @@ vm_map_create(struct vm_map **mapp) goto error_pmap; } - vm_map_init(map, pmap, VM_MIN_ADDRESS, VM_MAX_ADDRESS); + vm_map_init(map, pmap, PMAP_MIN_ADDRESS, PMAP_MAX_ADDRESS); *mapp = map; return 0; diff --git a/vm/vm_map.h b/vm/vm_map.h index 1e17bbb4..653eabd3 100644 --- a/vm/vm_map.h +++ b/vm/vm_map.h @@ -23,6 +23,7 @@ #include <stdint.h> +#include <kern/init.h> #include <kern/list.h> #include <kern/mutex.h> #include <kern/rbtree.h> @@ -104,11 +105,6 @@ int vm_map_enter(struct vm_map *map, uintptr_t *startp, void vm_map_remove(struct vm_map *map, uintptr_t start, uintptr_t end); /* - * Set up the vm_map module. - */ -void vm_map_setup(void); - -/* * Create a VM map. */ int vm_map_create(struct vm_map **mapp); @@ -118,4 +114,17 @@ int vm_map_create(struct vm_map **mapp); */ void vm_map_info(struct vm_map *map); +/* + * This init operation provides : + * - kernel mapping operations + */ +INIT_OP_DECLARE(vm_map_bootstrap); + +/* + * This init operation provides : + * - VM map creation + * - module fully initialized + */ +INIT_OP_DECLARE(vm_map_setup); + #endif /* _VM_VM_MAP_H */ diff --git a/vm/vm_object.c b/vm/vm_object.c new file mode 100644 index 00000000..649d8577 --- /dev/null +++ b/vm/vm_object.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * This implementation is based on the paper "A lockless pagecache in Linux" + * by Nick Piggin. It allows looking up pages without contention on VM objects. + */ + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> + +#include <kern/init.h> +#include <kern/llsync.h> +#include <kern/mutex.h> +#include <kern/rdxtree.h> +#include <vm/vm_object.h> +#include <vm/vm_page.h> +#include <machine/page.h> + +static int __init +vm_object_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(vm_object_setup, + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(rdxtree_setup, true), + INIT_OP_DEP(vm_page_setup, true)); + +void __init +vm_object_init(struct vm_object *object, uint64_t size) +{ + assert(vm_page_aligned(size)); + + mutex_init(&object->lock); + rdxtree_init(&object->pages, 0); + object->size = size; + object->nr_pages = 0; +} + +int +vm_object_insert(struct vm_object *object, struct vm_page *page, + uint64_t offset) +{ + int error; + + assert(vm_page_aligned(offset)); + + /* + * The page may have no references. Add one before publishing + * so that concurrent lookups succeed. + */ + vm_page_ref(page); + + mutex_lock(&object->lock); + + if (offset >= object->size) { + error = ERROR_INVAL; + goto error; + } + + error = rdxtree_insert(&object->pages, vm_page_atop(offset), page); + + if (error) { + goto error; + } + + vm_page_link(page, object, offset); + object->nr_pages++; + assert(object->nr_pages != 0); + + mutex_unlock(&object->lock); + + return 0; + +error: + mutex_unlock(&object->lock); + + vm_page_unref(page); + + return error; +} + +void +vm_object_remove(struct vm_object *object, uint64_t start, uint64_t end) +{ + struct vm_page *page; + uint64_t offset; + + assert(vm_page_aligned(start)); + assert(vm_page_aligned(end)); + assert(start <= end); + + mutex_lock(&object->lock); + + for (offset = start; offset < end; offset += PAGE_SIZE) { + page = rdxtree_remove(&object->pages, vm_page_atop(offset)); + + if (page == NULL) { + continue; + } + + vm_page_unlink(page); + vm_page_unref(page); + assert(object->nr_pages != 0); + object->nr_pages--; + } + + mutex_unlock(&object->lock); +} + +struct vm_page * +vm_object_lookup(struct vm_object *object, uint64_t offset) +{ + struct vm_page *page; + int error; + + llsync_read_enter(); + + do { + page = rdxtree_lookup(&object->pages, vm_page_atop(offset)); + + if (page == NULL) { + break; + } + + error = vm_page_tryref(page); + } while (error); + + llsync_read_exit(); + + return page; +} diff --git a/vm/vm_object.h b/vm/vm_object.h new file mode 100644 index 00000000..9ffe711e --- /dev/null +++ b/vm/vm_object.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Virtual memory object. + * + * The purpose of VM objects is to track pages that are resident in + * physical memory. They collectively form the page cache. + */ + +#ifndef _VM_OBJECT_H +#define _VM_OBJECT_H + +#include <stdint.h> + +#include <kern/init.h> +#include <kern/rdxtree.h> +#include <vm/vm_object_types.h> +#include <vm/vm_page.h> + +struct vm_object; + +/* + * Initialize a VM object. + */ +void vm_object_init(struct vm_object *object, uint64_t size); + +/* + * Insert a page into a VM object. + * + * The offset must be page-aligned. + * + * The page becomes managed, and gains a reference. If successful, + * the reference is kept. Otherwise it's dropped. If the page had + * no references on entry, and a failure occurs, the page is freed. + */ +int vm_object_insert(struct vm_object *object, struct vm_page *page, + uint64_t offset); + +/* + * Remove pages from a VM object. + * + * The range boundaries must be page-aligned. + * + * Holes in the given range are silently skipped. Pages that are removed + * become unmanaged and lose a reference. + */ +void vm_object_remove(struct vm_object *object, uint64_t start, uint64_t end); + +/* + * Look up a page in a VM object. + * + * The offset must be page-aligned. + * + * If successful, the returned page gains a reference. Note that, if a valid + * page is returned, it may already have been removed from the object, or + * moved at a different offset. + */ +struct vm_page * vm_object_lookup(struct vm_object *object, uint64_t offset); + +/* + * This init operation provides : + * - module fully initialized + */ +INIT_OP_DECLARE(vm_object_setup); + +#endif /* _VM_OBJECT_H */ diff --git a/vm/vm_setup.c b/vm/vm_object_types.h index 0957815f..4026002e 100644 --- a/vm/vm_setup.c +++ b/vm/vm_object_types.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2014 Richard Braun. + * Copyright (c) 2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,24 +13,24 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Isolated type definition used to avoid inclusion circular dependencies. */ -#include <kern/init.h> -#include <kern/kmem.h> +#ifndef _VM_OBJECT_TYPES_H +#define _VM_OBJECT_TYPES_H + +#include <stdint.h> + +#include <kern/mutex.h> #include <kern/rdxtree.h> -#include <kern/percpu.h> -#include <machine/pmap.h> -#include <vm/vm_map.h> -#include <vm/vm_page.h> -#include <vm/vm_setup.h> -void __init -vm_setup(void) -{ - vm_page_setup(); - kmem_setup(); - vm_map_setup(); - rdxtree_setup(); - pmap_setup(); - percpu_setup(); -} +struct vm_object { + struct mutex lock; + struct rdxtree pages; + uint64_t size; + unsigned long nr_pages; +}; + +#endif /* _VM_OBJECT_TYPES_H */ diff --git a/vm/vm_page.c b/vm/vm_page.c index 284964a4..7b0eb3e0 100644 --- a/vm/vm_page.c +++ b/vm/vm_page.c @@ -29,27 +29,28 @@ * The symmetric case is handled likewise. */ +#include <assert.h> +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/init.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> +#include <kern/shell.h> #include <kern/thread.h> +#include <machine/boot.h> #include <machine/cpu.h> -#include <machine/pmap.h> +#include <machine/page.h> +#include <machine/pmem.h> #include <machine/types.h> #include <vm/vm_page.h> -#define DEBUG 0 - /* * Number of free block lists per zone. */ @@ -76,12 +77,12 @@ * Per-processor cache of pages. */ struct vm_page_cpu_pool { - struct mutex lock; + alignas(CPU_L1_SIZE) struct mutex lock; int size; int transfer_size; int nr_pages; struct list pages; -} __aligned(CPU_L1_SIZE); +}; /* * Special order value for pages that aren't in a free list. Such pages are @@ -146,12 +147,12 @@ static int vm_page_is_ready __read_mostly; * the direct physical mapping, DMA and DMA32 are aliases for DIRECTMAP, * in which case the zone table contains DIRECTMAP and HIGHMEM only. */ -static struct vm_page_zone vm_page_zones[VM_PAGE_MAX_ZONES]; +static struct vm_page_zone vm_page_zones[PMEM_MAX_ZONES]; /* * Bootstrap zone table. */ -static struct vm_page_boot_zone vm_page_boot_zones[VM_PAGE_MAX_ZONES] +static struct vm_page_boot_zone vm_page_boot_zones[PMEM_MAX_ZONES] __initdata; /* @@ -167,6 +168,10 @@ vm_page_init(struct vm_page *page, unsigned short zone_index, phys_addr_t pa) page->zone_index = zone_index; page->order = VM_PAGE_ORDER_UNLISTED; page->phys_addr = pa; + + page->nr_refs = 0; + + page->object = NULL; } void @@ -510,11 +515,9 @@ vm_page_load(unsigned int zone_index, phys_addr_t start, phys_addr_t end) zone->end = end; zone->heap_present = false; -#if DEBUG - printf("vm_page: load: %s: %llx:%llx\n", - vm_page_zone_name(zone_index), - (unsigned long long)start, (unsigned long long)end); -#endif + log_debug("vm_page: load: %s: %llx:%llx", + vm_page_zone_name(zone_index), + (unsigned long long)start, (unsigned long long)end); vm_page_zones_size++; } @@ -537,11 +540,9 @@ vm_page_load_heap(unsigned int zone_index, phys_addr_t start, phys_addr_t end) zone->avail_end = end; zone->heap_present = true; -#if DEBUG - printf("vm_page: heap: %s: %llx:%llx\n", - vm_page_zone_name(zone_index), - (unsigned long long)start, (unsigned long long)end); -#endif + log_debug("vm_page: heap: %s: %llx:%llx", + vm_page_zone_name(zone_index), + (unsigned long long)start, (unsigned long long)end); } int @@ -557,16 +558,16 @@ vm_page_select_alloc_zone(unsigned int selector) switch (selector) { case VM_PAGE_SEL_DMA: - zone_index = VM_PAGE_ZONE_DMA; + zone_index = PMEM_ZONE_DMA; break; case VM_PAGE_SEL_DMA32: - zone_index = VM_PAGE_ZONE_DMA32; + zone_index = PMEM_ZONE_DMA32; break; case VM_PAGE_SEL_DIRECTMAP: - zone_index = VM_PAGE_ZONE_DIRECTMAP; + zone_index = PMEM_ZONE_DIRECTMAP; break; case VM_PAGE_SEL_HIGHMEM: - zone_index = VM_PAGE_ZONE_HIGHMEM; + zone_index = PMEM_ZONE_HIGHMEM; break; default: panic("vm_page: invalid selector"); @@ -640,7 +641,59 @@ vm_page_bootalloc(size_t size) panic("vm_page: no physical memory available"); } -void __init +static void +vm_page_info_common(int (*print_fn)(const char *format, ...)) +{ + struct vm_page_zone *zone; + unsigned long pages; + unsigned int i; + + for (i = 0; i < vm_page_zones_size; i++) { + zone = &vm_page_zones[i]; + pages = (unsigned long)(zone->pages_end - zone->pages); + print_fn("vm_page: %s: pages: %lu (%luM), free: %lu (%luM)\n", + vm_page_zone_name(i), pages, pages >> (20 - PAGE_SHIFT), + zone->nr_free_pages, zone->nr_free_pages >> (20 - PAGE_SHIFT)); + } +} + +#ifdef X15_SHELL + +static void +vm_page_info(void) +{ + vm_page_info_common(printf); +} + +static void +vm_page_shell_info(int argc, char **argv) +{ + (void)argc; + (void)argv; + vm_page_info(); +} + +static struct shell_cmd vm_page_shell_cmds[] = { + SHELL_CMD_INITIALIZER("vm_page_info", vm_page_shell_info, + "vm_page_info", + "display information about physical memory"), +}; + +static int __init +vm_page_setup_shell(void) +{ + SHELL_REGISTER_CMDS(vm_page_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(vm_page_setup_shell, + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(vm_page_setup, true)); + +#endif /* X15_SHELL */ + +static int __init vm_page_setup(void) { struct vm_page_boot_zone *boot_zone; @@ -663,8 +716,8 @@ vm_page_setup(void) } table_size = vm_page_round(nr_pages * sizeof(struct vm_page)); - printf("vm_page: page table size: %zu entries (%zuk)\n", nr_pages, - table_size >> 10); + log_info("vm_page: page table size: %zu entries (%zuk)", nr_pages, + table_size >> 10); table = vm_page_bootalloc(table_size); va = (uintptr_t)table; @@ -700,8 +753,16 @@ vm_page_setup(void) } vm_page_is_ready = 1; + + return 0; } +INIT_OP_DEFINE(vm_page_setup, + INIT_OP_DEP(boot_load_vm_page_zones, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(printf_setup, true)); + +/* TODO Rename to avoid confusion with "managed pages" */ void __init vm_page_manage(struct vm_page *page) { @@ -729,6 +790,22 @@ vm_page_lookup(phys_addr_t pa) return NULL; } +static bool +vm_page_block_referenced(const struct vm_page *page, unsigned int order) +{ + unsigned int i, nr_pages; + + nr_pages = 1 << order; + + for (i = 0; i < nr_pages; i++) { + if (vm_page_referenced(&page[i])) { + return true; + } + } + + return false; +} + struct vm_page * vm_page_alloc(unsigned int order, unsigned int selector, unsigned short type) { @@ -739,6 +816,7 @@ vm_page_alloc(unsigned int order, unsigned int selector, unsigned short type) page = vm_page_zone_alloc(&vm_page_zones[i], order, type); if (page != NULL) { + assert(!vm_page_block_referenced(page, order)); return page; } } @@ -750,6 +828,7 @@ void vm_page_free(struct vm_page *page, unsigned int order) { assert(page->zone_index < ARRAY_SIZE(vm_page_zones)); + assert(!vm_page_block_referenced(page, order)); vm_page_zone_free(&vm_page_zones[page->zone_index], page, order); } @@ -758,13 +837,13 @@ const char * vm_page_zone_name(unsigned int zone_index) { /* Don't use a switch statement since zones can be aliased */ - if (zone_index == VM_PAGE_ZONE_HIGHMEM) { + if (zone_index == PMEM_ZONE_HIGHMEM) { return "HIGHMEM"; - } else if (zone_index == VM_PAGE_ZONE_DIRECTMAP) { + } else if (zone_index == PMEM_ZONE_DIRECTMAP) { return "DIRECTMAP"; - } else if (zone_index == VM_PAGE_ZONE_DMA32) { + } else if (zone_index == PMEM_ZONE_DMA32) { return "DMA32"; - } else if (zone_index == VM_PAGE_ZONE_DMA) { + } else if (zone_index == PMEM_ZONE_DMA) { return "DMA"; } else { panic("vm_page: invalid zone index"); @@ -772,17 +851,7 @@ vm_page_zone_name(unsigned int zone_index) } void -vm_page_info(void) +vm_page_log_info(void) { - struct vm_page_zone *zone; - unsigned long pages; - unsigned int i; - - for (i = 0; i < vm_page_zones_size; i++) { - zone = &vm_page_zones[i]; - pages = (unsigned long)(zone->pages_end - zone->pages); - printf("vm_page: %s: pages: %lu (%luM), free: %lu (%luM)\n", - vm_page_zone_name(i), pages, pages >> (20 - PAGE_SHIFT), - zone->nr_free_pages, zone->nr_free_pages >> (20 - PAGE_SHIFT)); - } + vm_page_info_common(log_info); } diff --git a/vm/vm_page.h b/vm/vm_page.h index 139f1bff..4ca34209 100644 --- a/vm/vm_page.h +++ b/vm/vm_page.h @@ -16,26 +16,36 @@ * * * Physical page management. + * + * A page is said to be managed if it's linked to a VM object, in which + * case there is at least one reference to it. */ #ifndef _VM_VM_PAGE_H #define _VM_VM_PAGE_H +#include <assert.h> +#include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> +#include <kern/atomic.h> +#include <kern/init.h> #include <kern/list.h> #include <kern/log2.h> #include <kern/macros.h> -#include <kern/param.h> +#include <machine/page.h> #include <machine/pmap.h> +#include <machine/pmem.h> #include <machine/types.h> +#include <vm/vm_object_types.h> /* * Address/page conversion and rounding macros (not inline functions to * be easily usable on both virtual and physical addresses, which may not * have the same type size). + * + * TODO Rename btop and ptob. */ #define vm_page_atop(addr) ((addr) >> PAGE_SHIFT) #define vm_page_ptoa(page) ((page) << PAGE_SHIFT) @@ -79,6 +89,12 @@ struct vm_page { unsigned short order; phys_addr_t phys_addr; void *priv; + + unsigned int nr_refs; + + /* VM object back reference */ + struct vm_object *object; + uint64_t offset; }; static inline unsigned short @@ -105,16 +121,16 @@ vm_page_to_pa(const struct vm_page *page) static inline uintptr_t vm_page_direct_va(phys_addr_t pa) { - assert(pa < VM_PAGE_DIRECTMAP_LIMIT); - return ((uintptr_t)pa + VM_MIN_DIRECTMAP_ADDRESS); + assert(pa < PMEM_DIRECTMAP_LIMIT); + return ((uintptr_t)pa + PMAP_MIN_DIRECTMAP_ADDRESS); } static inline phys_addr_t vm_page_direct_pa(uintptr_t va) { - assert(va >= VM_MIN_DIRECTMAP_ADDRESS); - assert(va < VM_MAX_DIRECTMAP_ADDRESS); - return (va - VM_MIN_DIRECTMAP_ADDRESS); + assert(va >= PMAP_MIN_DIRECTMAP_ADDRESS); + assert(va < PMAP_MAX_DIRECTMAP_ADDRESS); + return (va - PMAP_MIN_DIRECTMAP_ADDRESS); } static inline void * @@ -138,6 +154,21 @@ vm_page_get_priv(const struct vm_page *page) return page->priv; } +static inline void +vm_page_link(struct vm_page *page, struct vm_object *object, uint64_t offset) +{ + assert(object != NULL); + page->object = object; + page->offset = offset; +} + +static inline void +vm_page_unlink(struct vm_page *page) +{ + assert(page->object != NULL); + page->object = NULL; +} + /* * Load physical memory into the vm_page module at boot time. * @@ -162,19 +193,6 @@ void vm_page_load_heap(unsigned int zone_index, phys_addr_t start, int vm_page_ready(void); /* - * Set up the vm_page module. - * - * Architecture-specific code must have loaded zones before calling this - * function. Zones must comply with the selector-to-zone-list table, - * e.g. HIGHMEM is loaded if and only if DIRECTMAP, DMA32 and DMA are loaded, - * notwithstanding zone aliasing. - * - * Once this function returns, the vm_page module is ready, and normal - * allocation functions can be used. - */ -void vm_page_setup(void); - -/* * Make the given page managed by the vm_page module. * * If additional memory can be made usable after the VM system is initialized, @@ -192,12 +210,16 @@ struct vm_page * vm_page_lookup(phys_addr_t pa); * * The selector is used to determine the zones from which allocation can * be attempted. + * + * If successful, the returned pages have no references. */ struct vm_page * vm_page_alloc(unsigned int order, unsigned int selector, unsigned short type); /* * Release a block of 2^order physical pages. + * + * The pages must have no references. */ void vm_page_free(struct vm_page *page, unsigned int order); @@ -207,8 +229,60 @@ void vm_page_free(struct vm_page *page, unsigned int order); const char * vm_page_zone_name(unsigned int zone_index); /* - * Display internal information about the module. + * Log information about physical pages. + */ +void vm_page_log_info(void); + +static inline bool +vm_page_referenced(const struct vm_page *page) +{ + return atomic_load(&page->nr_refs, ATOMIC_RELAXED) != 0; +} + +static inline void +vm_page_ref(struct vm_page *page) +{ + unsigned int nr_refs; + + nr_refs = atomic_fetch_add(&page->nr_refs, 1, ATOMIC_RELAXED); + assert(nr_refs != (unsigned int)-1); +} + +static inline void +vm_page_unref(struct vm_page *page) +{ + unsigned int nr_refs; + + nr_refs = atomic_fetch_sub_acq_rel(&page->nr_refs, 1); + assert(nr_refs != 0); + + if (nr_refs == 1) { + vm_page_free(page, 0); + } +} + +static inline int +vm_page_tryref(struct vm_page *page) +{ + unsigned int nr_refs, prev; + + do { + nr_refs = atomic_load(&page->nr_refs, ATOMIC_RELAXED); + + if (nr_refs == 0) { + return ERROR_AGAIN; + } + + prev = atomic_cas_acquire(&page->nr_refs, nr_refs, nr_refs + 1); + } while (prev != nr_refs); + + return 0; +} + +/* + * This init operation provides : + * - module fully initialized */ -void vm_page_info(void); +INIT_OP_DECLARE(vm_page_setup); #endif /* _VM_VM_PAGE_H */ |