diff options
Diffstat (limited to 'kern')
86 files changed, 4434 insertions, 1251 deletions
@@ -15,14 +15,14 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> #include <string.h> #include <kern/arg.h> -#include <kern/assert.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> @@ -35,22 +35,17 @@ static char arg_cmdline[ARG_CMDLINE_MAX_SIZE] __initdata; static const char *arg_cmdline_end __initdata; void __init -arg_setup(const char *cmdline) +arg_set_cmdline(const char *cmdline) { - size_t i, length; - - if (cmdline == NULL) { - arg_cmdline[0] = '\0'; - return; - } - - length = strlen(cmdline); + strlcpy(arg_cmdline, cmdline, sizeof(arg_cmdline)); +} - if ((length + 1) > ARRAY_SIZE(arg_cmdline)) { - panic("arg: command line too long"); - } +static int __init +arg_setup(void) +{ + size_t i, length; - memcpy(arg_cmdline, cmdline, length + 1); + length = strlen(arg_cmdline); for (i = 0; i < length; i++) { if (arg_cmdline[i] == ' ') { @@ -59,12 +54,23 @@ arg_setup(const char *cmdline) } arg_cmdline_end = arg_cmdline + length; + return 0; } +INIT_OP_DEFINE(arg_setup); + void __init -arg_info(void) +arg_log_info(void) { - printf("arg: %s\n", arg_cmdline); + char cmdline[sizeof(arg_cmdline)]; + size_t i; + + for (i = 0; &arg_cmdline[i] < arg_cmdline_end; i++) { + cmdline[i] = (arg_cmdline[i] == '\0') ? ' ' : arg_cmdline[i]; + } + + cmdline[i] = '\0'; + log_info("arg: %s", cmdline); } static const char * __init @@ -27,17 +27,21 @@ #include <stdbool.h> +#include <kern/init.h> + #define ARG_CMDLINE_MAX_SIZE 256 /* - * Initialize the arg module. + * Set the command line string. + * + * This function must be called before calling the kernel main entry point. */ -void arg_setup(const char *cmdline); +void arg_set_cmdline(const char *cmdline); /* - * Display command line information. + * Log command line information. */ -void arg_info(void); +void arg_log_info(void); /* * Return true if an argument with the given name is present in the @@ -54,4 +58,10 @@ bool arg_present(const char *name); */ const char * arg_value(const char *name); +/* + * This init operation provides : + * - command line arguments can be retrieved + */ +INIT_OP_DECLARE(arg_setup); + #endif /* _KERN_ARG_H */ diff --git a/kern/assert.h b/kern/assert.h deleted file mode 100644 index cc8b80ea..00000000 --- a/kern/assert.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2010 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#ifndef _KERN_ASSERT_H -#define _KERN_ASSERT_H - -#define static_assert _Static_assert - -#ifdef NDEBUG -#define assert(expression) ((void)(expression)) -#else /* NDEBUG */ - -#include <kern/macros.h> -#include <kern/panic.h> - -/* - * Panic if the given expression is false. - */ -#define assert(expression) \ -MACRO_BEGIN \ - if (unlikely(!(expression))) { \ - panic("assertion (%s) failed in %s:%d, function %s()", \ - __QUOTE(expression), __FILE__, __LINE__, __func__); \ - } \ -MACRO_END - -#endif /* NDEBUG */ - -#endif /* _KERN_ASSERT_H */ diff --git a/kern/atomic.h b/kern/atomic.h index 63f0ac73..6c0105dd 100644 --- a/kern/atomic.h +++ b/kern/atomic.h @@ -135,7 +135,4 @@ MACRO_END #define atomic_fetch_sub_acq_rel(ptr, val) \ atomic_fetch_sub(ptr, val, ATOMIC_ACQ_REL) -#define atomic_or_acq_rel(ptr, val) atomic_or(ptr, val, ATOMIC_ACQ_REL) -#define atomic_and_acq_rel(ptr, val) atomic_and(ptr, val, ATOMIC_ACQ_REL) - #endif /* _KERN_ATOMIC_H */ diff --git a/kern/bitmap.c b/kern/bitmap.c index d270b9f5..97e497d6 100644 --- a/kern/bitmap.c +++ b/kern/bitmap.c @@ -15,11 +15,11 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <limits.h> #include <string.h> #include <kern/bitmap.h> #include <kern/bitmap_i.h> -#include <kern/limits.h> int bitmap_cmp(const unsigned long *a, const unsigned long *b, int nr_bits) @@ -78,7 +78,7 @@ bitmap_find_next_bit(const unsigned long *bm, int nr_bits, int bit, end = bm + BITMAP_LONGS(nr_bits); if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); nr_bits -= ((bm - start) * LONG_BIT); } diff --git a/kern/bitmap.h b/kern/bitmap.h index a10fb512..9a3d7a0b 100644 --- a/kern/bitmap.h +++ b/kern/bitmap.h @@ -24,11 +24,11 @@ #ifndef _KERN_BITMAP_H #define _KERN_BITMAP_H +#include <limits.h> #include <string.h> #include <kern/atomic.h> #include <kern/bitmap_i.h> -#include <kern/limits.h> #define BITMAP_DECLARE(name, nr_bits) unsigned long name[BITMAP_LONGS(nr_bits)] @@ -65,7 +65,7 @@ static inline void bitmap_set(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } *bm |= bitmap_mask(bit); @@ -75,17 +75,17 @@ static inline void bitmap_set_atomic(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } - atomic_or_acq_rel(bm, bitmap_mask(bit)); + atomic_or(bm, bitmap_mask(bit), ATOMIC_RELEASE); } static inline void bitmap_clear(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } *bm &= ~bitmap_mask(bit); @@ -95,22 +95,32 @@ static inline void bitmap_clear_atomic(unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } - atomic_and_acq_rel(bm, ~bitmap_mask(bit)); + atomic_and(bm, ~bitmap_mask(bit), ATOMIC_RELEASE); } static inline int bitmap_test(const unsigned long *bm, int bit) { if (bit >= LONG_BIT) { - bitmap_lookup(bm, bit); + bitmap_lookup(&bm, &bit); } return ((*bm & bitmap_mask(bit)) != 0); } +static inline int +bitmap_test_atomic(const unsigned long *bm, int bit) +{ + if (bit >= LONG_BIT) { + bitmap_lookup(&bm, &bit); + } + + return ((atomic_load(bm, ATOMIC_ACQUIRE) & bitmap_mask(bit)) != 0); +} + static inline void bitmap_and(unsigned long *a, const unsigned long *b, int nr_bits) { diff --git a/kern/bitmap_i.h b/kern/bitmap_i.h index 39a330c9..dc91a0ab 100644 --- a/kern/bitmap_i.h +++ b/kern/bitmap_i.h @@ -18,7 +18,8 @@ #ifndef _KERN_BITMAP_I_H #define _KERN_BITMAP_I_H -#include <kern/limits.h> +#include <limits.h> + #include <kern/macros.h> #define BITMAP_LONGS(nr_bits) DIV_CEIL(nr_bits, LONG_BIT) @@ -29,13 +30,13 @@ * * Implemented as a macro for const-correctness. */ -#define bitmap_lookup(bm, bit) \ +#define bitmap_lookup(bmp, bitp) \ MACRO_BEGIN \ int i; \ \ - i = BITMAP_LONGS((bit) + 1) - 1; \ - (bm) += i; \ - (bit) -= i * LONG_BIT; \ + i = BITMAP_LONGS(*(bitp) + 1) - 1; \ + *(bmp) += i; \ + *(bitp) -= i * LONG_BIT; \ MACRO_END static inline unsigned long diff --git a/kern/cbuf.c b/kern/cbuf.c index 17369afe..ea848c56 100644 --- a/kern/cbuf.c +++ b/kern/cbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Richard Braun. + * Copyright (c) 2015-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,16 +15,19 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <kern/assert.h> +#include <assert.h> +#include <stddef.h> +#include <string.h> + #include <kern/cbuf.h> #include <kern/error.h> #include <kern/macros.h> /* Negative close to 0 so that an overflow occurs early */ -#define CBUF_INIT_INDEX ((unsigned long)-500) +#define CBUF_INIT_INDEX ((size_t)-500) void -cbuf_init(struct cbuf *cbuf, char *buf, unsigned long capacity) +cbuf_init(struct cbuf *cbuf, char *buf, size_t capacity) { assert(ISP2(capacity)); @@ -34,24 +37,29 @@ cbuf_init(struct cbuf *cbuf, char *buf, unsigned long capacity) cbuf->end = cbuf->start; } -static unsigned long -cbuf_index(const struct cbuf *cbuf, unsigned long abs_index) +static size_t +cbuf_index(const struct cbuf *cbuf, size_t abs_index) { return abs_index & (cbuf->capacity - 1); } -void -cbuf_push(struct cbuf *cbuf, char byte) +static void +cbuf_update_start(struct cbuf *cbuf) { - cbuf->buf[cbuf_index(cbuf, cbuf->end)] = byte; - cbuf->end++; - /* Mind integer overflows */ if (cbuf_size(cbuf) > cbuf->capacity) { cbuf->start = cbuf->end - cbuf->capacity; } } +void +cbuf_push(struct cbuf *cbuf, char byte) +{ + cbuf->buf[cbuf_index(cbuf, cbuf->end)] = byte; + cbuf->end++; + cbuf_update_start(cbuf); +} + int cbuf_pop(struct cbuf *cbuf, char *bytep) { @@ -65,13 +73,76 @@ cbuf_pop(struct cbuf *cbuf, char *bytep) } int -cbuf_read(const struct cbuf *cbuf, unsigned long index, char *bytep) +cbuf_write(struct cbuf *cbuf, size_t index, const void *buf, size_t size) { - /* Mind integer overflows */ - if ((cbuf->end - index - 1) >= cbuf_size(cbuf)) { + char *start, *end, *buf_end; + size_t new_end, skip; + + if (!cbuf_range_valid(cbuf, index, cbuf->end)) { return ERROR_INVAL; } - *bytep = cbuf->buf[cbuf_index(cbuf, index)]; + new_end = index + size; + + if (!cbuf_range_valid(cbuf, cbuf->start, new_end)) { + cbuf->end = new_end; + + if (size > cbuf_capacity(cbuf)) { + skip = size - cbuf_capacity(cbuf); + buf += skip; + index += skip; + size = cbuf_capacity(cbuf); + } + } + + start = &cbuf->buf[cbuf_index(cbuf, index)]; + end = start + size; + buf_end = cbuf->buf + cbuf->capacity; + + if ((end <= cbuf->buf) || (end > buf_end)) { + skip = buf_end - start; + memcpy(start, buf, skip); + buf += skip; + start = cbuf->buf; + size -= skip; + } + + memcpy(start, buf, size); + cbuf_update_start(cbuf); + return 0; +} + +int +cbuf_read(const struct cbuf *cbuf, size_t index, void *buf, size_t *sizep) +{ + const char *start, *end, *buf_end; + size_t size; + + /* At least one byte must be available */ + if (!cbuf_range_valid(cbuf, index, index + 1)) { + return ERROR_INVAL; + } + + size = cbuf->end - index; + + if (*sizep > size) { + *sizep = size; + } + + start = &cbuf->buf[cbuf_index(cbuf, index)]; + end = start + *sizep; + buf_end = cbuf->buf + cbuf->capacity; + + if ((end > cbuf->buf) && (end <= buf_end)) { + size = *sizep; + } else { + size = buf_end - start; + memcpy(buf, start, size); + buf += size; + start = cbuf->buf; + size = *sizep - size; + } + + memcpy(buf, start, size); return 0; } diff --git a/kern/cbuf.h b/kern/cbuf.h index 3ea38418..4d69334d 100644 --- a/kern/cbuf.h +++ b/kern/cbuf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Richard Braun. + * Copyright (c) 2015-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,6 +21,9 @@ #ifndef _KERN_CBUF_H #define _KERN_CBUF_H +#include <stdbool.h> +#include <stddef.h> + /* * Circular buffer descriptor. * @@ -29,30 +32,30 @@ */ struct cbuf { char *buf; - unsigned long capacity; - unsigned long start; - unsigned long end; + size_t capacity; + size_t start; + size_t end; }; -static inline unsigned long +static inline size_t cbuf_capacity(const struct cbuf *cbuf) { return cbuf->capacity; } -static inline unsigned long +static inline size_t cbuf_start(const struct cbuf *cbuf) { return cbuf->start; } -static inline unsigned long +static inline size_t cbuf_end(const struct cbuf *cbuf) { return cbuf->end; } -static inline unsigned long +static inline size_t cbuf_size(const struct cbuf *cbuf) { return cbuf->end - cbuf->start; @@ -64,13 +67,21 @@ cbuf_clear(struct cbuf *cbuf) cbuf->start = cbuf->end; } +static inline bool +cbuf_range_valid(const struct cbuf *cbuf, size_t start, size_t end) +{ + return (((end - start) <= cbuf_size(cbuf)) + && ((start - cbuf->start) <= cbuf_size(cbuf)) + && ((cbuf->end - end) <= cbuf_size(cbuf))); +} + /* * Initialize a circular buffer. * * The descriptor is set to use the given buffer for storage. Capacity * must be a power-of-two. */ -void cbuf_init(struct cbuf *cbuf, char *buf, unsigned long capacity); +void cbuf_init(struct cbuf *cbuf, char *buf, size_t capacity); /* * Append a byte to a circular buffer. @@ -90,12 +101,25 @@ void cbuf_push(struct cbuf *cbuf, char byte); int cbuf_pop(struct cbuf *cbuf, char *bytep); /* - * Read a byte at a specific location. + * Write into a circular buffer at a specific location. + * + * If the given index is outside buffer boundaries, ERROR_INVAL is returned. + * Otherwise size bytes are copied into the circular buffer. If the range + * in the circular buffer goes beyond its end, the end index is updated as + * appropriate. If the buffer is full when extending its end, the oldest + * bytes are overwritten and the start index is updated accordingly. + */ +int cbuf_write(struct cbuf *cbuf, size_t index, const void *buf, size_t size); + +/* + * Read from a circular buffer at a specific location. * * If the given index is outside buffer boundaries, ERROR_INVAL is returned. - * Otherwise the byte is stored at the bytep address and 0 is returned. - * The buffer isn't changed by this operation. + * Otherwise at most *sizep bytes are copied into the given byte buffer, + * and *sizep is updated to the number of bytes actually copied. + * + * The circular buffer isn't changed by this operation. */ -int cbuf_read(const struct cbuf *cbuf, unsigned long index, char *bytep); +int cbuf_read(const struct cbuf *cbuf, size_t index, void *buf, size_t *sizep); #endif /* _KERN_CBUF_H */ diff --git a/kern/condition.c b/kern/condition.c index 4a837f4e..c8ea5f39 100644 --- a/kern/condition.c +++ b/kern/condition.c @@ -18,10 +18,10 @@ * Locking order : mutex -> sleep queue */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <kern/assert.h> #include <kern/condition.h> #include <kern/condition_types.h> #include <kern/mutex.h> diff --git a/kern/console.c b/kern/console.c index c82e7dc9..1f9a8a9c 100644 --- a/kern/console.c +++ b/kern/console.c @@ -15,20 +15,21 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> #include <string.h> #include <kern/arg.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/init.h> #include <kern/console.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/mutex.h> #include <kern/spinlock.h> #include <kern/thread.h> +#include <machine/boot.h> #include <machine/cpu.h> /* @@ -66,13 +67,28 @@ console_init(struct console *console, const char *name, strlcpy(console->name, name, sizeof(console->name)); } +static int +console_process_ctrl_char(struct console *console, char c) +{ + switch (c) { + case CONSOLE_SCROLL_UP: + case CONSOLE_SCROLL_DOWN: + break; + default: + return ERROR_INVAL; + } + + console->ops->putc(console, c); + return 0; +} + static void console_putc(struct console *console, char c) { unsigned long flags; spinlock_lock_intr_save(&console->lock, &flags); - console->ops->putc(console_dev, c); + console->ops->putc(console, c); spinlock_unlock_intr_restore(&console->lock, flags); } @@ -96,7 +112,11 @@ console_getc(struct console *console) error = cbuf_pop(&console->recvbuf, &c); if (!error) { - break; + error = console_process_ctrl_char(console, c); + + if (error) { + break; + } } thread_sleep(&console->lock, console, "consgetc"); @@ -110,13 +130,28 @@ out: return c; } -void __init -console_setup(void) +static int __init +console_bootstrap(void) { list_init(&console_devs); console_name = arg_value("console"); + return 0; +} + +INIT_OP_DEFINE(console_bootstrap, + INIT_OP_DEP(arg_setup, true), + INIT_OP_DEP(log_setup, true)); + +static int __init +console_setup(void) +{ + return 0; } +INIT_OP_DEFINE(console_setup, + INIT_OP_DEP(boot_setup_console, true), + INIT_OP_DEP(thread_setup, true)); + void __init console_register(struct console *console) { @@ -128,30 +163,35 @@ console_register(struct console *console) console_dev = console; } - printf("console: %s registered\n", console->name); + log_info("console: %s registered", console->name); if (console == console_dev) { - printf("console: %s selected as active console\n", console->name); + log_info("console: %s selected as active console", console->name); } } void -console_intr(struct console *console, char c) +console_intr(struct console *console, const char *s) { assert(!cpu_intr_enabled()); - spinlock_lock(&console->lock); - - if (cbuf_size(&console->recvbuf) == cbuf_capacity(&console->recvbuf)) { - goto out; + if (*s == '\0') { + return; } - cbuf_push(&console->recvbuf, c); + spinlock_lock(&console->lock); - if ((console->waiter != NULL) && (console->waiter != thread_self())) { - thread_wakeup(console->waiter); + while (*s != '\0') { + if (cbuf_size(&console->recvbuf) == cbuf_capacity(&console->recvbuf)) { + goto out; + } + + cbuf_push(&console->recvbuf, *s); + s++; } + thread_wakeup(console->waiter); + out: spinlock_unlock(&console->lock); } diff --git a/kern/console.h b/kern/console.h index b967c0dd..15789974 100644 --- a/kern/console.h +++ b/kern/console.h @@ -22,10 +22,14 @@ #define _KERN_CONSOLE_H #include <kern/cbuf.h> +#include <kern/init.h> #include <kern/list.h> #include <kern/spinlock.h> #include <kern/thread.h> +#define CONSOLE_SCROLL_UP 0x12 /* DC2 */ +#define CONSOLE_SCROLL_DOWN 0x14 /* DC4 */ + struct console; struct console_ops { @@ -59,11 +63,6 @@ void console_init(struct console *console, const char *name, const struct console_ops *ops); /* - * Initialize the console module. - */ -void console_setup(void); - -/* * Register a console device. * * The given console must be initialized before calling this function. @@ -81,7 +80,7 @@ void console_register(struct console *console); * * Interrupts must be disabled when calling this function. */ -void console_intr(struct console *console, char c); +void console_intr(struct console *console, const char *s); /* * Write/read a single character to all registered console devices. @@ -92,4 +91,17 @@ void console_intr(struct console *console, char c); void console_putchar(char c); char console_getchar(void); +/* + * This init operation provides : + * - registration of consoles + */ +INIT_OP_DECLARE(console_bootstrap); + +/* + * This init operation provides : + * - all consoles have been registered + * - module fully initialized + */ +INIT_OP_DECLARE(console_setup); + #endif /* _KERN_CONSOLE_H */ diff --git a/kern/cpumap.c b/kern/cpumap.c index 5582e681..d166b237 100644 --- a/kern/cpumap.c +++ b/kern/cpumap.c @@ -20,15 +20,16 @@ #include <kern/bitmap.h> #include <kern/cpumap.h> #include <kern/error.h> +#include <kern/init.h> #include <kern/kmem.h> -#include <kern/param.h> +#include <kern/macros.h> #include <machine/cpu.h> static struct cpumap cpumap_active_cpus __read_mostly = { { 1 } }; static struct kmem_cache cpumap_cache; -void +static int __init cpumap_setup(void) { unsigned int i, nr_cpus; @@ -40,8 +41,14 @@ cpumap_setup(void) for (i = 0; i < nr_cpus; i++) { cpumap_set(&cpumap_active_cpus, i); } + + return 0; } +INIT_OP_DEFINE(cpumap_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(cpu_mp_probe, true)); + const struct cpumap * cpumap_all(void) { diff --git a/kern/cpumap.h b/kern/cpumap.h index 1843a99a..fd07afc1 100644 --- a/kern/cpumap.h +++ b/kern/cpumap.h @@ -27,6 +27,7 @@ #define _KERN_CPUMAP_H #include <kern/bitmap.h> +#include <kern/init.h> struct cpumap { BITMAP_DECLARE(cpus, X15_MAX_CPUS); @@ -135,11 +136,6 @@ cpumap_find_first_zero(const struct cpumap *cpumap) bitmap_for_each_zero((cpumap)->cpus, X15_MAX_CPUS, index) /* - * Initialize the cpumap module. - */ -void cpumap_setup(void); - -/* * Return a cpumap representing all active processors. * * Until the cpumap module is initialized, the cpumap returned by this @@ -167,4 +163,11 @@ void cpumap_destroy(struct cpumap *cpumap); */ int cpumap_check(const struct cpumap *cpumap); +/* + * This init operation provides : + * - cpumap creation + * - module fully initialized + */ +INIT_OP_DECLARE(cpumap_setup); + #endif /* _KERN_CPUMAP_H */ diff --git a/kern/error.c b/kern/error.c index 02a26d0c..f5d43e48 100644 --- a/kern/error.c +++ b/kern/error.c @@ -15,6 +15,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <stddef.h> + #include <kern/error.h> #include <kern/panic.h> @@ -33,13 +35,17 @@ error_str(int error) case ERROR_BUSY: return "device or resource busy"; case ERROR_FAULT: - return "Bad address"; + return "bad address"; case ERROR_NODEV: - return "No such device"; + return "no such device"; case ERROR_EXIST: - return "Entry exists"; + return "entry exists"; case ERROR_IO: - return "Input/output error"; + return "input/output error"; + case ERROR_SRCH: + return "no such process"; + case ERROR_TIMEDOUT: + return "timeout error"; default: return "unknown error"; } diff --git a/kern/error.h b/kern/error.h index 96a26206..a77db973 100644 --- a/kern/error.h +++ b/kern/error.h @@ -18,14 +18,16 @@ #ifndef _KERN_ERROR_H #define _KERN_ERROR_H -#define ERROR_NOMEM 1 -#define ERROR_AGAIN 2 -#define ERROR_INVAL 3 -#define ERROR_BUSY 4 -#define ERROR_FAULT 5 -#define ERROR_NODEV 6 -#define ERROR_EXIST 7 -#define ERROR_IO 8 +#define ERROR_NOMEM 1 +#define ERROR_AGAIN 2 +#define ERROR_INVAL 3 +#define ERROR_BUSY 4 +#define ERROR_FAULT 5 +#define ERROR_NODEV 6 +#define ERROR_EXIST 7 +#define ERROR_IO 8 +#define ERROR_SRCH 9 +#define ERROR_TIMEDOUT 10 /* * Return a string describing the given error. diff --git a/kern/fmt.c b/kern/fmt.c new file mode 100644 index 00000000..d1067e5c --- /dev/null +++ b/kern/fmt.c @@ -0,0 +1,1467 @@ +/* + * Copyright (c) 2010-2017 Richard Braun. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * + * Upstream site with license notes : + * http://git.sceen.net/rbraun/librbraun.git/ + */ + +#include <assert.h> +#include <limits.h> +#include <stdbool.h> +#include <stdarg.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> + +#include <kern/error.h> +#include <kern/fmt.h> +#include <kern/macros.h> +#include <kern/types.h> + +/* + * Size for the temporary number buffer. The minimum base is 8 so 3 bits + * are consumed per digit. Add one to round up. The conversion algorithm + * doesn't use the null byte. + */ +#define FMT_MAX_NUM_SIZE (((sizeof(unsigned long long) * CHAR_BIT) / 3) + 1) + +/* + * Special size for fmt_vsnprintf(), used when the buffer size is unknown. + */ +#define FMT_NOLIMIT ((size_t)-1) + +/* + * Formatting flags. + * + * FMT_FORMAT_LOWER must be 0x20 as it is OR'd with fmt_digits, eg. + * '0': 0x30 | 0x20 => 0x30 ('0') + * 'A': 0x41 | 0x20 => 0x61 ('a') + */ +#define FMT_FORMAT_ALT_FORM 0x0001 /* "Alternate form" */ +#define FMT_FORMAT_ZERO_PAD 0x0002 /* Zero padding on the left */ +#define FMT_FORMAT_LEFT_JUSTIFY 0x0004 /* Align text on the left */ +#define FMT_FORMAT_BLANK 0x0008 /* Blank space before positive number */ +#define FMT_FORMAT_SIGN 0x0010 /* Always place a sign (either + or -) */ +#define FMT_FORMAT_LOWER 0x0020 /* To lowercase (for %x) */ +#define FMT_FORMAT_CONV_SIGNED 0x0040 /* Format specifies signed conversion */ +#define FMT_FORMAT_DISCARD 0x0080 /* Discard output (scanf) */ +#define FMT_FORMAT_CHECK_WIDTH 0x0100 /* Check field width (scanf) */ + +enum { + FMT_MODIFIER_NONE, + FMT_MODIFIER_CHAR, + FMT_MODIFIER_SHORT, + FMT_MODIFIER_LONG, + FMT_MODIFIER_LONGLONG, + FMT_MODIFIER_PTR, /* Used only for %p */ + FMT_MODIFIER_SIZE, + FMT_MODIFIER_PTRDIFF, +}; + +enum { + FMT_SPECIFIER_INVALID, + FMT_SPECIFIER_INT, + FMT_SPECIFIER_CHAR, + FMT_SPECIFIER_STR, + FMT_SPECIFIER_NRCHARS, + FMT_SPECIFIER_PERCENT, +}; + +/* + * Note that copies of the original va_list object are made, because va_arg() + * may not reliably be used by different callee functions, and despite the + * standard explicitely allowing pointers to va_list objects, it's apparently + * very difficult for implementations to provide and is best avoided. + */ + +struct fmt_sprintf_state { + const char *format; + va_list ap; + unsigned int flags; + int width; + int precision; + unsigned int modifier; + unsigned int specifier; + unsigned int base; + char *str; + char *start; + char *end; +}; + +struct fmt_sscanf_state { + const char *str; + const char *start; + const char *format; + va_list ap; + unsigned int flags; + int width; + int precision; + unsigned int modifier; + unsigned int specifier; + unsigned int base; + int nr_convs; +}; + +static const char fmt_digits[] = "0123456789ABCDEF"; + +static char +fmt_consume(const char **strp) +{ + char c; + + c = **strp; + (*strp)++; + return c; +} + +static void +fmt_restore(const char **strp) +{ + (*strp)--; +} + +static void +fmt_vsnprintf_produce(char **strp, char *end, char c) +{ + if (*strp < end) { + **strp = c; + } + + (*strp)++; +} + +static bool +fmt_isdigit(char c) +{ + return (c >= '0') && (c <= '9'); +} + +static bool +fmt_isxdigit(char c) +{ + return fmt_isdigit(c) + || ((c >= 'a') && (c <= 'f')) + || ((c >= 'A') && (c <= 'F')); +} + +static void +fmt_sprintf_state_init(struct fmt_sprintf_state *state, + char *str, size_t size, + const char *format, va_list ap) +{ + state->format = format; + va_copy(state->ap, ap); + state->str = str; + state->start = str; + + if (size == 0) { + state->end = NULL; + } else if (size == FMT_NOLIMIT) { + state->end = (char *)-1; + } else { + state->end = state->start + size - 1; + } +} + +static int +fmt_sprintf_state_finalize(struct fmt_sprintf_state *state) +{ + va_end(state->ap); + + if (state->str < state->end) { + *state->str = '\0'; + } else if (state->end != NULL) { + *state->end = '\0'; + } + + return state->str - state->start; +} + +static char +fmt_sprintf_state_consume_format(struct fmt_sprintf_state *state) +{ + return fmt_consume(&state->format); +} + +static void +fmt_sprintf_state_restore_format(struct fmt_sprintf_state *state) +{ + fmt_restore(&state->format); +} + +static void +fmt_sprintf_state_consume_flags(struct fmt_sprintf_state *state) +{ + bool found; + char c; + + found = true; + state->flags = 0; + + do { + c = fmt_sprintf_state_consume_format(state); + + switch (c) { + case '#': + state->flags |= FMT_FORMAT_ALT_FORM; + break; + case '0': + state->flags |= FMT_FORMAT_ZERO_PAD; + break; + case '-': + state->flags |= FMT_FORMAT_LEFT_JUSTIFY; + break; + case ' ': + state->flags |= FMT_FORMAT_BLANK; + break; + case '+': + state->flags |= FMT_FORMAT_SIGN; + break; + default: + found = false; + break; + } + } while (found); + + fmt_sprintf_state_restore_format(state); +} + +static void +fmt_sprintf_state_consume_width(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_sprintf_state_consume_format(state); + + if (fmt_isdigit(c)) { + state->width = 0; + + do { + state->width = state->width * 10 + (c - '0'); + c = fmt_sprintf_state_consume_format(state); + } while (fmt_isdigit(c)); + + fmt_sprintf_state_restore_format(state); + } else if (c == '*') { + state->width = va_arg(state->ap, int); + + if (state->width < 0) { + state->flags |= FMT_FORMAT_LEFT_JUSTIFY; + state->width = -state->width; + } + } else { + state->width = 0; + fmt_sprintf_state_restore_format(state); + } +} + +static void +fmt_sprintf_state_consume_precision(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_sprintf_state_consume_format(state); + + if (c == '.') { + c = fmt_sprintf_state_consume_format(state); + + if (fmt_isdigit(c)) { + state->precision = 0; + + do { + state->precision = state->precision * 10 + (c - '0'); + c = fmt_sprintf_state_consume_format(state); + } while (fmt_isdigit(c)); + + fmt_sprintf_state_restore_format(state); + } else if (c == '*') { + state->precision = va_arg(state->ap, int); + + if (state->precision < 0) { + state->precision = 0; + } + } else { + state->precision = 0; + fmt_sprintf_state_restore_format(state); + } + } else { + /* precision is >= 0 only if explicit */ + state->precision = -1; + fmt_sprintf_state_restore_format(state); + } +} + +static void +fmt_sprintf_state_consume_modifier(struct fmt_sprintf_state *state) +{ + char c, c2; + + c = fmt_sprintf_state_consume_format(state); + + switch (c) { + case 'h': + case 'l': + c2 = fmt_sprintf_state_consume_format(state); + + if (c == c2) { + state->modifier = (c == 'h') ? FMT_MODIFIER_CHAR + : FMT_MODIFIER_LONGLONG; + } else { + state->modifier = (c == 'h') ? FMT_MODIFIER_SHORT + : FMT_MODIFIER_LONG; + fmt_sprintf_state_restore_format(state); + } + + break; + case 'z': + state->modifier = FMT_MODIFIER_SIZE; + case 't': + state->modifier = FMT_MODIFIER_PTRDIFF; + break; + default: + state->modifier = FMT_MODIFIER_NONE; + fmt_sprintf_state_restore_format(state); + break; + } +} + +static void +fmt_sprintf_state_consume_specifier(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_sprintf_state_consume_format(state); + + switch (c) { + case 'd': + case 'i': + state->flags |= FMT_FORMAT_CONV_SIGNED; + case 'u': + state->base = 10; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'o': + state->base = 8; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'p': + state->flags |= FMT_FORMAT_ALT_FORM; + state->modifier = FMT_MODIFIER_PTR; + case 'x': + state->flags |= FMT_FORMAT_LOWER; + case 'X': + state->base = 16; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'c': + state->specifier = FMT_SPECIFIER_CHAR; + break; + case 's': + state->specifier = FMT_SPECIFIER_STR; + break; + case 'n': + state->specifier = FMT_SPECIFIER_NRCHARS; + break; + case '%': + state->specifier = FMT_SPECIFIER_PERCENT; + break; + default: + state->specifier = FMT_SPECIFIER_INVALID; + fmt_sprintf_state_restore_format(state); + break; + } +} + +static void +fmt_sprintf_state_produce_raw_char(struct fmt_sprintf_state *state, char c) +{ + fmt_vsnprintf_produce(&state->str, state->end, c); +} + +static int +fmt_sprintf_state_consume(struct fmt_sprintf_state *state) +{ + char c; + + c = fmt_consume(&state->format); + + if (c == '\0') { + return ERROR_IO; + } + + if (c != '%') { + fmt_sprintf_state_produce_raw_char(state, c); + return ERROR_AGAIN; + } + + fmt_sprintf_state_consume_flags(state); + fmt_sprintf_state_consume_width(state); + fmt_sprintf_state_consume_precision(state); + fmt_sprintf_state_consume_modifier(state); + fmt_sprintf_state_consume_specifier(state); + return 0; +} + +static void +fmt_sprintf_state_produce_int(struct fmt_sprintf_state *state) +{ + char c, sign, tmp[FMT_MAX_NUM_SIZE]; + unsigned int r, mask, shift; + unsigned long long n; + int i; + + switch (state->modifier) { + case FMT_MODIFIER_CHAR: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = (signed char)va_arg(state->ap, int); + } else { + n = (unsigned char)va_arg(state->ap, int); + } + + break; + case FMT_MODIFIER_SHORT: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = (short)va_arg(state->ap, int); + } else { + n = (unsigned short)va_arg(state->ap, int); + } + + break; + case FMT_MODIFIER_LONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, long); + } else { + n = va_arg(state->ap, unsigned long); + } + + break; + case FMT_MODIFIER_LONGLONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, long long); + } else { + n = va_arg(state->ap, unsigned long long); + } + + break; + case FMT_MODIFIER_PTR: + n = (uintptr_t)va_arg(state->ap, void *); + break; + case FMT_MODIFIER_SIZE: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, ssize_t); + } else { + n = va_arg(state->ap, size_t); + } + + break; + case FMT_MODIFIER_PTRDIFF: + n = va_arg(state->ap, ptrdiff_t); + break; + default: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + n = va_arg(state->ap, int); + } else { + n = va_arg(state->ap, unsigned int); + } + + break; + } + + if ((state->flags & FMT_FORMAT_LEFT_JUSTIFY) || (state->precision >= 0)) { + state->flags &= ~FMT_FORMAT_ZERO_PAD; + } + + sign = '\0'; + + if (state->flags & FMT_FORMAT_ALT_FORM) { + /* '0' for octal */ + state->width--; + + /* '0x' or '0X' for hexadecimal */ + if (state->base == 16) { + state->width--; + } + } else if (state->flags & FMT_FORMAT_CONV_SIGNED) { + if ((long long)n < 0) { + sign = '-'; + state->width--; + n = -(long long)n; + } else if (state->flags & FMT_FORMAT_SIGN) { + /* FMT_FORMAT_SIGN must precede FMT_FORMAT_BLANK. */ + sign = '+'; + state->width--; + } else if (state->flags & FMT_FORMAT_BLANK) { + sign = ' '; + state->width--; + } + } + + /* Conversion, in reverse order */ + + i = 0; + + if (n == 0) { + if (state->precision != 0) { + tmp[i] = '0'; + i++; + } + } else if (state->base == 10) { + /* + * Try to avoid 64 bits operations if the processor doesn't + * support them. Note that even when using modulus and + * division operators close to each other, the compiler may + * forge two functions calls to compute the quotient and the + * remainder, whereas processor instructions are generally + * correctly used once, giving both results at once, through + * plain or reciprocal division. + */ +#ifndef __LP64__ + if (state->modifier == FMT_MODIFIER_LONGLONG) { +#endif /* __LP64__ */ + do { + r = n % 10; + n /= 10; + tmp[i] = fmt_digits[r]; + i++; + } while (n != 0); +#ifndef __LP64__ + } else { + unsigned long m; + + m = (unsigned long)n; + + do { + r = m % 10; + m /= 10; + tmp[i] = fmt_digits[r]; + i++; + } while (m != 0); + } +#endif /* __LP64__ */ + } else { + mask = state->base - 1; + shift = (state->base == 8) ? 3 : 4; + + do { + r = n & mask; + n >>= shift; + tmp[i] = fmt_digits[r] | (state->flags & FMT_FORMAT_LOWER); + i++; + } while (n != 0); + } + + if (i > state->precision) { + state->precision = i; + } + + state->width -= state->precision; + + if (!(state->flags & (FMT_FORMAT_LEFT_JUSTIFY | FMT_FORMAT_ZERO_PAD))) { + while (state->width > 0) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } + + state->width--; + } + + if (state->flags & FMT_FORMAT_ALT_FORM) { + fmt_sprintf_state_produce_raw_char(state, '0'); + + if (state->base == 16) { + c = 'X' | (state->flags & FMT_FORMAT_LOWER); + fmt_sprintf_state_produce_raw_char(state, c); + } + } else if (sign != '\0') { + fmt_sprintf_state_produce_raw_char(state, sign); + } + + if (!(state->flags & FMT_FORMAT_LEFT_JUSTIFY)) { + c = (state->flags & FMT_FORMAT_ZERO_PAD) ? '0' : ' '; + + while (state->width > 0) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, c); + } + + state->width--; + } + + while (i < state->precision) { + state->precision--; + fmt_sprintf_state_produce_raw_char(state, '0'); + } + + state->precision--; + + while (i > 0) { + i--; + fmt_sprintf_state_produce_raw_char(state, tmp[i]); + } + + while (state->width > 0) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } + + state->width--; +} + +static void +fmt_sprintf_state_produce_char(struct fmt_sprintf_state *state) +{ + char c; + + c = va_arg(state->ap, int); + + if (!(state->flags & FMT_FORMAT_LEFT_JUSTIFY)) { + for (;;) { + state->width--; + + if (state->width <= 0) { + break; + } + + fmt_sprintf_state_produce_raw_char(state, ' '); + } + } + + fmt_sprintf_state_produce_raw_char(state, c); + + for (;;) { + state->width--; + + if (state->width <= 0) { + break; + } + + fmt_sprintf_state_produce_raw_char(state, ' '); + } +} + +static void +fmt_sprintf_state_produce_str(struct fmt_sprintf_state *state) +{ + int i, len; + char *s; + + s = va_arg(state->ap, char *); + + if (s == NULL) { + s = "(null)"; + } + + len = 0; + + for (len = 0; s[len] != '\0'; len++) { + if (len == state->precision) { + break; + } + } + + if (!(state->flags & FMT_FORMAT_LEFT_JUSTIFY)) { + while (len < state->width) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } + } + + for (i = 0; i < len; i++) { + fmt_sprintf_state_produce_raw_char(state, *s); + s++; + } + + while (len < state->width) { + state->width--; + fmt_sprintf_state_produce_raw_char(state, ' '); + } +} + +static void +fmt_sprintf_state_produce_nrchars(struct fmt_sprintf_state *state) +{ + if (state->modifier == FMT_MODIFIER_CHAR) { + signed char *ptr = va_arg(state->ap, signed char *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_SHORT) { + short *ptr = va_arg(state->ap, short *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_LONG) { + long *ptr = va_arg(state->ap, long *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_LONGLONG) { + long long *ptr = va_arg(state->ap, long long *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_SIZE) { + ssize_t *ptr = va_arg(state->ap, ssize_t *); + *ptr = state->str - state->start; + } else if (state->modifier == FMT_MODIFIER_PTRDIFF) { + ptrdiff_t *ptr = va_arg(state->ap, ptrdiff_t *); + *ptr = state->str - state->start; + } else { + int *ptr = va_arg(state->ap, int *); + *ptr = state->str - state->start; + } +} + +static void +fmt_sprintf_state_produce(struct fmt_sprintf_state *state) +{ + switch (state->specifier) { + case FMT_SPECIFIER_INT: + fmt_sprintf_state_produce_int(state); + break; + case FMT_SPECIFIER_CHAR: + fmt_sprintf_state_produce_char(state); + break; + case FMT_SPECIFIER_STR: + fmt_sprintf_state_produce_str(state); + break; + case FMT_SPECIFIER_NRCHARS: + fmt_sprintf_state_produce_nrchars(state); + break; + case FMT_SPECIFIER_PERCENT: + case FMT_SPECIFIER_INVALID: + fmt_sprintf_state_produce_raw_char(state, '%'); + break; + } +} + +int +fmt_sprintf(char *str, const char *format, ...) +{ + va_list ap; + int length; + + va_start(ap, format); + length = fmt_vsprintf(str, format, ap); + va_end(ap); + + return length; +} + +int +fmt_vsprintf(char *str, const char *format, va_list ap) +{ + return fmt_vsnprintf(str, FMT_NOLIMIT, format, ap); +} + +int +fmt_snprintf(char *str, size_t size, const char *format, ...) +{ + va_list ap; + int length; + + va_start(ap, format); + length = fmt_vsnprintf(str, size, format, ap); + va_end(ap); + + return length; +} + +int +fmt_vsnprintf(char *str, size_t size, const char *format, va_list ap) +{ + struct fmt_sprintf_state state; + int error; + + fmt_sprintf_state_init(&state, str, size, format, ap); + + for (;;) { + error = fmt_sprintf_state_consume(&state); + + if (error == ERROR_AGAIN) { + continue; + } else if (error) { + break; + } + + fmt_sprintf_state_produce(&state); + } + + return fmt_sprintf_state_finalize(&state); +} + +static char +fmt_atoi(char c) +{ + assert(fmt_isxdigit(c)); + + if (fmt_isdigit(c)) { + return c - '0'; + } else if (c >= 'a' && c <= 'f') { + return 10 + (c - 'a'); + } else { + return 10 + (c - 'A'); + } +} + +static bool +fmt_isspace(char c) +{ + if (c == ' ') { + return true; + } + + if ((c >= '\t') && (c <= '\f')) { + return true; + } + + return false; +} + +static void +fmt_skip(const char **strp) +{ + while (fmt_isspace(**strp)) { + (*strp)++; + } +} + +static void +fmt_sscanf_state_init(struct fmt_sscanf_state *state, const char *str, + const char *format, va_list ap) +{ + state->str = str; + state->start = str; + state->format = format; + state->flags = 0; + state->width = 0; + va_copy(state->ap, ap); + state->nr_convs = 0; +} + +static int +fmt_sscanf_state_finalize(struct fmt_sscanf_state *state) +{ + va_end(state->ap); + return state->nr_convs; +} + +static void +fmt_sscanf_state_report_conv(struct fmt_sscanf_state *state) +{ + if (state->nr_convs == EOF) { + state->nr_convs = 1; + return; + } + + state->nr_convs++; +} + +static void +fmt_sscanf_state_report_error(struct fmt_sscanf_state *state) +{ + if (state->nr_convs != 0) { + return; + } + + state->nr_convs = EOF; +} + +static void +fmt_sscanf_state_skip_space(struct fmt_sscanf_state *state) +{ + fmt_skip(&state->str); +} + +static char +fmt_sscanf_state_consume_string(struct fmt_sscanf_state *state) +{ + char c; + + c = fmt_consume(&state->str); + + if (state->flags & FMT_FORMAT_CHECK_WIDTH) { + if (state->width == 0) { + c = EOF; + } else { + state->width--; + } + } + + return c; +} + +static void +fmt_sscanf_state_restore_string(struct fmt_sscanf_state *state) +{ + fmt_restore(&state->str); +} + +static char +fmt_sscanf_state_consume_format(struct fmt_sscanf_state *state) +{ + return fmt_consume(&state->format); +} + +static void +fmt_sscanf_state_restore_format(struct fmt_sscanf_state *state) +{ + fmt_restore(&state->format); +} + +static void +fmt_sscanf_state_consume_flags(struct fmt_sscanf_state *state) +{ + bool found; + char c; + + found = true; + state->flags = 0; + + do { + c = fmt_sscanf_state_consume_format(state); + + switch (c) { + case '*': + state->flags |= FMT_FORMAT_DISCARD; + break; + default: + found = false; + break; + } + } while (found); + + fmt_sscanf_state_restore_format(state); +} + +static void +fmt_sscanf_state_consume_width(struct fmt_sscanf_state *state) +{ + char c; + + state->width = 0; + + for (;;) { + c = fmt_sscanf_state_consume_format(state); + + if (!fmt_isdigit(c)) { + break; + } + + state->width = state->width * 10 + (c - '0'); + } + + if (state->width != 0) { + state->flags |= FMT_FORMAT_CHECK_WIDTH; + } + + fmt_sscanf_state_restore_format(state); +} + +static void +fmt_sscanf_state_consume_modifier(struct fmt_sscanf_state *state) +{ + char c, c2; + + c = fmt_sscanf_state_consume_format(state); + + switch (c) { + case 'h': + case 'l': + c2 = fmt_sscanf_state_consume_format(state); + + if (c == c2) { + state->modifier = (c == 'h') ? FMT_MODIFIER_CHAR + : FMT_MODIFIER_LONGLONG; + } else { + state->modifier = (c == 'h') ? FMT_MODIFIER_SHORT + : FMT_MODIFIER_LONG; + fmt_sscanf_state_restore_format(state); + } + + break; + case 'z': + state->modifier = FMT_MODIFIER_SIZE; + break; + case 't': + state->modifier = FMT_MODIFIER_PTRDIFF; + break; + default: + state->modifier = FMT_MODIFIER_NONE; + fmt_sscanf_state_restore_format(state); + break; + } +} + +static void +fmt_sscanf_state_consume_specifier(struct fmt_sscanf_state *state) +{ + char c; + + c = fmt_sscanf_state_consume_format(state); + + switch (c) { + case 'i': + state->base = 0; + state->flags |= FMT_FORMAT_CONV_SIGNED; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'd': + state->flags |= FMT_FORMAT_CONV_SIGNED; + case 'u': + state->base = 10; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'o': + state->base = 8; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'p': + state->modifier = FMT_MODIFIER_PTR; + case 'x': + case 'X': + state->base = 16; + state->specifier = FMT_SPECIFIER_INT; + break; + case 'c': + state->specifier = FMT_SPECIFIER_CHAR; + break; + case 's': + state->specifier = FMT_SPECIFIER_STR; + break; + case 'n': + state->specifier = FMT_SPECIFIER_NRCHARS; + break; + case '%': + state->specifier = FMT_SPECIFIER_PERCENT; + break; + default: + state->specifier = FMT_SPECIFIER_INVALID; + fmt_sscanf_state_restore_format(state); + break; + } +} + +static int +fmt_sscanf_state_discard_char(struct fmt_sscanf_state *state, char c) +{ + char c2; + + if (fmt_isspace(c)) { + fmt_sscanf_state_skip_space(state); + return 0; + } + + c2 = fmt_sscanf_state_consume_string(state); + + if (c != c2) { + if ((c2 == '\0') && (state->nr_convs == 0)) { + state->nr_convs = EOF; + } + + return ERROR_INVAL; + } + + return 0; +} + +static int +fmt_sscanf_state_consume(struct fmt_sscanf_state *state) +{ + int error; + char c; + + state->flags = 0; + + c = fmt_consume(&state->format); + + if (c == '\0') { + return ERROR_IO; + } + + if (c != '%') { + error = fmt_sscanf_state_discard_char(state, c); + + if (error) { + return error; + } + + return ERROR_AGAIN; + } + + fmt_sscanf_state_consume_flags(state); + fmt_sscanf_state_consume_width(state); + fmt_sscanf_state_consume_modifier(state); + fmt_sscanf_state_consume_specifier(state); + return 0; +} + +static int +fmt_sscanf_state_produce_int(struct fmt_sscanf_state *state) +{ + unsigned long long n, m, tmp; + char c, buf[FMT_MAX_NUM_SIZE]; + bool negative; + size_t i; + + negative = 0; + + fmt_sscanf_state_skip_space(state); + c = fmt_sscanf_state_consume_string(state); + + if (c == '-') { + negative = true; + c = fmt_sscanf_state_consume_string(state); + } + + if (c == '0') { + c = fmt_sscanf_state_consume_string(state); + + if ((c == 'x') || (c == 'X')) { + if (state->base == 0) { + state->base = 16; + } + + if (state->base == 16) { + c = fmt_sscanf_state_consume_string(state); + } else { + fmt_sscanf_state_restore_string(state); + c = '0'; + } + } else { + if (state->base == 0) { + state->base = 8; + } + + if (state->base != 8) { + fmt_sscanf_state_restore_string(state); + c = '0'; + } + } + } + + i = 0; + + while (c != '\0') { + if (state->base == 8) { + if (!((c >= '0') && (c <= '7'))) { + break; + } + } else if (state->base == 16) { + if (!fmt_isxdigit(c)) { + break; + } + } else { + if (!fmt_isdigit(c)) { + break; + } + } + + /* XXX Standard sscanf provides no way to cleanly handle overflows */ + if (i < (ARRAY_SIZE(buf) - 1)) { + buf[i] = c; + } else if (i == (ARRAY_SIZE(buf) - 1)) { + strcpy(buf, "1"); + negative = true; + } + + i++; + c = fmt_sscanf_state_consume_string(state); + } + + fmt_sscanf_state_restore_string(state); + + if (state->flags & FMT_FORMAT_DISCARD) { + return 0; + } + + if (i == 0) { + if (c == '\0') { + fmt_sscanf_state_report_error(state); + return ERROR_INVAL; + } + + buf[0] = '0'; + i = 1; + } + + if (i < ARRAY_SIZE(buf)) { + buf[i] = '\0'; + i--; + } else { + i = strlen(buf) - 1; + } + + n = 0; + +#ifndef __LP64__ + if (state->modifier == FMT_MODIFIER_LONGLONG) { +#endif /* __LP64__ */ + m = 1; + tmp = 0; + + while (&buf[i] >= buf) { + tmp += fmt_atoi(buf[i]) * m; + + if (tmp < n) { + n = 1; + negative = true; + break; + } + + n = tmp; + m *= state->base; + i--; + } +#ifndef __LP64__ + } else { + unsigned long _n, _m, _tmp; + + _n = 0; + _m = 1; + _tmp = 0; + + while (&buf[i] >= buf) { + _tmp += fmt_atoi(buf[i]) * _m; + + if (_tmp < _n) { + _n = 1; + negative = true; + break; + } + + _n = _tmp; + _m *= state->base; + i--; + } + + n = _n; + } +#endif /* __LP64__ */ + + if (negative) { + n = -n; + } + + switch (state->modifier) { + case FMT_MODIFIER_CHAR: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, char *) = n; + } else { + *va_arg(state->ap, unsigned char *) = n; + } + + break; + case FMT_MODIFIER_SHORT: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, short *) = n; + } else { + *va_arg(state->ap, unsigned short *) = n; + } + + break; + case FMT_MODIFIER_LONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, long *) = n; + } else { + *va_arg(state->ap, unsigned long *) = n; + } + + break; + case FMT_MODIFIER_LONGLONG: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, long long *) = n; + } else { + *va_arg(state->ap, unsigned long long *) = n; + } + + break; + case FMT_MODIFIER_PTR: + *va_arg(state->ap, uintptr_t *) = n; + break; + case FMT_MODIFIER_SIZE: + *va_arg(state->ap, size_t *) = n; + break; + case FMT_MODIFIER_PTRDIFF: + *va_arg(state->ap, ptrdiff_t *) = n; + break; + default: + if (state->flags & FMT_FORMAT_CONV_SIGNED) { + *va_arg(state->ap, int *) = n; + } else { + *va_arg(state->ap, unsigned int *) = n; + } + } + + fmt_sscanf_state_report_conv(state); + return 0; +} + +static int +fmt_sscanf_state_produce_char(struct fmt_sscanf_state *state) +{ + char c, *dest; + int i, width; + + if (state->flags & FMT_FORMAT_DISCARD) { + dest = NULL; + } else { + dest = va_arg(state->ap, char *); + } + + if (state->flags & FMT_FORMAT_CHECK_WIDTH) { + width = state->width; + } else { + width = 1; + } + + for (i = 0; i < width; i++) { + c = fmt_sscanf_state_consume_string(state); + + if ((c == '\0') || (c == EOF)) { + break; + } + + if (dest != NULL) { + *dest = c; + dest++; + } + } + + if (i < width) { + fmt_sscanf_state_restore_string(state); + } + + if ((dest != NULL) && (i != 0)) { + fmt_sscanf_state_report_conv(state); + } + + return 0; +} + +static int +fmt_sscanf_state_produce_str(struct fmt_sscanf_state *state) +{ + const char *orig; + char c, *dest; + + orig = state->str; + + fmt_sscanf_state_skip_space(state); + + if (state->flags & FMT_FORMAT_DISCARD) { + dest = NULL; + } else { + dest = va_arg(state->ap, char *); + } + + for (;;) { + c = fmt_sscanf_state_consume_string(state); + + if ((c == '\0') || (c == ' ') || (c == EOF)) { + break; + } + + if (dest != NULL) { + *dest = c; + dest++; + } + } + + fmt_sscanf_state_restore_string(state); + + if (state->str == orig) { + fmt_sscanf_state_report_error(state); + return ERROR_INVAL; + } + + if (dest != NULL) { + *dest = '\0'; + fmt_sscanf_state_report_conv(state); + } + + return 0; +} + +static int +fmt_sscanf_state_produce_nrchars(struct fmt_sscanf_state *state) +{ + *va_arg(state->ap, int *) = state->str - state->start; + return 0; +} + +static int +fmt_sscanf_state_produce(struct fmt_sscanf_state *state) +{ + switch (state->specifier) { + case FMT_SPECIFIER_INT: + return fmt_sscanf_state_produce_int(state); + case FMT_SPECIFIER_CHAR: + return fmt_sscanf_state_produce_char(state); + case FMT_SPECIFIER_STR: + return fmt_sscanf_state_produce_str(state); + case FMT_SPECIFIER_NRCHARS: + return fmt_sscanf_state_produce_nrchars(state); + case FMT_SPECIFIER_PERCENT: + fmt_sscanf_state_skip_space(state); + return fmt_sscanf_state_discard_char(state, '%'); + default: + fmt_sscanf_state_report_error(state); + return ERROR_INVAL; + } +} + +int +fmt_sscanf(const char *str, const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = fmt_vsscanf(str, format, ap); + va_end(ap); + + return ret; +} + +int +fmt_vsscanf(const char *str, const char *format, va_list ap) +{ + struct fmt_sscanf_state state; + int error; + + fmt_sscanf_state_init(&state, str, format, ap); + + for (;;) { + error = fmt_sscanf_state_consume(&state); + + if (error == ERROR_AGAIN) { + continue; + } else if (error) { + break; + } + + error = fmt_sscanf_state_produce(&state); + + if (error) { + break; + } + } + + return fmt_sscanf_state_finalize(&state); +} diff --git a/kern/fmt.h b/kern/fmt.h new file mode 100644 index 00000000..a339ed46 --- /dev/null +++ b/kern/fmt.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2010-2017 Richard Braun. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * + * Formatted string functions. + * + * The functions provided by this module implement a subset of the standard + * sprintf- and sscanf-like functions. + * + * sprintf: + * - flags: # 0 - ' ' (space) + + * - field width is supported + * - precision is supported + * + * sscanf: + * - flags: * + * - field width is supported + * + * common: + * - modifiers: hh h l ll z t + * - specifiers: d i o u x X c s p n % + * + * + * Upstream site with license notes : + * http://git.sceen.net/rbraun/librbraun.git/ + */ + +#ifndef _FMT_H +#define _FMT_H + +#include <stdarg.h> +#include <stddef.h> + +int fmt_sprintf(char *str, const char *format, ...) + __attribute__((format(printf, 2, 3))); + +int fmt_vsprintf(char *str, const char *format, va_list ap) + __attribute__((format(printf, 2, 0))); + +int fmt_snprintf(char *str, size_t size, const char *format, ...) + __attribute__((format(printf, 3, 4))); + +int fmt_vsnprintf(char *str, size_t size, const char *format, va_list ap) + __attribute__((format(printf, 3, 0))); + +int fmt_sscanf(const char *str, const char *format, ...) + __attribute__((format(scanf, 2, 3))); + +int fmt_vsscanf(const char *str, const char *format, va_list ap) + __attribute__((format(scanf, 2, 0))); + +#endif /* _FMT_H */ diff --git a/kern/hash.h b/kern/hash.h index 108d4b46..db399960 100644 --- a/kern/hash.h +++ b/kern/hash.h @@ -39,11 +39,10 @@ #ifndef _KERN_HASH_H #define _KERN_HASH_H +#include <assert.h> #include <stdint.h> #include <string.h> -#include <kern/assert.h> - #ifdef __LP64__ #define HASH_ALLBITS 64 #define hash_long(n, bits) hash_int64(n, bits) diff --git a/kern/init.c b/kern/init.c new file mode 100644 index 00000000..1dc51144 --- /dev/null +++ b/kern/init.c @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * The main purpose of the init module is to provide a convenient interface + * to declare initialization operations and their dependency, infer an order + * of execution based on the resulting graph (which must be acyclic), and + * run the operations in that order. Topological sorting is achieved using + * Kahn's algorithm. + */ + +#include <assert.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <kern/error.h> +#include <kern/init.h> +#include <kern/list.h> +#include <kern/macros.h> +#include <machine/cpu.h> + +#define INIT_DEBUG 1 + +extern struct init_op _init_ops; +extern struct init_op _init_ops_end; + +static_assert(sizeof(struct init_op) == INIT_OP_ALIGN, "invalid init_op size"); + +/* + * List of initialization operations. + * + * This type is used for the output of the topological sort. + */ +struct init_ops_list { + struct list ops; + size_t size; +}; + +static void __init +init_ops_list_init(struct init_ops_list *list) +{ + list_init(&list->ops); + list->size = 0; +} + +static size_t __init +init_ops_list_size(const struct init_ops_list *list) +{ + return list->size; +} + +static void __init +init_ops_list_push(struct init_ops_list *list, struct init_op *op) +{ + list_insert_tail(&list->ops, &op->list_node); + list->size++; +} + +static struct init_op * __init +init_ops_list_pop(struct init_ops_list *list) +{ + struct init_op *op; + + if (list->size == 0) { + return NULL; + } + + op = list_last_entry(&list->ops, struct init_op, list_node); + list_remove(&op->list_node); + list->size--; + return op; +} + +/* + * Stack of initialization operations. + * + * This type is used internally by the topological sort algorithm. + */ +struct init_ops_stack { + struct list ops; +}; + +static void __init +init_ops_stack_init(struct init_ops_stack *stack) +{ + list_init(&stack->ops); +} + +static void __init +init_ops_stack_push(struct init_ops_stack *stack, struct init_op *op) +{ + list_insert_tail(&stack->ops, &op->stack_node); +} + +static struct init_op * __init +init_ops_stack_pop(struct init_ops_stack *stack) +{ + struct init_op *op; + + if (list_empty(&stack->ops)) { + return NULL; + } + + op = list_last_entry(&stack->ops, struct init_op, stack_node); + list_remove(&op->stack_node); + return op; +} + +static struct init_op_dep * __init +init_op_get_dep(const struct init_op *op, size_t index) +{ + assert(index < op->nr_deps); + return &op->deps[index]; +} + +static void __init +init_op_init(struct init_op *op) +{ + struct init_op_dep *dep; + + for (size_t i = 0; i < op->nr_deps; i++) { + dep = init_op_get_dep(op, i); + dep->op->nr_parents++; + assert(dep->op->nr_parents != 0); + } +} + +static bool __init +init_op_orphan(const struct init_op *op) +{ + return (op->nr_parents == 0); +} + +static bool __init +init_op_pending(const struct init_op *op) +{ + return op->state == INIT_OP_STATE_PENDING; +} + +static void __init +init_op_set_pending(struct init_op *op) +{ + assert(op->state == INIT_OP_STATE_UNLINKED); + op->state = INIT_OP_STATE_PENDING; +} + +static bool __init +init_op_complete(const struct init_op *op) +{ + return op->state == INIT_OP_STATE_COMPLETE; +} + +static void __init +init_op_set_complete(struct init_op *op) +{ + assert(init_op_pending(op)); + op->state = INIT_OP_STATE_COMPLETE; +} + +static bool __init +init_op_ready(const struct init_op *op) +{ + const struct init_op_dep *dep; + size_t i; + + for (i = 0; i < op->nr_deps; i++) { + dep = init_op_get_dep(op, i); + + if (!init_op_complete(dep->op) + || (dep->required && dep->op->error)) { + return false; + } + } + + return true; +} + +static void __init +init_op_run(struct init_op *op) +{ + if (init_op_ready(op)) { + op->error = op->fn(); + } + + init_op_set_complete(op); +} + +#if INIT_DEBUG + +#define INIT_DEBUG_LOG_BUFFER_SIZE 8192 + +struct init_debug_log { + char buffer[INIT_DEBUG_LOG_BUFFER_SIZE]; + size_t index; +}; + +/* + * Buffers used to store an easy-to-dump text representation of init operations. + * + * These buffers are meant to be retrieved through a debugging interface such + * as JTAG. + */ +struct init_debug_logs { + struct init_debug_log roots; /* graph roots */ + struct init_debug_log cycles; /* operations with dependency cycles */ + struct init_debug_log pending; /* operations successfully sorted */ + struct init_debug_log complete; /* executed operations */ +}; + +static struct init_debug_logs init_debug_logs; + +static void __init +init_debug_log_append(struct init_debug_log *log, const char *name) +{ + size_t size; + + if (log->index == sizeof(log->buffer)) { + return; + } + + size = sizeof(log->buffer) - log->index; + log->index += snprintf(log->buffer + log->index, size, "%s ", name); + + if (log->index >= sizeof(log->buffer)) { + log->index = sizeof(log->buffer); + } +} + +static void __init +init_debug_append_root(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.roots, op->name); +} + +static void __init +init_debug_append_cycle(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.cycles, op->name); +} + +static void __init +init_debug_append_pending(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.pending, op->name); +} + +static void __init +init_debug_append_complete(const struct init_op *op) +{ + init_debug_log_append(&init_debug_logs.complete, op->name); +} + +static void __init +init_debug_scan_not_pending(void) +{ + const struct init_op *op; + + for (op = &_init_ops; op < &_init_ops_end; op++) { + if (!init_op_pending(op)) { + init_debug_append_cycle(op); + } + } +} + +#else /* INIT_DEBUG */ +#define init_debug_append_root(roots) +#define init_debug_append_pending(op) +#define init_debug_append_complete(op) +#define init_debug_scan_not_pending() +#endif /* INIT_DEBUG */ + +static void __init +init_add_pending_op(struct init_ops_list *pending_ops, struct init_op *op) +{ + assert(!init_op_pending(op)); + + init_op_set_pending(op); + init_ops_list_push(pending_ops, op); + init_debug_append_pending(op); +} + +static void __init +init_op_visit(struct init_op *op, struct init_ops_stack *stack) +{ + struct init_op_dep *dep; + + for (size_t i = 0; i < op->nr_deps; i++) { + dep = init_op_get_dep(op, i); + assert(dep->op->nr_parents != 0); + dep->op->nr_parents--; + + if (init_op_orphan(dep->op)) { + init_ops_stack_push(stack, dep->op); + } + } +} + +static void __init +init_check_ops_alignment(void) +{ + uintptr_t start, end; + + start = (uintptr_t)&_init_ops; + end = (uintptr_t)&_init_ops_end; + + if (((end - start) % INIT_OP_ALIGN) != 0) { + cpu_halt(); + } +} + +static void __init +init_bootstrap(void) +{ + struct init_op *op; + + init_check_ops_alignment(); + + for (op = &_init_ops; op < &_init_ops_end; op++) { + init_op_init(op); + } +} + +static void __init +init_scan_roots(struct init_ops_stack *stack) +{ + struct init_op *op; + + init_ops_stack_init(stack); + + for (op = &_init_ops; op < &_init_ops_end; op++) { + if (init_op_orphan(op)) { + init_ops_stack_push(stack, op); + init_debug_append_root(op); + } + } +} + +static void __init +init_scan_ops(struct init_ops_list *pending_ops) +{ + struct init_ops_stack stack; + struct init_op *op; + size_t nr_ops; + + init_scan_roots(&stack); + + for (;;) { + op = init_ops_stack_pop(&stack); + + if (op == NULL) { + break; + } + + init_add_pending_op(pending_ops, op); + init_op_visit(op, &stack); + } + + init_debug_scan_not_pending(); + + nr_ops = &_init_ops_end - &_init_ops; + + if (init_ops_list_size(pending_ops) != nr_ops) { + cpu_halt(); + } +} + +static void __init +init_run_ops(struct init_ops_list *pending_ops) +{ + struct init_op *op; + + for (;;) { + op = init_ops_list_pop(pending_ops); + + if (op == NULL) { + break; + } + + init_op_run(op); + init_debug_append_complete(op); + } +} + +void __init +init_setup(void) +{ + struct init_ops_list pending_ops; + + init_ops_list_init(&pending_ops); + + init_bootstrap(); + init_scan_ops(&pending_ops); + init_run_ops(&pending_ops); +} diff --git a/kern/init.h b/kern/init.h index 62fecf40..d15d9503 100644 --- a/kern/init.h +++ b/kern/init.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2012 Richard Braun. + * Copyright (c) 2010-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,6 +13,11 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * Init sections and operations. + * + * TODO Make the x86 linker script use macros from this header. */ #ifndef _KERN_INIT_H @@ -25,8 +30,15 @@ #define INIT_SECTION .init.text #define INIT_DATA_SECTION .init.data +/* + * This section must only contain init operation structures, and must be + * located inside the .init section. + */ +#define INIT_OPS_SECTION .init.ops + #ifndef __ASSEMBLER__ +#include <kern/error.h> #include <kern/macros.h> #define __init __section(QUOTE(INIT_SECTION)) @@ -38,6 +50,60 @@ extern char _init; extern char _init_end; +/* + * Type for initialization operation functions. + */ +typedef int (*init_op_fn_t)(void); + +#include <kern/init_i.h> + +/* + * Forge an init operation declaration. + */ +#define INIT_OP_DECLARE(fn) extern struct init_op INIT_OP(fn) + +/* + * Foge an entry suitable as an init operation dependency. + * + * If a dependency isn't required, it's still used to determine run + * order, but its result is ignored, and the operation depending on it + * behaves as if that dependency succeeded. + */ +#define INIT_OP_DEP(fn, required) { &INIT_OP(fn), required } + +/* + * Init operation definition macro. + * + * This macro is used to define a structure named after the given function. + * Init operations are placed in a specific section which doesn't contain + * any other object type, making it a system-wide array of init operations. + * There is no need to actively register init operations; this module finds + * them all from their section. Dependencies are given as a variable-length + * argument list of entries built with the INIT_OP_DEP() macro. + */ +#define INIT_OP_DEFINE(_fn, ...) \ + static struct init_op_dep INIT_OP_DEPS(_fn)[] __initdata = { \ + __VA_ARGS__ \ + }; \ + \ + struct init_op INIT_OP(_fn) __initop = { \ + .name = QUOTE(_fn), \ + .fn = _fn, \ + .deps = INIT_OP_DEPS(_fn), \ + .error = ERROR_AGAIN, \ + .state = INIT_OP_STATE_UNLINKED, \ + .nr_deps = ARRAY_SIZE(INIT_OP_DEPS(_fn)), \ + .nr_parents = 0, \ + } + +/* + * Initialize the init module. + * + * Scan the section containing init operations, resolve all dependencies, + * and run operations in an appropriate order. + */ +void init_setup(void); + #endif /* __ASSEMBLER__ */ #endif /* _KERN_INIT_H */ diff --git a/kern/init_i.h b/kern/init_i.h new file mode 100644 index 00000000..be23073c --- /dev/null +++ b/kern/init_i.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _KERN_INIT_I_H +#define _KERN_INIT_I_H + +/* + * Alignment is important to make sure initialization operations are + * stored as a C array in the reserved init op section. + */ +#define INIT_OP_ALIGN 64 + +#include <stdalign.h> +#include <stdbool.h> +#include <stddef.h> + +#include <kern/list_types.h> +#include <kern/macros.h> + +#define __initop __section(QUOTE(INIT_OPS_SECTION)) + +#define INIT_OP_STATE_UNLINKED 0 +#define INIT_OP_STATE_PENDING 1 +#define INIT_OP_STATE_COMPLETE 2 + +struct init_op { + alignas(INIT_OP_ALIGN) struct list list_node; + struct list stack_node; + const char *name; + init_op_fn_t fn; + struct init_op_dep *deps; + int error; + unsigned char state; + unsigned char nr_deps; + unsigned char nr_parents; +}; + +struct init_op_dep { + struct init_op *op; + bool required; +}; + +#define __INIT_OP_DEPS(fn) fn ## _init_op_deps +#define INIT_OP_DEPS(fn) __INIT_OP_DEPS(fn) + +#define __INIT_OP(fn) fn ## _init_op +#define INIT_OP(fn) __INIT_OP(fn) + +#endif /* _KERN_INIT_I_H */ diff --git a/kern/intr.c b/kern/intr.c index fe69e3e9..8579b744 100644 --- a/kern/intr.c +++ b/kern/intr.c @@ -22,27 +22,29 @@ * Shared interrupts are supported. */ +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> #include <kern/atomic.h> #include <kern/kmem.h> #include <kern/init.h> #include <kern/intr.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/spinlock.h> #include <kern/thread.h> +#include <machine/boot.h> #include <machine/cpu.h> +#include <machine/trap.h> struct intr_handler { - struct list node; + alignas(CPU_L1_SIZE) struct list node; intr_handler_fn_t fn; void *arg; -} __aligned(CPU_L1_SIZE); +}; /* * Interrupt controller. @@ -67,16 +69,16 @@ struct intr_ctl { * span a cache line to avoid false sharing. */ struct intr_entry { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; struct intr_ctl *ctl; unsigned int cpu; struct list handlers; -} __aligned(CPU_L1_SIZE); +}; /* * Interrupt table. */ -static struct intr_entry intr_table[INTR_TABLE_SIZE]; +static struct intr_entry intr_table[TRAP_INTR_TABLE_SIZE]; /* * List of registered controllers. @@ -314,8 +316,8 @@ intr_get_entry(unsigned int intr) return &intr_table[intr]; } -void __init -intr_setup(void) +static int __init +intr_bootstrap(void) { unsigned int i; @@ -327,8 +329,27 @@ intr_setup(void) for (i = 0; i < ARRAY_SIZE(intr_table); i++) { intr_entry_init(intr_get_entry(i)); } + + return 0; } +INIT_OP_DEFINE(intr_bootstrap, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(thread_bootstrap, true)); + +static int __init +intr_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(intr_setup, + INIT_OP_DEP(boot_setup_intr, true), + INIT_OP_DEP(intr_bootstrap, true)); + static void __init intr_check_range(unsigned int first_intr, unsigned int last_intr) { @@ -388,7 +409,7 @@ intr_unregister(unsigned int intr, intr_handler_fn_t fn) handler = intr_entry_remove(intr_get_entry(intr), fn); if (handler == NULL) { - printf("intr: warning: attempting to unregister unknown handler\n"); + log_warning("intr: attempting to unregister unknown handler"); return; } @@ -410,7 +431,7 @@ intr_handle(unsigned int intr) spinlock_lock(&entry->lock); if (intr_entry_empty(entry)) { - printf("intr: spurious interrupt %u\n", intr); + log_warning("intr: spurious interrupt %u", intr); goto out; } diff --git a/kern/intr.h b/kern/intr.h index 1cdab5ef..ac3f8cf2 100644 --- a/kern/intr.h +++ b/kern/intr.h @@ -21,6 +21,8 @@ #ifndef _KERN_INTR_H #define _KERN_INTR_H +#include <kern/init.h> + /* * Type for interrupt handler functions. * @@ -42,11 +44,6 @@ struct intr_ops { }; /* - * Initialize the intr module. - */ -void intr_setup(void); - -/* * Register an interrupt controller. * * This function isn't thread-safe and can only be called during system @@ -66,4 +63,17 @@ void intr_unregister(unsigned int intr, intr_handler_fn_t fn); */ void intr_handle(unsigned int intr); +/* + * This init operation provides : + * - registration of interrupt controllers and handlers + */ +INIT_OP_DECLARE(intr_bootstrap); + +/* + * This init operation provides : + * - all interrupt controllers have been registered + * - module fully initialized + */ +INIT_OP_DECLARE(intr_setup); + #endif /* _KERN_INTR_H */ diff --git a/kern/kernel.c b/kern/kernel.c index 05cf8392..434297d3 100644 --- a/kern/kernel.c +++ b/kern/kernel.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2014 Richard Braun. + * Copyright (c) 2011-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,47 +15,19 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <kern/cpumap.h> #include <kern/init.h> #include <kern/kernel.h> -#include <kern/llsync.h> -#include <kern/percpu.h> -#include <kern/shell.h> -#include <kern/sleepq.h> -#include <kern/sref.h> -#include <kern/task.h> #include <kern/thread.h> -#include <kern/turnstile.h> -#include <kern/work.h> -#include <kern/xcall.h> #include <machine/cpu.h> #include <vm/vm_page.h> -#ifdef X15_RUN_TEST_MODULE -#include <test/test.h> -#endif /* X15_RUN_TEST_MODULE */ - void __init kernel_main(void) { assert(!cpu_intr_enabled()); - percpu_cleanup(); - cpumap_setup(); - xcall_setup(); - task_setup(); - sleepq_setup(); - turnstile_setup(); - thread_setup(); - work_setup(); - llsync_setup(); - sref_setup(); - shell_setup(); - vm_page_info(); - -#ifdef X15_RUN_TEST_MODULE - test_setup(); -#endif /* X15_RUN_TEST_MODULE */ + init_setup(); + vm_page_log_info(); /* * Enabling application processors is done late in the boot process for diff --git a/kern/kernel.h b/kern/kernel.h index 868066e6..22cae43b 100644 --- a/kern/kernel.h +++ b/kern/kernel.h @@ -18,7 +18,7 @@ #ifndef _KERN_KERNEL_H #define _KERN_KERNEL_H -#include <kern/macros.h> +#include <stdnoreturn.h> /* * Kernel properties. @@ -31,13 +31,13 @@ * * Interrupts must be disabled when calling this function. */ -void __noreturn kernel_main(void); +noreturn void kernel_main(void); /* * Entry point for APs. * * Interrupts must be disabled when calling this function. */ -void __noreturn kernel_ap_main(void); +noreturn void kernel_ap_main(void); #endif /* _KERN_KERNEL_H */ diff --git a/kern/kmem.c b/kern/kmem.c index 5cce5589..d98be0d6 100644 --- a/kern/kmem.c +++ b/kern/kmem.c @@ -41,15 +41,15 @@ * TODO Rework the CPU pool layer to use the SLQB algorithm by Nick Piggin. */ +#include <assert.h> +#include <limits.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> #include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/init.h> -#include <kern/limits.h> #include <kern/list.h> #include <kern/log2.h> #include <kern/kmem.h> @@ -57,9 +57,10 @@ #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> +#include <kern/shell.h> #include <kern/thread.h> #include <machine/cpu.h> +#include <machine/page.h> #include <machine/pmap.h> #include <vm/vm_kmem.h> #include <vm/vm_page.h> @@ -1092,41 +1093,28 @@ kmem_cache_info(struct kmem_cache *cache) { char flags_str[64]; - if (cache == NULL) { - mutex_lock(&kmem_cache_list_lock); - - list_for_each_entry(&kmem_cache_list, cache, node) { - kmem_cache_info(cache); - } - - mutex_unlock(&kmem_cache_list_lock); - - return; - } - snprintf(flags_str, sizeof(flags_str), "%s%s", (cache->flags & KMEM_CF_SLAB_EXTERNAL) ? " SLAB_EXTERNAL" : "", (cache->flags & KMEM_CF_VERIFY) ? " VERIFY" : ""); mutex_lock(&cache->lock); - printf("kmem: name: %s\n" - "kmem: flags: 0x%x%s\n" - "kmem: obj_size: %zu\n" - "kmem: align: %zu\n" - "kmem: buf_size: %zu\n" - "kmem: bufctl_dist: %zu\n" - "kmem: slab_size: %zu\n" - "kmem: color_max: %zu\n" + printf("kmem: flags: 0x%x%s\n" + "kmem: obj_size: %zu\n" + "kmem: align: %zu\n" + "kmem: buf_size: %zu\n" + "kmem: bufctl_dist: %zu\n" + "kmem: slab_size: %zu\n" + "kmem: color_max: %zu\n" "kmem: bufs_per_slab: %lu\n" - "kmem: nr_objs: %lu\n" - "kmem: nr_bufs: %lu\n" - "kmem: nr_slabs: %lu\n" + "kmem: nr_objs: %lu\n" + "kmem: nr_bufs: %lu\n" + "kmem: nr_slabs: %lu\n" "kmem: nr_free_slabs: %lu\n" - "kmem: buftag_dist: %zu\n" - "kmem: redzone_pad: %zu\n" - "kmem: cpu_pool_size: %d\n", cache->name, cache->flags, flags_str, - cache->obj_size, cache->align, cache->buf_size, cache->bufctl_dist, + "kmem: buftag_dist: %zu\n" + "kmem: redzone_pad: %zu\n" + "kmem: cpu_pool_size: %d\n", cache->flags, flags_str, cache->obj_size, + cache->align, cache->buf_size, cache->bufctl_dist, cache->slab_size, cache->color_max, cache->bufs_per_slab, cache->nr_objs, cache->nr_bufs, cache->nr_slabs, cache->nr_free_slabs, cache->buftag_dist, cache->redzone_pad, @@ -1135,8 +1123,71 @@ kmem_cache_info(struct kmem_cache *cache) mutex_unlock(&cache->lock); } -void __init -kmem_setup(void) +#ifdef X15_SHELL + +static struct kmem_cache * +kmem_lookup_cache(const char *name) +{ + struct kmem_cache *cache; + + mutex_lock(&kmem_cache_list_lock); + + list_for_each_entry(&kmem_cache_list, cache, node) { + if (strcmp(cache->name, name) == 0) { + goto out; + } + } + + cache = NULL; + +out: + mutex_unlock(&kmem_cache_list_lock); + + return cache; +} + +static void +kmem_shell_info(int argc, char **argv) +{ + struct kmem_cache *cache; + + if (argc < 2) { + kmem_info(); + } else { + cache = kmem_lookup_cache(argv[1]); + + if (cache == NULL) { + printf("kmem: info: invalid argument\n"); + return; + } + + kmem_cache_info(cache); + } +} + +static struct shell_cmd kmem_shell_cmds[] = { + SHELL_CMD_INITIALIZER("kmem_info", kmem_shell_info, + "kmem_info [<cache_name>]", + "display information about kernel memory and caches"), +}; + +static int __init +kmem_setup_shell(void) +{ + SHELL_REGISTER_CMDS(kmem_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(kmem_setup_shell, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(thread_setup, true)); + +#endif /* X15_SHELL */ + +static int __init +kmem_bootstrap(void) { struct kmem_cpu_pool_type *cpu_pool_type; char name[KMEM_NAME_SIZE]; @@ -1170,8 +1221,24 @@ kmem_setup(void) kmem_cache_init(&kmem_caches[i], name, size, 0, NULL, 0); size <<= 1; } + + return 0; +} + +INIT_OP_DEFINE(kmem_bootstrap, + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(vm_page_setup, true)); + +static int __init +kmem_setup(void) +{ + return 0; } +INIT_OP_DEFINE(kmem_setup, + INIT_OP_DEP(kmem_bootstrap, true), + INIT_OP_DEP(vm_kmem_setup, true)); + static inline size_t kmem_get_index(unsigned long size) { @@ -1282,8 +1349,11 @@ kmem_free(void *ptr, size_t size) void kmem_info(void) { + size_t mem_usage, mem_reclaimable, total, total_reclaimable; struct kmem_cache *cache; - size_t mem_usage, mem_reclaimable; + + total = 0; + total_reclaimable = 0; printf("kmem: cache obj slab bufs objs bufs " " total reclaimable\n" @@ -1297,6 +1367,8 @@ kmem_info(void) mem_usage = (cache->nr_slabs * cache->slab_size) >> 10; mem_reclaimable = (cache->nr_free_slabs * cache->slab_size) >> 10; + total += mem_usage; + total_reclaimable += mem_reclaimable; printf("kmem: %-19s %6zu %3zuk %4lu %6lu %6lu %7zuk %10zuk\n", cache->name, cache->obj_size, cache->slab_size >> 10, @@ -1307,4 +1379,6 @@ kmem_info(void) } mutex_unlock(&kmem_cache_list_lock); + + printf("total: %zuk reclaimable: %zuk\n", total, total_reclaimable); } diff --git a/kern/kmem.h b/kern/kmem.h index 8f6c773c..3296ddb4 100644 --- a/kern/kmem.h +++ b/kern/kmem.h @@ -23,6 +23,8 @@ #include <stddef.h> +#include <kern/init.h> + /* * Object cache. */ @@ -77,14 +79,6 @@ void kmem_cache_free(struct kmem_cache *cache, void *obj); void kmem_cache_info(struct kmem_cache *cache); /* - * Set up the kernel memory allocator module. - * - * This function should only be called by the VM system. Once it returns, - * caches can be initialized. - */ -void kmem_setup(void); - -/* * Allocate size bytes of uninitialized memory. */ void * kmem_alloc(size_t size); @@ -106,4 +100,16 @@ void kmem_free(void *ptr, size_t size); */ void kmem_info(void); +/* + * This init operation provides : + * - allocation from caches backed by the page allocator + */ +INIT_OP_DECLARE(kmem_bootstrap); + +/* + * This init operation provides : + * - allocation from all caches + */ +INIT_OP_DECLARE(kmem_setup); + #endif /* _KERN_KMEM_H */ diff --git a/kern/kmem_i.h b/kern/kmem_i.h index f3ad8228..beae6c45 100644 --- a/kern/kmem_i.h +++ b/kern/kmem_i.h @@ -18,11 +18,12 @@ #ifndef _KERN_KMEM_I_H #define _KERN_KMEM_I_H +#include <stdalign.h> #include <stddef.h> #include <kern/list.h> #include <kern/mutex.h> -#include <kern/param.h> +#include <machine/cpu.h> /* * Per-processor cache of pre-constructed objects. @@ -30,13 +31,13 @@ * The flags member is a read-only CPU-local copy of the parent cache flags. */ struct kmem_cpu_pool { - struct mutex lock; + alignas(CPU_L1_SIZE) struct mutex lock; int flags; int size; int transfer_size; int nr_objs; void **array; -} __aligned(CPU_L1_SIZE); +}; /* * When a cache is created, its CPU pool type is determined from the buffer @@ -70,17 +71,17 @@ union kmem_bufctl { * Redzone guard word. */ #ifdef __LP64__ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_REDZONE_WORD 0xfeedfacefeedfaceUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_REDZONE_WORD 0xcefaedfecefaedfeUL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #else /* __LP64__ */ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_REDZONE_WORD 0xfeedfaceUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_REDZONE_WORD 0xcefaedfeUL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #endif /* __LP64__ */ /* @@ -104,21 +105,21 @@ struct kmem_buftag { * Values the buftag state member can take. */ #ifdef __LP64__ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_BUFTAG_ALLOC 0xa110c8eda110c8edUL #define KMEM_BUFTAG_FREE 0xf4eeb10cf4eeb10cUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_BUFTAG_ALLOC 0xedc810a1edc810a1UL #define KMEM_BUFTAG_FREE 0x0cb1eef40cb1eef4UL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #else /* __LP64__ */ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_BUFTAG_ALLOC 0xa110c8edUL #define KMEM_BUFTAG_FREE 0xf4eeb10cUL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_BUFTAG_ALLOC 0xedc810a1UL #define KMEM_BUFTAG_FREE 0x0cb1eef4UL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #endif /* __LP64__ */ /* @@ -127,13 +128,13 @@ struct kmem_buftag { * These values are unconditionally 64-bit wide since buffers are at least * 8-byte aligned. */ -#ifdef __BIG_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define KMEM_FREE_PATTERN 0xdeadbeefdeadbeefULL #define KMEM_UNINIT_PATTERN 0xbaddcafebaddcafeULL -#else /* __BIG_ENDIAN__ */ +#else /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ #define KMEM_FREE_PATTERN 0xefbeaddeefbeaddeULL #define KMEM_UNINIT_PATTERN 0xfecaddbafecaddbaULL -#endif /* __BIG_ENDIAN__ */ +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ /* * Page-aligned collection of unconstructed buffers. diff --git a/kern/limits.h b/kern/limits.h deleted file mode 100644 index fa468537..00000000 --- a/kern/limits.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2010-2014 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#ifndef _KERN_LIMITS_H -#define _KERN_LIMITS_H - -#define CHAR_BIT 8 - -#ifdef __LP64__ -#define LONG_BIT 64 -#else /* __LP64__ */ -#define LONG_BIT 32 -#endif /* __LP64__ */ - -#endif /* _KERN_LIMITS_H */ diff --git a/kern/llsync.c b/kern/llsync.c index 1942576d..c432fc68 100644 --- a/kern/llsync.c +++ b/kern/llsync.c @@ -32,24 +32,24 @@ * TODO Gracefully handle large amounts of deferred works. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/condition.h> #include <kern/cpumap.h> #include <kern/init.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/llsync.h> #include <kern/llsync_i.h> #include <kern/macros.h> #include <kern/mutex.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/syscnt.h> #include <kern/work.h> +#include <kern/thread.h> #include <machine/cpu.h> /* @@ -82,7 +82,7 @@ llsync_ready(void) return llsync_is_ready; } -void __init +static int __init llsync_setup(void) { struct llsync_cpu_data *cpu_data; @@ -104,9 +104,18 @@ llsync_setup(void) work_queue_init(&cpu_data->queue0); } - llsync_is_ready = true; + return 0; } +INIT_OP_DEFINE(llsync_setup, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(work_setup, true)); + static void llsync_process_global_checkpoint(void) { @@ -122,7 +131,7 @@ llsync_process_global_checkpoint(void) /* TODO Handle hysteresis */ if (!llsync_data.no_warning && (nr_works >= LLSYNC_NR_PENDING_WORKS_WARN)) { llsync_data.no_warning = 1; - printf("llsync: warning: large number of pending works\n"); + log_warning("llsync: large number of pending works\n"); } if (llsync_data.nr_registered_cpus == 0) { @@ -182,6 +191,10 @@ llsync_register(void) unsigned long flags; unsigned int cpu; + if (!llsync_is_ready) { + llsync_is_ready = true; + } + cpu = cpu_id(); cpu_data = llsync_get_cpu_data(); diff --git a/kern/llsync.h b/kern/llsync.h index 6736d6f9..3852b0b9 100644 --- a/kern/llsync.h +++ b/kern/llsync.h @@ -123,11 +123,6 @@ llsync_read_exit(void) bool llsync_ready(void); /* - * Initialize the llsync module. - */ -void llsync_setup(void); - -/* * Manage registration of the current processor. * * The caller must not be allowed to migrate when calling these functions. diff --git a/kern/llsync_i.h b/kern/llsync_i.h index 6bc1bf1d..e043eb9d 100644 --- a/kern/llsync_i.h +++ b/kern/llsync_i.h @@ -18,10 +18,11 @@ #ifndef _KERN_LLSYNC_I_H #define _KERN_LLSYNC_I_H -#include <kern/assert.h> +#include <assert.h> +#include <stdalign.h> + #include <kern/cpumap.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/spinlock.h> #include <kern/syscnt.h> #include <kern/work.h> @@ -57,7 +58,7 @@ struct llsync_data { * - apply optimistic accesses to reduce contention */ struct { - volatile unsigned int value __aligned(CPU_L1_SIZE); + alignas(CPU_L1_SIZE) volatile unsigned int value; } gcid; }; diff --git a/kern/log.c b/kern/log.c new file mode 100644 index 00000000..3d6ee378 --- /dev/null +++ b/kern/log.c @@ -0,0 +1,672 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <assert.h> +#include <limits.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdint.h> +#include <string.h> + +#include <kern/arg.h> +#include <kern/cbuf.h> +#include <kern/init.h> +#include <kern/log.h> +#include <kern/macros.h> +#include <kern/panic.h> +#include <kern/shell.h> +#include <kern/spinlock.h> +#include <kern/thread.h> +#include <machine/boot.h> +#include <machine/cpu.h> + +#define LOG_BUFFER_SIZE 16384 + +#if !ISP2(LOG_BUFFER_SIZE) +#error "log buffer size must be a power-of-two" +#endif + +#define LOG_MSG_SIZE 128 + +#define LOG_MARKER 0x1 + +static struct thread *log_thread; + +static struct cbuf log_cbuf; +static char log_buffer[LOG_BUFFER_SIZE]; + +/* + * This index is used by the log thread to report what it has consumed + * so that producers can detect overruns. + */ +static size_t log_index; + +static size_t log_nr_overruns; + +static unsigned int log_print_level; + +/* + * Global lock. + * + * Interrupts must be disabled when holding this lock. + */ +static struct spinlock log_lock; + +/* + * A record starts with a marker byte and ends with a null terminating byte. + * + * There must be no null byte inside the header, and the buffer is expected + * to be a standard null-terminated string. + */ +struct log_record { + uint8_t mark; + uint8_t level; + char buffer[LOG_MSG_SIZE]; +}; + +/* + * A consumer context allows faster, buffered reads of the circular buffer, + * by retaining bytes that weren't consumed in case a read spans multiple + * records. + */ +struct log_consume_ctx { + struct cbuf *cbuf; + size_t cbuf_index; + char buf[LOG_MSG_SIZE]; + size_t index; + size_t size; +}; + +static void +log_consume_ctx_init(struct log_consume_ctx *ctx, struct cbuf *cbuf) +{ + ctx->cbuf = cbuf; + ctx->cbuf_index = cbuf_start(cbuf); + ctx->index = 0; + ctx->size = 0; +} + +static size_t +log_consume_ctx_index(const struct log_consume_ctx *ctx) +{ + return ctx->cbuf_index; +} + +static void +log_consume_ctx_set_index(struct log_consume_ctx *ctx, size_t index) +{ + ctx->cbuf_index = index; +} + +static bool +log_consume_ctx_empty(const struct log_consume_ctx *ctx) +{ + return ctx->cbuf_index == cbuf_end(ctx->cbuf); +} + +static int +log_consume_ctx_pop(struct log_consume_ctx *ctx, char *byte) +{ + int error; + + if (ctx->index >= ctx->size) { + ctx->index = 0; + ctx->size = sizeof(ctx->buf); + error = cbuf_read(ctx->cbuf, ctx->cbuf_index, ctx->buf, &ctx->size); + + if (error) { + ctx->cbuf_index = cbuf_start(ctx->cbuf); + return error; + } + } + + ctx->cbuf_index++; + *byte = ctx->buf[ctx->index]; + ctx->index++; + return 0; +} + +static const char * +log_level2str(unsigned int level) +{ + switch (level) { + case LOG_EMERG: + return "emerg"; + case LOG_ALERT: + return "alert"; + case LOG_CRIT: + return "crit"; + case LOG_ERR: + return "error"; + case LOG_WARNING: + return "warning"; + case LOG_NOTICE: + return "notice"; + case LOG_INFO: + return "info"; + case LOG_DEBUG: + return "debug"; + default: + return NULL; + } +} + +static char +log_level2char(unsigned int level) +{ + assert(level < LOG_NR_LEVELS); + return '0' + level; +} + +static uint8_t +log_char2level(char c) +{ + uint8_t level; + + level = c - '0'; + assert(level < LOG_NR_LEVELS); + return level; +} + +static void +log_record_init_produce(struct log_record *record, unsigned int level) +{ + record->mark = LOG_MARKER; + record->level = log_level2char(level); +} + +static void +log_record_consume(struct log_record *record, char c, size_t *sizep) +{ + char *ptr; + + assert(*sizep < sizeof(*record)); + + ptr = (char *)record; + ptr[*sizep] = c; + (*sizep)++; +} + +static int +log_record_init_consume(struct log_record *record, struct log_consume_ctx *ctx) +{ + bool marker_found; + size_t size; + int error; + char c; + + marker_found = false; + size = 0; + + for (;;) { + if (log_consume_ctx_empty(ctx)) { + if (!marker_found) { + return ERROR_INVAL; + } + + break; + } + + error = log_consume_ctx_pop(ctx, &c); + + if (error) { + continue; + } + + if (!marker_found) { + if (c != LOG_MARKER) { + continue; + } + + marker_found = true; + log_record_consume(record, c, &size); + continue; + } else if (size == offsetof(struct log_record, level)) { + record->level = log_char2level(c); + size++; + continue; + } + + log_record_consume(record, c, &size); + + if (c == '\0') { + break; + } + } + + return 0; +} + +static void +log_record_print(const struct log_record *record, unsigned int level) +{ + if (record->level > level) { + return; + } + + if (record->level <= LOG_WARNING) { + printf("%7s %s\n", log_level2str(record->level), record->buffer); + } else { + printf("%s\n", record->buffer); + } +} + +static void +log_run(void *arg) +{ + unsigned long flags, nr_overruns; + struct log_consume_ctx ctx; + struct log_record record; + bool start_shell; + int error; + + (void)arg; + + nr_overruns = 0; + start_shell = true; + + spinlock_lock_intr_save(&log_lock, &flags); + + log_consume_ctx_init(&ctx, &log_cbuf); + + for (;;) { + while (log_consume_ctx_empty(&ctx)) { + /* + * Starting the shell after the log thread sleeps for the first + * time cleanly serializes log messages and shell prompt, making + * a clean ordered output. + */ + if (start_shell) { + spinlock_unlock_intr_restore(&log_lock, flags); + shell_start(); + start_shell = false; + spinlock_lock_intr_save(&log_lock, &flags); + } + + log_index = log_consume_ctx_index(&ctx); + + thread_sleep(&log_lock, &log_cbuf, "log_cbuf"); + + log_consume_ctx_set_index(&ctx, log_index); + } + + error = log_record_init_consume(&record, &ctx); + + /* Drain the log buffer before reporting overruns */ + if (log_consume_ctx_empty(&ctx)) { + nr_overruns = log_nr_overruns; + log_nr_overruns = 0; + } + + log_index = log_consume_ctx_index(&ctx); + + spinlock_unlock_intr_restore(&log_lock, flags); + + if (!error) { + log_record_print(&record, log_print_level); + } + + if (nr_overruns != 0) { + log_msg(LOG_ERR, "log: buffer overruns, %lu bytes dropped", + nr_overruns); + nr_overruns = 0; + } + + spinlock_lock_intr_save(&log_lock, &flags); + + log_consume_ctx_set_index(&ctx, log_index); + } +} + +#ifdef X15_SHELL + +static void +log_dump(unsigned int level) +{ + struct log_consume_ctx ctx; + struct log_record record; + unsigned long flags; + int error; + + spinlock_lock_intr_save(&log_lock, &flags); + + log_consume_ctx_init(&ctx, &log_cbuf); + + for (;;) { + error = log_record_init_consume(&record, &ctx); + + if (error) { + break; + } + + spinlock_unlock_intr_restore(&log_lock, flags); + + log_record_print(&record, level); + + spinlock_lock_intr_save(&log_lock, &flags); + } + + spinlock_unlock_intr_restore(&log_lock, flags); +} + +static void +log_shell_dump(int argc, char **argv) +{ + unsigned int level; + int ret; + + if (argc != 2) { + level = log_print_level; + } else { + ret = sscanf(argv[1], "%u", &level); + + if ((ret != 1) || (level >= LOG_NR_LEVELS)) { + printf("log: dump: invalid arguments\n"); + return; + } + } + + log_dump(level); +} + +static struct shell_cmd log_shell_cmds[] = { + SHELL_CMD_INITIALIZER2("log_dump", log_shell_dump, + "log_dump [<level>]", + "dump the log buffer", + "Only records of level less than or equal to the given level" + " are printed. Level may be one of :\n" + " 0: emergency\n" + " 1: alert\n" + " 2: critical\n" + " 3: error\n" + " 4: warning\n" + " 5: notice\n" + " 6: info\n" + " 7: debug"), +}; + +static int __init +log_setup_shell(void) +{ + SHELL_REGISTER_CMDS(log_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(log_setup_shell, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(shell_setup, true)); + +#endif /* X15_SHELL */ + +static int __init +log_setup(void) +{ + cbuf_init(&log_cbuf, log_buffer, sizeof(log_buffer)); + log_index = cbuf_start(&log_cbuf); + spinlock_init(&log_lock); + log_print_level = LOG_INFO; + + boot_log_info(); + arg_log_info(); + cpu_log_info(cpu_current()); + + return 0; +} + +INIT_OP_DEFINE(log_setup, + INIT_OP_DEP(arg_setup, true), + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(spinlock_setup, true)); + +static int __init +log_start(void) +{ + struct thread_attr attr; + int error; + + thread_attr_init(&attr, THREAD_KERNEL_PREFIX "log_run"); + thread_attr_set_detached(&attr); + error = thread_create(&log_thread, &attr, log_run, NULL); + + if (error) { + panic("log: unable to create thread"); + } + + return 0; +} + +INIT_OP_DEFINE(log_start, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(thread_setup, true)); + +static void +log_write(const void *s, size_t size) +{ + int error; + + error = cbuf_write(&log_cbuf, cbuf_end(&log_cbuf), s, size); + assert(!error); + + if (!cbuf_range_valid(&log_cbuf, log_index, log_index + 1)) { + log_nr_overruns += cbuf_start(&log_cbuf) - log_index; + log_index = cbuf_start(&log_cbuf); + } +} + +int +log_msg(unsigned int level, const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vmsg(level, format, ap); + va_end(ap); + + return ret; +} + +int +log_vmsg(unsigned int level, const char *format, va_list ap) +{ + struct log_record record; + unsigned long flags; + int nr_chars; + size_t size; + char *ptr; + + log_record_init_produce(&record, level); + nr_chars = vsnprintf(record.buffer, sizeof(record.buffer), format, ap); + + if ((unsigned int)nr_chars >= sizeof(record.buffer)) { + log_msg(LOG_ERR, "log: message too large"); + goto out; + } + + ptr = strchr(record.buffer, '\n'); + + if (ptr != NULL) { + *ptr = '\0'; + nr_chars = ptr - record.buffer; + } + + assert(nr_chars >= 0); + size = offsetof(struct log_record, buffer) + nr_chars + 1; + + spinlock_lock_intr_save(&log_lock, &flags); + log_write(&record, size); + thread_wakeup(log_thread); + spinlock_unlock_intr_restore(&log_lock, flags); + +out: + return nr_chars; +} + +int +log_emerg(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vemerg(format, ap); + va_end(ap); + + return ret; +} + +int +log_alert(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_valert(format, ap); + va_end(ap); + + return ret; +} + +int +log_crit(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vcrit(format, ap); + va_end(ap); + + return ret; +} + +int +log_err(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_verr(format, ap); + va_end(ap); + + return ret; +} + +int +log_warning(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vwarning(format, ap); + va_end(ap); + + return ret; +} + +int +log_notice(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vnotice(format, ap); + va_end(ap); + + return ret; +} + +int +log_info(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vinfo(format, ap); + va_end(ap); + + return ret; +} + +int +log_debug(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = log_vdebug(format, ap); + va_end(ap); + + return ret; +} + +int +log_vemerg(const char *format, va_list ap) +{ + return log_vmsg(LOG_EMERG, format, ap); +} + +int +log_valert(const char *format, va_list ap) +{ + return log_vmsg(LOG_ALERT, format, ap); +} + +int +log_vcrit(const char *format, va_list ap) +{ + return log_vmsg(LOG_CRIT, format, ap); +} + +int +log_verr(const char *format, va_list ap) +{ + return log_vmsg(LOG_ERR, format, ap); +} + +int +log_vwarning(const char *format, va_list ap) +{ + return log_vmsg(LOG_WARNING, format, ap); +} + +int +log_vnotice(const char *format, va_list ap) +{ + return log_vmsg(LOG_NOTICE, format, ap); +} + +int +log_vinfo(const char *format, va_list ap) +{ + return log_vmsg(LOG_INFO, format, ap); +} + +int +log_vdebug(const char *format, va_list ap) +{ + return log_vmsg(LOG_DEBUG, format, ap); +} diff --git a/kern/log.h b/kern/log.h new file mode 100644 index 00000000..581a7840 --- /dev/null +++ b/kern/log.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * System logging. + */ + +#ifndef _KERN_LOG_H +#define _KERN_LOG_H + +#include <stdarg.h> + + +#include <kern/init.h> +enum { + LOG_EMERG, + LOG_ALERT, + LOG_CRIT, + LOG_ERR, + LOG_WARNING, + LOG_NOTICE, + LOG_INFO, + LOG_DEBUG, + LOG_NR_LEVELS, +}; + +/* + * Generate a message and send it to the log thread. + * + * The arguments and return value are similar to printf(), with + * these exceptions : + * - a level is associated to each log message + * - processing stops at the first terminating null byte or newline + * character, whichever occurs first + * + * This function may safely be called in interrupt context. + */ +int log_msg(unsigned int level, const char *format, ...) + __attribute__((format(printf, 2, 3))); + +int log_vmsg(unsigned int level, const char *format, va_list ap) + __attribute__((format(printf, 2, 0))); + +/* + * Convenience wrappers. + */ + +int log_emerg(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_alert(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_crit(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_err(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_warning(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_notice(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_info(const char *format, ...) __attribute__((format(printf, 1, 2))); +int log_debug(const char *format, ...) __attribute__((format(printf, 1, 2))); + +int log_vemerg(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_valert(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vcrit(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_verr(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vwarning(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vnotice(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vinfo(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); +int log_vdebug(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); + +/* + * This init operation provides : + * - message logging + * + * The log thread isn't yet started and messages are merely stored in an + * internal buffer. + */ +INIT_OP_DECLARE(log_setup); + +#endif /* _KERN_LOG_H */ diff --git a/kern/log2.h b/kern/log2.h index 0a3768a7..ed12b441 100644 --- a/kern/log2.h +++ b/kern/log2.h @@ -21,8 +21,8 @@ #ifndef _KERN_LOG2_H #define _KERN_LOG2_H -#include <kern/assert.h> -#include <kern/limits.h> +#include <assert.h> +#include <limits.h> static inline unsigned int ilog2(unsigned long x) diff --git a/kern/macros.h b/kern/macros.h index a5b7b032..6b203e52 100644 --- a/kern/macros.h +++ b/kern/macros.h @@ -16,14 +16,22 @@ * * * Helper macros. + * + * This file is a top header in the inclusion hierarchy, and shouldn't include + * other headers that may cause circular dependencies. */ #ifndef _KERN_MACROS_H #define _KERN_MACROS_H -#ifndef __ASSEMBLER__ -#include <stddef.h> -#endif /* __ASSEMBLER__ */ +#if !defined(__GNUC__) || (__GNUC__ < 4) +#error "GCC 4+ required" +#endif + +/* + * Attributes for variables that are mostly read and seldom changed. + */ +#define __read_mostly __section(".data.read_mostly") #define MACRO_BEGIN ({ #define MACRO_END }) @@ -35,6 +43,16 @@ #define DECL_CONST(x, s) x #else /* __ASSEMBLER__ */ #define __DECL_CONST(x, s) x##s +void cga_putc(char c); + +static inline void +moo_print(const char *s) +{ + while (*s != '\0') { + cga_putc(*s); + s++; + } +} #define DECL_CONST(x, s) __DECL_CONST(x, s) #endif /* __ASSEMBLER__ */ @@ -48,32 +66,36 @@ #define P2ALIGNED(x, a) (((x) & ((a) - 1)) == 0) #define ISP2(x) P2ALIGNED(x, x) -#define P2ALIGN(x, a) ((x) & -(a)) -#define P2ROUND(x, a) (-(-(x) & -(a))) -#define P2END(x, a) (-(~(x) & -(a))) +#define P2ALIGN(x, a) ((x) & -(a)) /* decreases if not aligned */ +#define P2ROUND(x, a) (-(-(x) & -(a))) /* increases if not aligned */ +#define P2END(x, a) (-(~(x) & -(a))) /* always increases */ #define structof(ptr, type, member) \ ((type *)((char *)(ptr) - offsetof(type, member))) -#define alignof(x) __alignof__(x) - #define likely(expr) __builtin_expect(!!(expr), 1) #define unlikely(expr) __builtin_expect(!!(expr), 0) #define barrier() asm volatile("" : : : "memory") -#define __noreturn __attribute__((noreturn)) -#define __aligned(x) __attribute__((aligned(x))) +/* + * The following macros may be provided by the C environment. + */ + +#ifndef __noinline +#define __noinline __attribute__((noinline)) +#endif #ifndef __always_inline #define __always_inline inline __attribute__((always_inline)) -#endif /* __attribute__ */ +#endif +#ifndef __section #define __section(x) __attribute__((section(x))) -#define __packed __attribute__((packed)) -#define __alias(x) __attribute__((alias(x))) +#endif -#define __format_printf(fmt, args) \ - __attribute__((format(printf, fmt, args))) +#ifndef __packed +#define __packed __attribute__((packed)) +#endif #endif /* _KERN_MACROS_H */ diff --git a/kern/param.h b/kern/mutex.c index d0061728..62609768 100644 --- a/kern/param.h +++ b/kern/mutex.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Richard Braun. + * Copyright (c) 2013-2017 Richard Braun. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,9 +15,14 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef _KERN_PARAM_H -#define _KERN_PARAM_H +#include <kern/init.h> +#include <kern/thread.h> -#include <machine/param.h> +static int __init +mutex_setup(void) +{ + return 0; +} -#endif /* _KERN_PARAM_H */ +INIT_OP_DEFINE(mutex_setup, + INIT_OP_DEP(thread_setup_booter, true)); diff --git a/kern/mutex.h b/kern/mutex.h index 2936d82f..f192a70a 100644 --- a/kern/mutex.h +++ b/kern/mutex.h @@ -35,6 +35,7 @@ #include <kern/mutex/mutex_plain_i.h> #endif +#include <kern/init.h> #include <kern/mutex_types.h> #include <kern/thread.h> @@ -93,4 +94,12 @@ mutex_unlock(struct mutex *mutex) thread_wakeup_last_cond(); } +/* + * This init operation provides : + * - uncontended mutex locking + * + * Contended locking may only occur after starting the scheduler. + */ +INIT_OP_DECLARE(mutex_setup); + #endif /* _KERN_MUTEX_H */ diff --git a/kern/mutex/mutex_adaptive.c b/kern/mutex/mutex_adaptive.c index f4274ce1..ffc47169 100644 --- a/kern/mutex/mutex_adaptive.c +++ b/kern/mutex/mutex_adaptive.c @@ -35,7 +35,7 @@ mutex_adaptive_get_thread(uintptr_t owner) static void mutex_adaptive_set_contended(struct mutex *mutex) { - atomic_or_acq_rel(&mutex->owner, MUTEX_ADAPTIVE_CONTENDED); + atomic_or(&mutex->owner, MUTEX_ADAPTIVE_CONTENDED, ATOMIC_RELEASE); } static inline bool diff --git a/kern/mutex/mutex_adaptive_i.h b/kern/mutex/mutex_adaptive_i.h index 2dd06792..b9952ec6 100644 --- a/kern/mutex/mutex_adaptive_i.h +++ b/kern/mutex/mutex_adaptive_i.h @@ -23,9 +23,9 @@ " use <kern/mutex.h> instead" #endif +#include <assert.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/error.h> #include <kern/macros.h> diff --git a/kern/mutex/mutex_plain_i.h b/kern/mutex/mutex_plain_i.h index 9e41ff27..4f112b89 100644 --- a/kern/mutex/mutex_plain_i.h +++ b/kern/mutex/mutex_plain_i.h @@ -23,7 +23,8 @@ " use <kern/mutex.h> instead" #endif -#include <kern/assert.h> +#include <assert.h> + #include <kern/atomic.h> #include <kern/error.h> #include <kern/mutex_types.h> diff --git a/kern/panic.c b/kern/panic.c index 9e7d1d53..c610e3fa 100644 --- a/kern/panic.c +++ b/kern/panic.c @@ -19,6 +19,7 @@ #include <stdio.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/panic.h> #include <machine/cpu.h> #include <machine/strace.h> @@ -54,3 +55,14 @@ panic(const char *format, ...) * Never reached. */ } + +static int __init +panic_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(panic_setup, + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(strace_setup, true)); diff --git a/kern/panic.h b/kern/panic.h index 9d1d31e0..e6cbe8ba 100644 --- a/kern/panic.h +++ b/kern/panic.h @@ -18,11 +18,20 @@ #ifndef _KERN_PANIC_H #define _KERN_PANIC_H -#include <kern/macros.h> +#include <stdnoreturn.h> + +#include <kern/init.h> /* * Print the given message and halt the system immediately. */ -void __noreturn panic(const char *format, ...) __format_printf(1, 2); +noreturn void panic(const char *format, ...) + __attribute__((format(printf, 1, 2))); + +/* + * This init operation provides : + * - module fully initialized + */ +INIT_OP_DECLARE(panic_setup); #endif /* _KERN_PANIC_H */ diff --git a/kern/percpu.c b/kern/percpu.c index 5e4ff827..7621bb29 100644 --- a/kern/percpu.c +++ b/kern/percpu.c @@ -15,17 +15,16 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <stdio.h> #include <string.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> #include <machine/cpu.h> #include <vm/vm_kmem.h> @@ -37,25 +36,28 @@ static void *percpu_area_content __initdata; static size_t percpu_area_size __initdata; static int percpu_skip_warning __initdata; -void __init +static int __init percpu_bootstrap(void) { percpu_areas[0] = &_percpu; + return 0; } -void __init +INIT_OP_DEFINE(percpu_bootstrap); + +static int __init percpu_setup(void) { struct vm_page *page; unsigned int order; percpu_area_size = &_percpu_end - &_percpu; - printf("percpu: max_cpus: %u, section size: %zuk\n", X15_MAX_CPUS, - percpu_area_size >> 10); + log_info("percpu: max_cpus: %u, section size: %zuk", X15_MAX_CPUS, + percpu_area_size >> 10); assert(vm_page_aligned(percpu_area_size)); if (percpu_area_size == 0) { - return; + return 0; } order = vm_page_order(percpu_area_size); @@ -67,8 +69,13 @@ percpu_setup(void) percpu_area_content = vm_page_direct_ptr(page); memcpy(percpu_area_content, &_percpu, percpu_area_size); + return 0; } +INIT_OP_DEFINE(percpu_setup, + INIT_OP_DEP(percpu_bootstrap, true), + INIT_OP_DEP(vm_page_setup, true)); + int __init percpu_add(unsigned int cpu) { @@ -77,8 +84,8 @@ percpu_add(unsigned int cpu) if (cpu >= ARRAY_SIZE(percpu_areas)) { if (!percpu_skip_warning) { - printf("percpu: ignoring processor beyond id %zu\n", - ARRAY_SIZE(percpu_areas) - 1); + log_warning("percpu: ignoring processor beyond id %zu", + ARRAY_SIZE(percpu_areas) - 1); percpu_skip_warning = 1; } @@ -86,7 +93,7 @@ percpu_add(unsigned int cpu) } if (percpu_areas[cpu] != NULL) { - printf("percpu: error: id %u ignored, already registered\n", cpu); + log_err("percpu: id %u ignored, already registered", cpu); return ERROR_INVAL; } @@ -98,7 +105,7 @@ percpu_add(unsigned int cpu) page = vm_page_alloc(order, VM_PAGE_SEL_DIRECTMAP, VM_PAGE_KERNEL); if (page == NULL) { - printf("percpu: error: unable to allocate percpu area\n"); + log_err("percpu: unable to allocate percpu area"); return ERROR_NOMEM; } @@ -109,7 +116,7 @@ out: return 0; } -void +static int __init percpu_cleanup(void) { struct vm_page *page; @@ -118,4 +125,10 @@ percpu_cleanup(void) va = (uintptr_t)percpu_area_content; page = vm_page_lookup(vm_page_direct_pa(va)); vm_page_free(page, vm_page_order(percpu_area_size)); + return 0; } + +INIT_OP_DEFINE(percpu_cleanup, + INIT_OP_DEP(cpu_mp_probe, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(vm_page_setup, true)); diff --git a/kern/percpu.h b/kern/percpu.h index 59959518..879767ff 100644 --- a/kern/percpu.h +++ b/kern/percpu.h @@ -53,9 +53,11 @@ #ifndef _KERN_PERCPU_H #define _KERN_PERCPU_H +#include <assert.h> +#include <stddef.h> #include <stdint.h> -#include <kern/assert.h> +#include <kern/init.h> #include <kern/macros.h> #define PERCPU_SECTION .percpu @@ -94,25 +96,6 @@ percpu_area(unsigned int cpu) } /* - * Early initialization of the percpu module. - * - * This function registers the percpu section as the percpu area of the - * BSP. If a percpu variable is modified before calling percpu_setup(), - * the modification will be part of the percpu section and propagated to - * new percpu areas. - */ -void percpu_bootstrap(void); - -/* - * Complete initialization of the percpu module. - * - * The BSP keeps using the percpu section, but its content is copied to a - * dedicated block of memory used as a template for subsequently added - * processors. - */ -void percpu_setup(void); - -/* * Register a processor. * * This function creates a percpu area from kernel virtual memory for the @@ -122,8 +105,16 @@ void percpu_setup(void); int percpu_add(unsigned int cpu); /* - * Release init data allocated for setup. + * This init operation provides : + * - percpu section is registered as the BSP percpu area + */ +INIT_OP_DECLARE(percpu_bootstrap); + +/* + * This init operation provides : + * - percpu section is copied to a kernel buffer subsequently used as + * the template for future percpu areas */ -void percpu_cleanup(void); +INIT_OP_DECLARE(percpu_setup); #endif /* _KERN_PERCPU_H */ diff --git a/kern/printf.c b/kern/printf.c index f657fdfc..68cf4ba9 100644 --- a/kern/printf.c +++ b/kern/printf.c @@ -15,10 +15,11 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <stdio.h> - #include <kern/console.h> +#include <kern/fmt.h> +#include <kern/init.h> #include <kern/spinlock.h> +#include <machine/boot.h> #include <machine/cpu.h> /* @@ -51,7 +52,7 @@ vprintf(const char *format, va_list ap) spinlock_lock_intr_save(&printf_lock, &flags); - length = vsnprintf(printf_buffer, sizeof(printf_buffer), format, ap); + length = fmt_vsnprintf(printf_buffer, sizeof(printf_buffer), format, ap); for (ptr = printf_buffer; *ptr != '\0'; ptr++) { console_putchar(*ptr); @@ -62,8 +63,14 @@ vprintf(const char *format, va_list ap) return length; } -void +static int __init printf_setup(void) { spinlock_init(&printf_lock); + return 0; } + +INIT_OP_DEFINE(printf_setup, + INIT_OP_DEP(boot_bootstrap_console, true), + INIT_OP_DEP(console_bootstrap, true), + INIT_OP_DEP(spinlock_setup, true)); diff --git a/kern/printf.h b/kern/printf.h index 6643d664..8185c735 100644 --- a/kern/printf.h +++ b/kern/printf.h @@ -33,10 +33,18 @@ #include <stdarg.h> -#include <kern/macros.h> +#include <kern/init.h> -int printf(const char *format, ...) __format_printf(1, 2); -int vprintf(const char *format, va_list ap) __format_printf(1, 0); -void printf_setup(void); +int printf(const char *format, ...) + __attribute__((format(printf, 1, 2))); + +int vprintf(const char *format, va_list ap) + __attribute__((format(printf, 1, 0))); + +/* + * This init operation provides : + * - printf is usable + */ +INIT_OP_DECLARE(printf_setup); #endif /* _KERN_PRINTF_H */ diff --git a/kern/rbtree.c b/kern/rbtree.c index e2ea54ad..adce033a 100644 --- a/kern/rbtree.c +++ b/kern/rbtree.c @@ -15,10 +15,10 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/macros.h> #include <kern/rbtree.h> #include <kern/rbtree_i.h> diff --git a/kern/rbtree.h b/kern/rbtree.h index 38083427..4ae8353f 100644 --- a/kern/rbtree.h +++ b/kern/rbtree.h @@ -21,10 +21,10 @@ #ifndef _KERN_RBTREE_H #define _KERN_RBTREE_H +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/macros.h> /* diff --git a/kern/rbtree_i.h b/kern/rbtree_i.h index 99722977..944148e0 100644 --- a/kern/rbtree_i.h +++ b/kern/rbtree_i.h @@ -18,10 +18,10 @@ #ifndef _KERN_RBTREE_I_H #define _KERN_RBTREE_I_H +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/macros.h> /* diff --git a/kern/rdxtree.c b/kern/rdxtree.c index 788f3ba8..77005b64 100644 --- a/kern/rdxtree.c +++ b/kern/rdxtree.c @@ -15,15 +15,16 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <limits.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> #include <string.h> -#include <kern/assert.h> #include <kern/error.h> +#include <kern/init.h> #include <kern/kmem.h> -#include <kern/limits.h> #include <kern/llsync.h> #include <kern/macros.h> #include <kern/rdxtree.h> @@ -895,10 +896,14 @@ rdxtree_remove_all(struct rdxtree *tree) } } -void +static int __init rdxtree_setup(void) { kmem_cache_init(&rdxtree_node_cache, "rdxtree_node", sizeof(struct rdxtree_node), 0, - rdxtree_node_ctor, 0); + rdxtree_node_ctor, KMEM_CACHE_PAGE_ONLY); + return 0; } + +INIT_OP_DEFINE(rdxtree_setup, + INIT_OP_DEP(kmem_bootstrap, true)); diff --git a/kern/rdxtree.h b/kern/rdxtree.h index e3a2ba06..6ae1e6e7 100644 --- a/kern/rdxtree.h +++ b/kern/rdxtree.h @@ -29,7 +29,10 @@ #include <stddef.h> #include <stdint.h> -typedef uint32_t rdxtree_key_t; +#include <kern/init.h> +#include <kern/llsync.h> + +typedef uint64_t rdxtree_key_t; /* * Radix tree initialization flags. @@ -154,6 +157,12 @@ rdxtree_lookup_slot(const struct rdxtree *tree, rdxtree_key_t key) return rdxtree_lookup_common(tree, key, 1); } +static inline void * +rdxtree_load_slot(void **slot) +{ + return llsync_read_ptr(*slot); +} + /* * Replace a pointer in a tree. * @@ -192,8 +201,9 @@ rdxtree_iter_key(const struct rdxtree_iter *iter) void rdxtree_remove_all(struct rdxtree *tree); /* - * Initialize the rdxtree module. + * This init operation provides : + * - module fully initialized */ -void rdxtree_setup(void); +INIT_OP_DECLARE(rdxtree_setup); #endif /* _KERN_RDXTREE_H */ diff --git a/kern/rtmutex.c b/kern/rtmutex.c index 6909ce18..fcf5f358 100644 --- a/kern/rtmutex.c +++ b/kern/rtmutex.c @@ -15,10 +15,10 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/rtmutex.h> #include <kern/rtmutex_i.h> @@ -29,7 +29,7 @@ static void rtmutex_set_contended(struct rtmutex *rtmutex) { - atomic_or_acq_rel(&rtmutex->owner, RTMUTEX_CONTENDED); + atomic_or(&rtmutex->owner, RTMUTEX_CONTENDED, ATOMIC_RELEASE); } void @@ -64,8 +64,7 @@ rtmutex_lock_slow(struct rtmutex *rtmutex) turnstile_own(turnstile); if (turnstile_empty(turnstile)) { - /* TODO Review memory order */ - prev_owner = atomic_swap_acquire(&rtmutex->owner, owner); + prev_owner = atomic_swap(&rtmutex->owner, owner, ATOMIC_RELAXED); assert(prev_owner == (owner | bits)); } diff --git a/kern/rtmutex.h b/kern/rtmutex.h index f64274d7..ec79afa9 100644 --- a/kern/rtmutex.h +++ b/kern/rtmutex.h @@ -24,9 +24,9 @@ #ifndef _KERN_RTMUTEX_H #define _KERN_RTMUTEX_H +#include <assert.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/error.h> #include <kern/macros.h> #include <kern/rtmutex_i.h> diff --git a/kern/rtmutex_i.h b/kern/rtmutex_i.h index 2f2cc17f..984cfd16 100644 --- a/kern/rtmutex_i.h +++ b/kern/rtmutex_i.h @@ -18,10 +18,10 @@ #ifndef _KERN_RTMUTEX_I_H #define _KERN_RTMUTEX_I_H +#include <assert.h> #include <stdbool.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/rtmutex_types.h> #include <kern/thread.h> diff --git a/kern/semaphore.h b/kern/semaphore.h index d7219b4f..e08927ec 100644 --- a/kern/semaphore.h +++ b/kern/semaphore.h @@ -32,7 +32,8 @@ #ifndef _KERN_SEMAPHORE_H #define _KERN_SEMAPHORE_H -#include <kern/assert.h> +#include <assert.h> + #include <kern/atomic.h> #include <kern/error.h> #include <kern/macros.h> diff --git a/kern/semaphore_i.h b/kern/semaphore_i.h index 54985062..acd7cd48 100644 --- a/kern/semaphore_i.h +++ b/kern/semaphore_i.h @@ -18,7 +18,8 @@ #ifndef _KERN_SEMAPHORE_I_H #define _KERN_SEMAPHORE_I_H -#include <kern/assert.h> +#include <assert.h> + #include <kern/atomic.h> struct semaphore { diff --git a/kern/shell.c b/kern/shell.c index c4675bf4..a27c53f3 100644 --- a/kern/shell.c +++ b/kern/shell.c @@ -18,10 +18,10 @@ #include <stdio.h> #include <string.h> -#include <kern/console.h> #include <kern/error.h> -#include <kern/init.h> #include <kern/hash.h> +#include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/mutex.h> #include <kern/shell.h> @@ -224,6 +224,10 @@ shell_cmd_lookup(const char *name) } /* + * Look up the first command that matches a given string. + * + * The input string is defined by the given string pointer and size. + * * The global lock must be acquired before calling this function. */ static const struct shell_cmd * @@ -242,6 +246,21 @@ shell_cmd_match(const struct shell_cmd *cmd, const char *str, } /* + * Attempt command auto-completion. + * + * The given string is the beginning of a command, or the empty string. + * The sizep parameter initially points to the size of the given string. + * If the string matches any registered command, the cmdp pointer is + * updated to point to the first matching command in the sorted list of + * commands, and sizep is updated to the number of characters in the + * command name that are common in subsequent commands. The command + * pointer and the returned size can be used to print a list of commands + * eligible for completion. + * + * If there is a single match for the given string, return 0. If there + * are more than one match, return ERROR_AGAIN. If there is no match, + * return ERROR_INVAL. + * * The global lock must be acquired before calling this function. */ static int @@ -309,6 +328,10 @@ shell_cmd_complete(const char *str, unsigned long *sizep, } /* + * Print a list of commands eligible for completion, starting at the + * given command. Other eligible commands share the same prefix, as + * defined by the size argument. + * * The global lock must be acquired before calling this function. */ static void @@ -422,7 +445,7 @@ shell_cmd_add(struct shell_cmd *cmd) for (;;) { if (strcmp(cmd->name, tmp->name) == 0) { - printf("shell: %s: shell command name collision", cmd->name); + log_err("shell: %s: shell command name collision", cmd->name); return ERROR_EXIST; } @@ -644,7 +667,7 @@ shell_cmd_help(int argc, char *argv[]) shell_cmd_acquire(); for (cmd = shell_list; cmd != NULL; cmd = cmd->ls_next) { - printf("%13s %s\n", cmd->name, cmd->short_desc); + printf("%19s %s\n", cmd->name, cmd->short_desc); } shell_cmd_release(); @@ -668,7 +691,7 @@ shell_cmd_history(int argc, char *argv[]) static struct shell_cmd shell_default_cmds[] = { SHELL_CMD_INITIALIZER("help", shell_cmd_help, "help [command]", - "obtain help about shell commands"), + "display help about shell commands"), SHELL_CMD_INITIALIZER("history", shell_cmd_history, "history", "display history list"), @@ -811,6 +834,10 @@ shell_process_raw_char(char c) goto out; } + /* + * This assumes that the backspace character only moves the cursor + * without erasing characters. + */ printf("%s", shell_line_str(current_line) + shell_cursor - 1); remaining_chars = shell_line_size(current_line) - shell_cursor; @@ -1155,11 +1182,9 @@ shell_run(void *arg) } } -void __init +static int __init shell_setup(void) { - struct thread_attr attr; - struct thread *thread; unsigned long i; int error; @@ -1170,6 +1195,21 @@ shell_setup(void) error_check(error, "shell_cmd_register"); } + return 0; +} + +INIT_OP_DEFINE(shell_setup, + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(printf_setup, true)); + +void __init +shell_start(void) +{ + struct thread_attr attr; + struct thread *thread; + int error; + thread_attr_init(&attr, THREAD_KERNEL_PREFIX "shell"); thread_attr_set_detached(&attr); error = thread_create(&thread, &attr, shell_run, NULL); diff --git a/kern/shell.h b/kern/shell.h index 5ff4920d..cf56cebf 100644 --- a/kern/shell.h +++ b/kern/shell.h @@ -21,6 +21,23 @@ #ifndef _KERN_SHELL_H #define _KERN_SHELL_H +#include <kern/init.h> +#include <kern/error.h> +#include <kern/macros.h> + +#ifdef X15_SHELL + +#define SHELL_REGISTER_CMDS(cmds) \ +MACRO_BEGIN \ + size_t ___i; \ + int ___error; \ + \ + for (___i = 0; ___i < ARRAY_SIZE(cmds); ___i++) { \ + ___error = shell_cmd_register(&(cmds)[___i]); \ + error_check(___error, __func__); \ + } \ +MACRO_END + typedef void (*shell_fn_t)(int argc, char *argv[]); struct shell_cmd { @@ -49,11 +66,9 @@ void shell_cmd_init(struct shell_cmd *cmd, const char *name, const char *short_desc, const char *long_desc); /* - * Initialize the shell module. - * - * On return, shell commands can be registered. + * Start the shell thread. */ -void shell_setup(void); +void shell_start(void); /* * Register a shell command. @@ -66,4 +81,17 @@ void shell_setup(void); */ int shell_cmd_register(struct shell_cmd *cmd); +#else /* X15_SHELL */ +#define SHELL_REGISTER_CMDS(cmds) +#define shell_setup() +#define shell_start() +#endif /* X15_SHELL */ + +/* + * This init operation provides : + * - commands can be registered + * - module fully initialized + */ +INIT_OP_DECLARE(shell_setup); + #endif /* _KERN_SHELL_H */ diff --git a/kern/shutdown.c b/kern/shutdown.c new file mode 100644 index 00000000..1c950415 --- /dev/null +++ b/kern/shutdown.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <stdio.h> + +#include <kern/init.h> +#include <kern/plist.h> +#include <kern/shell.h> +#include <kern/shutdown.h> +#include <machine/boot.h> +#include <machine/cpu.h> + +static struct plist shutdown_ops_list; + +#ifdef X15_SHELL + +static void +shutdown_shell_halt(int argc, char **argv) +{ + (void)argc; + (void)argv; + + shutdown_halt(); +} + +static void +shutdown_shell_reboot(int argc, char **argv) +{ + (void)argc; + (void)argv; + + shutdown_reboot(); +} + +static struct shell_cmd shutdown_shell_cmds[] = { + SHELL_CMD_INITIALIZER("shutdown_halt", shutdown_shell_halt, + "shutdown_halt", + "halt the system"), + SHELL_CMD_INITIALIZER("shutdown_reboot", shutdown_shell_reboot, + "shutdown_reboot", + "reboot the system"), +}; + +static int __init +shutdown_setup_shell(void) +{ + SHELL_REGISTER_CMDS(shutdown_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(shutdown_setup_shell, + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(shutdown_setup, true)); + +#endif /* X15_SHELL */ + +static int __init +shutdown_bootstrap(void) +{ + plist_init(&shutdown_ops_list); + return 0; +} + +INIT_OP_DEFINE(shutdown_bootstrap); + +static int __init +shutdown_setup(void) +{ + return 0; +} + +INIT_OP_DEFINE(shutdown_setup, + INIT_OP_DEP(boot_setup_shutdown, true), + INIT_OP_DEP(cpu_setup, true), + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shutdown_bootstrap, true)); + +void __init +shutdown_register(struct shutdown_ops *ops, unsigned int priority) +{ + plist_node_init(&ops->node, priority); + plist_add(&shutdown_ops_list, &ops->node); +} + +static void +shutdown_halt_other_cpus(void) +{ + cpu_intr_disable(); + cpu_halt_broadcast(); +} + +void +shutdown_halt(void) +{ + shutdown_halt_other_cpus(); + printf("shutdown: system halted\n"); + cpu_halt(); +} + +void +shutdown_reboot(void) +{ + struct shutdown_ops *ops; + + if (plist_empty(&shutdown_ops_list)) { + printf("shutdown: no reset operation available, halting\n"); + shutdown_halt(); + } + + shutdown_halt_other_cpus(); + printf("shutdown: rebooting...\n"); + + plist_for_each_entry_reverse(&shutdown_ops_list, ops, node) { + ops->reset(); + } + + cpu_halt(); +} diff --git a/kern/shutdown.h b/kern/shutdown.h new file mode 100644 index 00000000..f61079b3 --- /dev/null +++ b/kern/shutdown.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 Richard Braun. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _KERN_SHUTDOWN_H +#define _KERN_SHUTDOWN_H + +#include <stdnoreturn.h> + +#include <kern/init.h> +#include <kern/plist.h> + +struct shutdown_ops { + struct plist_node node; + void (*reset)(void); +}; + +void shutdown_register(struct shutdown_ops *ops, unsigned int priority); + +noreturn void shutdown_halt(void); +noreturn void shutdown_reboot(void); + +/* + * This init operation provides : + * - registration of shutdown operations + */ +INIT_OP_DECLARE(shutdown_bootstrap); + +/* + * This init operation provides : + * - all shutdown operations have been registered + * - module fully initialized + */ +INIT_OP_DECLARE(shutdown_setup); + +#endif /* _KERN_SHUTDOWN_H */ diff --git a/kern/sleepq.c b/kern/sleepq.c index 5af3d064..44ad9962 100644 --- a/kern/sleepq.c +++ b/kern/sleepq.c @@ -18,24 +18,25 @@ * TODO Analyse hash parameters. */ +#include <assert.h> +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/sleepq.h> #include <kern/spinlock.h> #include <kern/thread.h> +#include <machine/cpu.h> struct sleepq_bucket { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; struct list list; -} __aligned(CPU_L1_SIZE); +}; struct sleepq { struct sleepq_bucket *bucket; @@ -192,7 +193,7 @@ sleepq_ctor(void *ptr) sleepq->next_free = NULL; } -void __init +static int __init sleepq_bootstrap(void) { unsigned int i; @@ -204,15 +205,24 @@ sleepq_bootstrap(void) for (i = 0; i < ARRAY_SIZE(sleepq_cond_htable); i++) { sleepq_bucket_init(&sleepq_cond_htable[i]); } + + return 0; } -void __init +INIT_OP_DEFINE(sleepq_bootstrap); + +static int __init sleepq_setup(void) { kmem_cache_init(&sleepq_cache, "sleepq", sizeof(struct sleepq), CPU_L1_SIZE, sleepq_ctor, 0); + return 0; } +INIT_OP_DEFINE(sleepq_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(sleepq_bootstrap, true)); + struct sleepq * sleepq_create(void) { diff --git a/kern/sleepq.h b/kern/sleepq.h index f55d8c4b..651b3e7c 100644 --- a/kern/sleepq.h +++ b/kern/sleepq.h @@ -38,21 +38,9 @@ #include <stdbool.h> -struct sleepq; +#include <kern/init.h> -/* - * Early initialization of the sleepq module. - * - * This module is initialized by architecture-specific code. It should - * be one of the first modules to be initialized since it's used by - * synchronization objects that may be accessed very early. - */ -void sleepq_bootstrap(void); - -/* - * Initialize the sleepq module. - */ -void sleepq_setup(void); +struct sleepq; /* * Create/destroy a sleep queue. @@ -142,4 +130,17 @@ void sleepq_wait(struct sleepq *sleepq, const char *wchan); */ void sleepq_signal(struct sleepq *sleepq); +/* + * This init operation provides : + * - ? TODO Review + */ +INIT_OP_DECLARE(sleepq_bootstrap); + +/* + * This init operation provides : + * - sleepq creation + * - module fully initialized + */ +INIT_OP_DECLARE(sleepq_setup); + #endif /* _KERN_SLEEPQ_H */ diff --git a/kern/spinlock.c b/kern/spinlock.c index 3ec36b36..fcb7c7b6 100644 --- a/kern/spinlock.c +++ b/kern/spinlock.c @@ -51,13 +51,25 @@ * and fast paths operations fail. The lock operation must make sure * that the lock value is restored to SPINLOCK_LOCKED if there is no * more contention, an operation called downgrading. + * + * In order to enforce correct visibility of the critical section, + * acquire and release atomic operations are used when accessing a + * qnode lock. Similarly, acquire and release semantics are also used + * when accessing the lock word which stores the first and last QIDs, + * so that the memory operations on the qnodes referenced by those QIDs + * are correctly enforced. Accessing the next QID in a qnode however + * is performed with no memory order constraint, because the qnodes + * referenced by those QID are never accessed unless they are the first + * or the last, cases which do enforce ordering. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/error.h> +#include <kern/init.h> #include <kern/macros.h> #include <kern/percpu.h> #include <kern/spinlock.h> @@ -118,7 +130,7 @@ struct spinlock_qnode { #endif struct spinlock_cpu_data { - struct spinlock_qnode qnodes[SPINLOCK_NR_CTXS - 1] __aligned(CPU_L1_SIZE); + alignas(CPU_L1_SIZE) struct spinlock_qnode qnodes[SPINLOCK_NR_CTXS - 1]; }; static struct spinlock_cpu_data spinlock_cpu_data __percpu; @@ -204,7 +216,7 @@ spinlock_load_first_qid(const struct spinlock *lock) { unsigned int value; - value = atomic_load(&lock->value, ATOMIC_ACQUIRE); + value = atomic_load_acquire(&lock->value); return (value >> SPINLOCK_QID_MAX_BITS) & SPINLOCK_QID_MASK; } @@ -268,7 +280,7 @@ spinlock_lock_slow(struct spinlock *lock) } for (;;) { - locked = atomic_load(&qnode->locked, ATOMIC_ACQUIRE); + locked = atomic_load_acquire(&qnode->locked); if (!locked) { break; @@ -316,5 +328,14 @@ spinlock_unlock_slow(struct spinlock *lock) } next_qnode = spinlock_get_remote_qnode(next_qid); - atomic_store(&next_qnode->locked, false, ATOMIC_RELEASE); + atomic_store_release(&next_qnode->locked, false); +} + +static int __init +spinlock_setup(void) +{ + return 0; } + +INIT_OP_DEFINE(spinlock_setup, + INIT_OP_DEP(thread_setup_booter, true)); diff --git a/kern/spinlock.h b/kern/spinlock.h index 4dc4c83e..d4105da0 100644 --- a/kern/spinlock.h +++ b/kern/spinlock.h @@ -26,6 +26,7 @@ #ifndef _KERN_SPINLOCK_H #define _KERN_SPINLOCK_H +#include <kern/init.h> #include <kern/macros.h> #include <kern/spinlock_i.h> #include <kern/spinlock_types.h> @@ -162,4 +163,12 @@ spinlock_unlock_intr_restore(struct spinlock *lock, unsigned long flags) thread_preempt_enable(); } +/* + * This init operation provides : + * - uncontended spinlock locking + * + * Contended locking may only occur after starting APs. + */ +INIT_OP_DECLARE(spinlock_setup); + #endif /* _KERN_SPINLOCK_H */ diff --git a/kern/spinlock_i.h b/kern/spinlock_i.h index 45018033..c9dcdd10 100644 --- a/kern/spinlock_i.h +++ b/kern/spinlock_i.h @@ -18,10 +18,10 @@ #ifndef _KERN_SPINLOCK_I_H #define _KERN_SPINLOCK_I_H +#include <assert.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/error.h> #include <kern/macros.h> diff --git a/kern/sprintf.c b/kern/sprintf.c deleted file mode 100644 index a606d866..00000000 --- a/kern/sprintf.c +++ /dev/null @@ -1,577 +0,0 @@ -/* - * Copyright (c) 2010 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <stdarg.h> -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> - -#include <kern/limits.h> -#include <kern/types.h> - -/* - * Formatting flags. - * - * FORMAT_LOWER must be 0x20 as it is OR'd with digits, eg. - * '0': 0x30 | 0x20 => 0x30 ('0') - * 'A': 0x41 | 0x20 => 0x61 ('a') - */ -#define SPRINTF_FORMAT_ALT_FORM 0x01 -#define SPRINTF_FORMAT_ZERO_PAD 0x02 -#define SPRINTF_FORMAT_LEFT_JUSTIFY 0x04 -#define SPRINTF_FORMAT_BLANK 0x08 -#define SPRINTF_FORMAT_SIGN 0x10 -#define SPRINTF_FORMAT_LOWER 0x20 -#define SPRINTF_FORMAT_CONV_SIGNED 0x40 - -enum { - SPRINTF_MODIFIER_NONE, - SPRINTF_MODIFIER_CHAR, - SPRINTF_MODIFIER_SHORT, - SPRINTF_MODIFIER_LONG, - SPRINTF_MODIFIER_LONGLONG, - SPRINTF_MODIFIER_PTR, /* Used only for %p */ - SPRINTF_MODIFIER_SIZE, - SPRINTF_MODIFIER_PTRDIFF -}; - -enum { - SPRINTF_SPECIFIER_INVALID, - SPRINTF_SPECIFIER_INT, - SPRINTF_SPECIFIER_CHAR, - SPRINTF_SPECIFIER_STR, - SPRINTF_SPECIFIER_NRCHARS, - SPRINTF_SPECIFIER_PERCENT -}; - -/* - * Size for the temporary number buffer. The minimum base is 8 so 3 bits - * are consumed per digit. Add one to round up. The conversion algorithm - * doesn't use the null byte. - */ -#define SPRINTF_MAX_NUM_SIZE (((sizeof(uint64_t) * CHAR_BIT) / 3) + 1) - -/* - * Special size for vsnprintf(), used by sprintf()/vsprintf() when the - * buffer size is unknown. - */ -#define SPRINTF_NOLIMIT ((size_t)-1) - -static const char sprintf_digits[] = "0123456789ABCDEF"; - -static inline char * -sprintf_putchar(char *str, char *end, char c) -{ - if (str < end) { - *str = c; - } - - str++; - - return str; -} - -static inline int -sprintf_isdigit(char c) -{ - return (c >= '0') && (c <= '9'); -} - -int -sprintf(char *str, const char *format, ...) -{ - va_list ap; - int length; - - va_start(ap, format); - length = vsprintf(str, format, ap); - va_end(ap); - - return length; -} - -int -vsprintf(char *str, const char *format, va_list ap) -{ - return vsnprintf(str, SPRINTF_NOLIMIT, format, ap); -} - -int -snprintf(char *str, size_t size, const char *format, ...) -{ - va_list ap; - int length; - - va_start(ap, format); - length = vsnprintf(str, size, format, ap); - va_end(ap); - - return length; -} - -int -vsnprintf(char *str, size_t size, const char *format, va_list ap) -{ - unsigned long long n; - int i, len, found, flags, width, precision, modifier, specifier, shift; - unsigned char r, base, mask; - char c, *s, *start, *end, sign, tmp[SPRINTF_MAX_NUM_SIZE]; - - start = str; - - if (size == 0) { - end = NULL; - } else if (size == SPRINTF_NOLIMIT) { - end = (char *)-1; - } else { - end = start + size - 1; - } - - while ((c = *format) != '\0') { - if (c != '%') { - str = sprintf_putchar(str, end, c); - format++; - continue; - } - - /* Flags */ - - found = 1; - flags = 0; - - do { - format++; - c = *format; - - switch (c) { - case '#': - flags |= SPRINTF_FORMAT_ALT_FORM; - break; - case '0': - flags |= SPRINTF_FORMAT_ZERO_PAD; - break; - case '-': - flags |= SPRINTF_FORMAT_LEFT_JUSTIFY; - break; - case ' ': - flags |= SPRINTF_FORMAT_BLANK; - break; - case '+': - flags |= SPRINTF_FORMAT_SIGN; - break; - default: - found = 0; - break; - } - } while (found); - - /* Width */ - - if (sprintf_isdigit(c)) { - width = 0; - - while (sprintf_isdigit(c)) { - width = width * 10 + (c - '0'); - format++; - c = *format; - } - } else if (c == '*') { - width = va_arg(ap, int); - - if (width < 0) { - flags |= SPRINTF_FORMAT_LEFT_JUSTIFY; - width = -width; - } - - format++; - c = *format; - } else { - width = 0; - } - - /* Precision */ - - if (c == '.') { - format++; - c = *format; - - if (sprintf_isdigit(c)) { - precision = 0; - - while (sprintf_isdigit(c)) { - precision = precision * 10 + (c - '0'); - format++; - c = *format; - } - } else if (c == '*') { - precision = va_arg(ap, int); - - if (precision < 0) { - precision = 0; - } - - format++; - c = *format; - } else { - precision = 0; - } - } else { - /* precision is >= 0 only if explicit */ - precision = -1; - } - - /* Length modifier */ - - switch (c) { - case 'h': - case 'l': - format++; - - if (c == *format) { - modifier = (c == 'h') - ? SPRINTF_MODIFIER_CHAR - : SPRINTF_MODIFIER_LONGLONG; - goto skip_modifier; - } else { - modifier = (c == 'h') - ? SPRINTF_MODIFIER_SHORT - : SPRINTF_MODIFIER_LONG; - c = *format; - } - - break; - case 'z': - modifier = SPRINTF_MODIFIER_SIZE; - goto skip_modifier; - case 't': - modifier = SPRINTF_MODIFIER_PTRDIFF; -skip_modifier: - format++; - c = *format; - break; - default: - modifier = SPRINTF_MODIFIER_NONE; - break; - } - - /* Specifier */ - - switch (c) { - case 'd': - case 'i': - flags |= SPRINTF_FORMAT_CONV_SIGNED; - case 'u': - base = 10; - goto integer; - case 'o': - base = 8; - goto integer; - case 'p': - flags |= SPRINTF_FORMAT_ALT_FORM; - modifier = SPRINTF_MODIFIER_PTR; - case 'x': - flags |= SPRINTF_FORMAT_LOWER; - case 'X': - base = 16; -integer: - specifier = SPRINTF_SPECIFIER_INT; - break; - case 'c': - specifier = SPRINTF_SPECIFIER_CHAR; - break; - case 's': - specifier = SPRINTF_SPECIFIER_STR; - break; - case 'n': - specifier = SPRINTF_SPECIFIER_NRCHARS; - break; - case '%': - specifier = SPRINTF_SPECIFIER_PERCENT; - break; - default: - specifier = SPRINTF_SPECIFIER_INVALID; - break; - } - - /* Output */ - - switch (specifier) { - case SPRINTF_SPECIFIER_INT: - switch (modifier) { - case SPRINTF_MODIFIER_CHAR: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = (signed char)va_arg(ap, int); - } else { - n = (unsigned char)va_arg(ap, int); - } - break; - case SPRINTF_MODIFIER_SHORT: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = (short)va_arg(ap, int); - } else { - n = (unsigned short)va_arg(ap, int); - } - break; - case SPRINTF_MODIFIER_LONG: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, long); - } else { - n = va_arg(ap, unsigned long); - } - break; - case SPRINTF_MODIFIER_LONGLONG: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, long long); - } else { - n = va_arg(ap, unsigned long long); - } - break; - case SPRINTF_MODIFIER_PTR: - n = (uintptr_t)va_arg(ap, void *); - break; - case SPRINTF_MODIFIER_SIZE: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, ssize_t); - } else { - n = va_arg(ap, size_t); - } - break; - case SPRINTF_MODIFIER_PTRDIFF: - n = va_arg(ap, ptrdiff_t); - break; - default: - if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - n = va_arg(ap, int); - } else { - n = va_arg(ap, unsigned int); - } - break; - } - - if ((flags & SPRINTF_FORMAT_LEFT_JUSTIFY) || (precision >= 0)) { - flags &= ~SPRINTF_FORMAT_ZERO_PAD; - } - - sign = 0; - - if (flags & SPRINTF_FORMAT_ALT_FORM) { - /* '0' for octal */ - width--; - - /* '0x' or '0X' for hexadecimal */ - if (base == 16) { - width--; - } - } else if (flags & SPRINTF_FORMAT_CONV_SIGNED) { - if ((long long)n < 0) { - sign = '-'; - width--; - n = -(long long)n; - } else if (flags & SPRINTF_FORMAT_SIGN) { - /* SPRINTF_FORMAT_SIGN must precede SPRINTF_FORMAT_BLANK. */ - sign = '+'; - width--; - } else if (flags & SPRINTF_FORMAT_BLANK) { - sign = ' '; - width--; - } - } - - /* Conversion, in reverse order */ - - i = 0; - - if (n == 0) { - if (precision != 0) { - tmp[i++] = '0'; - } - } else if (base == 10) { - /* - * Try to avoid 64 bits operations if the processor doesn't - * support them. Note that even when using modulus and - * division operators close to each other, the compiler will - * forge two calls to __udivdi3() and __umoddi3() instead of - * one to __udivmoddi3(), whereas processor instructions are - * generally correctly used once, giving both the remainder - * and the quotient, through plain or reciprocal division. - */ -#ifndef __LP64__ - if (modifier == SPRINTF_MODIFIER_LONGLONG) { -#endif /* __LP64__ */ - do { - r = n % 10; - n /= 10; - tmp[i++] = sprintf_digits[r]; - } while (n != 0); -#ifndef __LP64__ - } else { - unsigned long m; - - m = (unsigned long)n; - - do { - r = m % 10; - m /= 10; - tmp[i++] = sprintf_digits[r]; - } while (m != 0); - } -#endif /* __LP64__ */ - } else { - mask = base - 1; - shift = (base == 8) ? 3 : 4; - - do { - r = (unsigned char)n & mask; - n >>= shift; - tmp[i++] = sprintf_digits[r] - | (flags & SPRINTF_FORMAT_LOWER); - } while (n != 0); - } - - if (i > precision) { - precision = i; - } - - width -= precision; - - if (!(flags & (SPRINTF_FORMAT_LEFT_JUSTIFY - | SPRINTF_FORMAT_ZERO_PAD))) - while (width-- > 0) { - str = sprintf_putchar(str, end, ' '); - } - - if (flags & SPRINTF_FORMAT_ALT_FORM) { - str = sprintf_putchar(str, end, '0'); - - if (base == 16) - str = sprintf_putchar(str, end, - 'X' | (flags & SPRINTF_FORMAT_LOWER)); - } else if (sign) { - str = sprintf_putchar(str, end, sign); - } - - if (!(flags & SPRINTF_FORMAT_LEFT_JUSTIFY)) { - c = (flags & SPRINTF_FORMAT_ZERO_PAD) ? '0' : ' '; - - while (width-- > 0) { - str = sprintf_putchar(str, end, c); - } - } - - while (i < precision--) { - str = sprintf_putchar(str, end, '0'); - } - - while (i-- > 0) { - str = sprintf_putchar(str, end, tmp[i]); - } - - while (width-- > 0) { - str = sprintf_putchar(str, end, ' '); - } - - break; - case SPRINTF_SPECIFIER_CHAR: - c = (unsigned char)va_arg(ap, int); - - if (!(flags & SPRINTF_FORMAT_LEFT_JUSTIFY)) - while (--width > 0) { - str = sprintf_putchar(str, end, ' '); - } - - str = sprintf_putchar(str, end, c); - - while (--width > 0) { - str = sprintf_putchar(str, end, ' '); - } - - break; - case SPRINTF_SPECIFIER_STR: - s = va_arg(ap, char *); - - if (s == NULL) { - s = "(null)"; - } - - len = 0; - - for (len = 0; s[len] != '\0'; len++) - if (len == precision) { - break; - } - - if (!(flags & SPRINTF_FORMAT_LEFT_JUSTIFY)) - while (len < width--) { - str = sprintf_putchar(str, end, ' '); - } - - for (i = 0; i < len; i++) { - str = sprintf_putchar(str, end, *s); - s++; - } - - while (len < width--) { - str = sprintf_putchar(str, end, ' '); - } - - break; - case SPRINTF_SPECIFIER_NRCHARS: - if (modifier == SPRINTF_MODIFIER_CHAR) { - signed char *ptr = va_arg(ap, signed char *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_SHORT) { - short *ptr = va_arg(ap, short *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_LONG) { - long *ptr = va_arg(ap, long *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_LONGLONG) { - long long *ptr = va_arg(ap, long long *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_SIZE) { - ssize_t *ptr = va_arg(ap, ssize_t *); - *ptr = str - start; - } else if (modifier == SPRINTF_MODIFIER_PTRDIFF) { - ptrdiff_t *ptr = va_arg(ap, ptrdiff_t *); - *ptr = str - start; - } else { - int *ptr = va_arg(ap, int *); - *ptr = str - start; - } - - break; - case SPRINTF_SPECIFIER_PERCENT: - case SPRINTF_SPECIFIER_INVALID: - str = sprintf_putchar(str, end, '%'); - break; - default: - break; - } - - if (specifier != SPRINTF_SPECIFIER_INVALID) { - format++; - } - } - - if (str < end) { - *str = '\0'; - } else if (end != NULL) { - *end = '\0'; - } - - return str - start; -} diff --git a/kern/sprintf.h b/kern/sprintf.h deleted file mode 100644 index 1cb51e7d..00000000 --- a/kern/sprintf.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2010 Richard Braun. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * - * Formatted string functions. - * - * The functions provided by this module implement a subset of the C99 - * sprintf() like functions, mostly centered around character, string, and - * integer conversions. - * - * The supported specifiers are: d i o u x X c s p n % - * The supported length modifiers are: hh h l ll z t - */ - -#ifndef _KERN_SPRINTF_H -#define _KERN_SPRINTF_H - -#ifndef _STDIO_H -#error "do not use <kern/sprintf.h> directly; include <stdio.h> instead" -#endif /* _STDIO_H */ - -#include <stdarg.h> - -#include <kern/macros.h> - -int sprintf(char *str, const char *format, ...) __format_printf(2, 3); -int vsprintf(char *str, const char *format, va_list ap) __format_printf(2, 0); - -int snprintf(char *str, size_t size, const char *format, ...) - __format_printf(3, 4); -int vsnprintf(char *str, size_t size, const char *format, va_list ap) - __format_printf(3, 0); - -#endif /* _KERN_SPRINTF_H */ diff --git a/kern/sref.c b/kern/sref.c index 3f399a30..9c49a425 100644 --- a/kern/sref.c +++ b/kern/sref.c @@ -41,19 +41,18 @@ * TODO Reconsider whether it's possible to bring back local review queues. */ +#include <assert.h> #include <stdbool.h> #include <stddef.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/condition.h> #include <kern/cpumap.h> #include <kern/error.h> #include <kern/init.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/sref.h> @@ -259,13 +258,13 @@ sref_weakref_init(struct sref_weakref *weakref, struct sref_counter *counter) static void sref_weakref_mark_dying(struct sref_weakref *weakref) { - atomic_or_acq_rel(&weakref->addr, SREF_WEAKREF_DYING); + atomic_or(&weakref->addr, SREF_WEAKREF_DYING, ATOMIC_RELEASE); } static void sref_weakref_clear_dying(struct sref_weakref *weakref) { - atomic_and_acq_rel(&weakref->addr, SREF_WEAKREF_MASK); + atomic_and(&weakref->addr, SREF_WEAKREF_MASK, ATOMIC_RELEASE); } static int @@ -273,8 +272,8 @@ sref_weakref_kill(struct sref_weakref *weakref) { uintptr_t addr, oldval; - addr = weakref->addr | SREF_WEAKREF_DYING; - oldval = atomic_cas_release(&weakref->addr, addr, (uintptr_t)NULL); + addr = atomic_load(&weakref->addr, ATOMIC_RELAXED) | SREF_WEAKREF_DYING; + oldval = atomic_cas(&weakref->addr, addr, (uintptr_t)NULL, ATOMIC_RELAXED); if (oldval != addr) { assert((oldval & SREF_WEAKREF_MASK) == (addr & SREF_WEAKREF_MASK)); @@ -289,8 +288,9 @@ sref_weakref_tryget(struct sref_weakref *weakref) { uintptr_t addr, oldval, newval; + /* TODO Review */ do { - addr = weakref->addr; + addr = atomic_load(&weakref->addr, ATOMIC_RELAXED); newval = addr & SREF_WEAKREF_MASK; oldval = atomic_cas_acquire(&weakref->addr, addr, newval); } while (oldval != addr); @@ -503,7 +503,7 @@ sref_end_epoch(struct sref_queue *queue) if (!sref_data.no_warning && (sref_review_queue_size() >= SREF_NR_COUNTERS_WARN)) { sref_data.no_warning = 1; - printf("sref: warning: large number of counters in review queue\n"); + log_warning("sref: large number of counters in review queue"); } if (sref_data.nr_registered_cpus == 1) { @@ -816,7 +816,7 @@ sref_manage(void *arg) /* Never reached */ } -void __init +static int __init sref_bootstrap(void) { spinlock_init(&sref_data.lock); @@ -828,8 +828,13 @@ sref_bootstrap(void) syscnt_register(&sref_data.sc_true_zeroes, "sref_true_zeroes"); sref_cache_init(sref_cache_get(), 0); + + return 0; } +INIT_OP_DEFINE(sref_bootstrap, + INIT_OP_DEP(syscnt_setup, true)); + static void __init sref_setup_manager(struct sref_cache *cache, unsigned int cpu) { @@ -861,7 +866,7 @@ sref_setup_manager(struct sref_cache *cache, unsigned int cpu) cache->manager = manager; } -void __init +static int __init sref_setup(void) { unsigned int i; @@ -873,8 +878,20 @@ sref_setup(void) for (i = 0; i < cpu_count(); i++) { sref_setup_manager(percpu_ptr(sref_cache, i), i); } + + return 0; } +INIT_OP_DEFINE(sref_setup, + INIT_OP_DEP(cpu_mp_probe, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(mutex_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(sref_bootstrap, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_setup, true)); + void sref_register(void) { @@ -976,8 +993,7 @@ sref_report_periodic_event(void) cache = sref_cache_get(); - if (!sref_cache_is_registered(cache) - || (cache->manager == thread_self())) { + if (!sref_cache_is_registered(cache)) { return; } diff --git a/kern/sref.h b/kern/sref.h index ac16b13d..5299b9d2 100644 --- a/kern/sref.h +++ b/kern/sref.h @@ -49,20 +49,6 @@ typedef void (*sref_noref_fn_t)(struct sref_counter *); #include <kern/sref_i.h> /* - * Early initialization of the sref module. - * - * This function depends on the availability of percpu variables. - */ -void sref_bootstrap(void); - -/* - * Initialize the sref module. - * - * This function mostly takes care of setting up periodic maintenance. - */ -void sref_setup(void); - -/* * Manage registration of the current processor. * * Registering tells the sref module that the current processor reports diff --git a/kern/string.c b/kern/string.c index 584030b9..6c9c8abc 100644 --- a/kern/string.c +++ b/kern/string.c @@ -21,9 +21,10 @@ #include <stddef.h> #include <string.h> -#include <kern/param.h> +#include <kern/macros.h> +#include <machine/string.h> -#ifndef ARCH_STRING_MEMCPY +#ifndef STRING_ARCH_MEMCPY void * memcpy(void *dest, const void *src, size_t n) { @@ -35,14 +36,16 @@ memcpy(void *dest, const void *src, size_t n) src_ptr = src; for (i = 0; i < n; i++) { - *dest_ptr++ = *src_ptr++; + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; } return dest; } -#endif /* ARCH_STRING_MEMCPY */ +#endif /* STRING_ARCH_MEMCPY */ -#ifndef ARCH_STRING_MEMMOVE +#ifndef STRING_ARCH_MEMMOVE void * memmove(void *dest, const void *src, size_t n) { @@ -55,22 +58,26 @@ memmove(void *dest, const void *src, size_t n) src_ptr = src; for (i = 0; i < n; i++) { - *dest_ptr++ = *src_ptr++; + *dest_ptr = *src_ptr; + dest_ptr++; + src_ptr++; } } else { dest_ptr = dest + n - 1; src_ptr = src + n - 1; for (i = 0; i < n; i++) { - *dest_ptr-- = *src_ptr--; + *dest_ptr = *src_ptr; + dest_ptr--; + src_ptr--; } } return dest; } -#endif /* ARCH_STRING_MEMMOVE */ +#endif /* STRING_ARCH_MEMMOVE */ -#ifndef ARCH_STRING_MEMSET +#ifndef STRING_ARCH_MEMSET void * memset(void *s, int c, size_t n) { @@ -85,9 +92,9 @@ memset(void *s, int c, size_t n) return s; } -#endif /* ARCH_STRING_MEMSET */ +#endif /* STRING_ARCH_MEMSET */ -#ifndef ARCH_STRING_MEMCMP +#ifndef STRING_ARCH_MEMCMP int memcmp(const void *s1, const void *s2, size_t n) { @@ -97,32 +104,33 @@ memcmp(const void *s1, const void *s2, size_t n) a1 = s1; a2 = s2; - for (i = 0; i < n; i++) + for (i = 0; i < n; i++) { if (a1[i] != a2[i]) { return (int)a1[i] - (int)a2[i]; } + } return 0; } -#endif /* ARCH_STRING_MEMCMP */ +#endif /* STRING_ARCH_MEMCMP */ -#ifndef ARCH_STRING_STRLEN +#ifndef STRING_ARCH_STRLEN size_t strlen(const char *s) { - size_t i; + const char *start; - i = 0; + start = s; - while (*s++ != '\0') { - i++; + while (*s != '\0') { + s++; } - return i; + return (s - start); } -#endif /* ARCH_STRING_STRLEN */ +#endif /* STRING_ARCH_STRLEN */ -#ifndef ARCH_STRING_STRCPY +#ifndef STRING_ARCH_STRCPY char * strcpy(char *dest, const char *src) { @@ -137,7 +145,7 @@ strcpy(char *dest, const char *src) return tmp; } -#endif /* ARCH_STRING_STRCPY */ +#endif /* STRING_ARCH_STRCPY */ size_t strlcpy(char *dest, const char *src, size_t n) @@ -158,7 +166,7 @@ out: return len; } -#ifndef ARCH_STRING_STRCMP +#ifndef STRING_ARCH_STRCMP int strcmp(const char *s1, const char *s2) { @@ -175,16 +183,17 @@ strcmp(const char *s1, const char *s2) return (int)c1 - (int)c2; } -#endif /* ARCH_STRING_STRCMP */ +#endif /* STRING_ARCH_STRCMP */ -#ifndef ARCH_STRING_STRNCMP +#ifndef STRING_ARCH_STRNCMP int strncmp(const char *s1, const char *s2, size_t n) { char c1, c2; - c1 = '\0'; - c2 = '\0'; + if (unlikely(n == 0)) { + return 0; + } while ((n != 0) && (c1 = *s1) == (c2 = *s2)) { if (c1 == '\0') { @@ -198,9 +207,9 @@ strncmp(const char *s1, const char *s2, size_t n) return (int)c1 - (int)c2; } -#endif /* ARCH_STRING_STRNCMP */ +#endif /* STRING_ARCH_STRNCMP */ -#ifndef ARCH_STRING_STRCHR +#ifndef STRING_ARCH_STRCHR char * strchr(const char *s, int c) { @@ -214,4 +223,4 @@ strchr(const char *s, int c) s++; } } -#endif /* ARCH_STRING_STRCHR */ +#endif /* STRING_ARCH_STRCHR */ diff --git a/kern/string.h b/kern/string.h index 35490ffa..7cd79a60 100644 --- a/kern/string.h +++ b/kern/string.h @@ -20,13 +20,13 @@ #include <stddef.h> -void * memcpy(void *dest, const void *src, size_t n); +void * memcpy(void * restrict dest, const void * restrict src, size_t n); void * memmove(void *dest, const void *src, size_t n); void * memset(void *s, int c, size_t n); int memcmp(const void *s1, const void *s2, size_t n); size_t strlen(const char *s); -char * strcpy(char *dest, const char *src); -size_t strlcpy(char *dest, const char *src, size_t n); +char * strcpy(char * restrict dest, const char *restrict src); +size_t strlcpy(char * restrict dest, const char * restrict src, size_t n); int strcmp(const char *s1, const char *s2); int strncmp(const char *s1, const char *s2, size_t n); char * strchr(const char *s, int c); diff --git a/kern/syscnt.c b/kern/syscnt.c index 5b479f36..7cceabac 100644 --- a/kern/syscnt.c +++ b/kern/syscnt.c @@ -22,6 +22,7 @@ #include <kern/init.h> #include <kern/list.h> #include <kern/mutex.h> +#include <kern/shell.h> #include <kern/spinlock.h> #include <kern/syscnt.h> @@ -31,13 +32,47 @@ static struct list syscnt_list; static struct mutex syscnt_lock; -void __init +#ifdef X15_SHELL + +static void +syscnt_shell_info(int argc, char **argv) +{ + char *prefix; + + prefix = (argc >= 2) ? argv[1] : NULL; + syscnt_info(prefix); +} + +static struct shell_cmd syscnt_shell_cmds[] = { + SHELL_CMD_INITIALIZER("syscnt_info", syscnt_shell_info, + "syscnt_info [<prefix>]", + "display information about system counters"), +}; + +static int __init +syscnt_setup_shell(void) +{ + SHELL_REGISTER_CMDS(syscnt_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(syscnt_setup_shell, + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(syscnt_setup, true)); + +#endif /* X15_SHELL */ + +static int __init syscnt_setup(void) { list_init(&syscnt_list); mutex_init(&syscnt_lock); + return 0; } +INIT_OP_DEFINE(syscnt_setup, + INIT_OP_DEP(mutex_setup, true)); + void __init syscnt_register(struct syscnt *syscnt, const char *name) { @@ -61,8 +96,6 @@ syscnt_info(const char *prefix) prefix_length = (prefix == NULL) ? 0 : strlen(prefix); - printf("syscnt: name value\n"); - mutex_lock(&syscnt_lock); list_for_each_entry(&syscnt_list, syscnt, node) { @@ -77,7 +110,7 @@ syscnt_info(const char *prefix) value = syscnt_read(syscnt); - printf("syscnt: %-30s %17llu\n", syscnt->name, + printf("syscnt: %40s %20llu\n", syscnt->name, (unsigned long long)value); } diff --git a/kern/syscnt.h b/kern/syscnt.h index a4bae3d7..816aa4b9 100644 --- a/kern/syscnt.h +++ b/kern/syscnt.h @@ -27,6 +27,7 @@ #include <stdint.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/macros.h> #include <kern/spinlock.h> @@ -43,14 +44,6 @@ struct syscnt; /* - * Initialize the syscnt module. - * - * This module is initialized by architecture-specific code. It is normally - * safe to call this function very early at boot time. - */ -void syscnt_setup(void); - -/* * Initialize and register the given counter. * * The counter is set to 0. @@ -118,4 +111,10 @@ syscnt_dec(struct syscnt *syscnt) */ void syscnt_info(const char *prefix); +/* + * This init operation provides : + * - registration of system counters + */ +INIT_OP_DECLARE(syscnt_setup); + #endif /* _KERN_SYSCNT_H */ diff --git a/kern/task.c b/kern/task.c index 7f1c53d2..ab82ad83 100644 --- a/kern/task.c +++ b/kern/task.c @@ -23,7 +23,8 @@ #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> -#include <kern/param.h> +#include <kern/macros.h> +#include <kern/shell.h> #include <kern/spinlock.h> #include <kern/task.h> #include <kern/thread.h> @@ -56,13 +57,64 @@ static struct spinlock task_list_lock; static void task_init(struct task *task, const char *name, struct vm_map *map) { + task->nr_refs = 1; spinlock_init(&task->lock); list_init(&task->threads); task->map = map; strlcpy(task->name, name, sizeof(task->name)); } -void __init +#ifdef X15_SHELL + +static void +task_shell_info(int argc, char *argv[]) +{ + struct task *task; + int error; + + if (argc == 1) { + task_info(NULL); + return; + } else { + task = task_lookup(argv[1]); + + if (task == NULL) { + error = ERROR_INVAL; + goto error; + } + + task_info(task); + task_unref(task); + } + + return; + +error: + printf("task: info: %s\n", error_str(error)); +} + +static struct shell_cmd task_shell_cmds[] = { + SHELL_CMD_INITIALIZER("task_info", task_shell_info, + "task_info [<task_name>]", + "display tasks and threads"), +}; + +static int __init +task_setup_shell(void) +{ + SHELL_REGISTER_CMDS(task_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(task_setup_shell, + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(task_setup, true), + INIT_OP_DEP(thread_setup, true)); + +#endif /* X15_SHELL */ + +static int __init task_setup(void) { kmem_cache_init(&task_cache, "task", sizeof(struct task), 0, NULL, 0); @@ -70,8 +122,14 @@ task_setup(void) spinlock_init(&task_list_lock); task_init(kernel_task, "x15", kernel_map); list_insert_head(&task_list, &kernel_task->node); + return 0; } +INIT_OP_DEFINE(task_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(vm_map_setup, true)); + int task_create(struct task **taskp, const char *name) { @@ -107,6 +165,31 @@ error_task: return error; } +struct task * +task_lookup(const char *name) +{ + struct task *task; + + spinlock_lock(&task_list_lock); + + list_for_each_entry(&task_list, task, node) { + spinlock_lock(&task->lock); + + if (strcmp(task->name, name) == 0) { + task_ref(task); + spinlock_unlock(&task->lock); + spinlock_unlock(&task_list_lock); + return task; + } + + spinlock_unlock(&task->lock); + } + + spinlock_unlock(&task_list_lock); + + return NULL; +} + void task_add_thread(struct task *task, struct thread *thread) { @@ -123,6 +206,26 @@ task_remove_thread(struct task *task, struct thread *thread) spinlock_unlock(&task->lock); } +struct thread * +task_lookup_thread(struct task *task, const char *name) +{ + struct thread *thread; + + spinlock_lock(&task->lock); + + list_for_each_entry(&task->threads, thread, task_node) { + if (strcmp(thread->name, name) == 0) { + thread_ref(thread); + spinlock_unlock(&task->lock); + return thread; + } + } + + spinlock_unlock(&task->lock); + + return NULL; +} + void task_info(struct task *task) { @@ -132,7 +235,7 @@ task_info(struct task *task) spinlock_lock(&task_list_lock); list_for_each_entry(&task_list, task, node) { - printf("task: %s\n", task->name); + task_info(task); } spinlock_unlock(&task_list_lock); @@ -149,6 +252,8 @@ task_info(struct task *task) * can be used to debug in the middle of most critical sections. * Threads are only destroyed after being removed from their task * so holding the task lock is enough to guarantee existence. + * + * TODO Handle tasks and threads names modifications. */ list_for_each_entry(&task->threads, thread, task_node) { printf(TASK_INFO_ADDR_FMT " %c %8s:" TASK_INFO_ADDR_FMT diff --git a/kern/task.h b/kern/task.h index 67599399..1711fbee 100644 --- a/kern/task.h +++ b/kern/task.h @@ -18,6 +18,8 @@ #ifndef _KERN_TASK_H #define _KERN_TASK_H +#include <kern/atomic.h> +#include <kern/init.h> #include <kern/list.h> #include <kern/spinlock.h> #include <kern/thread.h> @@ -32,6 +34,7 @@ * Task structure. */ struct task { + unsigned long nr_refs; struct spinlock lock; struct list node; struct list threads; @@ -44,10 +47,33 @@ struct task { */ extern struct task *kernel_task; -/* - * Initialize the task module. - */ -void task_setup(void); +static inline void +task_ref(struct task *task) +{ + unsigned long nr_refs; + + nr_refs = atomic_fetch_add(&task->nr_refs, 1, ATOMIC_RELAXED); + assert(nr_refs != (unsigned long)-1); +} + +static inline void +task_unref(struct task *task) +{ + unsigned long nr_refs; + + nr_refs = atomic_fetch_sub_acq_rel(&task->nr_refs, 1); + assert(nr_refs != 0); + + if (nr_refs == 1) { + /* TODO Task destruction */ + } +} + +static inline struct vm_map * +task_get_vm_map(const struct task *task) +{ + return task->map; +} /* * Create a task. @@ -55,6 +81,15 @@ void task_setup(void); int task_create(struct task **taskp, const char *name); /* + * Look up a task from its name. + * + * If a task is found, it gains a reference. Otherwise, NULL is returned. + * + * This function is meant for debugging only. + */ +struct task * task_lookup(const char *name); + +/* * Add a thread to a task. */ void task_add_thread(struct task *task, struct thread *thread); @@ -65,10 +100,26 @@ void task_add_thread(struct task *task, struct thread *thread); void task_remove_thread(struct task *task, struct thread *thread); /* + * Look up a thread in a task from its name. + * + * If a thread is found, it gains a reference, Otherwise, NULL is returned. + * + * This function is meant for debugging only. + */ +struct thread * task_lookup_thread(struct task *task, const char *name); + +/* * Display task information. * * If task is NULL, this function displays all tasks. */ void task_info(struct task *task); +/* + * This init operation provides : + * - task creation + * - module fully initialized + */ +INIT_OP_DECLARE(task_setup); + #endif /* _KERN_TASK_H */ diff --git a/kern/thread.c b/kern/thread.c index 7ce22fb7..8e5b2b52 100644 --- a/kern/thread.c +++ b/kern/thread.c @@ -54,7 +54,7 @@ * * A few terms are used by both papers with slightly different meanings. Here * are the definitions used in this implementation : - * - The time unit is the system timer period (1 / HZ) + * - The time unit is the system timer period (1 / tick frequency) * - Work is the amount of execution time units consumed * - Weight is the amount of execution time units allocated * - A round is the shortest period during which all threads in a run queue @@ -81,13 +81,15 @@ * weights in a smoother way than a raw scaling). */ +#include <assert.h> +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> #include <stdio.h> +#include <stdnoreturn.h> #include <string.h> -#include <kern/assert.h> #include <kern/atomic.h> #include <kern/condition.h> #include <kern/cpumap.h> @@ -99,8 +101,8 @@ #include <kern/macros.h> #include <kern/mutex.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> +#include <kern/shell.h> #include <kern/sleepq.h> #include <kern/spinlock.h> #include <kern/sref.h> @@ -110,6 +112,7 @@ #include <kern/turnstile.h> #include <kern/work.h> #include <machine/cpu.h> +#include <machine/page.h> #include <machine/pmap.h> #include <machine/tcb.h> #include <vm/vm_map.h> @@ -158,7 +161,7 @@ /* * Default time slice for real-time round-robin scheduling. */ -#define THREAD_DEFAULT_RR_TIME_SLICE (HZ / 10) +#define THREAD_DEFAULT_RR_TIME_SLICE (THREAD_TICK_FREQ / 10) /* * Maximum number of threads which can be pulled from a remote run queue @@ -169,7 +172,7 @@ /* * Delay (in ticks) between two balance attempts when a run queue is idle. */ -#define THREAD_IDLE_BALANCE_TICKS (HZ / 2) +#define THREAD_IDLE_BALANCE_TICKS (THREAD_TICK_FREQ / 2) /* * Run queue properties for real-time threads. @@ -189,7 +192,7 @@ struct thread_rt_runq { /* * Round slice base unit for fair-scheduling threads. */ -#define THREAD_FS_ROUND_SLICE_BASE (HZ / 10) +#define THREAD_FS_ROUND_SLICE_BASE (THREAD_TICK_FREQ / 10) /* * Group of threads sharing the same weight. @@ -226,7 +229,7 @@ struct thread_fs_runq { * return path) may violate the locking order. */ struct thread_runq { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; unsigned int cpu; unsigned int nr_threads; struct thread *current; @@ -257,7 +260,7 @@ struct thread_runq { struct syscnt sc_schedule_intrs; struct syscnt sc_tick_intrs; struct syscnt sc_boosts; -} __aligned(CPU_L1_SIZE); +}; /* * Operations of a scheduling class. @@ -321,7 +324,7 @@ static struct cpumap thread_idle_runqs; * There can be moderate bouncing on this word so give it its own cache line. */ static struct { - volatile unsigned long value __aligned(CPU_L1_SIZE); + alignas(CPU_L1_SIZE) volatile unsigned long value; } thread_fs_highest_round_struct; #define thread_fs_highest_round (thread_fs_highest_round_struct.value) @@ -1514,7 +1517,7 @@ thread_sched_idle_select_runq(struct thread *thread) panic("thread: idler threads cannot be awaken"); } -static void __noreturn +static noreturn void thread_sched_idle_panic(void) { panic("thread: only idle threads are allowed in the idle class"); @@ -1684,12 +1687,10 @@ thread_reset_real_priority(struct thread *thread) } static void __init -thread_bootstrap_common(unsigned int cpu) +thread_init_booter(unsigned int cpu) { struct thread *booter; - cpumap_set(&thread_active_runqs, cpu); - /* Initialize only what's needed during bootstrap */ booter = &thread_booters[cpu]; booter->nr_refs = 0; /* Make sure booters aren't destroyed */ @@ -1705,10 +1706,20 @@ thread_bootstrap_common(unsigned int cpu) booter->task = kernel_task; snprintf(booter->name, sizeof(booter->name), THREAD_KERNEL_PREFIX "thread_boot/%u", cpu); - thread_runq_init(percpu_ptr(thread_runq, cpu), cpu, booter); } -void __init +static int __init +thread_setup_booter(void) +{ + tcb_set_current(&thread_booters[0].tcb); + thread_init_booter(0); + return 0; +} + +INIT_OP_DEFINE(thread_setup_booter, + INIT_OP_DEP(tcb_setup, true)); + +static int __init thread_bootstrap(void) { cpumap_zero(&thread_active_runqs); @@ -1716,18 +1727,17 @@ thread_bootstrap(void) thread_fs_highest_round = THREAD_FS_INITIAL_ROUND; - tcb_set_current(&thread_booters[0].tcb); - thread_bootstrap_common(0); + cpumap_set(&thread_active_runqs, 0); + thread_runq_init(cpu_local_ptr(thread_runq), 0, &thread_booters[0]); + return 0; } -void __init -thread_ap_bootstrap(void) -{ - tcb_set_current(&thread_booters[cpu_id()].tcb); -} +INIT_OP_DEFINE(thread_bootstrap, + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_setup_booter, true)); -static void -thread_main(void) +void +thread_main(void (*fn)(void *), void *arg) { struct thread *thread; @@ -1741,7 +1751,7 @@ thread_main(void) cpu_intr_enable(); thread_preempt_enable(); - thread->fn(thread->arg); + fn(arg); thread_exit(); } @@ -1828,14 +1838,12 @@ thread_init(struct thread *thread, void *stack, thread->task = task; thread->stack = stack; strlcpy(thread->name, attr->name, sizeof(thread->name)); - thread->fn = fn; - thread->arg = arg; if (attr->flags & THREAD_ATTR_DETACHED) { thread->flags |= THREAD_DETACHED; } - error = tcb_init(&thread->tcb, stack, thread_main); + error = tcb_init(&thread->tcb, stack, fn, arg); if (error) { goto error_tcb; @@ -1894,7 +1902,7 @@ thread_alloc_stack(void) void *mem; int error; - stack_size = vm_page_round(STACK_SIZE); + stack_size = vm_page_round(TCB_STACK_SIZE); mem = vm_kmem_alloc((PAGE_SIZE * 2) + stack_size); if (mem == NULL) { @@ -1923,24 +1931,18 @@ thread_alloc_stack(void) pmap_remove(kernel_pmap, va + PAGE_SIZE + stack_size, cpumap_all()); pmap_update(kernel_pmap); - vm_page_free(first_page, 0); - vm_page_free(last_page, 0); - - return (char *)va + PAGE_SIZE; + return (void *)va + PAGE_SIZE; } static void thread_free_stack(void *stack) { size_t stack_size; - char *va; - - stack_size = vm_page_round(STACK_SIZE); - va = (char *)stack - PAGE_SIZE; + void *va; - vm_kmem_free_va(va, PAGE_SIZE); - vm_kmem_free(va + PAGE_SIZE, stack_size); - vm_kmem_free_va(va + PAGE_SIZE + stack_size, PAGE_SIZE); + stack_size = vm_page_round(TCB_STACK_SIZE); + va = (void *)stack - PAGE_SIZE; + vm_kmem_free(va, (PAGE_SIZE * 2) + stack_size); } #else /* X15_THREAD_STACK_GUARD */ @@ -2228,20 +2230,107 @@ thread_setup_runq(struct thread_runq *runq) thread_setup_idler(runq); } -void __init +#ifdef X15_SHELL + +/* + * This function is meant for debugging only. As a result, it uses a weak + * locking policy which allows tracing threads which state may mutate during + * tracing. + */ +static void +thread_shell_trace(int argc, char *argv[]) +{ + const char *task_name, *thread_name; + struct thread_runq *runq; + struct thread *thread; + unsigned long flags; + struct task *task; + int error; + + if (argc != 3) { + error = ERROR_INVAL; + goto error; + } + + task_name = argv[1]; + thread_name = argv[2]; + + task = task_lookup(task_name); + + if (task == NULL) { + error = ERROR_SRCH; + goto error; + } + + thread = task_lookup_thread(task, thread_name); + task_unref(task); + + if (thread == NULL) { + error = ERROR_SRCH; + goto error; + } + + runq = thread_lock_runq(thread, &flags); + + if (thread == runq->current) { + printf("thread: trace: thread is running\n"); + } else { + tcb_trace(&thread->tcb); + } + + thread_unlock_runq(runq, flags); + + thread_unref(thread); + return; + +error: + printf("thread: trace: %s\n", error_str(error)); +} + +static struct shell_cmd thread_shell_cmds[] = { + SHELL_CMD_INITIALIZER("thread_trace", thread_shell_trace, + "thread_trace <task_name> <thread_name>", + "display the stack trace of a given thread"), +}; + +static int __init +thread_setup_shell(void) +{ + SHELL_REGISTER_CMDS(thread_shell_cmds); + return 0; +} + +INIT_OP_DEFINE(thread_setup_shell, + INIT_OP_DEP(printf_setup, true), + INIT_OP_DEP(shell_setup, true), + INIT_OP_DEP(task_setup, true), + INIT_OP_DEP(thread_setup, true)); + +#endif /* X15_SHELL */ + +static void __init +thread_setup_common(unsigned int cpu) +{ + assert(cpu != 0); + cpumap_set(&thread_active_runqs, cpu); + thread_init_booter(cpu); + thread_runq_init(percpu_ptr(thread_runq, cpu), cpu, &thread_booters[cpu]); +} + +static int __init thread_setup(void) { int cpu; for (cpu = 1; (unsigned int)cpu < cpu_count(); cpu++) { - thread_bootstrap_common(cpu); + thread_setup_common(cpu); } kmem_cache_init(&thread_cache, "thread", sizeof(struct thread), CPU_L1_SIZE, NULL, 0); #ifndef X15_THREAD_STACK_GUARD - kmem_cache_init(&thread_stack_cache, "thread_stack", STACK_SIZE, - DATA_ALIGN, NULL, 0); + kmem_cache_init(&thread_stack_cache, "thread_stack", TCB_STACK_SIZE, + CPU_DATA_ALIGN, NULL, 0); #endif /* X15_THREAD_STACK_GUARD */ thread_setup_reaper(); @@ -2249,6 +2338,29 @@ thread_setup(void) cpumap_for_each(&thread_active_runqs, cpu) { thread_setup_runq(percpu_ptr(thread_runq, cpu)); } + + return 0; +} + +INIT_OP_DEFINE(thread_setup, + INIT_OP_DEP(cpumap_setup, true), + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(pmap_setup, true), + INIT_OP_DEP(sleepq_setup, true), + INIT_OP_DEP(task_setup, true), + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(turnstile_setup, true), +#ifdef X15_THREAD_STACK_GUARD + INIT_OP_DEP(vm_kmem_setup, true), + INIT_OP_DEP(vm_map_setup, true), + INIT_OP_DEP(vm_page_setup, true), +#endif + ); + +void __init +thread_ap_setup(void) +{ + tcb_set_current(&thread_booters[cpu_id()].tcb); } int @@ -2395,6 +2507,10 @@ thread_wakeup(struct thread *thread) struct thread_runq *runq; unsigned long flags; + if ((thread == NULL) || (thread == thread_self())) { + return; + } + /* * There is at most one reference on threads that were never dispatched, * in which case there is no need to lock anything. diff --git a/kern/thread.h b/kern/thread.h index 4d85c59d..43a98e87 100644 --- a/kern/thread.h +++ b/kern/thread.h @@ -33,20 +33,29 @@ #ifndef _KERN_THREAD_H #define _KERN_THREAD_H +#include <assert.h> #include <stdbool.h> #include <stddef.h> +#include <stdnoreturn.h> -#include <kern/assert.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/condition.h> #include <kern/cpumap.h> -#include <kern/macros.h> #include <kern/spinlock_types.h> #include <kern/turnstile_types.h> #include <machine/cpu.h> #include <machine/tcb.h> /* + * Scheduler tick frequency. + * + * The selected value of 200 translates to a period of 5ms, small enough to + * provide low latency, and is practical as both a dividend and divisor. + */ +#define THREAD_TICK_FREQ 200 + +/* * Thread structure. */ struct thread; @@ -162,18 +171,16 @@ thread_attr_set_priority(struct thread_attr *attr, unsigned short priority) } /* - * Early initialization of the thread module. + * Thread entry point. * - * These function make it possible to use migration and preemption control - * operations (and in turn, spin locks) during bootstrap. + * Loaded TCBs are expected to call this function with interrupts disabled. */ -void thread_bootstrap(void); -void thread_ap_bootstrap(void); +void thread_main(void (*fn)(void *), void *arg); /* - * Initialize the thread module. + * Initialization of the thread module on APs. */ -void thread_setup(void); +void thread_ap_setup(void); /* * Create a thread. @@ -188,7 +195,7 @@ int thread_create(struct thread **threadp, const struct thread_attr *attr, /* * Terminate the calling thread. */ -void __noreturn thread_exit(void); +noreturn void thread_exit(void); /* * Wait for the given thread to terminate and release its resources. @@ -219,7 +226,8 @@ void thread_sleep(struct spinlock *interlock, const void *wchan_addr, /* * Schedule a thread for execution on a processor. * - * No action is performed if the target thread is already in the running state. + * No action is performed if the target thread is NULL, the calling thread, + * or already in the running state. */ void thread_wakeup(struct thread *thread); @@ -228,7 +236,7 @@ void thread_wakeup(struct thread *thread); * * Interrupts must be disabled when calling this function. */ -void __noreturn thread_run_scheduler(void); +noreturn void thread_run_scheduler(void); /* * Make the calling thread release the processor. @@ -732,4 +740,24 @@ thread_get_specific(unsigned int key) */ bool thread_is_running(const struct thread *thread); +/* + * This init operation provides : + * - a dummy thread context for the BSP, allowing the use of thread_self() + */ +INIT_OP_DECLARE(thread_setup_booter); + +/* + * This init operation provides : + * - same as thread_setup_booter + * - BSP run queue initialization + */ +INIT_OP_DECLARE(thread_bootstrap); + +/* + * This init operation provides : + * - thread creation + * - module fully initialized + */ +INIT_OP_DECLARE(thread_setup); + #endif /* _KERN_THREAD_H */ diff --git a/kern/thread_i.h b/kern/thread_i.h index 2e1b88aa..3f350889 100644 --- a/kern/thread_i.h +++ b/kern/thread_i.h @@ -18,16 +18,16 @@ #ifndef _KERN_THREAD_I_H #define _KERN_THREAD_I_H +#include <stdalign.h> #include <stdbool.h> #include <kern/atomic.h> #include <kern/condition_types.h> #include <kern/cpumap.h> #include <kern/list_types.h> -#include <kern/macros.h> #include <kern/mutex_types.h> -#include <kern/param.h> #include <kern/turnstile_types.h> +#include <machine/cpu.h> #include <machine/tcb.h> /* @@ -97,7 +97,7 @@ struct thread_fs_data { * ( ) read-only */ struct thread { - struct tcb tcb; /* (r) */ + alignas(CPU_L1_SIZE) struct tcb tcb; /* (r) */ unsigned long nr_refs; /* (a) */ unsigned long flags; /* (a) */ @@ -178,11 +178,7 @@ struct thread { struct list task_node; /* (T) */ void *stack; /* (-) */ char name[THREAD_NAME_SIZE]; /* ( ) */ - - /* TODO Move out of the structure and make temporary */ - void (*fn)(void *); - void *arg; -} __aligned(CPU_L1_SIZE); +}; #define THREAD_ATTR_DETACHED 0x1 @@ -195,13 +191,13 @@ void thread_destroy(struct thread *thread); static inline void thread_set_flag(struct thread *thread, unsigned long flag) { - atomic_or_acq_rel(&thread->flags, flag); + atomic_or(&thread->flags, flag, ATOMIC_RELEASE); } static inline void thread_clear_flag(struct thread *thread, unsigned long flag) { - atomic_and_acq_rel(&thread->flags, ~flag); + atomic_and(&thread->flags, ~flag, ATOMIC_RELEASE); } static inline int diff --git a/kern/turnstile.c b/kern/turnstile.c index 7b27d841..e724c1e9 100644 --- a/kern/turnstile.c +++ b/kern/turnstile.c @@ -43,30 +43,31 @@ * TODO Analyse hash parameters. */ +#include <assert.h> +#include <stdalign.h> #include <stdbool.h> #include <stddef.h> #include <stdint.h> -#include <kern/assert.h> #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/plist.h> #include <kern/spinlock.h> #include <kern/thread.h> #include <kern/turnstile.h> #include <kern/turnstile_types.h> +#include <machine/cpu.h> /* * Locking keys : * (b) bucket */ struct turnstile_bucket { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; struct list list; /* (b) */ -} __aligned(CPU_L1_SIZE); +}; /* * Adding/removing waiters to/from a turnstile are performed while @@ -499,7 +500,7 @@ turnstile_ctor(void *ptr) turnstile->owner = NULL; } -void __init +static int __init turnstile_bootstrap(void) { unsigned int i; @@ -507,15 +508,24 @@ turnstile_bootstrap(void) for (i = 0; i < ARRAY_SIZE(turnstile_htable); i++) { turnstile_bucket_init(&turnstile_htable[i]); } + + return 0; } -void __init +INIT_OP_DEFINE(turnstile_bootstrap); + +static int __init turnstile_setup(void) { kmem_cache_init(&turnstile_cache, "turnstile", sizeof(struct turnstile), CPU_L1_SIZE, turnstile_ctor, 0); + return 0; } +INIT_OP_DEFINE(turnstile_setup, + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(turnstile_bootstrap, true)); + struct turnstile * turnstile_create(void) { diff --git a/kern/turnstile.h b/kern/turnstile.h index 4bc8f74d..74512fa1 100644 --- a/kern/turnstile.h +++ b/kern/turnstile.h @@ -29,6 +29,7 @@ #include <stdbool.h> #include <stddef.h> +#include <kern/init.h> #include <kern/plist.h> #include <kern/spinlock.h> #include <kern/thread.h> @@ -101,20 +102,6 @@ turnstile_td_get_turnstile(const struct turnstile_td *td) void turnstile_td_propagate_priority(struct turnstile_td *td); /* - * Early initialization of the turnstile module. - * - * This module is initialized by architecture-specific code. It should - * be one of the first modules to be initialized since it's used by - * synchronization objects that may be accessed very early. - */ -void turnstile_bootstrap(void); - -/* - * Initialize the turnstile module. - */ -void turnstile_setup(void); - -/* * Create/destroy a turnstile. */ struct turnstile * turnstile_create(void); @@ -197,4 +184,17 @@ void turnstile_signal(struct turnstile *turnstile); void turnstile_own(struct turnstile *turnstile); void turnstile_disown(struct turnstile *turnstile); +/* + * This init operation provides : + * - ? TODO Review + */ +INIT_OP_DECLARE(turnstile_bootstrap); + +/* + * This init operation provides : + * - turnstile creation + * - module fully initialized + */ +INIT_OP_DECLARE(turnstile_setup); + #endif /* _KERN_TURNSTILE_H */ diff --git a/kern/work.c b/kern/work.c index 714af35c..9aa3f40e 100644 --- a/kern/work.c +++ b/kern/work.c @@ -15,18 +15,18 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> -#include <stdio.h> -#include <kern/assert.h> #include <kern/bitmap.h> #include <kern/error.h> #include <kern/init.h> #include <kern/kmem.h> #include <kern/list.h> +#include <kern/log.h> #include <kern/macros.h> #include <kern/panic.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/syscnt.h> @@ -85,7 +85,7 @@ struct work_thread { * only use one queue. */ struct work_pool { - struct spinlock lock; + alignas(CPU_L1_SIZE) struct spinlock lock; int flags; struct work_queue queue0; struct work_queue queue1; @@ -98,7 +98,7 @@ struct work_pool { struct list available_threads; struct list dead_threads; BITMAP_DECLARE(bitmap, WORK_MAX_THREADS); -} __aligned(CPU_L1_SIZE); +}; static int work_thread_create(struct work_pool *pool, unsigned int id); static void work_thread_destroy(struct work_thread *worker); @@ -259,7 +259,7 @@ work_pool_wakeup_manager(struct work_pool *pool) return; } - if ((pool->manager != NULL) && (pool->manager->thread != thread_self())) { + if (pool->manager != NULL) { thread_wakeup(pool->manager->thread); } } @@ -374,7 +374,7 @@ work_process(void *arg) if (error) { work_pool_free_id(pool, id); - printf("work: warning: unable to create worker thread\n"); + log_warning("work: unable to create worker thread"); } } } @@ -472,7 +472,7 @@ work_thread_destroy(struct work_thread *worker) kmem_cache_free(&work_thread_cache, worker); } -void __init +static int __init work_setup(void) { unsigned int i; @@ -490,11 +490,23 @@ work_setup(void) work_pool_init(&work_pool_highprio, WORK_INVALID_CPU, WORK_PF_GLOBAL | WORK_PF_HIGHPRIO); - printf("work: threads per pool (per-cpu/global): %u/%u, spare: %u\n", - percpu_var(work_pool_cpu_main.max_threads, 0), - work_pool_main.max_threads, WORK_THREADS_SPARE); + log_info("work: threads per pool (per-cpu/global): %u/%u, spare: %u", + percpu_var(work_pool_cpu_main.max_threads, 0), + work_pool_main.max_threads, WORK_THREADS_SPARE); + + return 0; } +INIT_OP_DEFINE(work_setup, + INIT_OP_DEP(cpu_mp_probe, true), + INIT_OP_DEP(kmem_setup, true), + INIT_OP_DEP(log_setup, true), + INIT_OP_DEP(panic_setup, true), + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(spinlock_setup, true), + INIT_OP_DEP(syscnt_setup, true), + INIT_OP_DEP(thread_bootstrap, true)); + void work_schedule(struct work *work, int flags) { diff --git a/kern/work.h b/kern/work.h index 6e3876f0..a8df1f7e 100644 --- a/kern/work.h +++ b/kern/work.h @@ -26,6 +26,8 @@ #ifndef _KERN_WORK_H #define _KERN_WORK_H +#include <kern/init.h> + /* * Work scheduling flags. */ @@ -134,11 +136,6 @@ work_init(struct work *work, work_fn_t fn) } /* - * Initialize the work module. - */ -void work_setup(void); - -/* * Schedule work for deferred processing. * * This function may be called from interrupt context. @@ -156,4 +153,11 @@ void work_queue_schedule(struct work_queue *queue, int flags); */ void work_report_periodic_event(void); +/* + * This init operation provides : + * - works can be scheduled + * - module fully initialized + */ +INIT_OP_DECLARE(work_setup); + #endif /* _KERN_WORK_H */ diff --git a/kern/xcall.c b/kern/xcall.c index 364ad0be..cccb9373 100644 --- a/kern/xcall.c +++ b/kern/xcall.c @@ -15,12 +15,13 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdalign.h> #include <stddef.h> -#include <kern/assert.h> #include <kern/atomic.h> +#include <kern/init.h> #include <kern/macros.h> -#include <kern/param.h> #include <kern/percpu.h> #include <kern/spinlock.h> #include <kern/thread.h> @@ -28,9 +29,9 @@ #include <machine/cpu.h> struct xcall { - xcall_fn_t fn; + alignas(CPU_L1_SIZE) xcall_fn_t fn; void *arg; -} __aligned(CPU_L1_SIZE); +}; /* * Per-CPU data. @@ -49,11 +50,11 @@ struct xcall { * between multiple cross-calls. */ struct xcall_cpu_data { - struct xcall send_calls[X15_MAX_CPUS]; + alignas(CPU_L1_SIZE) struct xcall send_calls[X15_MAX_CPUS]; struct xcall *recv_call; struct spinlock lock; -} __aligned(CPU_L1_SIZE); +}; static struct xcall_cpu_data xcall_cpu_data __percpu; @@ -92,7 +93,8 @@ xcall_cpu_data_get_recv_call(const struct xcall_cpu_data *cpu_data) } static void -xcall_cpu_data_set_recv_call(struct xcall_cpu_data *cpu_data, struct xcall *call) +xcall_cpu_data_set_recv_call(struct xcall_cpu_data *cpu_data, + struct xcall *call) { atomic_store(&cpu_data->recv_call, call, ATOMIC_RELEASE); } @@ -103,7 +105,7 @@ xcall_cpu_data_clear_recv_call(struct xcall_cpu_data *cpu_data) xcall_cpu_data_set_recv_call(cpu_data, NULL); } -void +static int __init xcall_setup(void) { unsigned int i; @@ -111,8 +113,15 @@ xcall_setup(void) for (i = 0; i < cpu_count(); i++) { xcall_cpu_data_init(percpu_ptr(xcall_cpu_data, i)); } + + return 0; } +INIT_OP_DEFINE(xcall_setup, + INIT_OP_DEP(percpu_setup, true), + INIT_OP_DEP(thread_bootstrap, true), + INIT_OP_DEP(spinlock_setup, true)); + void xcall_call(xcall_fn_t fn, void *arg, unsigned int cpu) { diff --git a/kern/xcall.h b/kern/xcall.h index 37c85867..27b6af25 100644 --- a/kern/xcall.h +++ b/kern/xcall.h @@ -30,11 +30,6 @@ typedef void (*xcall_fn_t)(void *arg); /* - * Initialize the xcall module. - */ -void xcall_setup(void); - -/* * Run the given cross-call function on a specific processor. * * The operation is completely synchronous, returning only when the function |