kern/{kernel,thread}: rework initialization

Make cpu_count() available on kernel entry so that modules (and in particular
the thread module) can allocate per-CPU resources from the BSP. This makes
the initial state stable and simplifies code (no need to check for a transient
early initialization state).

5 files changed, 197 insertions, 136 deletions

diff --git a/arch/x86/machine/boot.c b/arch/x86/machine/boot.c
index 58b675f9..f42f9bbf 100644
--- a/arch/x86/machine/boot.c
+++ b/arch/x86/machine/boot.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2012 Richard Braun.
+ * Copyright (c) 2010, 2012, 2013 Richard Braun.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -284,6 +284,7 @@ boot_main(void)
     vm_phys_info();
     pic_setup();
     pit_setup();
+    cpu_mp_setup();
     kernel_main();
 
     /* Never reached */
diff --git a/arch/x86/machine/cpu.c b/arch/x86/machine/cpu.c
index a5a62499..0d1a9f38 100644
--- a/arch/x86/machine/cpu.c
+++ b/arch/x86/machine/cpu.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2011, 2012 Richard Braun.
+ * Copyright (c) 2010, 2011, 2012, 2013 Richard Braun.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -79,6 +79,11 @@ static unsigned int cpu_boot_array_size __initdata;
 unsigned int cpu_array_size;
 
 /*
+ * Barrier for processor synchronization on kernel entry.
+ */
+static unsigned int cpu_mp_synced __initdata;
+
+/*
  * Interrupt descriptor table.
  */
 static struct cpu_gate_desc cpu_idt[CPU_IDT_SIZE] __aligned(8);
@@ -555,6 +560,12 @@ cpu_mp_setup(void)
 }
 
 void __init
+cpu_mp_sync(void)
+{
+    cpu_mp_synced = 1;
+}
+
+void __init
 cpu_ap_setup(void)
 {
     cpu_init(&cpu_array[boot_ap_id]);
@@ -565,7 +576,7 @@ cpu_ap_setup(void)
 void __init
 cpu_ap_sync(void)
 {
-    while (cpu_count() == 1)
+    while (!cpu_mp_synced)
         cpu_pause();
 }
 
diff --git a/arch/x86/machine/cpu.h b/arch/x86/machine/cpu.h
index 9e846e34..24d4ef48 100644
--- a/arch/x86/machine/cpu.h
+++ b/arch/x86/machine/cpu.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2011, 2012 Richard Braun.
+ * Copyright (c) 2010, 2011, 2012, 2013 Richard Braun.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -576,22 +576,23 @@ void cpu_mp_register_lapic(unsigned int apic_id, int is_bsp);
 
 /*
  * Probe application processors and start them.
+ *
+ * On return, cpu_count() gives the actual number of managed processors.
  */
 void cpu_mp_setup(void);
 
 /*
+ * Synchronize with APs on kernel entry.
+ */
+void cpu_mp_sync(void);
+
+/*
  * CPU initialization on APs.
  */
 void cpu_ap_setup(void);
 
 /*
- * Synchronize processors on kernel entry.
- *
- * Wait for all processors to reach a proper state when entering the kernel,
- * so that memory allocations can proceed and thread scheduling started.
- *
- * Once this function returns, cpu_count can be used reliably to know if there
- * are more than one processors, and how many.
+ * Synchronize with BSP on kernel entry.
  */
 void cpu_ap_sync(void);
 
diff --git a/kern/kernel.c b/kern/kernel.c
index 07a0f660..93c0c19c 100644
--- a/kern/kernel.c
+++ b/kern/kernel.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2012 Richard Braun.
+ * Copyright (c) 2011, 2012, 2013 Richard Braun.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -27,12 +27,17 @@ kernel_main(void)
 {
     assert(!cpu_intr_enabled());
 
+    /* Enable interrupts to allow inter-processor pmap updates */
+    cpu_intr_enable();
+
+    /* Initialize the kernel */
     task_setup();
     thread_setup();
 
-    cpu_intr_enable();
-    cpu_mp_setup();
+    /* Rendezvous with APs */
+    cpu_mp_sync();
 
+    /* Run the scheduler */
     thread_run();
 
     /* Never reached */
@@ -43,9 +48,16 @@ kernel_ap_main(void)
 {
     assert(!cpu_intr_enabled());
 
+    /*
+     * Enable interrupts to allow inter-processor pmap updates while the BSP
+     * is initializing the kernel.
+     */
     cpu_intr_enable();
+
+    /* Wait for the BSP to complete kernel initialization */
     cpu_ap_sync();
 
+    /* Run the scheduler */
     thread_run();
 
     /* Never reached */
diff --git a/kern/thread.c b/kern/thread.c
index 03a41d70..00c3dc64 100644
--- a/kern/thread.c
+++ b/kern/thread.c
@@ -743,9 +743,6 @@ thread_sched_ts_wakeup_balancer(struct thread_runq *runq)
 {
     unsigned long on_rq;
 
-    if (runq->balancer == NULL)
-        return;
-
     on_rq = atomic_cas(&runq->balancer->on_rq, 0, 1);
 
     if (on_rq)
@@ -1165,24 +1162,8 @@ thread_sched_idle_tick(struct thread_runq *runq, struct thread *thread)
 void __init
 thread_bootstrap(void)
 {
-    size_t i;
-
-    for (i = 0; i < ARRAY_SIZE(thread_runqs); i++)
-        thread_runq_init(&thread_runqs[i]);
-
-    tcb_set_current(&thread_idlers[0].tcb);
-}
-
-void __init
-thread_ap_bootstrap(void)
-{
-    tcb_set_current(&thread_idlers[cpu_id()].tcb);
-}
-
-void __init
-thread_setup(void)
-{
     struct thread_sched_ops *ops;
+    size_t i;
 
     thread_policy_table[THREAD_SCHED_POLICY_FIFO] = THREAD_SCHED_CLASS_RT;
     thread_policy_table[THREAD_SCHED_POLICY_RR] = THREAD_SCHED_CLASS_RT;
@@ -1220,10 +1201,16 @@ thread_setup(void)
 
     thread_ts_highest_round = THREAD_TS_INITIAL_ROUND;
 
-    kmem_cache_init(&thread_cache, "thread", sizeof(struct thread),
-                    CPU_L1_SIZE, NULL, NULL, NULL, 0);
-    kmem_cache_init(&thread_stack_cache, "thread_stack", STACK_SIZE,
-                    DATA_ALIGN, NULL, NULL, NULL, 0);
+    for (i = 0; i < ARRAY_SIZE(thread_runqs); i++)
+        thread_runq_init(&thread_runqs[i]);
+
+    tcb_set_current(&thread_idlers[0].tcb);
+}
+
+void __init
+thread_ap_bootstrap(void)
+{
+    tcb_set_current(&thread_idlers[cpu_id()].tcb);
 }
 
 static void
@@ -1302,6 +1289,152 @@ thread_init(struct thread *thread, void *stack, const struct thread_attr *attr,
     task_add_thread(task, thread);
 }
 
+static void
+thread_balancer(void *arg)
+{
+    struct thread_runq *runq;
+
+    runq = arg;
+
+    for (;;) {
+        /*
+         * Start by balancing in case the first threads were not evenly
+         * dispatched.
+         */
+        thread_sched_ts_balance(runq);
+        thread_sleep();
+    }
+}
+
+static void __init
+thread_setup_balancer(struct thread_runq *runq)
+{
+    char name[THREAD_NAME_SIZE];
+    struct thread_runq *local_runq;
+    struct thread_attr attr;
+    struct thread *balancer;
+    int error;
+
+    snprintf(name, sizeof(name), "x15_balancer/%u", thread_runq_id(runq));
+    attr.task = kernel_task;
+    attr.name = name;
+    attr.sched_policy = THREAD_SCHED_CLASS_RT;
+    attr.priority = THREAD_SCHED_RT_PRIO_MIN;
+    error = thread_create(&balancer, &attr, thread_balancer, runq);
+
+    if (error)
+        panic("thread: unable to create balancer thread");
+
+    runq->balancer = balancer;
+
+    /*
+     * XXX Real-time threads are currently dispatched on the creator's run
+     * queue.
+     *
+     * TODO Implement processor affinity and remove this kludge.
+     */
+    local_runq = thread_runq_local();
+
+    if (runq != local_runq) {
+        unsigned long flags;
+
+        spinlock_lock_intr_save(&local_runq->lock, &flags);
+        thread_runq_remove(local_runq, balancer);
+        spinlock_unlock_intr_restore(&local_runq->lock, flags);
+
+        spinlock_lock(&runq->lock);
+        thread_runq_add(runq, balancer);
+        spinlock_unlock(&runq->lock);
+    }
+}
+
+static void
+thread_idler(void *arg)
+{
+    (void)arg;
+
+    for (;;)
+        cpu_idle();
+}
+
+static void __init
+thread_setup_idler(struct thread_runq *runq)
+{
+    char name[THREAD_NAME_SIZE];
+    struct thread_attr attr;
+    struct thread *idler;
+    unsigned long flags = flags;
+    unsigned long preempt;
+    void *stack;
+
+    stack = kmem_cache_alloc(&thread_stack_cache);
+
+    if (stack == NULL)
+        panic("thread: unable to allocate idler thread stack");
+
+    snprintf(name, sizeof(name), "x15_idler/%u", thread_runq_id(runq));
+    attr.task = kernel_task;
+    attr.name = name;
+    attr.sched_policy = THREAD_SCHED_POLICY_IDLE;
+    idler = &thread_idlers[thread_runq_id(runq)];
+
+    /*
+     * When a processor enters the scheduler, the idler thread is the current
+     * one. Initializing it puts it in the state of a thread waiting to be
+     * dispatched, although it's actually still running. This only affects
+     * the preemption counter. Save it now and restore it once initialized.
+     */
+    preempt = idler->preempt;
+
+    /*
+     * Interrupts are already enabled on every processor, invoking the
+     * scheduler on their return path. Lock run queues while initializing
+     * idler threads to avoid unpleasant surprises.
+     */
+    if (runq == thread_runq_local())
+        spinlock_lock_intr_save(&runq->lock, &flags);
+    else
+        spinlock_lock(&runq->lock);
+
+    thread_init(idler, stack, &attr, thread_idler, NULL);
+    idler->state = THREAD_RUNNING;
+    idler->cpu = thread_runq_id(runq);
+
+    /*
+     * The initial preemption counter should never be less than 2, otherwise
+     * preemption will be reenabled when unlocking the run queue.
+     */
+    assert(idler->preempt > 1);
+
+    if (runq == thread_runq_local())
+        spinlock_unlock_intr_restore(&runq->lock, flags);
+    else
+        spinlock_unlock(&runq->lock);
+
+    idler->preempt = preempt;
+}
+
+static void __init
+thread_setup_runq(struct thread_runq *runq)
+{
+    thread_setup_balancer(runq);
+    thread_setup_idler(runq);
+}
+
+void __init
+thread_setup(void)
+{
+    size_t i;
+
+    kmem_cache_init(&thread_cache, "thread", sizeof(struct thread),
+                    CPU_L1_SIZE, NULL, NULL, NULL, 0);
+    kmem_cache_init(&thread_stack_cache, "thread_stack", STACK_SIZE,
+                    DATA_ALIGN, NULL, NULL, NULL, 0);
+
+    for (i = 0; i < cpu_count(); i++)
+        thread_setup_runq(&thread_runqs[i]);
+}
+
 int
 thread_create(struct thread **threadp, const struct thread_attr *attr,
               void (*fn)(void *), void *arg)
@@ -1371,93 +1504,6 @@ thread_wakeup(struct thread *thread)
     thread_preempt_enable();
 }
 
-static void
-thread_balancer(void *arg)
-{
-    struct thread_runq *runq;
-
-    runq = arg;
-
-    for (;;) {
-        /*
-         * Start by balancing in case the first threads were not evenly
-         * dispatched.
-         */
-        thread_sched_ts_balance(runq);
-        thread_sleep();
-    }
-}
-
-static void __init
-thread_setup_balancer(void)
-{
-    char name[THREAD_NAME_SIZE];
-    struct thread_runq *runq;
-    struct thread_attr attr;
-    struct thread *balancer;
-    int error;
-
-    runq = thread_runq_local();
-
-    /*
-     * Real-time threads are currently dispatched on the caller's run queue.
-     *
-     * TODO CPU affinity
-     */
-    snprintf(name, sizeof(name), "x15_balancer/%u", cpu_id());
-    attr.task = kernel_task;
-    attr.name = name;
-    attr.sched_policy = THREAD_SCHED_CLASS_RT;
-    attr.priority = THREAD_SCHED_RT_PRIO_MIN;
-    error = thread_create(&balancer, &attr, thread_balancer, runq);
-
-    if (error)
-        panic("thread: unable to create balancer thread");
-
-    runq->balancer = balancer;
-}
-
-static void
-thread_idler(void *arg)
-{
-    (void)arg;
-
-    for (;;)
-        cpu_idle();
-}
-
-static void __init
-thread_setup_idler(void)
-{
-    char name[THREAD_NAME_SIZE];
-    struct thread_attr attr;
-    struct thread *idler;
-    unsigned int cpu;
-    void *stack;
-
-    stack = kmem_cache_alloc(&thread_stack_cache);
-
-    if (stack == NULL)
-        panic("thread: unable to allocate idler thread stack");
-
-    /*
-     * Having interrupts enabled was required to allocate the stack, but
-     * at this stage, the idler thread is still the current thread, so disable
-     * interrupts while initializing it.
-     */
-    cpu_intr_disable();
-
-    cpu = cpu_id();
-    snprintf(name, sizeof(name), "x15_idler/%u", cpu);
-    attr.task = kernel_task;
-    attr.name = name;
-    attr.sched_policy = THREAD_SCHED_POLICY_IDLE;
-    idler = &thread_idlers[cpu];
-    thread_init(idler, stack, &attr, thread_idler, NULL);
-    idler->state = THREAD_RUNNING;
-    idler->cpu = cpu;
-}
-
 void __init
 thread_run(void)
 {
@@ -1466,21 +1512,11 @@ thread_run(void)
 
     assert(cpu_intr_enabled());
 
-    thread_setup_balancer();
-
-    /* This call disables interrupts */
-    thread_setup_idler();
-
+    cpu_intr_disable();
     runq = thread_runq_local();
     spinlock_lock(&runq->lock);
     thread = thread_runq_get_next(thread_runq_local());
 
-    /*
-     * Locking the run queue increased the preemption counter to 3.
-     * Artificially reduce it to the expected value.
-     */
-    thread_preempt_enable_no_resched();
-
     if (thread->task != kernel_task)
         pmap_load(thread->task->map->pmap);