/*
 * Copyright (c) 2014-2018 Remy Noel.
 * Copyright (c) 2014-2015 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * TODO Description.
 *
 * Locking order : interrupts -> thread runq -> grouplist -> group
 *
 * TODO API to differenciate user and kernel events.
 */

#include <assert.h>
#include <stddef.h>
#include <stdint.h>

#include <kern/error.h>
#include <kern/init.h>
#include <kern/kmem.h>
#include <kern/list.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <kern/panic.h>
#include <kern/percpu.h>
#include <kern/perfmon.h>
#include <kern/perfmon_i.h>
#include <kern/spinlock.h>
#include <kern/thread.h>
#include <kern/xcall.h>
#include <machine/cpu.h>
#include <machine/pmu.h>
#include <machine/trap.h>

/*
 * Performance monitoring event.
 *
 * When a group is attached, each of its events is associated to a PMC,
 * adding a reference in the process.
 */
struct perfmon_event {
    uint64_t count;
    uint64_t prev;
    uint64_t overflow_id;
    struct list node;
    int flags;
    unsigned int type;
    unsigned int id;
    unsigned int pmc_index;
#ifdef CONFIG_PERFMON_TEST
    uint64_t value;
    bool set_value;
#endif
};

#define PERFMON_INVALID_CPU ((unsigned int)-1)

/*
 * Group types.
 */
#define PERFMON_GT_UNKNOWN  0
#define PERFMON_GT_CPU      1
#define PERFMON_GT_THREAD   2

/*
 * Group States flags.
 */
#define PERFMON_GF_ATTACHED         1
#define PERFMON_GF_ENABLED          2
#define PERFMON_GF_LOADED           4
#define PERFMON_GF_PENDING_DISABLE  8

/*
 * Group possible states are handled through the flags attribute.
 *  - A group can either be unattached or attached(to a thread or a cpu)
 *  - An attached group may be enabled or not.
 *  - An enabled group may be loaded or not e.g. have actual running
 * performance counters on a CPU.
 *
 * When a group is attached, some ressources are reserved for it so it can be
 * monitored at any time.
 * When a group is enabled it will get loaded when needed:
 *   -Cpu groups stay loaded as long as they are enabled.
 *   -Thread groups are loaded when running.
 * When a group is loaded its performance counters are currently enabled.
 *
 * The PENDING_DISABLE is here so that a remote thread can be disabled when it
 * unscedule itself.
 *
 * Note that a non-attached group can only be referenced from the api. Since
 * manipulating the same group from different threads as the same time is not
 * supported, the code does not bother to lock it when the group is assumed
 * un-attached.
 *
 * About thread-type group counters synchronization:
 *      - The groups are synchronized when their thread is unscheduled which
 *        means their counter value is updated and pending counter changes
 *        (like unloading) are performed.
 *      - Since all operations requires the group to be locked, it is mandatory
 *        to unlock the group before xcalling any remote operation in order to
 *        avoid a deadlock.
 *      - Any remote thread operation that gets executed after the thread got
 *        unscheduled will have nothing to do if the current thread is not the
 *        target one since the target thread have been unloaded inbetween.
 */
struct perfmon_group {
    struct list node;
    struct list events;
    struct thread *thread;
    struct spinlock lock;
    unsigned int cpu;
    short flags;
    unsigned short type;
};

/*
 * List of all groups attached to a single monitored object, either a CPU
 * or a thread.
 */
struct perfmon_grouplist {
    struct list groups;
    struct spinlock lock;
};

/*
 * Maximum number of supported hardware counters.
 */
#define PERFMON_MAX_PMCS 64

/*
 * Performance monitoring counter.
 *
 * When a PMC is valid, it maps a raw event to a hardware counter.
 * A PMC is valid if and only if its reference count isn't zero.
 */
struct perfmon_pmc {
    unsigned int nr_refs;
    unsigned int raw_event_id;
    unsigned int id;
};

/*
 * Performance monitoring unit.
 *
 * There is a single system-wide logical PMU, used to globally allocate
 * PMCs. Reserving a counter across the entire system ensures thread
 * migration isn't hindered by performance monitoring.
 */
struct perfmon_pmu {
    struct spinlock lock;
    unsigned int nr_pmcs;
    struct perfmon_pmc pmcs[PERFMON_MAX_PMCS];
};

/*
 * Per-CPU performance monitoring counter.
 *
 * When a group is loaded on a processor, the per-CPU PMCs of its events
 * get referenced. When a per-CPU PMC is referenced, its underlying hardware
 * counter is active.
 *
 * Preemption must be disabled on access.
 */
struct perfmon_cpu_pmc {
    unsigned int nr_refs;
    uint64_t prev_value;
    uint64_t overflow_id;
};

/*
 * Per-CPU performance monitoring unit.
 *
 * The per-CPU PMCs are indexed the same way as the global PMCs.
 *
 * Preemption must be disabled on access.
 */
struct perfmon_cpu_pmu {
    struct perfmon_cpu_pmc pmcs[PERFMON_MAX_PMCS];
};

static struct perfmon_pmu_ops pmu_driver __read_mostly;

static struct perfmon_pmu perfmon_pmu;
static unsigned int perfmon_pmc_id_to_index[PERFMON_MAX_PMCS];

static struct perfmon_cpu_pmu perfmon_cpu_pmu __percpu;

/*
 * Cache of thread-specific group lists.
 */
static struct kmem_cache perfmon_grouplist_cache;

/*
 * CPU specific group lists.
 */
static struct perfmon_grouplist *perfmon_cpu_grouplist  __percpu;

static inline int
perfmon_translate(unsigned int *raw_event_idp, unsigned int event_type,
                  unsigned int event_id)
{
    switch (event_type) {
    case PERFMON_ET_RAW:
        *raw_event_idp = event_id;
        return 0;
    case PERFMON_ET_GENERIC:
        return pmu_driver.translate(raw_event_idp, event_id);
    default:
        panic("perfmon: unsupported event type");
    }
}

static int
perfmon_pmc_alloc(struct perfmon_pmc **pmcp, unsigned int raw_event_id)
{
    struct perfmon_pmc *pmc;
    size_t i;
    int error;

    if (perfmon_pmu.nr_pmcs == ARRAY_SIZE(perfmon_pmu.pmcs)) {
        return EAGAIN;
    }

    for (i = 0; i < ARRAY_SIZE(perfmon_pmu.pmcs); i++) {
        pmc = &perfmon_pmu.pmcs[i];

        if (pmc->nr_refs == 0) {
            break;
        }
    }

    assert(i < ARRAY_SIZE(perfmon_pmu.pmcs));
    error = pmu_driver.alloc(&pmc->id, raw_event_id);

    if (error) {
        return error;
    }

    pmc->raw_event_id = raw_event_id;
    perfmon_pmu.nr_pmcs++;
    *pmcp = pmc;

    return 0;
}

static struct perfmon_pmc *
perfmon_pmc_lookup(unsigned int raw_event_id)
{
    struct perfmon_pmc *pmc;
    size_t i;

    if (perfmon_pmu.nr_pmcs == 0) {
        return NULL;
    }

    for (i = 0; i < ARRAY_SIZE(perfmon_pmu.pmcs); i++) {
        pmc = &perfmon_pmu.pmcs[i];

        if ((pmc->nr_refs != 0) && (pmc->raw_event_id == raw_event_id)) {
            return pmc;
        }
    }

    return NULL;
}

static inline unsigned int
perfmon_pmc_index(const struct perfmon_pmc *pmc)
{
    unsigned int index;

    index = pmc - perfmon_pmu.pmcs;
    assert(index < ARRAY_SIZE(perfmon_pmu.pmcs));
    return index;
}

/*
 * Obtain a reference on a PMC for the given event.
 *
 * If there is no existing PMC suitable for this event, allocate one.
 */
static int
perfmon_pmc_get(struct perfmon_pmc **pmcp, const struct perfmon_event *event)
{
    struct perfmon_pmc *pmc;
    unsigned int raw_event_id;
    unsigned int pmc_index;
    int error;

    error = perfmon_translate(&raw_event_id, event->type, event->id);

    if (error) {
        return error;
    }

    spinlock_lock(&perfmon_pmu.lock);

    pmc = perfmon_pmc_lookup(raw_event_id);

    if (pmc == NULL) {
        error = perfmon_pmc_alloc(&pmc, raw_event_id);

        if (error) {
            goto out;
        }
        pmc_index = perfmon_pmc_index(pmc);
        assert(perfmon_pmc_id_to_index[pmc->id] == UINT32_MAX);
        perfmon_pmc_id_to_index[pmc->id] = pmc_index;
    }

    pmc->nr_refs++;

out:
    spinlock_unlock(&perfmon_pmu.lock);

    if (error) {
        return error;
    }
    *pmcp = pmc;

    return 0;
}

/*
 * Release a reference on a PMC.
 */
static void
perfmon_pmc_put(struct perfmon_pmc *pmc)
{
    spinlock_lock(&perfmon_pmu.lock);

    assert(pmc->nr_refs != 0);
    pmc->nr_refs--;

    if (pmc->nr_refs == 0) {
        pmu_driver.free(pmc->id);
        assert(perfmon_pmc_id_to_index[pmc->id] != UINT32_MAX);
        perfmon_pmc_id_to_index[pmc->id] = UINT32_MAX;
    }

    spinlock_unlock(&perfmon_pmu.lock);
}

static inline struct perfmon_pmc *
perfmon_pmc_from_index(unsigned int index)
{
    assert(index < ARRAY_SIZE(perfmon_pmu.pmcs));
    return &perfmon_pmu.pmcs[index];
}

static void
perfmon_grouplist_ctor(void *arg)
{
    struct perfmon_grouplist *grouplist;

    grouplist = arg;

    list_init(&grouplist->groups);
    spinlock_init(&grouplist->lock);
}

static struct perfmon_grouplist *
perfmon_grouplist_create(void)
{
    return kmem_cache_alloc(&perfmon_grouplist_cache);
}

static void
perfmon_grouplist_destroy(struct perfmon_grouplist *grouplist)
{
    kmem_cache_free(&perfmon_grouplist_cache, grouplist);
}

static void __init
perfmon_cpu_pmu_init(struct perfmon_cpu_pmu *cpu_pmu)
{
    unsigned int i;

    for (i = 0; i < ARRAY_SIZE(cpu_pmu->pmcs); i++) {
        struct perfmon_cpu_pmc *pmc;

        pmc = &cpu_pmu->pmcs[i];

        pmc->nr_refs = 0;
        pmc->prev_value = pmu_driver.read(perfmon_pmu.pmcs[i].id);
        pmc->overflow_id = 0;

    }
}

static struct perfmon_cpu_pmc *
perfmon_cpu_pmu_get_pmc_from_id(unsigned int pmc_id)
{
    unsigned int pmc_index;
    struct perfmon_cpu_pmu *cpu_pmu;
    struct perfmon_cpu_pmc *cpu_pmc;

    assert(perfmon_pmc_id_to_index[pmc_id] != UINT32_MAX);
    pmc_index = perfmon_pmc_id_to_index[pmc_id];

    /* TODO: this may be called many times in a row. We may want to have it
     * passed to the function.
     */
    cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);
    cpu_pmc = &cpu_pmu->pmcs[pmc_index];

    assert(cpu_pmc->nr_refs != 0);

    return cpu_pmc;
}

uint64_t
perfmon_cpu_pmc_get_prev(unsigned int pmc_id)
{
    struct perfmon_cpu_pmc *cpu_pmc;

    cpu_pmc = perfmon_cpu_pmu_get_pmc_from_id(pmc_id);

    return cpu_pmc->prev_value;
}

void
perfmon_cpu_pmc_set_prev(unsigned int pmc_id, uint64_t prev)
{
    struct perfmon_cpu_pmc *cpu_pmc;

    cpu_pmc = perfmon_cpu_pmu_get_pmc_from_id(pmc_id);

    cpu_pmc->prev_value = prev;
}

void
perfmon_cpu_pmc_inc_of(unsigned int pmc_id)
{
    struct perfmon_cpu_pmc *cpu_pmc;

    cpu_pmc = perfmon_cpu_pmu_get_pmc_from_id(pmc_id);
    cpu_pmc->overflow_id++;
}

static void
perfmon_cpu_pmu_load(struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_index)
{
    struct perfmon_cpu_pmc *cpu_pmc;

    cpu_pmc = &cpu_pmu->pmcs[pmc_index];

    if (cpu_pmc->nr_refs == 0) {
        pmu_driver.start(perfmon_pmu.pmcs[pmc_index].id,
                          perfmon_pmu.pmcs[pmc_index].raw_event_id);
    }

    cpu_pmc->nr_refs++;
}

static void
perfmon_cpu_pmu_unload(struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_index)
{
    struct perfmon_cpu_pmc *cpu_pmc;

    cpu_pmc = &cpu_pmu->pmcs[pmc_index];
    assert(cpu_pmc->nr_refs != 0);
    cpu_pmc->nr_refs--;

    if (cpu_pmc->nr_refs == 0) {
        pmu_driver.stop(perfmon_pmu.pmcs[pmc_index].id);
    }
}

void
perfmon_of_intr(void)
{
    pmu_driver.handle_of_intr();
}

int
perfmon_pmu_register(struct perfmon_pmu_ops *driver)
{
    assert(driver->info && driver->translate && driver->alloc
           && driver->free && driver->start && driver->stop
           && driver->get_pmc_width);

    if (pmu_driver.info) {
        /* Already initialized */
        assert(0);
        return EINVAL;
    }
    pmu_driver = *driver;

    return 0;
}

static int __init
perfmon_bootstrap(void)
{
    kmem_cache_init(&perfmon_grouplist_cache, "perfmon_grouplist",
                    sizeof(struct perfmon_grouplist), 0,
                    perfmon_grouplist_ctor, 0);

    return 0;
}

INIT_OP_DEFINE(perfmon_bootstrap,
               INIT_OP_DEP(kmem_setup, true));

static int __init
perfmon_setup(void)
{
    struct perfmon_grouplist *grouplist;
    unsigned int i;

    spinlock_init(&perfmon_pmu.lock);
    perfmon_pmu.nr_pmcs = 0;

    for (i = 0; i < ARRAY_SIZE(perfmon_pmu.pmcs); i++) {
        perfmon_pmu.pmcs[i].nr_refs = 0;
    }
    for (i = 0; i < ARRAY_SIZE(perfmon_pmc_id_to_index); i++) {
        perfmon_pmc_id_to_index[i] = UINT32_MAX;
    }

    for (i = 0; i < cpu_count(); i++) {
        perfmon_cpu_pmu_init(percpu_ptr(perfmon_cpu_pmu, i));
    }

    for (i = 0; i < cpu_count(); i++) {
        grouplist = perfmon_grouplist_create();

        if (grouplist == NULL) {
            panic("perfmon: unable to create cpu grouplists");
        }

        percpu_var(perfmon_cpu_grouplist, i) = grouplist;
    }

    if (!pmu_driver.info) {
        log_err("unable to start perfmon: no compatible pmu driver available");
        return ENODEV;
    }
    pmu_driver.info();
    if (pmu_driver.handle_of_intr) {
        trap_register(TRAP_LAPIC_PMC_OF, lapic_pmc_of_intr);
    } else {
        log_warning("registered pmu does not handle overflow\n");
    }

    return 0;
}

INIT_OP_DEFINE(perfmon_setup,
               INIT_OP_DEP(cpu_setup, true),
               INIT_OP_DEP(kmem_setup, true),
               INIT_OP_DEP(panic_setup, true),
               INIT_OP_DEP(percpu_setup, true),
               INIT_OP_DEP(perfmon_bootstrap, true),
               INIT_OP_DEP(pmu_amd_setup, false),
               INIT_OP_DEP(pmu_intel_setup, false),
               INIT_OP_DEP(spinlock_setup, true),
               INIT_OP_DEP(thread_setup, true),
               INIT_OP_DEP(trap_setup, true));

static void
perfmon_check_event_args(unsigned int type, unsigned int id, int flags)
{
    (void)type;
    (void)id;
    (void)flags;
    assert((type == PERFMON_ET_RAW) || (type == PERFMON_ET_GENERIC));
    assert((type != PERFMON_ET_GENERIC) || (id < PERFMON_NR_GENERIC_EVENTS));
    assert((flags & PERFMON_EF_MASK) == flags);
    assert((flags & (PERFMON_EF_KERN | PERFMON_EF_USER)));
}

int
perfmon_event_create(struct perfmon_event **eventp, unsigned int type,
                     unsigned int id, int flags)
{
    struct perfmon_event *event;

    perfmon_check_event_args(type, id, flags);

    event = kmem_alloc(sizeof(*event));

    if (event == NULL) {
        return ENOMEM;
    }

    event->count = 0;
    list_node_init(&event->node);
    event->flags = flags;
    event->type = type;
    event->id = id;
    *eventp = event;
    return 0;
}

void
perfmon_event_destroy(struct perfmon_event *event)
{
    kmem_free(event, sizeof(*event));
}

uint64_t
perfmon_event_read(const struct perfmon_event *event)
{
    return event->count;
}

#ifdef CONFIG_PERFMON_TEST

int
perfmon_event_write(struct perfmon_event *event, uint64_t value)
{
    if (!pmu_driver.write) {
        return ENODEV;
    }
    event->value = value;
    event->set_value = true;

    return 0;
}

int
perfmon_get_pmc_width(void)
{
    return pmu_driver.get_pmc_width();
}

#endif /* CONFIG_PERFMON_TEST */

void
perfmon_event_reset(struct perfmon_event *event)
{
    event->count = 0;
}

static void
perfmon_event_sync(struct perfmon_cpu_pmu *cpu_pmu,
                   struct perfmon_event *event)
{
    struct perfmon_pmc *pmc;
    struct perfmon_cpu_pmc *cpu_pmc;
    uint64_t count;
    int diff;

    pmc = perfmon_pmc_from_index(event->pmc_index);
    cpu_pmc = &cpu_pmu->pmcs[event->pmc_index];
    count = pmu_driver.read(pmc->id);

    if (unlikely(event->overflow_id != cpu_pmc->overflow_id)) {
        assert(cpu_pmc->overflow_id > event->overflow_id);

        diff = cpu_pmc->overflow_id > event->overflow_id;
        /* diff is very likely 1. */
        event->count += (1UL <<  pmu_driver.get_pmc_width()) * diff
                        - event->prev + count;
        event->overflow_id = cpu_pmc->overflow_id;
    } else {
        event->count += count - event->prev;
    }
    event->prev = count;
}

static inline int
perfmon_group_attached(const struct perfmon_group *group)
{
    return group->flags & PERFMON_GF_ATTACHED;
}

static inline int
perfmon_group_enabled(const struct perfmon_group *group)
{
    return group->flags & PERFMON_GF_ENABLED;
}

static inline int
perfmon_group_loaded(const struct perfmon_group *group)
{
    return group->flags & PERFMON_GF_LOADED;
}

static inline int
perfmon_group_stopping(const struct perfmon_group *group)
{
    return group->flags & PERFMON_GF_PENDING_DISABLE;
}

int
perfmon_group_create(struct perfmon_group **groupp)
{
    struct perfmon_group *group;

    group = kmem_alloc(sizeof(*group));

    if (group == NULL) {
        return ENOMEM;
    }

    list_init(&group->events);
    spinlock_init(&group->lock);
    group->cpu = PERFMON_INVALID_CPU;
    group->flags = 0;
    group->type = PERFMON_GT_UNKNOWN;
    *groupp = group;
    return 0;
}

int
perfmon_group_destroy(struct perfmon_group *group)
{
    struct perfmon_event *event;

    if (perfmon_group_attached(group)) {
        return EINVAL;
    }
    assert (!perfmon_group_enabled(group));

    while (!list_empty(&group->events)) {
        event = list_first_entry(&group->events, struct perfmon_event, node);
        list_remove(&event->node);
        perfmon_event_destroy(event);
    }

    kmem_free(group, sizeof(*group));
    return 0;
}

void
perfmon_group_add(struct perfmon_group *group, struct perfmon_event *event)
{
    assert(list_node_unlinked(&event->node));
    assert(!perfmon_group_attached(group));

    /* TODO: check that we we do not have the same event twice. */

    list_insert_tail(&group->events, &event->node);
}

/*
 * Attach a group to the global logical PMU.
 *
 * For each event in the group, obtain a reference on a PMC.
 */
static int
perfmon_group_attach_pmu(struct perfmon_group *group)
{
    struct perfmon_event *event, *tmp;
    struct perfmon_pmc *pmc = NULL;
    int error;

    assert(!perfmon_group_attached(group));

    list_for_each_entry(&group->events, event, node) {
        error = perfmon_pmc_get(&pmc, event);

        if (error) {
            goto error_pmc;
        }

        event->pmc_index = perfmon_pmc_index(pmc);
    }

    return 0;

error_pmc:
    list_for_each_entry(&group->events, tmp, node) {
        if (tmp == event) {
            break;
        }

        perfmon_pmc_put(perfmon_pmc_from_index(tmp->pmc_index));
    }

    return error;
}

static void
perfmon_group_detach_pmu(struct perfmon_group *group)
{
    struct perfmon_event *event;

    assert(perfmon_group_attached(group));

    list_for_each_entry(&group->events, event, node) {
        perfmon_pmc_put(perfmon_pmc_from_index(event->pmc_index));
    }
}

int
perfmon_group_attach(struct perfmon_group *group, struct thread *thread)
{
    struct perfmon_grouplist *grouplist;
    unsigned long flags;
    int error;

    assert(group->type == PERFMON_GT_UNKNOWN);

    error = perfmon_group_attach_pmu(group);

    if (error) {
        return error;
    }

    thread_ref(thread);
    group->thread = thread;
    group->type = PERFMON_GT_THREAD;
    group->flags |= PERFMON_GF_ATTACHED;

    grouplist = thread->perfmon_groups;

    spinlock_lock_intr_save(&grouplist->lock, &flags);
    list_insert_tail(&grouplist->groups, &group->node);
    spinlock_unlock_intr_restore(&grouplist->lock, flags);

    return 0;
}

int
perfmon_group_attach_cpu(struct perfmon_group *group, unsigned int cpu)
{
    int error;
    struct perfmon_grouplist *grouplist;

    assert(cpu < cpu_count());
    assert(group->type == PERFMON_GT_UNKNOWN);

    error = perfmon_group_attach_pmu(group);

    if (error) {
        return error;
    }

    group->cpu = cpu;
    group->type = PERFMON_GT_CPU;
    group->flags |= PERFMON_GF_ATTACHED;

    grouplist = percpu_var(perfmon_cpu_grouplist, cpu);

    spinlock_lock(&grouplist->lock);
    list_insert_tail(&grouplist->groups, &group->node);
    spinlock_unlock(&grouplist->lock);

    return 0;
}

int
perfmon_group_detach(struct perfmon_group *group)
{
    unsigned long flags;
    unsigned long grouplist_flags;
    struct perfmon_grouplist *grouplist;
    struct thread *prev_thread;
    unsigned int type;
    int ret;

    type = group->type;
    grouplist_flags = 0; /* silence Wmaybe-uninitialized warning. */
    ret = 0;
    prev_thread = NULL;

    switch (type) {
    case PERFMON_GT_THREAD:
        grouplist = group->thread->perfmon_groups;
        spinlock_lock_intr_save(&grouplist->lock, &grouplist_flags);
        prev_thread = group->thread;
        break;
    case PERFMON_GT_CPU:
        grouplist = percpu_var(perfmon_cpu_grouplist, group->cpu);
        spinlock_lock(&grouplist->lock);
        break;
    default:
        panic("perfmon: invalid group type on detach");
    }
    spinlock_lock_intr_save(&group->lock, &flags);


    if (perfmon_group_enabled(group)) {
        ret = EINVAL;
        goto out;
    }

    if (!perfmon_group_attached(group)) {
        goto out;
    }

    perfmon_group_detach_pmu(group);
    list_remove(&group->node);

    group->thread = NULL;
    group->cpu = PERFMON_INVALID_CPU;
    group->type = PERFMON_GT_UNKNOWN;
    group->flags &= ~PERFMON_GF_ATTACHED;
    assert(!group->flags);

    goto out;

out:
    spinlock_unlock_intr_restore(&group->lock, flags);
    switch (type) {
    case PERFMON_GT_THREAD:
        spinlock_unlock_intr_restore(&grouplist->lock, grouplist_flags);
        break;
    case PERFMON_GT_CPU:
        spinlock_unlock(&grouplist->lock);
        break;
    }

    if (prev_thread) {
        /* Late unref as it might destroy the thread and lock the runq. */
        thread_unref(prev_thread);
    }

    return ret;
}

static void
perfmon_group_load(struct perfmon_group *group)
{
    struct perfmon_cpu_pmu *cpu_pmu;
    struct perfmon_event *event;
#ifdef CONFIG_PERFMON_TEST
    struct perfmon_pmc *pmc;
#endif

    assert(!thread_preempt_enabled());
    assert(perfmon_group_enabled(group));
    assert(!perfmon_group_loaded(group));

    cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);

#ifdef CONFIG_PERFMON_TEST
    /* XXX: could be done in the loading loop, but performance does not
     * matters in the functional tests using this feature.
     */
    list_for_each_entry(&group->events, event, node) {
        if (!event->set_value) {
            continue;
        }
        pmc = perfmon_pmc_from_index(event->pmc_index);
        pmu_driver.write(pmc->id, event->value);
        event->set_value = false;
    }
#endif

    list_for_each_entry(&group->events, event, node) {
        perfmon_cpu_pmu_load(cpu_pmu, event->pmc_index);
        event->prev = pmu_driver.read(perfmon_pmu.pmcs[event->pmc_index].id);
        event->overflow_id = cpu_pmu->pmcs[event->pmc_index].overflow_id;
    }

    group->cpu = cpu_id();
    group->flags |= PERFMON_GF_LOADED;
}

static void
perfmon_cpu_load_remote(void *arg)
{
    struct perfmon_group *group;

    group = arg;
    assert (group->cpu == cpu_id());

    spinlock_lock(&group->lock);

    perfmon_group_load(group);

    spinlock_unlock(&group->lock);
}

static void
perfmon_group_unload(struct perfmon_group *group)
{
    struct perfmon_cpu_pmu *cpu_pmu;
    struct perfmon_event *event;

    assert(!thread_preempt_enabled());
    assert(perfmon_group_enabled(group));
    assert(perfmon_group_loaded(group));

    cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);

    list_for_each_entry(&group->events, event, node) {
        perfmon_cpu_pmu_unload(cpu_pmu, event->pmc_index);
        perfmon_event_sync(cpu_pmu, event);
    }

    group->flags &= ~PERFMON_GF_LOADED;
}

static void
perfmon_cpu_unload_remote(void *arg)
{
    struct perfmon_group *group;

    group = arg;
    assert (group->cpu == cpu_id());
    assert (perfmon_group_stopping(group));

    spinlock_lock(&group->lock);

    perfmon_group_unload(group);

    group->flags &= ~PERFMON_GF_PENDING_DISABLE;
    group->flags &= ~PERFMON_GF_ENABLED;

    spinlock_unlock(&group->lock);
}

static void
perfmon_thread_load_remote(void *arg)
{
    struct perfmon_group *group;
    struct thread *thread;

    assert (!cpu_intr_enabled());

    group = arg;
    thread = thread_self();

    if (thread != group->thread) {
        return;
    }

    spinlock_lock(&group->lock);

    if (perfmon_group_enabled(group) && !perfmon_group_loaded(group)) {
        perfmon_group_load(group);
    }

    spinlock_unlock(&group->lock);
}

static void
perfmon_thread_unload_remote(void *arg)
{
    struct perfmon_group *group;
    struct thread *thread;

    assert (!cpu_intr_enabled());

    group = arg;
    thread = thread_self();

    if (thread != group->thread) {
        return;
    }

    spinlock_lock(&group->lock);

    if (perfmon_group_enabled(group)) {
        assert (perfmon_group_stopping(group));
        if (perfmon_group_loaded(group)) {
            perfmon_group_unload(group);
        }
        group->flags &= ~PERFMON_GF_PENDING_DISABLE;
        group->flags &= ~PERFMON_GF_ENABLED;
    }

    spinlock_unlock(&group->lock);
}

int
perfmon_group_start(struct perfmon_group *group)
{
    unsigned long flags;
    unsigned int cpu;
    int ret;

    ret = 0;
    spinlock_lock_intr_save(&group->lock, &flags);

    if (!perfmon_group_attached(group) || perfmon_group_loaded(group)) {
        ret = EINVAL;
        goto end;
    }
    assert(!perfmon_group_enabled(group));

    group->flags |= PERFMON_GF_ENABLED;

    if (group->type == PERFMON_GT_CPU) {
        spinlock_unlock_intr_restore(&group->lock, flags);

        xcall_call(perfmon_cpu_load_remote, group, group->cpu);

        return 0;
    } else if (group->thread == thread_self()) {
        perfmon_group_load(group);
    } else if (group->thread->state == THREAD_RUNNING) {
        spinlock_unlock_intr_restore(&group->lock, flags);

        cpu = thread_cpu(group->thread);

        xcall_call(perfmon_thread_load_remote, group, cpu);

        return 0;
    }
end:
    spinlock_unlock_intr_restore(&group->lock, flags);

    return ret;
}

static void
perfmon_group_sync_local(struct perfmon_group *group)
{
    struct perfmon_event *event;
    struct perfmon_cpu_pmu *cpu_pmu;

    cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);

    /* The group sync duration *should be* limited as a group may only have a
     * limited amount of *different* events.
     */
    list_for_each_entry(&group->events, event, node) {
        perfmon_event_sync(cpu_pmu, event);
    }
}

static void
perfmon_cpu_sync_remote(void *arg)
{
    struct perfmon_group *group;

    group = arg;
    assert (group->type == PERFMON_GT_CPU);
    assert (group->cpu == cpu_id());

    perfmon_group_sync_local(group);
}

static void
perfmon_thread_sync_remote(void *arg)
{
    struct perfmon_group *group;
    unsigned long flags;

    group = arg;

    assert (group->type == PERFMON_GT_THREAD);
    if (thread_self() != group->thread) {
        return;
    }
    spinlock_lock_intr_save(&group->lock, &flags);

    perfmon_group_sync_local(group);

    spinlock_unlock_intr_restore(&group->lock, flags);
}

void
perfmon_group_update(struct perfmon_group *group)
{
    unsigned long flags;
    unsigned int cpu;

    assert(perfmon_group_enabled(group));

    spinlock_lock_intr_save(&group->lock, &flags);

    assert(perfmon_group_attached(group));
    assert(perfmon_group_enabled(group));

    if (!perfmon_group_loaded(group)) {
        goto end;
    }

    if (group->type == PERFMON_GT_CPU) {
        if (group->cpu == cpu_id())
            perfmon_group_sync_local(group);
        else {
            xcall_call(perfmon_cpu_sync_remote, group, group->cpu);
        }
    } else {
        if (group->thread == thread_self()) {
            assert (perfmon_group_loaded(group));
            perfmon_group_sync_local(group);
        } else if (group->thread->state == THREAD_RUNNING) {
            spinlock_unlock_intr_restore(&group->lock, flags);
            cpu = thread_cpu(group->thread);
            xcall_call(perfmon_thread_sync_remote, group, cpu);
            return;
        }
    }
end:
    spinlock_unlock_intr_restore(&group->lock, flags);
}

int
perfmon_group_stop(struct perfmon_group *group)
{
    int ret;
    unsigned long flags;
    unsigned int cpu;

    ret = 0;
    spinlock_lock_intr_save(&group->lock, &flags);

    if (!perfmon_group_attached(group) || !perfmon_group_enabled(group)) {
       ret = EINVAL;
       goto end;
    }

    if (!perfmon_group_loaded(group)) {
        goto disable;
    }

    group->flags |= PERFMON_GF_PENDING_DISABLE;

    if (group->type == PERFMON_GT_CPU) {
        spinlock_unlock_intr_restore(&group->lock, flags);

        xcall_call(perfmon_cpu_unload_remote, group, group->cpu);
        return 0;
    } else if (group->thread == thread_self()) {
        perfmon_group_unload(group);
    } else {
        /* If the thead is not running (but still loaded), the unload is
         * (probably) getting called when we release the group lock, but we
         * still need a blocking xcall to guarantee the group is disabled when
         * the function returns.
         */
        spinlock_unlock_intr_restore(&group->lock, flags);

        cpu = thread_cpu(group->thread);

        xcall_call(perfmon_thread_unload_remote, group, cpu);
        return 0;
    }

disable:
    group->flags &= ~PERFMON_GF_PENDING_DISABLE;
    group->flags &= ~PERFMON_GF_ENABLED;

end:
    spinlock_unlock_intr_restore(&group->lock, flags);
    return ret;
}

int
perfmon_thread_init(struct thread *thread)
{
    struct perfmon_grouplist *grouplist;

    grouplist = perfmon_grouplist_create();

    if (grouplist == NULL) {
        return ENOMEM;
    }

    thread->perfmon_groups = grouplist;
    return 0;
}

void
perfmon_thread_destroy(struct thread *thread)
{
    perfmon_grouplist_destroy(thread->perfmon_groups);
}

void
perfmon_thread_load(struct thread *thread)
{
    struct perfmon_grouplist *grouplist;
    struct perfmon_group *group;

    assert(!cpu_intr_enabled());
    assert(!thread_preempt_enabled());

    grouplist = thread->perfmon_groups;

    spinlock_lock(&grouplist->lock);

    list_for_each_entry(&grouplist->groups, group, node) {
        spinlock_lock(&group->lock);

        if (perfmon_group_enabled(group) && !perfmon_group_loaded(group)) {
            perfmon_group_load(group);
        }

        spinlock_unlock(&group->lock);
    }

    spinlock_unlock(&grouplist->lock);
}

void
perfmon_thread_unload(struct thread *thread)
{
    struct perfmon_grouplist *grouplist;
    struct perfmon_group *group;


    assert(!cpu_intr_enabled());
    assert(!thread_preempt_enabled());

    grouplist = thread->perfmon_groups;

    spinlock_lock(&grouplist->lock);

    list_for_each_entry(&grouplist->groups, group, node) {
        spinlock_lock(&group->lock);
        /* TODO: we may want to prevent long looping on the groups.
         * One way to do this would be to maintain events mapping in the
         * grouplist in order to have a finite operation upon schedueling.
         */

        if (perfmon_group_loaded(group)) {
            perfmon_group_unload(group);
            if (perfmon_group_stopping(group)) {
                group->flags &= ~PERFMON_GF_PENDING_DISABLE;
                group->flags &= ~PERFMON_GF_ENABLED;
            }
        }

        spinlock_unlock(&group->lock);
    }

    spinlock_unlock(&grouplist->lock);
}