/*
* Copyright (c) 2014-2018 Remy Noel.
* Copyright (c) 2014-2015 Richard Braun.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*
*
* The perfomance monitoring modules allows to manage performance monitoring as
* event groups. Each physical performance monitoring counter (pmc) may be
* referenced by perfmon events, which are theelves groupped in perfmon groups.
* Groups can then be attached to either threads or cpus into perfmon
* grouplists.
*
* In order to guarantee that thread relocation, is properly handled, events
* types are reseved on perfomance monitoring units (pmu) for all cpus for every
* event of a group when it is attached. Therefore a group attach may fail if no
* compatible pmc is available globally.
*
* Locking order : interrupts -> thread runq -> grouplist -> group
*
* TODO API to differenciate user and kernel events.
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
/*
* Performance monitoring event.
*
* When a group is attached, each of its events is associated to a PMC,
* adding a reference in the process.
*/
struct perfmon_event {
uint64_t count;
uint64_t prev;
uint64_t overflow_id;
struct list node;
int flags;
unsigned int type;
unsigned int id;
unsigned int pmc_index;
#ifdef CONFIG_PERFMON_TEST
uint64_t value;
bool set_value;
#endif
};
#define PERFMON_INVALID_CPU ((unsigned int)-1)
/*
* Group types.
*/
#define PERFMON_GT_UNKNOWN 0
#define PERFMON_GT_CPU 1
#define PERFMON_GT_THREAD 2
/*
* Group States flags.
*/
#define PERFMON_GF_ATTACHED 1
#define PERFMON_GF_ENABLED 2
#define PERFMON_GF_LOADED 4
#define PERFMON_GF_PENDING_DISABLE 8
/*
* Group possible states are handled through the flags attribute.
* - A group can either be unattached or attached(to a thread or a cpu)
* - An attached group may be enabled or not.
* - An enabled group may be loaded or not e.g. have actual running
* performance counters on a CPU.
*
* When a group is attached, some ressources are reserved for it so it can be
* monitored at any time.
* When a group is enabled it will get loaded when needed:
* -Cpu groups stay loaded as long as they are enabled.
* -Thread groups are loaded when running.
* When a group is loaded its performance counters are currently enabled.
*
* The PENDING_DISABLE is here so that a remote thread can be disabled when it
* unscedule itself.
*
* Note that a non-attached group can only be referenced from the api. Since
* manipulating the same group from different threads as the same time is not
* supported, the code does not bother to lock it when the group is assumed
* un-attached.
*
* About thread-type group counters synchronization:
* - The groups are synchronized when their thread is unscheduled which
* means their counter value is updated and pending counter changes
* (like unloading) are performed.
* - Since all operations requires the group to be locked, it is mandatory
* to unlock the group before xcalling any remote operation in order to
* avoid a deadlock.
* - Any remote thread operation that gets executed after the thread got
* unscheduled will have nothing to do if the current thread is not the
* target one since the target thread have been unloaded inbetween.
*/
struct perfmon_group {
struct list node;
struct list events;
struct thread *thread;
struct spinlock lock;
unsigned int cpu;
short flags;
unsigned short type;
};
/*
* List of all groups attached to a single monitored object, either a CPU
* or a thread.
*/
struct perfmon_grouplist {
struct list groups;
struct spinlock lock;
};
/*
* Maximum number of supported hardware counters.
*/
#define PERFMON_MAX_PMCS 64
/*
* Performance monitoring counter.
*
* When a PMC is valid, it maps a raw event to a hardware counter.
* A PMC is valid if and only if its reference count isn't zero.
*/
struct perfmon_pmc {
unsigned int nr_refs;
unsigned int raw_event_id;
unsigned int id;
};
/*
* Performance monitoring unit.
*
* There is a single system-wide logical PMU, used to globally allocate
* PMCs. Reserving a counter across the entire system ensures thread
* migration isn't hindered by performance monitoring.
*/
struct perfmon_pmu {
struct spinlock lock;
unsigned int nr_pmcs;
struct perfmon_pmc pmcs[PERFMON_MAX_PMCS];
};
/*
* Per-CPU performance monitoring counter.
*
* When a group is loaded on a processor, the per-CPU PMCs of its events
* get referenced. When a per-CPU PMC is referenced, its underlying hardware
* counter is active.
*
* Preemption must be disabled on access.
*/
struct perfmon_cpu_pmc {
unsigned int nr_refs;
uint64_t prev_value;
uint64_t overflow_id;
};
/*
* Per-CPU performance monitoring unit.
*
* The per-CPU PMCs are indexed the same way as the global PMCs.
*
* Preemption must be disabled on access.
*/
struct perfmon_cpu_pmu {
struct perfmon_cpu_pmc pmcs[PERFMON_MAX_PMCS];
struct timer of_timer;
unsigned int cpu_id;
};
static struct perfmon_pmu_driver pmu_driver __read_mostly;
static struct perfmon_pmu perfmon_pmu;
static unsigned int perfmon_pmc_id_to_index[PERFMON_MAX_PMCS];
static struct perfmon_cpu_pmu perfmon_cpu_pmu __percpu;
/*
* Cache of thread-specific group lists.
*/
static struct kmem_cache perfmon_grouplist_cache;
/*
* CPU specific group lists.
*/
static struct perfmon_grouplist *perfmon_cpu_grouplist __percpu;
static inline int
perfmon_translate(unsigned int *raw_event_idp, unsigned int event_type,
unsigned int event_id)
{
switch (event_type) {
case PERFMON_ET_RAW:
*raw_event_idp = event_id;
return 0;
case PERFMON_ET_GENERIC:
return pmu_driver.ops.translate(raw_event_idp, event_id);
default:
panic("perfmon: unsupported event type");
}
}
static int
perfmon_pmc_alloc(struct perfmon_pmc **pmcp, unsigned int raw_event_id)
{
struct perfmon_pmc *pmc;
size_t i;
int error;
if (perfmon_pmu.nr_pmcs == ARRAY_SIZE(perfmon_pmu.pmcs)) {
return EAGAIN;
}
for (i = 0; i < ARRAY_SIZE(perfmon_pmu.pmcs); i++) {
pmc = &perfmon_pmu.pmcs[i];
if (pmc->nr_refs == 0) {
break;
}
}
assert(i < ARRAY_SIZE(perfmon_pmu.pmcs));
error = pmu_driver.ops.alloc(&pmc->id, raw_event_id);
if (error) {
return error;
}
pmc->raw_event_id = raw_event_id;
perfmon_pmu.nr_pmcs++;
*pmcp = pmc;
return 0;
}
static struct perfmon_pmc *
perfmon_pmc_lookup(unsigned int raw_event_id)
{
struct perfmon_pmc *pmc;
size_t i;
if (perfmon_pmu.nr_pmcs == 0) {
return NULL;
}
for (i = 0; i < ARRAY_SIZE(perfmon_pmu.pmcs); i++) {
pmc = &perfmon_pmu.pmcs[i];
if ((pmc->nr_refs != 0) && (pmc->raw_event_id == raw_event_id)) {
return pmc;
}
}
return NULL;
}
static inline unsigned int
perfmon_pmc_index(const struct perfmon_pmc *pmc)
{
unsigned int index;
index = pmc - perfmon_pmu.pmcs;
assert(index < ARRAY_SIZE(perfmon_pmu.pmcs));
return index;
}
/*
* Obtain a reference on a PMC for the given event.
*
* If there is no existing PMC suitable for this event, allocate one.
*/
static int
perfmon_pmc_get(struct perfmon_pmc **pmcp, const struct perfmon_event *event)
{
struct perfmon_pmc *pmc;
unsigned int raw_event_id;
unsigned int pmc_index;
int error;
error = perfmon_translate(&raw_event_id, event->type, event->id);
if (error) {
return error;
}
spinlock_lock(&perfmon_pmu.lock);
pmc = perfmon_pmc_lookup(raw_event_id);
if (pmc == NULL) {
error = perfmon_pmc_alloc(&pmc, raw_event_id);
if (error) {
goto out;
}
pmc_index = perfmon_pmc_index(pmc);
assert(perfmon_pmc_id_to_index[pmc->id] == UINT32_MAX);
perfmon_pmc_id_to_index[pmc->id] = pmc_index;
}
pmc->nr_refs++;
out:
spinlock_unlock(&perfmon_pmu.lock);
if (error) {
return error;
}
*pmcp = pmc;
return 0;
}
/*
* Release a reference on a PMC.
*/
static void
perfmon_pmc_put(struct perfmon_pmc *pmc)
{
spinlock_lock(&perfmon_pmu.lock);
assert(pmc->nr_refs != 0);
pmc->nr_refs--;
if (pmc->nr_refs == 0) {
pmu_driver.ops.free(pmc->id);
assert(perfmon_pmc_id_to_index[pmc->id] != UINT32_MAX);
perfmon_pmc_id_to_index[pmc->id] = UINT32_MAX;
}
spinlock_unlock(&perfmon_pmu.lock);
}
static inline struct perfmon_pmc *
perfmon_pmc_from_index(unsigned int index)
{
assert(index < ARRAY_SIZE(perfmon_pmu.pmcs));
return &perfmon_pmu.pmcs[index];
}
static void
perfmon_grouplist_ctor(void *arg)
{
struct perfmon_grouplist *grouplist;
grouplist = arg;
list_init(&grouplist->groups);
spinlock_init(&grouplist->lock);
}
static struct perfmon_grouplist *
perfmon_grouplist_create(void)
{
return kmem_cache_alloc(&perfmon_grouplist_cache);
}
static void
perfmon_grouplist_destroy(struct perfmon_grouplist *grouplist)
{
kmem_cache_free(&perfmon_grouplist_cache, grouplist);
}
static void __init
perfmon_cpu_pmu_init(struct perfmon_cpu_pmu *cpu_pmu)
{
unsigned int i;
for (i = 0; i < ARRAY_SIZE(cpu_pmu->pmcs); i++) {
struct perfmon_cpu_pmc *pmc;
pmc = &cpu_pmu->pmcs[i];
pmc->nr_refs = 0;
pmc->prev_value = pmu_driver.ops.read(perfmon_pmu.pmcs[i].id);
pmc->overflow_id = 0;
}
}
static struct perfmon_cpu_pmc *
perfmon_cpu_pmu_get_pmc_from_id(unsigned int pmc_id)
{
unsigned int pmc_index;
struct perfmon_cpu_pmu *cpu_pmu;
struct perfmon_cpu_pmc *cpu_pmc;
assert(perfmon_pmc_id_to_index[pmc_id] != UINT32_MAX);
pmc_index = perfmon_pmc_id_to_index[pmc_id];
/* TODO: this may be called many times in a row. We may want to have it
* passed to the function.
*/
cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);
cpu_pmc = &cpu_pmu->pmcs[pmc_index];
assert(cpu_pmc->nr_refs != 0);
return cpu_pmc;
}
uint64_t
perfmon_cpu_pmc_get_prev(unsigned int pmc_id)
{
struct perfmon_cpu_pmc *cpu_pmc;
cpu_pmc = perfmon_cpu_pmu_get_pmc_from_id(pmc_id);
return cpu_pmc->prev_value;
}
void
perfmon_cpu_pmc_set_prev(unsigned int pmc_id, uint64_t prev)
{
struct perfmon_cpu_pmc *cpu_pmc;
cpu_pmc = perfmon_cpu_pmu_get_pmc_from_id(pmc_id);
cpu_pmc->prev_value = prev;
}
void
perfmon_cpu_pmc_inc_of(unsigned int pmc_id)
{
struct perfmon_cpu_pmc *cpu_pmc;
cpu_pmc = perfmon_cpu_pmu_get_pmc_from_id(pmc_id);
cpu_pmc->overflow_id++;
}
static void
perfmon_cpu_pmu_load(struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_index)
{
struct perfmon_cpu_pmc *cpu_pmc;
cpu_pmc = &cpu_pmu->pmcs[pmc_index];
if (cpu_pmc->nr_refs == 0) {
pmu_driver.ops.start(perfmon_pmu.pmcs[pmc_index].id,
perfmon_pmu.pmcs[pmc_index].raw_event_id);
}
cpu_pmc->nr_refs++;
}
static void
perfmon_cpu_pmu_unload(struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_index)
{
struct perfmon_cpu_pmc *cpu_pmc;
cpu_pmc = &cpu_pmu->pmcs[pmc_index];
assert(cpu_pmc->nr_refs != 0);
cpu_pmc->nr_refs--;
if (cpu_pmc->nr_refs == 0) {
pmu_driver.ops.stop(perfmon_pmu.pmcs[pmc_index].id);
}
}
void
perfmon_of_intr(void)
{
pmu_driver.ops.handle_of_intr();
}
int
perfmon_pmu_register(struct perfmon_pmu_driver *driver)
{
struct perfmon_pmu_ops *ops = &driver->ops;
assert(ops->info && ops->translate && ops->alloc
&& ops->free && ops->start && ops->stop);
assert(!ops->handle_of_intr != !driver->of_max_ticks);
if (pmu_driver.ops.info) {
/* Already initialized */
assert(0);
return EINVAL;
}
pmu_driver = *driver;
return 0;
}
static int __init
perfmon_bootstrap(void)
{
kmem_cache_init(&perfmon_grouplist_cache, "perfmon_grouplist",
sizeof(struct perfmon_grouplist), 0,
perfmon_grouplist_ctor, 0);
return 0;
}
INIT_OP_DEFINE(perfmon_bootstrap,
INIT_OP_DEP(kmem_setup, true));
static int __init
perfmon_setup(void)
{
struct perfmon_grouplist *grouplist;
unsigned int i;
spinlock_init(&perfmon_pmu.lock);
perfmon_pmu.nr_pmcs = 0;
for (i = 0; i < ARRAY_SIZE(perfmon_pmu.pmcs); i++) {
perfmon_pmu.pmcs[i].nr_refs = 0;
}
for (i = 0; i < ARRAY_SIZE(perfmon_pmc_id_to_index); i++) {
perfmon_pmc_id_to_index[i] = UINT32_MAX;
}
for (i = 0; i < cpu_count(); i++) {
perfmon_cpu_pmu_init(percpu_ptr(perfmon_cpu_pmu, i));
}
for (i = 0; i < cpu_count(); i++) {
grouplist = perfmon_grouplist_create();
if (grouplist == NULL) {
panic("perfmon: unable to create cpu grouplists");
}
percpu_var(perfmon_cpu_grouplist, i) = grouplist;
}
if (!pmu_driver.ops.info) {
log_err("unable to start perfmon: no compatible pmu driver available");
return ENODEV;
}
pmu_driver.ops.info();
if (pmu_driver.ops.handle_of_intr) {
/* FIXME: this should not require an architectural api call. */
trap_register(TRAP_LAPIC_PMC_OF, lapic_pmc_of_intr);
}
return 0;
}
INIT_OP_DEFINE(perfmon_setup,
INIT_OP_DEP(cpu_setup, true),
INIT_OP_DEP(kmem_setup, true),
INIT_OP_DEP(panic_setup, true),
INIT_OP_DEP(percpu_setup, true),
INIT_OP_DEP(perfmon_bootstrap, true),
INIT_OP_DEP(pmu_amd_setup, false),
INIT_OP_DEP(pmu_intel_setup, false),
INIT_OP_DEP(spinlock_setup, true),
INIT_OP_DEP(thread_setup, true),
INIT_OP_DEP(trap_setup, true));
static void
perfmon_check_event_args(unsigned int type, unsigned int id, int flags)
{
(void)type;
(void)id;
(void)flags;
assert((type == PERFMON_ET_RAW) || (type == PERFMON_ET_GENERIC));
assert((type != PERFMON_ET_GENERIC) || (id < PERFMON_NR_GENERIC_EVENTS));
assert((flags & PERFMON_EF_MASK) == flags);
assert((flags & (PERFMON_EF_KERN | PERFMON_EF_USER)));
}
int
perfmon_event_create(struct perfmon_event **eventp, unsigned int type,
unsigned int id, int flags)
{
struct perfmon_event *event;
perfmon_check_event_args(type, id, flags);
event = kmem_alloc(sizeof(*event));
if (event == NULL) {
return ENOMEM;
}
event->count = 0;
list_node_init(&event->node);
event->flags = flags;
event->type = type;
event->id = id;
*eventp = event;
return 0;
}
void
perfmon_event_destroy(struct perfmon_event *event)
{
kmem_free(event, sizeof(*event));
}
uint64_t
perfmon_event_read(const struct perfmon_event *event)
{
return event->count;
}
#ifdef CONFIG_PERFMON_TEST
int
perfmon_event_write(struct perfmon_event *event, uint64_t value)
{
if (!pmu_driver.ops.write) {
return ENODEV;
}
event->value = value;
event->set_value = true;
return 0;
}
int
perfmon_get_pmc_width(void)
{
return pmu_driver.pmc_width;
}
#endif /* CONFIG_PERFMON_TEST */
void
perfmon_event_reset(struct perfmon_event *event)
{
event->count = 0;
}
static void
perfmon_event_sync(struct perfmon_cpu_pmu *cpu_pmu,
struct perfmon_event *event)
{
struct perfmon_pmc *pmc;
struct perfmon_cpu_pmc *cpu_pmc;
uint64_t count;
int diff;
pmc = perfmon_pmc_from_index(event->pmc_index);
cpu_pmc = &cpu_pmu->pmcs[event->pmc_index];
count = pmu_driver.ops.read(pmc->id);
if (unlikely(event->overflow_id != cpu_pmc->overflow_id)) {
assert(cpu_pmc->overflow_id > event->overflow_id);
diff = cpu_pmc->overflow_id > event->overflow_id;
/* diff is very likely 1. */
event->count += (1UL << pmu_driver.pmc_width) * diff
- event->prev + count;
event->overflow_id = cpu_pmc->overflow_id;
} else {
event->count += count - event->prev;
}
event->prev = count;
}
static inline int
perfmon_group_attached(const struct perfmon_group *group)
{
return group->flags & PERFMON_GF_ATTACHED;
}
static inline int
perfmon_group_enabled(const struct perfmon_group *group)
{
return group->flags & PERFMON_GF_ENABLED;
}
static inline int
perfmon_group_loaded(const struct perfmon_group *group)
{
return group->flags & PERFMON_GF_LOADED;
}
static inline int
perfmon_group_stopping(const struct perfmon_group *group)
{
return group->flags & PERFMON_GF_PENDING_DISABLE;
}
int
perfmon_group_create(struct perfmon_group **groupp)
{
struct perfmon_group *group;
group = kmem_alloc(sizeof(*group));
if (group == NULL) {
return ENOMEM;
}
list_init(&group->events);
spinlock_init(&group->lock);
group->cpu = PERFMON_INVALID_CPU;
group->flags = 0;
group->type = PERFMON_GT_UNKNOWN;
*groupp = group;
return 0;
}
int
perfmon_group_destroy(struct perfmon_group *group)
{
struct perfmon_event *event;
if (perfmon_group_attached(group)) {
return EINVAL;
}
assert (!perfmon_group_enabled(group));
while (!list_empty(&group->events)) {
event = list_first_entry(&group->events, struct perfmon_event, node);
list_remove(&event->node);
perfmon_event_destroy(event);
}
kmem_free(group, sizeof(*group));
return 0;
}
void
perfmon_group_add(struct perfmon_group *group, struct perfmon_event *event)
{
assert(list_node_unlinked(&event->node));
assert(!perfmon_group_attached(group));
/* TODO: check that we we do not have the same event twice. */
list_insert_tail(&group->events, &event->node);
}
/*
* Attach a group to the global logical PMU.
*
* For each event in the group, obtain a reference on a PMC.
*/
static int
perfmon_group_attach_pmu(struct perfmon_group *group)
{
struct perfmon_event *event, *tmp;
struct perfmon_pmc *pmc = NULL;
int error;
assert(!perfmon_group_attached(group));
list_for_each_entry(&group->events, event, node) {
error = perfmon_pmc_get(&pmc, event);
if (error) {
goto error_pmc;
}
event->pmc_index = perfmon_pmc_index(pmc);
}
return 0;
error_pmc:
list_for_each_entry(&group->events, tmp, node) {
if (tmp == event) {
break;
}
perfmon_pmc_put(perfmon_pmc_from_index(tmp->pmc_index));
}
return error;
}
static void
perfmon_group_detach_pmu(struct perfmon_group *group)
{
struct perfmon_event *event;
assert(perfmon_group_attached(group));
list_for_each_entry(&group->events, event, node) {
perfmon_pmc_put(perfmon_pmc_from_index(event->pmc_index));
}
}
int
perfmon_group_attach(struct perfmon_group *group, struct thread *thread)
{
struct perfmon_grouplist *grouplist;
unsigned long flags;
int error;
assert(group->type == PERFMON_GT_UNKNOWN);
error = perfmon_group_attach_pmu(group);
if (error) {
return error;
}
thread_ref(thread);
group->thread = thread;
group->type = PERFMON_GT_THREAD;
group->flags |= PERFMON_GF_ATTACHED;
grouplist = thread->perfmon_groups;
spinlock_lock_intr_save(&grouplist->lock, &flags);
list_insert_tail(&grouplist->groups, &group->node);
spinlock_unlock_intr_restore(&grouplist->lock, flags);
return 0;
}
int
perfmon_group_attach_cpu(struct perfmon_group *group, unsigned int cpu)
{
int error;
struct perfmon_grouplist *grouplist;
assert(cpu < cpu_count());
assert(group->type == PERFMON_GT_UNKNOWN);
error = perfmon_group_attach_pmu(group);
if (error) {
return error;
}
group->cpu = cpu;
group->type = PERFMON_GT_CPU;
group->flags |= PERFMON_GF_ATTACHED;
grouplist = percpu_var(perfmon_cpu_grouplist, cpu);
spinlock_lock(&grouplist->lock);
list_insert_tail(&grouplist->groups, &group->node);
spinlock_unlock(&grouplist->lock);
return 0;
}
int
perfmon_group_detach(struct perfmon_group *group)
{
unsigned long flags;
unsigned long grouplist_flags;
struct perfmon_grouplist *grouplist;
struct thread *prev_thread;
unsigned int type;
int ret;
type = group->type;
grouplist_flags = 0; /* silence Wmaybe-uninitialized warning. */
ret = 0;
prev_thread = NULL;
switch (type) {
case PERFMON_GT_THREAD:
grouplist = group->thread->perfmon_groups;
spinlock_lock_intr_save(&grouplist->lock, &grouplist_flags);
prev_thread = group->thread;
break;
case PERFMON_GT_CPU:
grouplist = percpu_var(perfmon_cpu_grouplist, group->cpu);
spinlock_lock(&grouplist->lock);
break;
default:
panic("perfmon: invalid group type on detach");
}
spinlock_lock_intr_save(&group->lock, &flags);
if (perfmon_group_enabled(group)) {
ret = EINVAL;
goto out;
}
if (!perfmon_group_attached(group)) {
goto out;
}
perfmon_group_detach_pmu(group);
list_remove(&group->node);
group->thread = NULL;
group->cpu = PERFMON_INVALID_CPU;
group->type = PERFMON_GT_UNKNOWN;
group->flags &= ~PERFMON_GF_ATTACHED;
assert(!group->flags);
goto out;
out:
spinlock_unlock_intr_restore(&group->lock, flags);
switch (type) {
case PERFMON_GT_THREAD:
spinlock_unlock_intr_restore(&grouplist->lock, grouplist_flags);
break;
case PERFMON_GT_CPU:
spinlock_unlock(&grouplist->lock);
break;
}
if (prev_thread) {
/* Late unref as it might destroy the thread and lock the runq. */
thread_unref(prev_thread);
}
return ret;
}
static void
perfmon_group_load(struct perfmon_group *group)
{
struct perfmon_cpu_pmu *cpu_pmu;
struct perfmon_event *event;
#ifdef CONFIG_PERFMON_TEST
struct perfmon_pmc *pmc;
#endif
assert(!thread_preempt_enabled());
assert(perfmon_group_enabled(group));
assert(!perfmon_group_loaded(group));
cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);
#ifdef CONFIG_PERFMON_TEST
/* XXX: could be done in the loading loop, but performance does not
* matters in the functional tests using this feature.
*/
list_for_each_entry(&group->events, event, node) {
if (!event->set_value) {
continue;
}
pmc = perfmon_pmc_from_index(event->pmc_index);
pmu_driver.ops.write(pmc->id, event->value);
event->set_value = false;
}
#endif
list_for_each_entry(&group->events, event, node) {
perfmon_cpu_pmu_load(cpu_pmu, event->pmc_index);
event->prev = pmu_driver.ops.read(perfmon_pmu.pmcs[event->pmc_index].id);
event->overflow_id = cpu_pmu->pmcs[event->pmc_index].overflow_id;
}
group->cpu = cpu_id();
group->flags |= PERFMON_GF_LOADED;
}
static void
perfmon_cpu_load_remote(void *arg)
{
struct perfmon_group *group;
group = arg;
assert (group->cpu == cpu_id());
spinlock_lock(&group->lock);
perfmon_group_load(group);
spinlock_unlock(&group->lock);
}
static void
perfmon_group_unload(struct perfmon_group *group)
{
struct perfmon_cpu_pmu *cpu_pmu;
struct perfmon_event *event;
assert(!thread_preempt_enabled());
assert(perfmon_group_enabled(group));
assert(perfmon_group_loaded(group));
cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);
list_for_each_entry(&group->events, event, node) {
perfmon_cpu_pmu_unload(cpu_pmu, event->pmc_index);
perfmon_event_sync(cpu_pmu, event);
}
group->flags &= ~PERFMON_GF_LOADED;
}
static void
perfmon_cpu_unload_remote(void *arg)
{
struct perfmon_group *group;
group = arg;
assert (group->cpu == cpu_id());
assert (perfmon_group_stopping(group));
spinlock_lock(&group->lock);
perfmon_group_unload(group);
group->flags &= ~PERFMON_GF_PENDING_DISABLE;
group->flags &= ~PERFMON_GF_ENABLED;
spinlock_unlock(&group->lock);
}
static void
perfmon_thread_load_remote(void *arg)
{
struct perfmon_group *group;
struct thread *thread;
assert (!cpu_intr_enabled());
group = arg;
thread = thread_self();
if (thread != group->thread) {
return;
}
spinlock_lock(&group->lock);
if (perfmon_group_enabled(group) && !perfmon_group_loaded(group)) {
perfmon_group_load(group);
}
spinlock_unlock(&group->lock);
}
static void
perfmon_thread_unload_remote(void *arg)
{
struct perfmon_group *group;
struct thread *thread;
assert (!cpu_intr_enabled());
group = arg;
thread = thread_self();
if (thread != group->thread) {
return;
}
spinlock_lock(&group->lock);
if (perfmon_group_enabled(group)) {
assert (perfmon_group_stopping(group));
if (perfmon_group_loaded(group)) {
perfmon_group_unload(group);
}
group->flags &= ~PERFMON_GF_PENDING_DISABLE;
group->flags &= ~PERFMON_GF_ENABLED;
}
spinlock_unlock(&group->lock);
}
int
perfmon_group_start(struct perfmon_group *group)
{
unsigned long flags;
unsigned int cpu;
int ret;
ret = 0;
spinlock_lock_intr_save(&group->lock, &flags);
if (!perfmon_group_attached(group) || perfmon_group_loaded(group)) {
ret = EINVAL;
goto end;
}
assert(!perfmon_group_enabled(group));
group->flags |= PERFMON_GF_ENABLED;
if (group->type == PERFMON_GT_CPU) {
spinlock_unlock_intr_restore(&group->lock, flags);
xcall_call(perfmon_cpu_load_remote, group, group->cpu);
return 0;
} else if (group->thread == thread_self()) {
perfmon_group_load(group);
} else if (group->thread->state == THREAD_RUNNING) {
spinlock_unlock_intr_restore(&group->lock, flags);
cpu = thread_cpu(group->thread);
xcall_call(perfmon_thread_load_remote, group, cpu);
return 0;
}
end:
spinlock_unlock_intr_restore(&group->lock, flags);
return ret;
}
static void
perfmon_group_sync_local(struct perfmon_group *group)
{
struct perfmon_event *event;
struct perfmon_cpu_pmu *cpu_pmu;
cpu_pmu = cpu_local_ptr(perfmon_cpu_pmu);
/* The group sync duration *should be* limited as a group may only have a
* limited amount of *different* events.
*/
list_for_each_entry(&group->events, event, node) {
perfmon_event_sync(cpu_pmu, event);
}
}
static void
perfmon_cpu_sync_remote(void *arg)
{
struct perfmon_group *group;
group = arg;
assert (group->type == PERFMON_GT_CPU);
assert (group->cpu == cpu_id());
perfmon_group_sync_local(group);
}
static void
perfmon_thread_sync_remote(void *arg)
{
struct perfmon_group *group;
unsigned long flags;
group = arg;
assert (group->type == PERFMON_GT_THREAD);
if (thread_self() != group->thread) {
return;
}
spinlock_lock_intr_save(&group->lock, &flags);
perfmon_group_sync_local(group);
spinlock_unlock_intr_restore(&group->lock, flags);
}
void
perfmon_group_update(struct perfmon_group *group)
{
unsigned long flags;
unsigned int cpu;
assert(perfmon_group_enabled(group));
spinlock_lock_intr_save(&group->lock, &flags);
assert(perfmon_group_attached(group));
assert(perfmon_group_enabled(group));
if (!perfmon_group_loaded(group)) {
goto end;
}
if (group->type == PERFMON_GT_CPU) {
if (group->cpu == cpu_id())
perfmon_group_sync_local(group);
else {
xcall_call(perfmon_cpu_sync_remote, group, group->cpu);
}
} else {
if (group->thread == thread_self()) {
assert (perfmon_group_loaded(group));
perfmon_group_sync_local(group);
} else if (group->thread->state == THREAD_RUNNING) {
spinlock_unlock_intr_restore(&group->lock, flags);
cpu = thread_cpu(group->thread);
xcall_call(perfmon_thread_sync_remote, group, cpu);
return;
}
}
end:
spinlock_unlock_intr_restore(&group->lock, flags);
}
int
perfmon_group_stop(struct perfmon_group *group)
{
int ret;
unsigned long flags;
unsigned int cpu;
ret = 0;
spinlock_lock_intr_save(&group->lock, &flags);
if (!perfmon_group_attached(group) || !perfmon_group_enabled(group)) {
ret = EINVAL;
goto end;
}
if (!perfmon_group_loaded(group)) {
goto disable;
}
group->flags |= PERFMON_GF_PENDING_DISABLE;
if (group->type == PERFMON_GT_CPU) {
spinlock_unlock_intr_restore(&group->lock, flags);
xcall_call(perfmon_cpu_unload_remote, group, group->cpu);
return 0;
} else if (group->thread == thread_self()) {
perfmon_group_unload(group);
} else {
/* If the thead is not running (but still loaded), the unload is
* (probably) getting called when we release the group lock, but we
* still need a blocking xcall to guarantee the group is disabled when
* the function returns.
*/
spinlock_unlock_intr_restore(&group->lock, flags);
cpu = thread_cpu(group->thread);
xcall_call(perfmon_thread_unload_remote, group, cpu);
return 0;
}
disable:
group->flags &= ~PERFMON_GF_PENDING_DISABLE;
group->flags &= ~PERFMON_GF_ENABLED;
end:
spinlock_unlock_intr_restore(&group->lock, flags);
return ret;
}
int
perfmon_thread_init(struct thread *thread)
{
struct perfmon_grouplist *grouplist;
grouplist = perfmon_grouplist_create();
if (grouplist == NULL) {
return ENOMEM;
}
thread->perfmon_groups = grouplist;
return 0;
}
void
perfmon_thread_destroy(struct thread *thread)
{
perfmon_grouplist_destroy(thread->perfmon_groups);
}
void
perfmon_thread_load(struct thread *thread)
{
struct perfmon_grouplist *grouplist;
struct perfmon_group *group;
assert(!cpu_intr_enabled());
assert(!thread_preempt_enabled());
grouplist = thread->perfmon_groups;
spinlock_lock(&grouplist->lock);
list_for_each_entry(&grouplist->groups, group, node) {
spinlock_lock(&group->lock);
if (perfmon_group_enabled(group) && !perfmon_group_loaded(group)) {
perfmon_group_load(group);
}
spinlock_unlock(&group->lock);
}
spinlock_unlock(&grouplist->lock);
}
void
perfmon_thread_unload(struct thread *thread)
{
struct perfmon_grouplist *grouplist;
struct perfmon_group *group;
assert(!cpu_intr_enabled());
assert(!thread_preempt_enabled());
grouplist = thread->perfmon_groups;
spinlock_lock(&grouplist->lock);
list_for_each_entry(&grouplist->groups, group, node) {
spinlock_lock(&group->lock);
/* TODO: we may want to prevent long looping on the groups.
* One way to do this would be to maintain events mapping in the
* grouplist in order to have a finite operation upon schedueling.
*/
if (perfmon_group_loaded(group)) {
perfmon_group_unload(group);
if (perfmon_group_stopping(group)) {
group->flags &= ~PERFMON_GF_PENDING_DISABLE;
group->flags &= ~PERFMON_GF_ENABLED;
}
}
spinlock_unlock(&group->lock);
}
spinlock_unlock(&grouplist->lock);
}