diff options
-rw-r--r-- | arch/s390/kernel/perf_pai_crypto.c | 101 |
1 files changed, 46 insertions, 55 deletions
diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 0e296a1482bc9..c0b7f5422e5e7 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -24,6 +24,8 @@ static debug_info_t *cfm_dbg; static unsigned int paicrypt_cnt; /* Size of the mapped counter sets */ /* extracted with QPACI instruction */ +#define PAI_SWLIST(x) (&(x)->hw.tp_list) + DEFINE_STATIC_KEY_FALSE(pai_key); struct pai_userdata { @@ -36,8 +38,8 @@ struct paicrypt_map { struct pai_userdata *save; /* Page to store no-zero counters */ unsigned int active_events; /* # of PAI crypto users */ refcount_t refcnt; /* Reference count mapped buffers */ - enum paievt_mode mode; /* Type of event */ struct perf_event *event; /* Perf event for sampling */ + struct list_head syswide_list; /* List system-wide sampling events */ }; struct paicrypt_mapptr { @@ -90,13 +92,10 @@ static void paicrypt_event_destroy_cpu(struct perf_event *event, int cpu) struct paicrypt_map *cpump = mp->mapptr; mutex_lock(&pai_reserve_mutex); - debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d" - " mode %d refcnt %u\n", __func__, - event->attr.config, event->cpu, - cpump->active_events, cpump->mode, + debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d " + "refcnt %u\n", __func__, event->attr.config, + event->cpu, cpump->active_events, refcount_read(&cpump->refcnt)); - if (event->attr.sample_period) - cpump->mode &= ~PAI_MODE_SAMPLING; if (refcount_dec_and_test(&cpump->refcnt)) { debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", __func__, (unsigned long)cpump->page, @@ -172,12 +171,7 @@ static u64 paicrypt_getall(struct perf_event *event) return sum; } -/* Used to avoid races in checking concurrent access of counting and - * sampling for crypto events - * - * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is - * allowed. Several counting events are allowed in parallel. - * +/* Check concurrent access of counting and sampling for crypto events. * This function is called in process context and it is save to block. * When the event initialization functions fails, no other call back will * be invoked. @@ -186,7 +180,6 @@ static u64 paicrypt_getall(struct perf_event *event) */ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) { - struct perf_event_attr *a = &event->attr; struct paicrypt_map *cpump = NULL; struct paicrypt_mapptr *mp; int rc; @@ -207,25 +200,9 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) rc = -ENOMEM; goto free_root; } + INIT_LIST_HEAD(&cpump->syswide_list); } - if (a->sample_period) { /* Sampling requested */ - if (cpump->mode & PAI_MODE_SAMPLING) - rc = -EBUSY; /* ... and sampling active */ - else - cpump->mode |= PAI_MODE_SAMPLING; - } - - /* - * This error case triggers when there is a conflict: - * Either sampling requested and counting already active, or visa - * versa. Therefore the struct paicrypto_map for this CPU is - * needed or the error could not have occurred. Only adjust root - * node refcount. - */ - if (rc) - goto free_root; - /* Allocate memory for counter page and counter extraction. * Only the first counting event has to allocate a page. */ @@ -250,14 +227,14 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) rc = 0; refcount_set(&cpump->refcnt, 1); mp->mapptr = cpump; - debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" - " mode %d refcnt %u page %#lx save %p rc %d\n", - __func__, a->sample_period, cpump->active_events, - cpump->mode, refcount_read(&cpump->refcnt), + debug_sprintf_event(cfm_dbg, 5, "%s users %d refcnt %u page %#lx " + "save %p rc %d\n", __func__, cpump->active_events, + refcount_read(&cpump->refcnt), (unsigned long)cpump->page, cpump->save, rc); goto unlock; free_paicrypt_map: + /* Undo memory allocation */ kfree(cpump); mp->mapptr = NULL; free_root: @@ -315,11 +292,7 @@ static int paicrypt_event_init(struct perf_event *event) if (a->config < PAI_CRYPTO_BASE || a->config > PAI_CRYPTO_BASE + paicrypt_cnt) return -EINVAL; - /* Allow only CPU wide operation for sampling */ - if (a->sample_period && - ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)) - return -ENOENT; - /* Allow only CRYPTO_ALL for sampling. */ + /* Allow only CRYPTO_ALL for sampling */ if (a->sample_period && a->config != PAI_CRYPTO_BASE) return -EINVAL; /* Get a page to store last counter values for sampling */ @@ -382,9 +355,14 @@ static void paicrypt_start(struct perf_event *event, int flags) sum = paicrypt_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ - cpump->event = event; memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); - perf_sched_cb_inc(event->pmu); + /* Enable context switch callback for system-wide sampling */ + if (!(event->attach_state & PERF_ATTACH_TASK)) { + list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); + perf_sched_cb_inc(event->pmu); + } else { + cpump->event = event; + } } } @@ -405,6 +383,7 @@ static int paicrypt_add(struct perf_event *event, int flags) return 0; } +static void paicrypt_have_sample(struct perf_event *, struct paicrypt_map *); static void paicrypt_stop(struct perf_event *event, int flags) { struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); @@ -413,8 +392,13 @@ static void paicrypt_stop(struct perf_event *event, int flags) if (!event->attr.sample_period) { /* Counting */ paicrypt_read(event); } else { /* Sampling */ - perf_sched_cb_dec(event->pmu); - cpump->event = NULL; + if (!(event->attach_state & PERF_ATTACH_TASK)) { + perf_sched_cb_dec(event->pmu); + list_del(PAI_SWLIST(event)); + } else { + paicrypt_have_sample(event, cpump); + cpump->event = NULL; + } } event->hw.state = PERF_HES_STOPPED; } @@ -507,23 +491,30 @@ static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump, } /* Check if there is data to be saved on schedule out of a task. */ -static int paicrypt_have_sample(void) +static void paicrypt_have_sample(struct perf_event *event, + struct paicrypt_map *cpump) { - struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); - struct paicrypt_map *cpump = mp->mapptr; - struct perf_event *event = cpump->event; size_t rawsize; - int rc = 0; if (!event) /* No event active */ - return 0; + return; rawsize = paicrypt_copy(cpump->save, cpump->page, (unsigned long *)PAI_SAVE_AREA(event), - cpump->event->attr.exclude_user, - cpump->event->attr.exclude_kernel); + event->attr.exclude_user, + event->attr.exclude_kernel); if (rawsize) /* No incremented counters */ - rc = paicrypt_push_sample(rawsize, cpump, event); - return rc; + paicrypt_push_sample(rawsize, cpump, event); +} + +/* Check if there is data to be saved on schedule out of a task. */ +static void paicrypt_have_samples(void) +{ + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; + struct perf_event *event; + + list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) + paicrypt_have_sample(event, cpump); } /* Called on schedule-in and schedule-out. No access to event structure, @@ -535,7 +526,7 @@ static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sch * results on schedule_out and if page was dirty, save old values. */ if (!sched_in) - paicrypt_have_sample(); + paicrypt_have_samples(); } /* Attribute definitions for paicrypt interface. As with other CPU |