AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-11-30 07:26:46 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity
da477737c5
linux_dsm_epyc7002/arch/s390/oprofile/hwsampler.c
Hendrik Brueckner 02456169e0 s390/irq: split irq stats for cpu-measurement alert facilities 
CPU-measurement alerts are generated for different CPU-measurement
facilities, for example, the sampling and counter facilities.
Split the irq stats according to available facilities.

Signed-off-by: Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
2012-05-23 17:06:52 +02:00

1241 lines
25 KiB
C
Raw Blame History

/**
* arch/s390/oprofile/hwsampler.c
*
* Copyright IBM Corp. 2010
* Author: Heinz Graalfs <graalfs@de.ibm.com>
*/
#include <linux/kernel_stat.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/workqueue.h>
#include <linux/interrupt.h>
#include <linux/notifier.h>
#include <linux/cpu.h>
#include <linux/semaphore.h>
#include <linux/oom.h>
#include <linux/oprofile.h>
#include <asm/facility.h>
#include <asm/cpu_mf.h>
#include <asm/irq.h>
#include "hwsampler.h"
#include "op_counter.h"
#define MAX_NUM_SDB 511
#define MIN_NUM_SDB 1
#define ALERT_REQ_MASK 0x4000000000000000ul
#define BUFFER_FULL_MASK 0x8000000000000000ul
DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
struct hws_execute_parms {
void *buffer;
signed int rc;
};
DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
static DEFINE_MUTEX(hws_sem);
static DEFINE_MUTEX(hws_sem_oom);
static unsigned char hws_flush_all;
static unsigned int hws_oom;
static struct workqueue_struct *hws_wq;
static unsigned int hws_state;
enum {
HWS_INIT = 1,
HWS_DEALLOCATED,
HWS_STOPPED,
HWS_STARTED,
HWS_STOPPING };
/* set to 1 if called by kernel during memory allocation */
static unsigned char oom_killer_was_active;
/* size of SDBT and SDB as of allocate API */
static unsigned long num_sdbt = 100;
static unsigned long num_sdb = 511;
/* sampling interval (machine cycles) */
static unsigned long interval;
static unsigned long min_sampler_rate;
static unsigned long max_sampler_rate;
static int ssctl(void *buffer)
{
int cc;
/* set in order to detect a program check */
cc = 1;
asm volatile(
"0: .insn s,0xB2870000,0(%1)\n"
"1: ipm %0\n"
" srl %0,28\n"
"2:\n"
EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
: "+d" (cc), "+a" (buffer)
: "m" (*((struct hws_ssctl_request_block *)buffer))
: "cc", "memory");
return cc ? -EINVAL : 0 ;
}
static int qsi(void *buffer)
{
int cc;
cc = 1;
asm volatile(
"0: .insn s,0xB2860000,0(%1)\n"
"1: lhi %0,0\n"
"2:\n"
EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
: "=d" (cc), "+a" (buffer)
: "m" (*((struct hws_qsi_info_block *)buffer))
: "cc", "memory");
return cc ? -EINVAL : 0;
}
static void execute_qsi(void *parms)
{
struct hws_execute_parms *ep = parms;
ep->rc = qsi(ep->buffer);
}
static void execute_ssctl(void *parms)
{
struct hws_execute_parms *ep = parms;
ep->rc = ssctl(ep->buffer);
}
static int smp_ctl_ssctl_stop(int cpu)
{
int rc;
struct hws_execute_parms ep;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
cb->ssctl.es = 0;
cb->ssctl.cs = 0;
ep.buffer = &cb->ssctl;
smp_call_function_single(cpu, execute_ssctl, &ep, 1);
rc = ep.rc;
if (rc) {
printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
dump_stack();
}
ep.buffer = &cb->qsi;
smp_call_function_single(cpu, execute_qsi, &ep, 1);
if (cb->qsi.es || cb->qsi.cs) {
printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
dump_stack();
}
return rc;
}
static int smp_ctl_ssctl_deactivate(int cpu)
{
int rc;
struct hws_execute_parms ep;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
cb->ssctl.es = 1;
cb->ssctl.cs = 0;
ep.buffer = &cb->ssctl;
smp_call_function_single(cpu, execute_ssctl, &ep, 1);
rc = ep.rc;
if (rc)
printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
ep.buffer = &cb->qsi;
smp_call_function_single(cpu, execute_qsi, &ep, 1);
if (cb->qsi.cs)
printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
return rc;
}
static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
{
int rc;
struct hws_execute_parms ep;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
cb->ssctl.h = 1;
cb->ssctl.tear = cb->first_sdbt;
cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
cb->ssctl.interval = interval;
cb->ssctl.es = 1;
cb->ssctl.cs = 1;
ep.buffer = &cb->ssctl;
smp_call_function_single(cpu, execute_ssctl, &ep, 1);
rc = ep.rc;
if (rc)
printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
ep.buffer = &cb->qsi;
smp_call_function_single(cpu, execute_qsi, &ep, 1);
if (ep.rc)
printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
return rc;
}
static int smp_ctl_qsi(int cpu)
{
struct hws_execute_parms ep;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
ep.buffer = &cb->qsi;
smp_call_function_single(cpu, execute_qsi, &ep, 1);
return ep.rc;
}
static inline unsigned long *trailer_entry_ptr(unsigned long v)
{
void *ret;
ret = (void *)v;
ret += PAGE_SIZE;
ret -= sizeof(struct hws_trailer_entry);
return (unsigned long *) ret;
}
static void hws_ext_handler(struct ext_code ext_code,
unsigned int param32, unsigned long param64)
{
struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
if (!(param32 & CPU_MF_INT_SF_MASK))
return;
kstat_cpu(smp_processor_id()).irqs[EXTINT_CMS]++;
atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
if (hws_wq)
queue_work(hws_wq, &cb->worker);
}
static void worker(struct work_struct *work);
static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
unsigned long *dear);
static void init_all_cpu_buffers(void)
{
int cpu;
struct hws_cpu_buffer *cb;
for_each_online_cpu(cpu) {
cb = &per_cpu(sampler_cpu_buffer, cpu);
memset(cb, 0, sizeof(struct hws_cpu_buffer));
}
}
static int is_link_entry(unsigned long *s)
{
return *s & 0x1ul ? 1 : 0;
}
static unsigned long *get_next_sdbt(unsigned long *s)
{
return (unsigned long *) (*s & ~0x1ul);
}
static int prepare_cpu_buffers(void)
{
int cpu;
int rc;
struct hws_cpu_buffer *cb;
rc = 0;
for_each_online_cpu(cpu) {
cb = &per_cpu(sampler_cpu_buffer, cpu);
atomic_set(&cb->ext_params, 0);
cb->worker_entry = 0;
cb->sample_overflow = 0;
cb->req_alert = 0;
cb->incorrect_sdbt_entry = 0;
cb->invalid_entry_address = 0;
cb->loss_of_sample_data = 0;
cb->sample_auth_change_alert = 0;
cb->finish = 0;
cb->oom = 0;
cb->stop_mode = 0;
}
return rc;
}
/*
* allocate_sdbt() - allocate sampler memory
* @cpu: the cpu for which sampler memory is allocated
*
* A 4K page is allocated for each requested SDBT.
* A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
* Set ALERT_REQ mask in each SDBs trailer.
* Returns zero if successful, <0 otherwise.
*/
static int allocate_sdbt(int cpu)
{
int j, k, rc;
unsigned long *sdbt;
unsigned long sdb;
unsigned long *tail;
unsigned long *trailer;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
if (cb->first_sdbt)
return -EINVAL;
sdbt = NULL;
tail = sdbt;
for (j = 0; j < num_sdbt; j++) {
sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
mutex_lock(&hws_sem_oom);
/* OOM killer might have been activated */
barrier();
if (oom_killer_was_active || !sdbt) {
if (sdbt)
free_page((unsigned long)sdbt);
goto allocate_sdbt_error;
}
if (cb->first_sdbt == 0)
cb->first_sdbt = (unsigned long)sdbt;
/* link current page to tail of chain */
if (tail)
*tail = (unsigned long)(void *)sdbt + 1;
mutex_unlock(&hws_sem_oom);
for (k = 0; k < num_sdb; k++) {
/* get and set SDB page */
sdb = get_zeroed_page(GFP_KERNEL);
mutex_lock(&hws_sem_oom);
/* OOM killer might have been activated */
barrier();
if (oom_killer_was_active || !sdb) {
if (sdb)
free_page(sdb);
goto allocate_sdbt_error;
}
*sdbt = sdb;
trailer = trailer_entry_ptr(*sdbt);
*trailer = ALERT_REQ_MASK;
sdbt++;
mutex_unlock(&hws_sem_oom);
}
tail = sdbt;
}
mutex_lock(&hws_sem_oom);
if (oom_killer_was_active)
goto allocate_sdbt_error;
rc = 0;
if (tail)
*tail = (unsigned long)
((void *)cb->first_sdbt) + 1;
allocate_sdbt_exit:
mutex_unlock(&hws_sem_oom);
return rc;
allocate_sdbt_error:
rc = -ENOMEM;
goto allocate_sdbt_exit;
}
/*
* deallocate_sdbt() - deallocate all sampler memory
*
* For each online CPU all SDBT trees are deallocated.
* Returns the number of freed pages.
*/
static int deallocate_sdbt(void)
{
int cpu;
int counter;
counter = 0;
for_each_online_cpu(cpu) {
unsigned long start;
unsigned long sdbt;
unsigned long *curr;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
if (!cb->first_sdbt)
continue;
sdbt = cb->first_sdbt;
curr = (unsigned long *) sdbt;
start = sdbt;
/* we'll free the SDBT after all SDBs are processed... */
while (1) {
if (!*curr || !sdbt)
break;
/* watch for link entry reset if found */
if (is_link_entry(curr)) {
curr = get_next_sdbt(curr);
if (sdbt)
free_page(sdbt);
/* we are done if we reach the start */
if ((unsigned long) curr == start)
break;
else
sdbt = (unsigned long) curr;
} else {
/* process SDB pointer */
if (*curr) {
free_page(*curr);
curr++;
}
}
counter++;
}
cb->first_sdbt = 0;
}
return counter;
}
static int start_sampling(int cpu)
{
int rc;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
rc = smp_ctl_ssctl_enable_activate(cpu, interval);
if (rc) {
printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
goto start_exit;
}
rc = -EINVAL;
if (!cb->qsi.es) {
printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
goto start_exit;
}
if (!cb->qsi.cs) {
printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
goto start_exit;
}
printk(KERN_INFO
"hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
cpu, interval);
rc = 0;
start_exit:
return rc;
}
static int stop_sampling(int cpu)
{
unsigned long v;
int rc;
struct hws_cpu_buffer *cb;
rc = smp_ctl_qsi(cpu);
WARN_ON(rc);
cb = &per_cpu(sampler_cpu_buffer, cpu);
if (!rc && !cb->qsi.es)
printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
rc = smp_ctl_ssctl_stop(cpu);
if (rc) {
printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
cpu, rc);
goto stop_exit;
}
printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
stop_exit:
v = cb->req_alert;
if (v)
printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
" count=%lu.\n", cpu, v);
v = cb->loss_of_sample_data;
if (v)
printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
" count=%lu.\n", cpu, v);
v = cb->invalid_entry_address;
if (v)
printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
" count=%lu.\n", cpu, v);
v = cb->incorrect_sdbt_entry;
if (v)
printk(KERN_ERR
"hwsampler: CPU %d CPUMF Incorrect SDBT address,"
" count=%lu.\n", cpu, v);
v = cb->sample_auth_change_alert;
if (v)
printk(KERN_ERR
"hwsampler: CPU %d CPUMF Sample authorization change,"
" count=%lu.\n", cpu, v);
return rc;
}
static int check_hardware_prerequisites(void)
{
if (!test_facility(68))
return -EOPNOTSUPP;
return 0;
}
/*
* hws_oom_callback() - the OOM callback function
*
* In case the callback is invoked during memory allocation for the
* hw sampler, all obtained memory is deallocated and a flag is set
* so main sampler memory allocation can exit with a failure code.
* In case the callback is invoked during sampling the hw sampler
* is deactivated for all CPUs.
*/
static int hws_oom_callback(struct notifier_block *nfb,
unsigned long dummy, void *parm)
{
unsigned long *freed;
int cpu;
struct hws_cpu_buffer *cb;
freed = parm;
mutex_lock(&hws_sem_oom);
if (hws_state == HWS_DEALLOCATED) {
/* during memory allocation */
if (oom_killer_was_active == 0) {
oom_killer_was_active = 1;
*freed += deallocate_sdbt();
}
} else {
int i;
cpu = get_cpu();
cb = &per_cpu(sampler_cpu_buffer, cpu);
if (!cb->oom) {
for_each_online_cpu(i) {
smp_ctl_ssctl_deactivate(i);
cb->oom = 1;
}
cb->finish = 1;
printk(KERN_INFO
"hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
cpu);
}
}
mutex_unlock(&hws_sem_oom);
return NOTIFY_OK;
}
static struct notifier_block hws_oom_notifier = {
.notifier_call = hws_oom_callback
};
static int hws_cpu_callback(struct notifier_block *nfb,
unsigned long action, void *hcpu)
{
/* We do not have sampler space available for all possible CPUs.
All CPUs should be online when hw sampling is activated. */
return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
}
static struct notifier_block hws_cpu_notifier = {
.notifier_call = hws_cpu_callback
};
/**
* hwsampler_deactivate() - set hardware sampling temporarily inactive
* @cpu: specifies the CPU to be set inactive.
*
* Returns 0 on success, !0 on failure.
*/
int hwsampler_deactivate(unsigned int cpu)
{
/*
* Deactivate hw sampling temporarily and flush the buffer
* by pushing all the pending samples to oprofile buffer.
*
* This function can be called under one of the following conditions:
* Memory unmap, task is exiting.
*/
int rc;
struct hws_cpu_buffer *cb;
rc = 0;
mutex_lock(&hws_sem);
cb = &per_cpu(sampler_cpu_buffer, cpu);
if (hws_state == HWS_STARTED) {
rc = smp_ctl_qsi(cpu);
WARN_ON(rc);
if (cb->qsi.cs) {
rc = smp_ctl_ssctl_deactivate(cpu);
if (rc) {
printk(KERN_INFO
"hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
cb->finish = 1;
hws_state = HWS_STOPPING;
} else {
hws_flush_all = 1;
/* Add work to queue to read pending samples.*/
queue_work_on(cpu, hws_wq, &cb->worker);
}
}
}
mutex_unlock(&hws_sem);
if (hws_wq)
flush_workqueue(hws_wq);
return rc;
}
/**
* hwsampler_activate() - activate/resume hardware sampling which was deactivated
* @cpu: specifies the CPU to be set active.
*
* Returns 0 on success, !0 on failure.
*/
int hwsampler_activate(unsigned int cpu)
{
/*
* Re-activate hw sampling. This should be called in pair with
* hwsampler_deactivate().
*/
int rc;
struct hws_cpu_buffer *cb;
rc = 0;
mutex_lock(&hws_sem);
cb = &per_cpu(sampler_cpu_buffer, cpu);
if (hws_state == HWS_STARTED) {
rc = smp_ctl_qsi(cpu);
WARN_ON(rc);
if (!cb->qsi.cs) {
hws_flush_all = 0;
rc = smp_ctl_ssctl_enable_activate(cpu, interval);
if (rc) {
printk(KERN_ERR
"CPU %d, CPUMF activate sampling failed.\n",
cpu);
}
}
}
mutex_unlock(&hws_sem);
return rc;
}
static int check_qsi_on_setup(void)
{
int rc;
unsigned int cpu;
struct hws_cpu_buffer *cb;
for_each_online_cpu(cpu) {
cb = &per_cpu(sampler_cpu_buffer, cpu);
rc = smp_ctl_qsi(cpu);
WARN_ON(rc);
if (rc)
return -EOPNOTSUPP;
if (!cb->qsi.as) {
printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
return -EINVAL;
}
if (cb->qsi.es) {
printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
rc = smp_ctl_ssctl_stop(cpu);
if (rc)
return -EINVAL;
printk(KERN_INFO
"CPU %d, CPUMF Sampling stopped now.\n", cpu);
}
}
return 0;
}
static int check_qsi_on_start(void)
{
unsigned int cpu;
int rc;
struct hws_cpu_buffer *cb;
for_each_online_cpu(cpu) {
cb = &per_cpu(sampler_cpu_buffer, cpu);
rc = smp_ctl_qsi(cpu);
WARN_ON(rc);
if (!cb->qsi.as)
return -EINVAL;
if (cb->qsi.es)
return -EINVAL;
if (cb->qsi.cs)
return -EINVAL;
}
return 0;
}
static void worker_on_start(unsigned int cpu)
{
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
cb->worker_entry = cb->first_sdbt;
}
static int worker_check_error(unsigned int cpu, int ext_params)
{
int rc;
unsigned long *sdbt;
struct hws_cpu_buffer *cb;
rc = 0;
cb = &per_cpu(sampler_cpu_buffer, cpu);
sdbt = (unsigned long *) cb->worker_entry;
if (!sdbt || !*sdbt)
return -EINVAL;
if (ext_params & CPU_MF_INT_SF_PRA)
cb->req_alert++;
if (ext_params & CPU_MF_INT_SF_LSDA)
cb->loss_of_sample_data++;
if (ext_params & CPU_MF_INT_SF_IAE) {
cb->invalid_entry_address++;
rc = -EINVAL;
}
if (ext_params & CPU_MF_INT_SF_ISE) {
cb->incorrect_sdbt_entry++;
rc = -EINVAL;
}
if (ext_params & CPU_MF_INT_SF_SACA) {
cb->sample_auth_change_alert++;
rc = -EINVAL;
}
return rc;
}
static void worker_on_finish(unsigned int cpu)
{
int rc, i;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
if (cb->finish) {
rc = smp_ctl_qsi(cpu);
WARN_ON(rc);
if (cb->qsi.es) {
printk(KERN_INFO
"hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
cpu);
rc = smp_ctl_ssctl_stop(cpu);
if (rc)
printk(KERN_INFO
"hwsampler: CPU %d, CPUMF Deactivation failed.\n",
cpu);
for_each_online_cpu(i) {
if (i == cpu)
continue;
if (!cb->finish) {
cb->finish = 1;
queue_work_on(i, hws_wq,
&cb->worker);
}
}
}
}
}
static void worker_on_interrupt(unsigned int cpu)
{
unsigned long *sdbt;
unsigned char done;
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
sdbt = (unsigned long *) cb->worker_entry;
done = 0;
/* do not proceed if stop was entered,
* forget the buffers not yet processed */
while (!done && !cb->stop_mode) {
unsigned long *trailer;
struct hws_trailer_entry *te;
unsigned long *dear = 0;
trailer = trailer_entry_ptr(*sdbt);
/* leave loop if no more work to do */
if (!(*trailer & BUFFER_FULL_MASK)) {
done = 1;
if (!hws_flush_all)
continue;
}
te = (struct hws_trailer_entry *)trailer;
cb->sample_overflow += te->overflow;
add_samples_to_oprofile(cpu, sdbt, dear);
/* reset trailer */
xchg((unsigned char *) te, 0x40);
/* advance to next sdb slot in current sdbt */
sdbt++;
/* in case link bit is set use address w/o link bit */
if (is_link_entry(sdbt))
sdbt = get_next_sdbt(sdbt);
cb->worker_entry = (unsigned long)sdbt;
}
}
static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
unsigned long *dear)
{
struct hws_data_entry *sample_data_ptr;
unsigned long *trailer;
trailer = trailer_entry_ptr(*sdbt);
if (dear) {
if (dear > trailer)
return;
trailer = dear;
}
sample_data_ptr = (struct hws_data_entry *)(*sdbt);
while ((unsigned long *)sample_data_ptr < trailer) {
struct pt_regs *regs = NULL;
struct task_struct *tsk = NULL;
/*
* Check sampling mode, 1 indicates basic (=customer) sampling
* mode.
*/
if (sample_data_ptr->def != 1) {
/* sample slot is not yet written */
break;
} else {
/* make sure we don't use it twice,
* the next time the sampler will set it again */
sample_data_ptr->def = 0;
}
/* Get pt_regs. */
if (sample_data_ptr->P == 1) {
/* userspace sample */
unsigned int pid = sample_data_ptr->prim_asn;
if (!counter_config.user)
goto skip_sample;
rcu_read_lock();
tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
if (tsk)
regs = task_pt_regs(tsk);
rcu_read_unlock();
} else {
/* kernelspace sample */
if (!counter_config.kernel)
goto skip_sample;
regs = task_pt_regs(current);
}
mutex_lock(&hws_sem);
oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
!sample_data_ptr->P, tsk);
mutex_unlock(&hws_sem);
skip_sample:
sample_data_ptr++;
}
}
static void worker(struct work_struct *work)
{
unsigned int cpu;
int ext_params;
struct hws_cpu_buffer *cb;
cb = container_of(work, struct hws_cpu_buffer, worker);
cpu = smp_processor_id();
ext_params = atomic_xchg(&cb->ext_params, 0);
if (!cb->worker_entry)
worker_on_start(cpu);
if (worker_check_error(cpu, ext_params))
return;
if (!cb->finish)
worker_on_interrupt(cpu);
if (cb->finish)
worker_on_finish(cpu);
}
/**
* hwsampler_allocate() - allocate memory for the hardware sampler
* @sdbt: number of SDBTs per online CPU (must be > 0)
* @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
*
* Returns 0 on success, !0 on failure.
*/
int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
{
int cpu, rc;
mutex_lock(&hws_sem);
rc = -EINVAL;
if (hws_state != HWS_DEALLOCATED)
goto allocate_exit;
if (sdbt < 1)
goto allocate_exit;
if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
goto allocate_exit;
num_sdbt = sdbt;
num_sdb = sdb;
oom_killer_was_active = 0;
register_oom_notifier(&hws_oom_notifier);
for_each_online_cpu(cpu) {
if (allocate_sdbt(cpu)) {
unregister_oom_notifier(&hws_oom_notifier);
goto allocate_error;
}
}
unregister_oom_notifier(&hws_oom_notifier);
if (oom_killer_was_active)
goto allocate_error;
hws_state = HWS_STOPPED;
rc = 0;
allocate_exit:
mutex_unlock(&hws_sem);
return rc;
allocate_error:
rc = -ENOMEM;
printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
goto allocate_exit;
}
/**
* hwsampler_deallocate() - deallocate hardware sampler memory
*
* Returns 0 on success, !0 on failure.
*/
int hwsampler_deallocate(void)
{
int rc;
mutex_lock(&hws_sem);
rc = -EINVAL;
if (hws_state != HWS_STOPPED)
goto deallocate_exit;
measurement_alert_subclass_unregister();
deallocate_sdbt();
hws_state = HWS_DEALLOCATED;
rc = 0;
deallocate_exit:
mutex_unlock(&hws_sem);
return rc;
}
unsigned long hwsampler_query_min_interval(void)
{
return min_sampler_rate;
}
unsigned long hwsampler_query_max_interval(void)
{
return max_sampler_rate;
}
unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
{
struct hws_cpu_buffer *cb;
cb = &per_cpu(sampler_cpu_buffer, cpu);
return cb->sample_overflow;
}
int hwsampler_setup(void)
{
int rc;
int cpu;
struct hws_cpu_buffer *cb;
mutex_lock(&hws_sem);
rc = -EINVAL;
if (hws_state)
goto setup_exit;
hws_state = HWS_INIT;
init_all_cpu_buffers();
rc = check_hardware_prerequisites();
if (rc)
goto setup_exit;
rc = check_qsi_on_setup();
if (rc)
goto setup_exit;
rc = -EINVAL;
hws_wq = create_workqueue("hwsampler");
if (!hws_wq)
goto setup_exit;
register_cpu_notifier(&hws_cpu_notifier);
for_each_online_cpu(cpu) {
cb = &per_cpu(sampler_cpu_buffer, cpu);
INIT_WORK(&cb->worker, worker);
rc = smp_ctl_qsi(cpu);
WARN_ON(rc);
if (min_sampler_rate != cb->qsi.min_sampl_rate) {
if (min_sampler_rate) {
printk(KERN_WARNING
"hwsampler: different min sampler rate values.\n");
if (min_sampler_rate < cb->qsi.min_sampl_rate)
min_sampler_rate =
cb->qsi.min_sampl_rate;
} else
min_sampler_rate = cb->qsi.min_sampl_rate;
}
if (max_sampler_rate != cb->qsi.max_sampl_rate) {
if (max_sampler_rate) {
printk(KERN_WARNING
"hwsampler: different max sampler rate values.\n");
if (max_sampler_rate > cb->qsi.max_sampl_rate)
max_sampler_rate =
cb->qsi.max_sampl_rate;
} else
max_sampler_rate = cb->qsi.max_sampl_rate;
}
}
register_external_interrupt(0x1407, hws_ext_handler);
hws_state = HWS_DEALLOCATED;
rc = 0;
setup_exit:
mutex_unlock(&hws_sem);
return rc;
}
int hwsampler_shutdown(void)
{
int rc;
mutex_lock(&hws_sem);
rc = -EINVAL;
if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
mutex_unlock(&hws_sem);
if (hws_wq)
flush_workqueue(hws_wq);
mutex_lock(&hws_sem);
if (hws_state == HWS_STOPPED) {
measurement_alert_subclass_unregister();
deallocate_sdbt();
}
if (hws_wq) {
destroy_workqueue(hws_wq);
hws_wq = NULL;
}
unregister_external_interrupt(0x1407, hws_ext_handler);
hws_state = HWS_INIT;
rc = 0;
}
mutex_unlock(&hws_sem);
unregister_cpu_notifier(&hws_cpu_notifier);
return rc;
}
/**
* hwsampler_start_all() - start hardware sampling on all online CPUs
* @rate: specifies the used interval when samples are taken
*
* Returns 0 on success, !0 on failure.
*/
int hwsampler_start_all(unsigned long rate)
{
int rc, cpu;
mutex_lock(&hws_sem);
hws_oom = 0;
rc = -EINVAL;
if (hws_state != HWS_STOPPED)
goto start_all_exit;
interval = rate;
/* fail if rate is not valid */
if (interval < min_sampler_rate || interval > max_sampler_rate)
goto start_all_exit;
rc = check_qsi_on_start();
if (rc)
goto start_all_exit;
rc = prepare_cpu_buffers();
if (rc)
goto start_all_exit;
for_each_online_cpu(cpu) {
rc = start_sampling(cpu);
if (rc)
break;
}
if (rc) {
for_each_online_cpu(cpu) {
stop_sampling(cpu);
}
goto start_all_exit;
}
hws_state = HWS_STARTED;
rc = 0;
start_all_exit:
mutex_unlock(&hws_sem);
if (rc)
return rc;
register_oom_notifier(&hws_oom_notifier);
hws_oom = 1;
hws_flush_all = 0;
/* now let them in, 1407 CPUMF external interrupts */
measurement_alert_subclass_register();
return 0;
}
/**
* hwsampler_stop_all() - stop hardware sampling on all online CPUs
*
* Returns 0 on success, !0 on failure.
*/
int hwsampler_stop_all(void)
{
int tmp_rc, rc, cpu;
struct hws_cpu_buffer *cb;
mutex_lock(&hws_sem);
rc = 0;
if (hws_state == HWS_INIT) {
mutex_unlock(&hws_sem);
return rc;
}
hws_state = HWS_STOPPING;
mutex_unlock(&hws_sem);
for_each_online_cpu(cpu) {
cb = &per_cpu(sampler_cpu_buffer, cpu);
cb->stop_mode = 1;
tmp_rc = stop_sampling(cpu);
if (tmp_rc)
rc = tmp_rc;
}
if (hws_wq)
flush_workqueue(hws_wq);
mutex_lock(&hws_sem);
if (hws_oom) {
unregister_oom_notifier(&hws_oom_notifier);
hws_oom = 0;
}
hws_state = HWS_STOPPED;
mutex_unlock(&hws_sem);
return rc;
}
Reference in New Issue View Git Blame Copy Permalink