Commit c5617b20 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'perf-core-for-linus' of...

Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (61 commits)
  tracing: Add __used annotation to event variable
  perf, trace: Fix !x86 build bug
  perf report: Support multiple events on the TUI
  perf annotate: Fix up usage of the build id cache
  x86/mmiotrace: Remove redundant instruction prefix checks
  perf annotate: Add TUI interface
  perf tui: Remove annotate from popup menu after failure
  perf report: Don't start the TUI if -D is used
  perf: Fix getline undeclared
  perf: Optimize perf_tp_event_match()
  perf: Remove more code from the fastpath
  perf: Optimize the !vmalloc backed buffer
  perf: Optimize perf_output_copy()
  perf: Fix wakeup storm for RO mmap()s
  perf-record: Share per-cpu buffers
  perf-record: Remove -M
  perf: Ensure that IOC_OUTPUT isn't used to create multi-writer buffers
  perf, trace: Optimize tracepoints by using per-tracepoint-per-cpu hlist to track events
  perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction
  perf tui: Allow disabling the TUI on a per command basis in ~/.perfconfig
  ...
parents cad719d8 49c17746
...@@ -92,6 +92,8 @@ struct cpu_hw_events { ...@@ -92,6 +92,8 @@ struct cpu_hw_events {
/* Enabled/disable state. */ /* Enabled/disable state. */
int enabled; int enabled;
unsigned int group_flag;
}; };
DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, }; DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
...@@ -981,53 +983,6 @@ static int collect_events(struct perf_event *group, int max_count, ...@@ -981,53 +983,6 @@ static int collect_events(struct perf_event *group, int max_count,
return n; return n;
} }
static void event_sched_in(struct perf_event *event)
{
event->state = PERF_EVENT_STATE_ACTIVE;
event->oncpu = smp_processor_id();
event->tstamp_running += event->ctx->time - event->tstamp_stopped;
if (is_software_event(event))
event->pmu->enable(event);
}
int hw_perf_group_sched_in(struct perf_event *group_leader,
struct perf_cpu_context *cpuctx,
struct perf_event_context *ctx)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct perf_event *sub;
int n0, n;
if (!sparc_pmu)
return 0;
n0 = cpuc->n_events;
n = collect_events(group_leader, perf_max_events - n0,
&cpuc->event[n0], &cpuc->events[n0],
&cpuc->current_idx[n0]);
if (n < 0)
return -EAGAIN;
if (check_excludes(cpuc->event, n0, n))
return -EINVAL;
if (sparc_check_constraints(cpuc->event, cpuc->events, n + n0))
return -EAGAIN;
cpuc->n_events = n0 + n;
cpuc->n_added += n;
cpuctx->active_oncpu += n;
n = 1;
event_sched_in(group_leader);
list_for_each_entry(sub, &group_leader->sibling_list, group_entry) {
if (sub->state != PERF_EVENT_STATE_OFF) {
event_sched_in(sub);
n++;
}
}
ctx->nr_active += n;
return 1;
}
static int sparc_pmu_enable(struct perf_event *event) static int sparc_pmu_enable(struct perf_event *event)
{ {
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
...@@ -1045,11 +1000,20 @@ static int sparc_pmu_enable(struct perf_event *event) ...@@ -1045,11 +1000,20 @@ static int sparc_pmu_enable(struct perf_event *event)
cpuc->events[n0] = event->hw.event_base; cpuc->events[n0] = event->hw.event_base;
cpuc->current_idx[n0] = PIC_NO_INDEX; cpuc->current_idx[n0] = PIC_NO_INDEX;
/*
* If group events scheduling transaction was started,
* skip the schedulability test here, it will be peformed
* at commit time(->commit_txn) as a whole
*/
if (cpuc->group_flag & PERF_EVENT_TXN_STARTED)
goto nocheck;
if (check_excludes(cpuc->event, n0, 1)) if (check_excludes(cpuc->event, n0, 1))
goto out; goto out;
if (sparc_check_constraints(cpuc->event, cpuc->events, n0 + 1)) if (sparc_check_constraints(cpuc->event, cpuc->events, n0 + 1))
goto out; goto out;
nocheck:
cpuc->n_events++; cpuc->n_events++;
cpuc->n_added++; cpuc->n_added++;
...@@ -1129,11 +1093,61 @@ static int __hw_perf_event_init(struct perf_event *event) ...@@ -1129,11 +1093,61 @@ static int __hw_perf_event_init(struct perf_event *event)
return 0; return 0;
} }
/*
* Start group events scheduling transaction
* Set the flag to make pmu::enable() not perform the
* schedulability test, it will be performed at commit time
*/
static void sparc_pmu_start_txn(const struct pmu *pmu)
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
cpuhw->group_flag |= PERF_EVENT_TXN_STARTED;
}
/*
* Stop group events scheduling transaction
* Clear the flag and pmu::enable() will perform the
* schedulability test.
*/
static void sparc_pmu_cancel_txn(const struct pmu *pmu)
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
cpuhw->group_flag &= ~PERF_EVENT_TXN_STARTED;
}
/*
* Commit group events scheduling transaction
* Perform the group schedulability test as a whole
* Return 0 if success
*/
static int sparc_pmu_commit_txn(const struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
int n;
if (!sparc_pmu)
return -EINVAL;
cpuc = &__get_cpu_var(cpu_hw_events);
n = cpuc->n_events;
if (check_excludes(cpuc->event, 0, n))
return -EINVAL;
if (sparc_check_constraints(cpuc->event, cpuc->events, n))
return -EAGAIN;
return 0;
}
static const struct pmu pmu = { static const struct pmu pmu = {
.enable = sparc_pmu_enable, .enable = sparc_pmu_enable,
.disable = sparc_pmu_disable, .disable = sparc_pmu_disable,
.read = sparc_pmu_read, .read = sparc_pmu_read,
.unthrottle = sparc_pmu_unthrottle, .unthrottle = sparc_pmu_unthrottle,
.start_txn = sparc_pmu_start_txn,
.cancel_txn = sparc_pmu_cancel_txn,
.commit_txn = sparc_pmu_commit_txn,
}; };
const struct pmu *hw_perf_event_init(struct perf_event *event) const struct pmu *hw_perf_event_init(struct perf_event *event)
......
...@@ -89,7 +89,8 @@ ...@@ -89,7 +89,8 @@
P4_CCCR_ENABLE) P4_CCCR_ENABLE)
/* HT mask */ /* HT mask */
#define P4_CCCR_MASK_HT (P4_CCCR_MASK | P4_CCCR_THREAD_ANY) #define P4_CCCR_MASK_HT \
(P4_CCCR_MASK | P4_CCCR_OVF_PMI_T1 | P4_CCCR_THREAD_ANY)
#define P4_GEN_ESCR_EMASK(class, name, bit) \ #define P4_GEN_ESCR_EMASK(class, name, bit) \
class##__##name = ((1 << bit) << P4_ESCR_EVENTMASK_SHIFT) class##__##name = ((1 << bit) << P4_ESCR_EVENTMASK_SHIFT)
......
...@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski ...@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski
*/ */
regs->bp = rewind_frame_pointer(skip + 1); regs->bp = rewind_frame_pointer(skip + 1);
regs->cs = __KERNEL_CS; regs->cs = __KERNEL_CS;
local_save_flags(regs->flags); /*
* We abuse bit 3 to pass exact information, see perf_misc_flags
* and the comment with PERF_EFLAGS_EXACT.
*/
regs->flags = 0;
} }
unsigned long perf_instruction_pointer(struct pt_regs *regs) unsigned long perf_instruction_pointer(struct pt_regs *regs)
......
...@@ -465,15 +465,21 @@ out: ...@@ -465,15 +465,21 @@ out:
return rc; return rc;
} }
static inline void p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc) static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc)
{ {
unsigned long dummy; int overflow = 0;
u32 low, high;
rdmsrl(hwc->config_base + hwc->idx, dummy); rdmsr(hwc->config_base + hwc->idx, low, high);
if (dummy & P4_CCCR_OVF) {
/* we need to check high bit for unflagged overflows */
if ((low & P4_CCCR_OVF) || !(high & (1 << 31))) {
overflow = 1;
(void)checking_wrmsrl(hwc->config_base + hwc->idx, (void)checking_wrmsrl(hwc->config_base + hwc->idx,
((u64)dummy) & ~P4_CCCR_OVF); ((u64)low) & ~P4_CCCR_OVF);
} }
return overflow;
} }
static inline void p4_pmu_disable_event(struct perf_event *event) static inline void p4_pmu_disable_event(struct perf_event *event)
...@@ -584,21 +590,15 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) ...@@ -584,21 +590,15 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
WARN_ON_ONCE(hwc->idx != idx); WARN_ON_ONCE(hwc->idx != idx);
/* /* it might be unflagged overflow */
* FIXME: Redundant call, actually not needed handled = p4_pmu_clear_cccr_ovf(hwc);
* but just to check if we're screwed
*/
p4_pmu_clear_cccr_ovf(hwc);
val = x86_perf_event_update(event); val = x86_perf_event_update(event);
if (val & (1ULL << (x86_pmu.cntval_bits - 1))) if (!handled && (val & (1ULL << (x86_pmu.cntval_bits - 1))))
continue; continue;
/* /* event overflow for sure */
* event overflow data.period = event->hw.last_period;
*/
handled = 1;
data.period = event->hw.last_period;
if (!x86_perf_event_set_period(event)) if (!x86_perf_event_set_period(event))
continue; continue;
...@@ -670,7 +670,7 @@ static void p4_pmu_swap_config_ts(struct hw_perf_event *hwc, int cpu) ...@@ -670,7 +670,7 @@ static void p4_pmu_swap_config_ts(struct hw_perf_event *hwc, int cpu)
/* /*
* ESCR address hashing is tricky, ESCRs are not sequential * ESCR address hashing is tricky, ESCRs are not sequential
* in memory but all starts from MSR_P4_BSU_ESCR0 (0x03e0) and * in memory but all starts from MSR_P4_BSU_ESCR0 (0x03a0) and
* the metric between any ESCRs is laid in range [0xa0,0xe1] * the metric between any ESCRs is laid in range [0xa0,0xe1]
* *
* so we make ~70% filled hashtable * so we make ~70% filled hashtable
...@@ -735,8 +735,9 @@ static int p4_get_escr_idx(unsigned int addr) ...@@ -735,8 +735,9 @@ static int p4_get_escr_idx(unsigned int addr)
{ {
unsigned int idx = P4_ESCR_MSR_IDX(addr); unsigned int idx = P4_ESCR_MSR_IDX(addr);
if (unlikely(idx >= P4_ESCR_MSR_TABLE_SIZE || if (unlikely(idx >= P4_ESCR_MSR_TABLE_SIZE ||
!p4_escr_table[idx])) { !p4_escr_table[idx] ||
p4_escr_table[idx] != addr)) {
WARN_ONCE(1, "P4 PMU: Wrong address passed: %x\n", addr); WARN_ONCE(1, "P4 PMU: Wrong address passed: %x\n", addr);
return -1; return -1;
} }
...@@ -762,7 +763,7 @@ static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign ...@@ -762,7 +763,7 @@ static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign
{ {
unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
unsigned long escr_mask[BITS_TO_LONGS(P4_ESCR_MSR_TABLE_SIZE)]; unsigned long escr_mask[BITS_TO_LONGS(P4_ESCR_MSR_TABLE_SIZE)];
int cpu = raw_smp_processor_id(); int cpu = smp_processor_id();
struct hw_perf_event *hwc; struct hw_perf_event *hwc;
struct p4_event_bind *bind; struct p4_event_bind *bind;
unsigned int i, thread, num; unsigned int i, thread, num;
......
...@@ -34,7 +34,7 @@ ...@@ -34,7 +34,7 @@
/* IA32 Manual 3, 2-1 */ /* IA32 Manual 3, 2-1 */
static unsigned char prefix_codes[] = { static unsigned char prefix_codes[] = {
0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64, 0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64,
0x65, 0x2E, 0x3E, 0x66, 0x67 0x65, 0x66, 0x67
}; };
/* IA32 Manual 3, 3-432*/ /* IA32 Manual 3, 3-432*/
static unsigned int reg_rop[] = { static unsigned int reg_rop[] = {
......
...@@ -73,18 +73,25 @@ struct trace_iterator { ...@@ -73,18 +73,25 @@ struct trace_iterator {
}; };
struct trace_event;
typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter, typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter,
int flags); int flags, struct trace_event *event);
struct trace_event {
struct hlist_node node; struct trace_event_functions {
struct list_head list;
int type;
trace_print_func trace; trace_print_func trace;
trace_print_func raw; trace_print_func raw;
trace_print_func hex; trace_print_func hex;
trace_print_func binary; trace_print_func binary;
}; };
struct trace_event {
struct hlist_node node;
struct list_head list;
int type;
struct trace_event_functions *funcs;
};
extern int register_ftrace_event(struct trace_event *event); extern int register_ftrace_event(struct trace_event *event);
extern int unregister_ftrace_event(struct trace_event *event); extern int unregister_ftrace_event(struct trace_event *event);
...@@ -116,28 +123,70 @@ void tracing_record_cmdline(struct task_struct *tsk); ...@@ -116,28 +123,70 @@ void tracing_record_cmdline(struct task_struct *tsk);
struct event_filter; struct event_filter;
enum trace_reg {
TRACE_REG_REGISTER,
TRACE_REG_UNREGISTER,
TRACE_REG_PERF_REGISTER,
TRACE_REG_PERF_UNREGISTER,
};
struct ftrace_event_call;
struct ftrace_event_class {
char *system;
void *probe;
#ifdef CONFIG_PERF_EVENTS
void *perf_probe;
#endif
int (*reg)(struct ftrace_event_call *event,
enum trace_reg type);
int (*define_fields)(struct ftrace_event_call *);
struct list_head *(*get_fields)(struct ftrace_event_call *);
struct list_head fields;
int (*raw_init)(struct ftrace_event_call *);
};
enum {
TRACE_EVENT_FL_ENABLED_BIT,
TRACE_EVENT_FL_FILTERED_BIT,
};
enum {
TRACE_EVENT_FL_ENABLED = (1 << TRACE_EVENT_FL_ENABLED_BIT),
TRACE_EVENT_FL_FILTERED = (1 << TRACE_EVENT_FL_FILTERED_BIT),
};
struct ftrace_event_call { struct ftrace_event_call {
struct list_head list; struct list_head list;
struct ftrace_event_class *class;
char *name; char *name;
char *system;
struct dentry *dir; struct dentry *dir;
struct trace_event *event; struct trace_event event;
int enabled;
int (*regfunc)(struct ftrace_event_call *);
void (*unregfunc)(struct ftrace_event_call *);
int id;
const char *print_fmt; const char *print_fmt;
int (*raw_init)(struct ftrace_event_call *);
int (*define_fields)(struct ftrace_event_call *);
struct list_head fields;
int filter_active;
struct event_filter *filter; struct event_filter *filter;
void *mod; void *mod;
void *data; void *data;
/*
* 32 bit flags:
* bit 1: enabled
* bit 2: filter_active
*
* Changes to flags must hold the event_mutex.
*
* Note: Reads of flags do not hold the event_mutex since
* they occur in critical sections. But the way flags
* is currently used, these changes do no affect the code
* except that when a change is made, it may have a slight
* delay in propagating the changes to other CPUs due to
* caching and such.
*/
unsigned int flags;
#ifdef CONFIG_PERF_EVENTS
int perf_refcount; int perf_refcount;
int (*perf_event_enable)(struct ftrace_event_call *); struct hlist_head *perf_events;
void (*perf_event_disable)(struct ftrace_event_call *); #endif
}; };
#define PERF_MAX_TRACE_SIZE 2048 #define PERF_MAX_TRACE_SIZE 2048
...@@ -194,24 +243,22 @@ struct perf_event; ...@@ -194,24 +243,22 @@ struct perf_event;
DECLARE_PER_CPU(struct pt_regs, perf_trace_regs); DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
extern int perf_trace_enable(int event_id); extern int perf_trace_init(struct perf_event *event);
extern void perf_trace_disable(int event_id); extern void perf_trace_destroy(struct perf_event *event);
extern int ftrace_profile_set_filter(struct perf_event *event, int event_id, extern int perf_trace_enable(struct perf_event *event);
extern void perf_trace_disable(struct perf_event *event);
extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
char *filter_str); char *filter_str);
extern void ftrace_profile_free_filter(struct perf_event *event); extern void ftrace_profile_free_filter(struct perf_event *event);
extern void * extern void *perf_trace_buf_prepare(int size, unsigned short type,
perf_trace_buf_prepare(int size, unsigned short type, int *rctxp, struct pt_regs *regs, int *rctxp);
unsigned long *irq_flags);
static inline void static inline void
perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr, perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
u64 count, unsigned long irq_flags, struct pt_regs *regs) u64 count, struct pt_regs *regs, void *head)
{ {
struct trace_entry *entry = raw_data; perf_tp_event(addr, count, raw_data, size, regs, head);
perf_tp_event(entry->type, addr, count, raw_data, size, regs);
perf_swevent_put_recursion_context(rctx); perf_swevent_put_recursion_context(rctx);
local_irq_restore(irq_flags);
} }
#endif #endif
......
...@@ -485,6 +485,7 @@ struct perf_guest_info_callbacks { ...@@ -485,6 +485,7 @@ struct perf_guest_info_callbacks {
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/cpu.h> #include <linux/cpu.h>
#include <asm/atomic.h> #include <asm/atomic.h>
#include <asm/local.h>
#define PERF_MAX_STACK_DEPTH 255 #define PERF_MAX_STACK_DEPTH 255
...@@ -587,21 +588,19 @@ struct perf_mmap_data { ...@@ -587,21 +588,19 @@ struct perf_mmap_data {
struct rcu_head rcu_head; struct rcu_head rcu_head;
#ifdef CONFIG_PERF_USE_VMALLOC #ifdef CONFIG_PERF_USE_VMALLOC
struct work_struct work; struct work_struct work;
int page_order; /* allocation order */
#endif #endif
int data_order;
int nr_pages; /* nr of data pages */ int nr_pages; /* nr of data pages */
int writable; /* are we writable */ int writable; /* are we writable */
int nr_locked; /* nr pages mlocked */ int nr_locked; /* nr pages mlocked */
atomic_t poll; /* POLL_ for wakeups */ atomic_t poll; /* POLL_ for wakeups */
atomic_t events; /* event_id limit */
atomic_long_t head; /* write position */ local_t head; /* write position */
atomic_long_t done_head; /* completed head */ local_t nest; /* nested writers */
local_t events; /* event limit */
atomic_t lock; /* concurrent writes */ local_t wakeup; /* wakeup stamp */
atomic_t wakeup; /* needs a wakeup */ local_t lost; /* nr records lost */
atomic_t lost; /* nr records lost */
long watermark; /* wakeup watermark */ long watermark; /* wakeup watermark */
...@@ -728,6 +727,7 @@ struct perf_event { ...@@ -728,6 +727,7 @@ struct perf_event {
perf_overflow_handler_t overflow_handler; perf_overflow_handler_t overflow_handler;
#ifdef CONFIG_EVENT_TRACING #ifdef CONFIG_EVENT_TRACING
struct ftrace_event_call *tp_event;
struct event_filter *filter; struct event_filter *filter;
#endif #endif
...@@ -803,11 +803,12 @@ struct perf_cpu_context { ...@@ -803,11 +803,12 @@ struct perf_cpu_context {
struct perf_output_handle { struct perf_output_handle {
struct perf_event *event; struct perf_event *event;
struct perf_mmap_data *data; struct perf_mmap_data *data;
unsigned long head; unsigned long wakeup;
unsigned long offset; unsigned long size;
void *addr;
int page;
int nmi; int nmi;
int sample; int sample;
int locked;
}; };
#ifdef CONFIG_PERF_EVENTS #ifdef CONFIG_PERF_EVENTS
...@@ -993,8 +994,9 @@ static inline bool perf_paranoid_kernel(void) ...@@ -993,8 +994,9 @@ static inline bool perf_paranoid_kernel(void)
} }
extern void perf_event_init(void); extern void perf_event_init(void);
extern void perf_tp_event(int event_id, u64 addr, u64 count, void *record, extern void perf_tp_event(u64 addr, u64 count, void *record,
int entry_size, struct pt_regs *regs); int entry_size, struct pt_regs *regs,
struct hlist_head *head);
extern void perf_bp_event(struct perf_event *event, void *data); extern void perf_bp_event(struct perf_event *event, void *data);
#ifndef perf_misc_flags #ifndef perf_misc_flags
......
...@@ -103,22 +103,6 @@ struct perf_event_attr; ...