aboutsummaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorPeter Zijlstra <a.p.zijlstra@chello.nl>2009-11-23 11:37:29 +0100
committerIngo Molnar <mingo@elte.hu>2009-11-23 11:49:57 +0100
commit4ed7c92d68a5387ba5f7030dc76eab03558e27f5 (patch)
tree38d758819c41261275c2dbd6f64976f824c5fd27 /kernel
parentf67218c3e93abaf0f480bb94b53d234853ffe4de (diff)
perf_events: Undo some recursion damage
Make perf_swevent_get_recursion_context return a context number and disable preemption. This could be used to remove the IRQ disable from the trace bit and index the per-cpu buffer with. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <20091123103819.993226816@chello.nl> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/perf_event.c71
-rw-r--r--kernel/trace/trace_kprobe.c14
-rw-r--r--kernel/trace/trace_syscalls.c14
3 files changed, 48 insertions, 51 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 50f11b5f8c3..0b0d5f72fe7 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -3869,45 +3869,50 @@ static void perf_swevent_ctx_event(struct perf_event_context *ctx,
}
}
-/*
- * Must be called with preemption disabled
- */
-int perf_swevent_get_recursion_context(int **recursion)
+int perf_swevent_get_recursion_context(void)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
+ int rctx;
if (in_nmi())
- *recursion = &cpuctx->recursion[3];
+ rctx = 3;
else if (in_irq())
- *recursion = &cpuctx->recursion[2];
+ rctx = 2;
else if (in_softirq())
- *recursion = &cpuctx->recursion[1];
+ rctx = 1;
else
- *recursion = &cpuctx->recursion[0];
+ rctx = 0;
- if (**recursion)
+ if (cpuctx->recursion[rctx]) {
+ put_cpu_var(perf_cpu_context);
return -1;
+ }
- (**recursion)++;
+ cpuctx->recursion[rctx]++;
+ barrier();
- return 0;
+ return rctx;
}
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
-void perf_swevent_put_recursion_context(int *recursion)
+void perf_swevent_put_recursion_context(int rctx)
{
- (*recursion)--;
+ struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ barrier();
+ cpuctx->recursion[rctx]++;
+ put_cpu_var(perf_cpu_context);
}
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
-static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
- u64 nr, int nmi,
- struct perf_sample_data *data,
- struct pt_regs *regs)
+static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
+ u64 nr, int nmi,
+ struct perf_sample_data *data,
+ struct pt_regs *regs)
{
+ struct perf_cpu_context *cpuctx;
struct perf_event_context *ctx;
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ cpuctx = &__get_cpu_var(perf_cpu_context);
rcu_read_lock();
perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
nr, nmi, data, regs);
@@ -3921,34 +3926,22 @@ static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
rcu_read_unlock();
}
-static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
- u64 nr, int nmi,
- struct perf_sample_data *data,
- struct pt_regs *regs)
-{
- int *recursion;
-
- preempt_disable();
-
- if (perf_swevent_get_recursion_context(&recursion))
- goto out;
-
- __do_perf_sw_event(type, event_id, nr, nmi, data, regs);
-
- perf_swevent_put_recursion_context(recursion);
-out:
- preempt_enable();
-}
-
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
struct pt_regs *regs, u64 addr)
{
struct perf_sample_data data;
+ int rctx;
+
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
+ return;
data.addr = addr;
data.raw = NULL;
do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
+
+ perf_swevent_put_recursion_context(rctx);
}
static void perf_swevent_read(struct perf_event *event)
@@ -4172,7 +4165,7 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
regs = task_pt_regs(current);
/* Trace events already protected against recursion */
- __do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
+ do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
&data, regs);
}
EXPORT_SYMBOL_GPL(perf_tp_event);
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 22e6f68b05b..79ce6a2bd74 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -1213,7 +1213,7 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
unsigned long irq_flags;
char *trace_buf;
char *raw_data;
- int *recursion;
+ int rctx;
pc = preempt_count();
__size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args);
@@ -1229,7 +1229,8 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
*/
local_irq_save(irq_flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
__cpu = smp_processor_id();
@@ -1258,7 +1259,7 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
perf_tp_event(call->id, entry->ip, 1, entry, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(irq_flags);
@@ -1276,8 +1277,8 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
int size, __size, i, pc, __cpu;
unsigned long irq_flags;
char *trace_buf;
- int *recursion;
char *raw_data;
+ int rctx;
pc = preempt_count();
__size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args);
@@ -1293,7 +1294,8 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
*/
local_irq_save(irq_flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
__cpu = smp_processor_id();
@@ -1323,7 +1325,7 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
perf_tp_event(call->id, entry->ret_ip, 1, entry, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(irq_flags);
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 41b6dd963da..9189cbe8607 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -481,8 +481,8 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
unsigned long flags;
char *trace_buf;
char *raw_data;
- int *recursion;
int syscall_nr;
+ int rctx;
int size;
int cpu;
@@ -506,7 +506,8 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
/* Protect the per cpu buffer, begin the rcu read side */
local_irq_save(flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
cpu = smp_processor_id();
@@ -530,7 +531,7 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
perf_tp_event(sys_data->enter_id, 0, 1, rec, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(flags);
}
@@ -582,7 +583,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
int syscall_nr;
char *trace_buf;
char *raw_data;
- int *recursion;
+ int rctx;
int size;
int cpu;
@@ -609,7 +610,8 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
/* Protect the per cpu buffer, begin the rcu read side */
local_irq_save(flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
cpu = smp_processor_id();
@@ -634,7 +636,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
perf_tp_event(sys_data->exit_id, 0, 1, rec, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(flags);
}