mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-11-24 00:40:51 +07:00
tracepoint: Optimize using static_call()
Currently the tracepoint site will iterate a vector and issue indirect calls to however many handlers are registered (ie. the vector is long). Using static_call() it is possible to optimize this for the common case of only having a single handler registered. In this case the static_call() can directly call this handler. Otherwise, if the vector is longer than 1, call a function that iterates the whole vector like the current code. [peterz: updated to new interface] Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: https://lore.kernel.org/r/20200818135805.279421092@infradead.org
This commit is contained in:
Steven Rostedt (VMware)
Ingo Molnar
parent
a945c8345e
commit
d25e37d89d
@ -11,6 +11,8 @@
|
||||
#include <linux/atomic.h>
|
||||
#include <linux/static_key.h>
|
||||
|
||||
struct static_call_key;
|
||||
|
||||
struct trace_print_flags {
|
||||
unsigned long mask;
|
||||
const char *name;
|
||||
@ -30,6 +32,9 @@ struct tracepoint_func {
|
||||
struct tracepoint {
|
||||
const char *name; /* Tracepoint name */
|
||||
struct static_key key;
|
||||
struct static_call_key *static_call_key;
|
||||
void *static_call_tramp;
|
||||
void *iterator;
|
||||
int (*regfunc)(void);
|
||||
void (*unregfunc)(void);
|
||||
struct tracepoint_func __rcu *funcs;
|
||||
|
||||
@ -19,6 +19,7 @@
|
||||
#include <linux/cpumask.h>
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/tracepoint-defs.h>
|
||||
#include <linux/static_call.h>
|
||||
|
||||
struct module;
|
||||
struct tracepoint;
|
||||
@ -92,7 +93,9 @@ extern int syscall_regfunc(void);
|
||||
extern void syscall_unregfunc(void);
|
||||
#endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */
|
||||
|
||||
#ifndef PARAMS
|
||||
#define PARAMS(args...) args
|
||||
#endif
|
||||
|
||||
#define TRACE_DEFINE_ENUM(x)
|
||||
#define TRACE_DEFINE_SIZEOF(x)
|
||||
@ -148,6 +151,12 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
|
||||
|
||||
#ifdef TRACEPOINTS_ENABLED
|
||||
|
||||
#ifdef CONFIG_HAVE_STATIC_CALL
|
||||
#define __DO_TRACE_CALL(name) static_call(tp_func_##name)
|
||||
#else
|
||||
#define __DO_TRACE_CALL(name) __tracepoint_iter_##name
|
||||
#endif /* CONFIG_HAVE_STATIC_CALL */
|
||||
|
||||
/*
|
||||
* it_func[0] is never NULL because there is at least one element in the array
|
||||
* when the array itself is non NULL.
|
||||
@ -157,12 +166,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
|
||||
* has a "void" prototype, then it is invalid to declare a function
|
||||
* as "(void *, void)".
|
||||
*/
|
||||
#define __DO_TRACE(tp, proto, args, cond, rcuidle) \
|
||||
#define __DO_TRACE(name, proto, args, cond, rcuidle) \
|
||||
do { \
|
||||
struct tracepoint_func *it_func_ptr; \
|
||||
void *it_func; \
|
||||
void *__data; \
|
||||
int __maybe_unused __idx = 0; \
|
||||
void *__data; \
|
||||
\
|
||||
if (!(cond)) \
|
||||
return; \
|
||||
@ -182,14 +190,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
|
||||
rcu_irq_enter_irqson(); \
|
||||
} \
|
||||
\
|
||||
it_func_ptr = rcu_dereference_raw((tp)->funcs); \
|
||||
\
|
||||
it_func_ptr = \
|
||||
rcu_dereference_raw((&__tracepoint_##name)->funcs); \
|
||||
if (it_func_ptr) { \
|
||||
do { \
|
||||
it_func = (it_func_ptr)->func; \
|
||||
__data = (it_func_ptr)->data; \
|
||||
((void(*)(proto))(it_func))(args); \
|
||||
} while ((++it_func_ptr)->func); \
|
||||
__data = (it_func_ptr)->data; \
|
||||
__DO_TRACE_CALL(name)(args); \
|
||||
} \
|
||||
\
|
||||
if (rcuidle) { \
|
||||
@ -205,7 +210,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
|
||||
static inline void trace_##name##_rcuidle(proto) \
|
||||
{ \
|
||||
if (static_key_false(&__tracepoint_##name.key)) \
|
||||
__DO_TRACE(&__tracepoint_##name, \
|
||||
__DO_TRACE(name, \
|
||||
TP_PROTO(data_proto), \
|
||||
TP_ARGS(data_args), \
|
||||
TP_CONDITION(cond), 1); \
|
||||
@ -227,11 +232,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
|
||||
* poking RCU a bit.
|
||||
*/
|
||||
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
|
||||
extern int __tracepoint_iter_##name(data_proto); \
|
||||
DECLARE_STATIC_CALL(tp_func_##name, __tracepoint_iter_##name); \
|
||||
extern struct tracepoint __tracepoint_##name; \
|
||||
static inline void trace_##name(proto) \
|
||||
{ \
|
||||
if (static_key_false(&__tracepoint_##name.key)) \
|
||||
__DO_TRACE(&__tracepoint_##name, \
|
||||
__DO_TRACE(name, \
|
||||
TP_PROTO(data_proto), \
|
||||
TP_ARGS(data_args), \
|
||||
TP_CONDITION(cond), 0); \
|
||||
@ -277,21 +284,50 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
|
||||
* structures, so we create an array of pointers that will be used for iteration
|
||||
* on the tracepoints.
|
||||
*/
|
||||
#define DEFINE_TRACE_FN(name, reg, unreg) \
|
||||
static const char __tpstrtab_##name[] \
|
||||
__section(__tracepoints_strings) = #name; \
|
||||
struct tracepoint __tracepoint_##name __used \
|
||||
__section(__tracepoints) = \
|
||||
{ __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
|
||||
__TRACEPOINT_ENTRY(name);
|
||||
#define DEFINE_TRACE_FN(_name, _reg, _unreg, proto, args) \
|
||||
static const char __tpstrtab_##_name[] \
|
||||
__section(__tracepoints_strings) = #_name; \
|
||||
extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \
|
||||
int __tracepoint_iter_##_name(void *__data, proto); \
|
||||
struct tracepoint __tracepoint_##_name __used \
|
||||
__section(__tracepoints) = { \
|
||||
.name = __tpstrtab_##_name, \
|
||||
.key = STATIC_KEY_INIT_FALSE, \
|
||||
.static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \
|
||||
.static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \
|
||||
.iterator = &__tracepoint_iter_##_name, \
|
||||
.regfunc = _reg, \
|
||||
.unregfunc = _unreg, \
|
||||
.funcs = NULL }; \
|
||||
__TRACEPOINT_ENTRY(_name); \
|
||||
int __tracepoint_iter_##_name(void *__data, proto) \
|
||||
{ \
|
||||
struct tracepoint_func *it_func_ptr; \
|
||||
void *it_func; \
|
||||
\
|
||||
it_func_ptr = \
|
||||
rcu_dereference_raw((&__tracepoint_##_name)->funcs); \
|
||||
do { \
|
||||
it_func = (it_func_ptr)->func; \
|
||||
__data = (it_func_ptr)->data; \
|
||||
((void(*)(void *, proto))(it_func))(__data, args); \
|
||||
} while ((++it_func_ptr)->func); \
|
||||
return 0; \
|
||||
} \
|
||||
DEFINE_STATIC_CALL(tp_func_##_name, __tracepoint_iter_##_name);
|
||||
|
||||
#define DEFINE_TRACE(name) \
|
||||
DEFINE_TRACE_FN(name, NULL, NULL);
|
||||
#define DEFINE_TRACE(name, proto, args) \
|
||||
DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
|
||||
|
||||
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
|
||||
EXPORT_SYMBOL_GPL(__tracepoint_##name)
|
||||
EXPORT_SYMBOL_GPL(__tracepoint_##name); \
|
||||
EXPORT_SYMBOL_GPL(__tracepoint_iter_##name); \
|
||||
EXPORT_STATIC_CALL_GPL(tp_func_##name)
|
||||
#define EXPORT_TRACEPOINT_SYMBOL(name) \
|
||||
EXPORT_SYMBOL(__tracepoint_##name)
|
||||
EXPORT_SYMBOL(__tracepoint_##name); \
|
||||
EXPORT_SYMBOL(__tracepoint_iter_##name); \
|
||||
EXPORT_STATIC_CALL(tp_func_##name)
|
||||
|
||||
|
||||
#else /* !TRACEPOINTS_ENABLED */
|
||||
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
|
||||
@ -320,8 +356,8 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
|
||||
return false; \
|
||||
}
|
||||
|
||||
#define DEFINE_TRACE_FN(name, reg, unreg)
|
||||
#define DEFINE_TRACE(name)
|
||||
#define DEFINE_TRACE_FN(name, reg, unreg, proto, args)
|
||||
#define DEFINE_TRACE(name, proto, args)
|
||||
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
|
||||
#define EXPORT_TRACEPOINT_SYMBOL(name)
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@
|
||||
|
||||
#undef TRACE_EVENT
|
||||
#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
|
||||
DEFINE_TRACE(name)
|
||||
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
|
||||
|
||||
#undef TRACE_EVENT_CONDITION
|
||||
#define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
|
||||
@ -39,12 +39,12 @@
|
||||
#undef TRACE_EVENT_FN
|
||||
#define TRACE_EVENT_FN(name, proto, args, tstruct, \
|
||||
assign, print, reg, unreg) \
|
||||
DEFINE_TRACE_FN(name, reg, unreg)
|
||||
DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
|
||||
|
||||
#undef TRACE_EVENT_FN_COND
|
||||
#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \
|
||||
assign, print, reg, unreg) \
|
||||
DEFINE_TRACE_FN(name, reg, unreg)
|
||||
DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
|
||||
|
||||
#undef TRACE_EVENT_NOP
|
||||
#define TRACE_EVENT_NOP(name, proto, args, struct, assign, print)
|
||||
@ -54,15 +54,15 @@
|
||||
|
||||
#undef DEFINE_EVENT
|
||||
#define DEFINE_EVENT(template, name, proto, args) \
|
||||
DEFINE_TRACE(name)
|
||||
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
|
||||
|
||||
#undef DEFINE_EVENT_FN
|
||||
#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
|
||||
DEFINE_TRACE_FN(name, reg, unreg)
|
||||
DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
|
||||
|
||||
#undef DEFINE_EVENT_PRINT
|
||||
#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
|
||||
DEFINE_TRACE(name)
|
||||
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
|
||||
|
||||
#undef DEFINE_EVENT_CONDITION
|
||||
#define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
|
||||
@ -70,7 +70,7 @@
|
||||
|
||||
#undef DECLARE_TRACE
|
||||
#define DECLARE_TRACE(name, proto, args) \
|
||||
DEFINE_TRACE(name)
|
||||
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
|
||||
|
||||
#undef TRACE_INCLUDE
|
||||
#undef __TRACE_INCLUDE
|
||||
|
||||
@ -221,6 +221,20 @@ static void *func_remove(struct tracepoint_func **funcs,
|
||||
return old;
|
||||
}
|
||||
|
||||
static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
|
||||
{
|
||||
void *func = tp->iterator;
|
||||
|
||||
/* Synthetic events do not have static call sites */
|
||||
if (!tp->static_call_key)
|
||||
return;
|
||||
|
||||
if (!tp_funcs[1].func)
|
||||
func = tp_funcs[0].func;
|
||||
|
||||
__static_call_update(tp->static_call_key, tp->static_call_tramp, func);
|
||||
}
|
||||
|
||||
/*
|
||||
* Add the probe function to a tracepoint.
|
||||
*/
|
||||
@ -251,8 +265,9 @@ static int tracepoint_add_func(struct tracepoint *tp,
|
||||
* include/linux/tracepoint.h using rcu_dereference_sched().
|
||||
*/
|
||||
rcu_assign_pointer(tp->funcs, tp_funcs);
|
||||
if (!static_key_enabled(&tp->key))
|
||||
static_key_slow_inc(&tp->key);
|
||||
tracepoint_update_call(tp, tp_funcs);
|
||||
static_key_enable(&tp->key);
|
||||
|
||||
release_probes(old);
|
||||
return 0;
|
||||
}
|
||||
@ -281,9 +296,11 @@ static int tracepoint_remove_func(struct tracepoint *tp,
|
||||
if (tp->unregfunc && static_key_enabled(&tp->key))
|
||||
tp->unregfunc();
|
||||
|
||||
if (static_key_enabled(&tp->key))
|
||||
static_key_slow_dec(&tp->key);
|
||||
static_key_disable(&tp->key);
|
||||
} else {
|
||||
tracepoint_update_call(tp, tp_funcs);
|
||||
}
|
||||
|
||||
rcu_assign_pointer(tp->funcs, tp_funcs);
|
||||
release_probes(old);
|
||||
return 0;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user