2014-05-09 02:21:52 +07:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 2014 Steven Rostedt, Red Hat Inc
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
|
#include <asm/ptrace.h>
|
|
|
|
|
#include <asm/ftrace.h>
|
2016-01-11 23:04:34 +07:00
|
|
|
#include <asm/export.h>
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
.code64
|
|
|
|
|
.section .entry.text, "ax"
|
|
|
|
|
|
|
|
|
|
#ifdef CC_USING_FENTRY
|
|
|
|
|
# define function_hook __fentry__
|
2016-10-25 02:01:48 +07:00
|
|
|
EXPORT_SYMBOL(__fentry__)
|
2014-05-09 02:21:52 +07:00
|
|
|
#else
|
|
|
|
|
# define function_hook mcount
|
2016-10-25 02:01:48 +07:00
|
|
|
EXPORT_SYMBOL(mcount)
|
2014-05-09 02:21:52 +07:00
|
|
|
#endif
|
|
|
|
|
|
2014-11-25 06:08:48 +07:00
|
|
|
/* All cases save the original rbp (8 bytes) */
|
|
|
|
|
#ifdef CONFIG_FRAME_POINTER
|
|
|
|
|
# ifdef CC_USING_FENTRY
|
|
|
|
|
/* Save parent and function stack frames (rip and rbp) */
|
|
|
|
|
# define MCOUNT_FRAME_SIZE (8+16*2)
|
|
|
|
|
# else
|
|
|
|
|
/* Save just function stack frame (rip and rbp) */
|
|
|
|
|
# define MCOUNT_FRAME_SIZE (8+16)
|
|
|
|
|
# endif
|
|
|
|
|
#else
|
|
|
|
|
/* No need to save a stack frame */
|
|
|
|
|
# define MCOUNT_FRAME_SIZE 8
|
|
|
|
|
#endif /* CONFIG_FRAME_POINTER */
|
|
|
|
|
|
2014-11-25 02:26:38 +07:00
|
|
|
/* Size of stack used to save mcount regs in save_mcount_regs */
|
2014-11-25 06:08:48 +07:00
|
|
|
#define MCOUNT_REG_SIZE (SS+8 + MCOUNT_FRAME_SIZE)
|
2014-11-25 02:26:38 +07:00
|
|
|
|
2014-11-24 23:43:39 +07:00
|
|
|
/*
|
|
|
|
|
* gcc -pg option adds a call to 'mcount' in most functions.
|
|
|
|
|
* When -mfentry is used, the call is to 'fentry' and not 'mcount'
|
|
|
|
|
* and is done before the function's stack frame is set up.
|
|
|
|
|
* They both require a set of regs to be saved before calling
|
|
|
|
|
* any C code and restored before returning back to the function.
|
|
|
|
|
*
|
|
|
|
|
* On boot up, all these calls are converted into nops. When tracing
|
|
|
|
|
* is enabled, the call can jump to either ftrace_caller or
|
|
|
|
|
* ftrace_regs_caller. Callbacks (tracing functions) that require
|
|
|
|
|
* ftrace_regs_caller (like kprobes) need to have pt_regs passed to
|
|
|
|
|
* it. For this reason, the size of the pt_regs structure will be
|
|
|
|
|
* allocated on the stack and the required mcount registers will
|
|
|
|
|
* be saved in the locations that pt_regs has them in.
|
|
|
|
|
*/
|
|
|
|
|
|
2014-11-25 09:38:40 +07:00
|
|
|
/*
|
|
|
|
|
* @added: the amount of stack added before calling this
|
|
|
|
|
*
|
|
|
|
|
* After this is called, the following registers contain:
|
|
|
|
|
*
|
|
|
|
|
* %rdi - holds the address that called the trampoline
|
|
|
|
|
* %rsi - holds the parent function (traced function's return address)
|
|
|
|
|
* %rdx - holds the original %rbp
|
|
|
|
|
*/
|
2014-11-25 01:06:05 +07:00
|
|
|
.macro save_mcount_regs added=0
|
2014-11-25 06:08:48 +07:00
|
|
|
|
|
|
|
|
/* Always save the original rbp */
|
|
|
|
|
pushq %rbp
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_FRAME_POINTER
|
|
|
|
|
/*
|
|
|
|
|
* Stack traces will stop at the ftrace trampoline if the frame pointer
|
|
|
|
|
* is not set up properly. If fentry is used, we need to save a frame
|
|
|
|
|
* pointer for the parent as well as the function traced, because the
|
|
|
|
|
* fentry is called before the stack frame is set up, where as mcount
|
|
|
|
|
* is called afterward.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CC_USING_FENTRY
|
|
|
|
|
/* Save the parent pointer (skip orig rbp and our return address) */
|
|
|
|
|
pushq \added+8*2(%rsp)
|
|
|
|
|
pushq %rbp
|
|
|
|
|
movq %rsp, %rbp
|
|
|
|
|
/* Save the return address (now skip orig rbp, rbp and parent) */
|
|
|
|
|
pushq \added+8*3(%rsp)
|
|
|
|
|
#else
|
|
|
|
|
/* Can't assume that rip is before this (unless added was zero) */
|
|
|
|
|
pushq \added+8(%rsp)
|
|
|
|
|
#endif
|
|
|
|
|
pushq %rbp
|
|
|
|
|
movq %rsp, %rbp
|
|
|
|
|
#endif /* CONFIG_FRAME_POINTER */
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We add enough stack to save all regs.
|
|
|
|
|
*/
|
|
|
|
|
subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp
|
2014-11-24 23:30:58 +07:00
|
|
|
movq %rax, RAX(%rsp)
|
|
|
|
|
movq %rcx, RCX(%rsp)
|
|
|
|
|
movq %rdx, RDX(%rsp)
|
|
|
|
|
movq %rsi, RSI(%rsp)
|
|
|
|
|
movq %rdi, RDI(%rsp)
|
|
|
|
|
movq %r8, R8(%rsp)
|
|
|
|
|
movq %r9, R9(%rsp)
|
2014-11-25 06:08:48 +07:00
|
|
|
/*
|
|
|
|
|
* Save the original RBP. Even though the mcount ABI does not
|
|
|
|
|
* require this, it helps out callers.
|
|
|
|
|
*/
|
|
|
|
|
movq MCOUNT_REG_SIZE-8(%rsp), %rdx
|
|
|
|
|
movq %rdx, RBP(%rsp)
|
|
|
|
|
|
2014-11-25 09:38:40 +07:00
|
|
|
/* Copy the parent address into %rsi (second parameter) */
|
|
|
|
|
#ifdef CC_USING_FENTRY
|
|
|
|
|
movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
|
|
|
|
|
#else
|
|
|
|
|
/* %rdx contains original %rbp */
|
|
|
|
|
movq 8(%rdx), %rsi
|
|
|
|
|
#endif
|
|
|
|
|
|
2014-11-24 23:30:58 +07:00
|
|
|
/* Move RIP to its proper location */
|
2014-11-25 02:26:38 +07:00
|
|
|
movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
|
2014-11-25 01:21:09 +07:00
|
|
|
movq %rdi, RIP(%rsp)
|
2014-11-25 09:38:40 +07:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Now %rdi (the first parameter) has the return address of
|
|
|
|
|
* where ftrace_call returns. But the callbacks expect the
|
2014-11-25 09:00:34 +07:00
|
|
|
* address of the call itself.
|
2014-11-25 09:38:40 +07:00
|
|
|
*/
|
|
|
|
|
subq $MCOUNT_INSN_SIZE, %rdi
|
2014-11-24 23:30:58 +07:00
|
|
|
.endm
|
|
|
|
|
|
2014-11-25 01:06:05 +07:00
|
|
|
.macro restore_mcount_regs
|
2014-11-24 23:30:58 +07:00
|
|
|
movq R9(%rsp), %r9
|
|
|
|
|
movq R8(%rsp), %r8
|
|
|
|
|
movq RDI(%rsp), %rdi
|
|
|
|
|
movq RSI(%rsp), %rsi
|
|
|
|
|
movq RDX(%rsp), %rdx
|
|
|
|
|
movq RCX(%rsp), %rcx
|
|
|
|
|
movq RAX(%rsp), %rax
|
2014-11-25 06:08:48 +07:00
|
|
|
|
|
|
|
|
/* ftrace_regs_caller can modify %rbp */
|
|
|
|
|
movq RBP(%rsp), %rbp
|
|
|
|
|
|
2014-11-25 02:26:38 +07:00
|
|
|
addq $MCOUNT_REG_SIZE, %rsp
|
2014-11-25 06:08:48 +07:00
|
|
|
|
2014-11-24 23:30:58 +07:00
|
|
|
.endm
|
|
|
|
|
|
2014-11-25 02:54:27 +07:00
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
|
|
|
|
|
|
|
|
ENTRY(function_hook)
|
|
|
|
|
retq
|
|
|
|
|
END(function_hook)
|
|
|
|
|
|
2014-05-09 02:21:52 +07:00
|
|
|
ENTRY(ftrace_caller)
|
2014-11-25 09:38:40 +07:00
|
|
|
/* save_mcount_regs fills in first two parameters */
|
|
|
|
|
save_mcount_regs
|
|
|
|
|
|
|
|
|
|
GLOBAL(ftrace_caller_op_ptr)
|
|
|
|
|
/* Load the ftrace_ops into the 3rd parameter */
|
|
|
|
|
movq function_trace_op(%rip), %rdx
|
|
|
|
|
|
2014-05-09 02:21:52 +07:00
|
|
|
/* regs go into 4th parameter (but make it NULL) */
|
|
|
|
|
movq $0, %rcx
|
|
|
|
|
|
|
|
|
|
GLOBAL(ftrace_call)
|
|
|
|
|
call ftrace_stub
|
|
|
|
|
|
2014-11-24 23:43:39 +07:00
|
|
|
restore_mcount_regs
|
ftrace/x86: Add dynamic allocated trampoline for ftrace_ops
The current method of handling multiple function callbacks is to register
a list function callback that calls all the other callbacks based on
their hash tables and compare it to the function that the callback was
called on. But this is very inefficient.
For example, if you are tracing all functions in the kernel and then
add a kprobe to a function such that the kprobe uses ftrace, the
mcount trampoline will switch from calling the function trace callback
to calling the list callback that will iterate over all registered
ftrace_ops (in this case, the function tracer and the kprobes callback).
That means for every function being traced it checks the hash of the
ftrace_ops for function tracing and kprobes, even though the kprobes
is only set at a single function. The kprobes ftrace_ops is checked
for every function being traced!
Instead of calling the list function for functions that are only being
traced by a single callback, we can call a dynamically allocated
trampoline that calls the callback directly. The function graph tracer
already uses a direct call trampoline when it is being traced by itself
but it is not dynamically allocated. It's trampoline is static in the
kernel core. The infrastructure that called the function graph trampoline
can also be used to call a dynamically allocated one.
For now, only ftrace_ops that are not dynamically allocated can have
a trampoline. That is, users such as function tracer or stack tracer.
kprobes and perf allocate their ftrace_ops, and until there's a safe
way to free the trampoline, it can not be used. The dynamically allocated
ftrace_ops may, although, use the trampoline if the kernel is not
compiled with CONFIG_PREEMPT. But that will come later.
Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Tested-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2014-07-03 10:23:31 +07:00
|
|
|
|
|
|
|
|
/*
|
2016-02-16 15:43:21 +07:00
|
|
|
* The copied trampoline must call ftrace_epilogue as it
|
ftrace/x86: Add dynamic allocated trampoline for ftrace_ops
The current method of handling multiple function callbacks is to register
a list function callback that calls all the other callbacks based on
their hash tables and compare it to the function that the callback was
called on. But this is very inefficient.
For example, if you are tracing all functions in the kernel and then
add a kprobe to a function such that the kprobe uses ftrace, the
mcount trampoline will switch from calling the function trace callback
to calling the list callback that will iterate over all registered
ftrace_ops (in this case, the function tracer and the kprobes callback).
That means for every function being traced it checks the hash of the
ftrace_ops for function tracing and kprobes, even though the kprobes
is only set at a single function. The kprobes ftrace_ops is checked
for every function being traced!
Instead of calling the list function for functions that are only being
traced by a single callback, we can call a dynamically allocated
trampoline that calls the callback directly. The function graph tracer
already uses a direct call trampoline when it is being traced by itself
but it is not dynamically allocated. It's trampoline is static in the
kernel core. The infrastructure that called the function graph trampoline
can also be used to call a dynamically allocated one.
For now, only ftrace_ops that are not dynamically allocated can have
a trampoline. That is, users such as function tracer or stack tracer.
kprobes and perf allocate their ftrace_ops, and until there's a safe
way to free the trampoline, it can not be used. The dynamically allocated
ftrace_ops may, although, use the trampoline if the kernel is not
compiled with CONFIG_PREEMPT. But that will come later.
Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Tested-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2014-07-03 10:23:31 +07:00
|
|
|
* still may need to call the function graph tracer.
|
2016-02-16 15:43:21 +07:00
|
|
|
*
|
|
|
|
|
* The code up to this label is copied into trampolines so
|
|
|
|
|
* think twice before adding any new code or changing the
|
|
|
|
|
* layout here.
|
ftrace/x86: Add dynamic allocated trampoline for ftrace_ops
The current method of handling multiple function callbacks is to register
a list function callback that calls all the other callbacks based on
their hash tables and compare it to the function that the callback was
called on. But this is very inefficient.
For example, if you are tracing all functions in the kernel and then
add a kprobe to a function such that the kprobe uses ftrace, the
mcount trampoline will switch from calling the function trace callback
to calling the list callback that will iterate over all registered
ftrace_ops (in this case, the function tracer and the kprobes callback).
That means for every function being traced it checks the hash of the
ftrace_ops for function tracing and kprobes, even though the kprobes
is only set at a single function. The kprobes ftrace_ops is checked
for every function being traced!
Instead of calling the list function for functions that are only being
traced by a single callback, we can call a dynamically allocated
trampoline that calls the callback directly. The function graph tracer
already uses a direct call trampoline when it is being traced by itself
but it is not dynamically allocated. It's trampoline is static in the
kernel core. The infrastructure that called the function graph trampoline
can also be used to call a dynamically allocated one.
For now, only ftrace_ops that are not dynamically allocated can have
a trampoline. That is, users such as function tracer or stack tracer.
kprobes and perf allocate their ftrace_ops, and until there's a safe
way to free the trampoline, it can not be used. The dynamically allocated
ftrace_ops may, although, use the trampoline if the kernel is not
compiled with CONFIG_PREEMPT. But that will come later.
Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Tested-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2014-07-03 10:23:31 +07:00
|
|
|
*/
|
2016-02-16 15:43:21 +07:00
|
|
|
GLOBAL(ftrace_epilogue)
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
|
|
|
GLOBAL(ftrace_graph_call)
|
|
|
|
|
jmp ftrace_stub
|
|
|
|
|
#endif
|
|
|
|
|
|
2016-05-17 10:00:35 +07:00
|
|
|
/* This is weak to keep gas from relaxing the jumps */
|
|
|
|
|
WEAK(ftrace_stub)
|
2014-05-09 02:21:52 +07:00
|
|
|
retq
|
|
|
|
|
END(ftrace_caller)
|
|
|
|
|
|
|
|
|
|
ENTRY(ftrace_regs_caller)
|
2014-11-25 01:06:05 +07:00
|
|
|
/* Save the current flags before any operations that can change them */
|
2014-05-09 02:21:52 +07:00
|
|
|
pushfq
|
|
|
|
|
|
2014-11-25 01:06:05 +07:00
|
|
|
/* added 8 bytes to save flags */
|
2014-11-25 09:38:40 +07:00
|
|
|
save_mcount_regs 8
|
|
|
|
|
/* save_mcount_regs fills in first two parameters */
|
|
|
|
|
|
|
|
|
|
GLOBAL(ftrace_regs_caller_op_ptr)
|
|
|
|
|
/* Load the ftrace_ops into the 3rd parameter */
|
|
|
|
|
movq function_trace_op(%rip), %rdx
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
/* Save the rest of pt_regs */
|
|
|
|
|
movq %r15, R15(%rsp)
|
|
|
|
|
movq %r14, R14(%rsp)
|
|
|
|
|
movq %r13, R13(%rsp)
|
|
|
|
|
movq %r12, R12(%rsp)
|
|
|
|
|
movq %r11, R11(%rsp)
|
|
|
|
|
movq %r10, R10(%rsp)
|
|
|
|
|
movq %rbx, RBX(%rsp)
|
|
|
|
|
/* Copy saved flags */
|
2014-11-25 02:26:38 +07:00
|
|
|
movq MCOUNT_REG_SIZE(%rsp), %rcx
|
2014-05-09 02:21:52 +07:00
|
|
|
movq %rcx, EFLAGS(%rsp)
|
|
|
|
|
/* Kernel segments */
|
|
|
|
|
movq $__KERNEL_DS, %rcx
|
|
|
|
|
movq %rcx, SS(%rsp)
|
|
|
|
|
movq $__KERNEL_CS, %rcx
|
|
|
|
|
movq %rcx, CS(%rsp)
|
2014-11-25 01:06:05 +07:00
|
|
|
/* Stack - skipping return address and flags */
|
2014-11-25 02:26:38 +07:00
|
|
|
leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
|
2014-05-09 02:21:52 +07:00
|
|
|
movq %rcx, RSP(%rsp)
|
|
|
|
|
|
|
|
|
|
/* regs go into 4th parameter */
|
|
|
|
|
leaq (%rsp), %rcx
|
|
|
|
|
|
|
|
|
|
GLOBAL(ftrace_regs_call)
|
|
|
|
|
call ftrace_stub
|
|
|
|
|
|
|
|
|
|
/* Copy flags back to SS, to restore them */
|
|
|
|
|
movq EFLAGS(%rsp), %rax
|
2014-11-25 02:26:38 +07:00
|
|
|
movq %rax, MCOUNT_REG_SIZE(%rsp)
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
/* Handlers can change the RIP */
|
|
|
|
|
movq RIP(%rsp), %rax
|
2014-11-25 02:26:38 +07:00
|
|
|
movq %rax, MCOUNT_REG_SIZE+8(%rsp)
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
/* restore the rest of pt_regs */
|
|
|
|
|
movq R15(%rsp), %r15
|
|
|
|
|
movq R14(%rsp), %r14
|
|
|
|
|
movq R13(%rsp), %r13
|
|
|
|
|
movq R12(%rsp), %r12
|
|
|
|
|
movq R10(%rsp), %r10
|
|
|
|
|
movq RBX(%rsp), %rbx
|
|
|
|
|
|
2014-11-25 01:06:05 +07:00
|
|
|
restore_mcount_regs
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
/* Restore flags */
|
|
|
|
|
popfq
|
|
|
|
|
|
ftrace/x86: Add dynamic allocated trampoline for ftrace_ops
The current method of handling multiple function callbacks is to register
a list function callback that calls all the other callbacks based on
their hash tables and compare it to the function that the callback was
called on. But this is very inefficient.
For example, if you are tracing all functions in the kernel and then
add a kprobe to a function such that the kprobe uses ftrace, the
mcount trampoline will switch from calling the function trace callback
to calling the list callback that will iterate over all registered
ftrace_ops (in this case, the function tracer and the kprobes callback).
That means for every function being traced it checks the hash of the
ftrace_ops for function tracing and kprobes, even though the kprobes
is only set at a single function. The kprobes ftrace_ops is checked
for every function being traced!
Instead of calling the list function for functions that are only being
traced by a single callback, we can call a dynamically allocated
trampoline that calls the callback directly. The function graph tracer
already uses a direct call trampoline when it is being traced by itself
but it is not dynamically allocated. It's trampoline is static in the
kernel core. The infrastructure that called the function graph trampoline
can also be used to call a dynamically allocated one.
For now, only ftrace_ops that are not dynamically allocated can have
a trampoline. That is, users such as function tracer or stack tracer.
kprobes and perf allocate their ftrace_ops, and until there's a safe
way to free the trampoline, it can not be used. The dynamically allocated
ftrace_ops may, although, use the trampoline if the kernel is not
compiled with CONFIG_PREEMPT. But that will come later.
Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Tested-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2014-07-03 10:23:31 +07:00
|
|
|
/*
|
2016-02-16 15:43:21 +07:00
|
|
|
* As this jmp to ftrace_epilogue can be a short jump
|
ftrace/x86: Add dynamic allocated trampoline for ftrace_ops
The current method of handling multiple function callbacks is to register
a list function callback that calls all the other callbacks based on
their hash tables and compare it to the function that the callback was
called on. But this is very inefficient.
For example, if you are tracing all functions in the kernel and then
add a kprobe to a function such that the kprobe uses ftrace, the
mcount trampoline will switch from calling the function trace callback
to calling the list callback that will iterate over all registered
ftrace_ops (in this case, the function tracer and the kprobes callback).
That means for every function being traced it checks the hash of the
ftrace_ops for function tracing and kprobes, even though the kprobes
is only set at a single function. The kprobes ftrace_ops is checked
for every function being traced!
Instead of calling the list function for functions that are only being
traced by a single callback, we can call a dynamically allocated
trampoline that calls the callback directly. The function graph tracer
already uses a direct call trampoline when it is being traced by itself
but it is not dynamically allocated. It's trampoline is static in the
kernel core. The infrastructure that called the function graph trampoline
can also be used to call a dynamically allocated one.
For now, only ftrace_ops that are not dynamically allocated can have
a trampoline. That is, users such as function tracer or stack tracer.
kprobes and perf allocate their ftrace_ops, and until there's a safe
way to free the trampoline, it can not be used. The dynamically allocated
ftrace_ops may, although, use the trampoline if the kernel is not
compiled with CONFIG_PREEMPT. But that will come later.
Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Tested-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2014-07-03 10:23:31 +07:00
|
|
|
* it must not be copied into the trampoline.
|
|
|
|
|
* The trampoline will add the code to jump
|
|
|
|
|
* to the return.
|
|
|
|
|
*/
|
|
|
|
|
GLOBAL(ftrace_regs_caller_end)
|
|
|
|
|
|
2016-02-16 15:43:21 +07:00
|
|
|
jmp ftrace_epilogue
|
2014-06-25 22:59:45 +07:00
|
|
|
|
2014-05-09 02:21:52 +07:00
|
|
|
END(ftrace_regs_caller)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#else /* ! CONFIG_DYNAMIC_FTRACE */
|
|
|
|
|
|
|
|
|
|
ENTRY(function_hook)
|
|
|
|
|
cmpq $ftrace_stub, ftrace_trace_function
|
|
|
|
|
jnz trace
|
|
|
|
|
|
2014-11-25 02:58:17 +07:00
|
|
|
fgraph_trace:
|
2014-05-09 02:21:52 +07:00
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
|
|
|
cmpq $ftrace_stub, ftrace_graph_return
|
|
|
|
|
jnz ftrace_graph_caller
|
|
|
|
|
|
|
|
|
|
cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
|
|
|
|
|
jnz ftrace_graph_caller
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
GLOBAL(ftrace_stub)
|
|
|
|
|
retq
|
|
|
|
|
|
|
|
|
|
trace:
|
2014-11-25 09:38:40 +07:00
|
|
|
/* save_mcount_regs fills in first two parameters */
|
|
|
|
|
save_mcount_regs
|
2014-05-09 02:21:52 +07:00
|
|
|
|
2015-11-17 07:43:24 +07:00
|
|
|
/*
|
|
|
|
|
* When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not
|
|
|
|
|
* set (see include/asm/ftrace.h and include/linux/ftrace.h). Only the
|
|
|
|
|
* ip and parent ip are used and the list function is called when
|
|
|
|
|
* function tracing is enabled.
|
|
|
|
|
*/
|
2014-05-09 02:21:52 +07:00
|
|
|
call *ftrace_trace_function
|
|
|
|
|
|
2014-11-24 23:43:39 +07:00
|
|
|
restore_mcount_regs
|
2014-05-09 02:21:52 +07:00
|
|
|
|
2014-11-25 02:58:17 +07:00
|
|
|
jmp fgraph_trace
|
2014-05-09 02:21:52 +07:00
|
|
|
END(function_hook)
|
|
|
|
|
#endif /* CONFIG_DYNAMIC_FTRACE */
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
|
|
|
ENTRY(ftrace_graph_caller)
|
2014-11-25 09:00:34 +07:00
|
|
|
/* Saves rbp into %rdx and fills first parameter */
|
2014-11-24 23:43:39 +07:00
|
|
|
save_mcount_regs
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
#ifdef CC_USING_FENTRY
|
2014-11-25 09:00:34 +07:00
|
|
|
leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
|
2014-05-09 02:21:52 +07:00
|
|
|
movq $0, %rdx /* No framepointers needed */
|
|
|
|
|
#else
|
2014-11-25 09:38:40 +07:00
|
|
|
/* Save address of the return address of traced function */
|
2014-11-25 09:00:34 +07:00
|
|
|
leaq 8(%rdx), %rsi
|
2014-11-25 09:38:40 +07:00
|
|
|
/* ftrace does sanity checks against frame pointers */
|
2014-11-25 06:08:48 +07:00
|
|
|
movq (%rdx), %rdx
|
2014-05-09 02:21:52 +07:00
|
|
|
#endif
|
|
|
|
|
call prepare_ftrace_return
|
|
|
|
|
|
2014-11-24 23:43:39 +07:00
|
|
|
restore_mcount_regs
|
2014-05-09 02:21:52 +07:00
|
|
|
|
|
|
|
|
retq
|
|
|
|
|
END(ftrace_graph_caller)
|
|
|
|
|
|
|
|
|
|
GLOBAL(return_to_handler)
|
|
|
|
|
subq $24, %rsp
|
|
|
|
|
|
|
|
|
|
/* Save the return values */
|
|
|
|
|
movq %rax, (%rsp)
|
|
|
|
|
movq %rdx, 8(%rsp)
|
|
|
|
|
movq %rbp, %rdi
|
|
|
|
|
|
|
|
|
|
call ftrace_return_to_handler
|
|
|
|
|
|
|
|
|
|
movq %rax, %rdi
|
|
|
|
|
movq 8(%rsp), %rdx
|
|
|
|
|
movq (%rsp), %rax
|
|
|
|
|
addq $24, %rsp
|
|
|
|
|
jmp *%rdi
|
|
|
|
|
#endif
|