mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-12 18:46:46 +07:00
946c191161
With frame pointers, when a task is interrupted, its stack is no longer completely reliable because the function could have been interrupted before it had a chance to save the previous frame pointer on the stack. So the caller of the interrupted function could get skipped by a stack trace. This is problematic for live patching, which needs to know whether a stack trace of a sleeping task can be relied upon. There's currently no way to detect if a sleeping task was interrupted by a page fault exception or preemption before it went to sleep. Another issue is that when dumping the stack of an interrupted task, the unwinder has no way of knowing where the saved pt_regs registers are, so it can't print them. This solves those issues by encoding the pt_regs pointer in the frame pointer on entry from an interrupt or an exception. This patch also updates the unwinder to be able to decode it, because otherwise the unwinder would be broken by this change. Note that this causes a change in the behavior of the unwinder: each instance of a pt_regs on the stack is now considered a "frame". So callers of unwind_get_return_address() will now get an occasional 'regs->ip' address that would have previously been skipped over. Suggested-by: Andy Lutomirski <luto@amacapital.net> Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Brian Gerst <brgerst@gmail.com> Cc: Denys Vlasenko <dvlasenk@redhat.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/8b9f84a21e39d249049e0547b559ff8da0df0988.1476973742.git.jpoimboe@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
230 lines
6.4 KiB
C
230 lines
6.4 KiB
C
#include <linux/jump_label.h>
|
|
|
|
/*
|
|
|
|
x86 function call convention, 64-bit:
|
|
-------------------------------------
|
|
arguments | callee-saved | extra caller-saved | return
|
|
[callee-clobbered] | | [callee-clobbered] |
|
|
---------------------------------------------------------------------------
|
|
rdi rsi rdx rcx r8-9 | rbx rbp [*] r12-15 | r10-11 | rax, rdx [**]
|
|
|
|
( rsp is obviously invariant across normal function calls. (gcc can 'merge'
|
|
functions when it sees tail-call optimization possibilities) rflags is
|
|
clobbered. Leftover arguments are passed over the stack frame.)
|
|
|
|
[*] In the frame-pointers case rbp is fixed to the stack frame.
|
|
|
|
[**] for struct return values wider than 64 bits the return convention is a
|
|
bit more complex: up to 128 bits width we return small structures
|
|
straight in rax, rdx. For structures larger than that (3 words or
|
|
larger) the caller puts a pointer to an on-stack return struct
|
|
[allocated in the caller's stack frame] into the first argument - i.e.
|
|
into rdi. All other arguments shift up by one in this case.
|
|
Fortunately this case is rare in the kernel.
|
|
|
|
For 32-bit we have the following conventions - kernel is built with
|
|
-mregparm=3 and -freg-struct-return:
|
|
|
|
x86 function calling convention, 32-bit:
|
|
----------------------------------------
|
|
arguments | callee-saved | extra caller-saved | return
|
|
[callee-clobbered] | | [callee-clobbered] |
|
|
-------------------------------------------------------------------------
|
|
eax edx ecx | ebx edi esi ebp [*] | <none> | eax, edx [**]
|
|
|
|
( here too esp is obviously invariant across normal function calls. eflags
|
|
is clobbered. Leftover arguments are passed over the stack frame. )
|
|
|
|
[*] In the frame-pointers case ebp is fixed to the stack frame.
|
|
|
|
[**] We build with -freg-struct-return, which on 32-bit means similar
|
|
semantics as on 64-bit: edx can be used for a second return value
|
|
(i.e. covering integer and structure sizes up to 64 bits) - after that
|
|
it gets more complex and more expensive: 3-word or larger struct returns
|
|
get done in the caller's frame and the pointer to the return struct goes
|
|
into regparm0, i.e. eax - the other arguments shift up and the
|
|
function's register parameters degenerate to regparm=2 in essence.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
/*
|
|
* 64-bit system call stack frame layout defines and helpers,
|
|
* for assembly code:
|
|
*/
|
|
|
|
/* The layout forms the "struct pt_regs" on the stack: */
|
|
/*
|
|
* C ABI says these regs are callee-preserved. They aren't saved on kernel entry
|
|
* unless syscall needs a complete, fully filled "struct pt_regs".
|
|
*/
|
|
#define R15 0*8
|
|
#define R14 1*8
|
|
#define R13 2*8
|
|
#define R12 3*8
|
|
#define RBP 4*8
|
|
#define RBX 5*8
|
|
/* These regs are callee-clobbered. Always saved on kernel entry. */
|
|
#define R11 6*8
|
|
#define R10 7*8
|
|
#define R9 8*8
|
|
#define R8 9*8
|
|
#define RAX 10*8
|
|
#define RCX 11*8
|
|
#define RDX 12*8
|
|
#define RSI 13*8
|
|
#define RDI 14*8
|
|
/*
|
|
* On syscall entry, this is syscall#. On CPU exception, this is error code.
|
|
* On hw interrupt, it's IRQ number:
|
|
*/
|
|
#define ORIG_RAX 15*8
|
|
/* Return frame for iretq */
|
|
#define RIP 16*8
|
|
#define CS 17*8
|
|
#define EFLAGS 18*8
|
|
#define RSP 19*8
|
|
#define SS 20*8
|
|
|
|
#define SIZEOF_PTREGS 21*8
|
|
|
|
.macro ALLOC_PT_GPREGS_ON_STACK
|
|
addq $-(15*8), %rsp
|
|
.endm
|
|
|
|
.macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1
|
|
.if \r11
|
|
movq %r11, 6*8+\offset(%rsp)
|
|
.endif
|
|
.if \r8910
|
|
movq %r10, 7*8+\offset(%rsp)
|
|
movq %r9, 8*8+\offset(%rsp)
|
|
movq %r8, 9*8+\offset(%rsp)
|
|
.endif
|
|
.if \rax
|
|
movq %rax, 10*8+\offset(%rsp)
|
|
.endif
|
|
.if \rcx
|
|
movq %rcx, 11*8+\offset(%rsp)
|
|
.endif
|
|
movq %rdx, 12*8+\offset(%rsp)
|
|
movq %rsi, 13*8+\offset(%rsp)
|
|
movq %rdi, 14*8+\offset(%rsp)
|
|
.endm
|
|
.macro SAVE_C_REGS offset=0
|
|
SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
|
|
.endm
|
|
.macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0
|
|
SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1
|
|
.endm
|
|
.macro SAVE_C_REGS_EXCEPT_R891011
|
|
SAVE_C_REGS_HELPER 0, 1, 1, 0, 0
|
|
.endm
|
|
.macro SAVE_C_REGS_EXCEPT_RCX_R891011
|
|
SAVE_C_REGS_HELPER 0, 1, 0, 0, 0
|
|
.endm
|
|
.macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11
|
|
SAVE_C_REGS_HELPER 0, 0, 0, 1, 0
|
|
.endm
|
|
|
|
.macro SAVE_EXTRA_REGS offset=0
|
|
movq %r15, 0*8+\offset(%rsp)
|
|
movq %r14, 1*8+\offset(%rsp)
|
|
movq %r13, 2*8+\offset(%rsp)
|
|
movq %r12, 3*8+\offset(%rsp)
|
|
movq %rbp, 4*8+\offset(%rsp)
|
|
movq %rbx, 5*8+\offset(%rsp)
|
|
.endm
|
|
|
|
.macro RESTORE_EXTRA_REGS offset=0
|
|
movq 0*8+\offset(%rsp), %r15
|
|
movq 1*8+\offset(%rsp), %r14
|
|
movq 2*8+\offset(%rsp), %r13
|
|
movq 3*8+\offset(%rsp), %r12
|
|
movq 4*8+\offset(%rsp), %rbp
|
|
movq 5*8+\offset(%rsp), %rbx
|
|
.endm
|
|
|
|
.macro RESTORE_C_REGS_HELPER rstor_rax=1, rstor_rcx=1, rstor_r11=1, rstor_r8910=1, rstor_rdx=1
|
|
.if \rstor_r11
|
|
movq 6*8(%rsp), %r11
|
|
.endif
|
|
.if \rstor_r8910
|
|
movq 7*8(%rsp), %r10
|
|
movq 8*8(%rsp), %r9
|
|
movq 9*8(%rsp), %r8
|
|
.endif
|
|
.if \rstor_rax
|
|
movq 10*8(%rsp), %rax
|
|
.endif
|
|
.if \rstor_rcx
|
|
movq 11*8(%rsp), %rcx
|
|
.endif
|
|
.if \rstor_rdx
|
|
movq 12*8(%rsp), %rdx
|
|
.endif
|
|
movq 13*8(%rsp), %rsi
|
|
movq 14*8(%rsp), %rdi
|
|
.endm
|
|
.macro RESTORE_C_REGS
|
|
RESTORE_C_REGS_HELPER 1,1,1,1,1
|
|
.endm
|
|
.macro RESTORE_C_REGS_EXCEPT_RAX
|
|
RESTORE_C_REGS_HELPER 0,1,1,1,1
|
|
.endm
|
|
.macro RESTORE_C_REGS_EXCEPT_RCX
|
|
RESTORE_C_REGS_HELPER 1,0,1,1,1
|
|
.endm
|
|
.macro RESTORE_C_REGS_EXCEPT_R11
|
|
RESTORE_C_REGS_HELPER 1,1,0,1,1
|
|
.endm
|
|
.macro RESTORE_C_REGS_EXCEPT_RCX_R11
|
|
RESTORE_C_REGS_HELPER 1,0,0,1,1
|
|
.endm
|
|
|
|
.macro REMOVE_PT_GPREGS_FROM_STACK addskip=0
|
|
subq $-(15*8+\addskip), %rsp
|
|
.endm
|
|
|
|
.macro icebp
|
|
.byte 0xf1
|
|
.endm
|
|
|
|
/*
|
|
* This is a sneaky trick to help the unwinder find pt_regs on the stack. The
|
|
* frame pointer is replaced with an encoded pointer to pt_regs. The encoding
|
|
* is just setting the LSB, which makes it an invalid stack address and is also
|
|
* a signal to the unwinder that it's a pt_regs pointer in disguise.
|
|
*
|
|
* NOTE: This macro must be used *after* SAVE_EXTRA_REGS because it corrupts
|
|
* the original rbp.
|
|
*/
|
|
.macro ENCODE_FRAME_POINTER ptregs_offset=0
|
|
#ifdef CONFIG_FRAME_POINTER
|
|
.if \ptregs_offset
|
|
leaq \ptregs_offset(%rsp), %rbp
|
|
.else
|
|
mov %rsp, %rbp
|
|
.endif
|
|
orq $0x1, %rbp
|
|
#endif
|
|
.endm
|
|
|
|
#endif /* CONFIG_X86_64 */
|
|
|
|
/*
|
|
* This does 'call enter_from_user_mode' unless we can avoid it based on
|
|
* kernel config or using the static jump infrastructure.
|
|
*/
|
|
.macro CALL_enter_from_user_mode
|
|
#ifdef CONFIG_CONTEXT_TRACKING
|
|
#ifdef HAVE_JUMP_LABEL
|
|
STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0
|
|
#endif
|
|
call enter_from_user_mode
|
|
.Lafter_call_\@:
|
|
#endif
|
|
.endm
|