mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-28 11:18:45 +07:00
0474d5d9d2
As described in:
77b0bf55bc: ("kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs")
GCC's inlining heuristics are broken with common asm() patterns used in
kernel code, resulting in the effective disabling of inlining.
The workaround is to set an assembly macro and call it from the inline
assembly block - which is also a minor cleanup for the exception table
code.
Text size goes up a bit:
text data bss dec hex filename
18162555 10226288 2957312 31346155 1de4deb ./vmlinux before
18162879 10226256 2957312 31346447 1de4f0f ./vmlinux after (+292)
But this allows the inlining of functions such as nested_vmx_exit_reflected(),
set_segment_reg(), __copy_xstate_to_user() which is a net benefit.
Tested-by: Kees Cook <keescook@chromium.org>
Signed-off-by: Nadav Amit <namit@vmware.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20181005202718.229565-2-namit@vmware.com
Link: https://lore.kernel.org/lkml/20181003213100.189959-9-namit@vmware.com/T/#u
Signed-off-by: Ingo Molnar <mingo@kernel.org>
202 lines
4.7 KiB
C
202 lines
4.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_X86_ASM_H
|
|
#define _ASM_X86_ASM_H
|
|
|
|
#ifdef __ASSEMBLY__
|
|
# define __ASM_FORM(x) x
|
|
# define __ASM_FORM_RAW(x) x
|
|
# define __ASM_FORM_COMMA(x) x,
|
|
#else
|
|
# define __ASM_FORM(x) " " #x " "
|
|
# define __ASM_FORM_RAW(x) #x
|
|
# define __ASM_FORM_COMMA(x) " " #x ","
|
|
#endif
|
|
|
|
#ifndef __x86_64__
|
|
/* 32 bit */
|
|
# define __ASM_SEL(a,b) __ASM_FORM(a)
|
|
# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(a)
|
|
#else
|
|
/* 64 bit */
|
|
# define __ASM_SEL(a,b) __ASM_FORM(b)
|
|
# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(b)
|
|
#endif
|
|
|
|
#define __ASM_SIZE(inst, ...) __ASM_SEL(inst##l##__VA_ARGS__, \
|
|
inst##q##__VA_ARGS__)
|
|
#define __ASM_REG(reg) __ASM_SEL_RAW(e##reg, r##reg)
|
|
|
|
#define _ASM_PTR __ASM_SEL(.long, .quad)
|
|
#define _ASM_ALIGN __ASM_SEL(.balign 4, .balign 8)
|
|
|
|
#define _ASM_MOV __ASM_SIZE(mov)
|
|
#define _ASM_INC __ASM_SIZE(inc)
|
|
#define _ASM_DEC __ASM_SIZE(dec)
|
|
#define _ASM_ADD __ASM_SIZE(add)
|
|
#define _ASM_SUB __ASM_SIZE(sub)
|
|
#define _ASM_XADD __ASM_SIZE(xadd)
|
|
#define _ASM_MUL __ASM_SIZE(mul)
|
|
|
|
#define _ASM_AX __ASM_REG(ax)
|
|
#define _ASM_BX __ASM_REG(bx)
|
|
#define _ASM_CX __ASM_REG(cx)
|
|
#define _ASM_DX __ASM_REG(dx)
|
|
#define _ASM_SP __ASM_REG(sp)
|
|
#define _ASM_BP __ASM_REG(bp)
|
|
#define _ASM_SI __ASM_REG(si)
|
|
#define _ASM_DI __ASM_REG(di)
|
|
|
|
#ifndef __x86_64__
|
|
/* 32 bit */
|
|
|
|
#define _ASM_ARG1 _ASM_AX
|
|
#define _ASM_ARG2 _ASM_DX
|
|
#define _ASM_ARG3 _ASM_CX
|
|
|
|
#define _ASM_ARG1L eax
|
|
#define _ASM_ARG2L edx
|
|
#define _ASM_ARG3L ecx
|
|
|
|
#define _ASM_ARG1W ax
|
|
#define _ASM_ARG2W dx
|
|
#define _ASM_ARG3W cx
|
|
|
|
#define _ASM_ARG1B al
|
|
#define _ASM_ARG2B dl
|
|
#define _ASM_ARG3B cl
|
|
|
|
#else
|
|
/* 64 bit */
|
|
|
|
#define _ASM_ARG1 _ASM_DI
|
|
#define _ASM_ARG2 _ASM_SI
|
|
#define _ASM_ARG3 _ASM_DX
|
|
#define _ASM_ARG4 _ASM_CX
|
|
#define _ASM_ARG5 r8
|
|
#define _ASM_ARG6 r9
|
|
|
|
#define _ASM_ARG1Q rdi
|
|
#define _ASM_ARG2Q rsi
|
|
#define _ASM_ARG3Q rdx
|
|
#define _ASM_ARG4Q rcx
|
|
#define _ASM_ARG5Q r8
|
|
#define _ASM_ARG6Q r9
|
|
|
|
#define _ASM_ARG1L edi
|
|
#define _ASM_ARG2L esi
|
|
#define _ASM_ARG3L edx
|
|
#define _ASM_ARG4L ecx
|
|
#define _ASM_ARG5L r8d
|
|
#define _ASM_ARG6L r9d
|
|
|
|
#define _ASM_ARG1W di
|
|
#define _ASM_ARG2W si
|
|
#define _ASM_ARG3W dx
|
|
#define _ASM_ARG4W cx
|
|
#define _ASM_ARG5W r8w
|
|
#define _ASM_ARG6W r9w
|
|
|
|
#define _ASM_ARG1B dil
|
|
#define _ASM_ARG2B sil
|
|
#define _ASM_ARG3B dl
|
|
#define _ASM_ARG4B cl
|
|
#define _ASM_ARG5B r8b
|
|
#define _ASM_ARG6B r9b
|
|
|
|
#endif
|
|
|
|
/*
|
|
* Macros to generate condition code outputs from inline assembly,
|
|
* The output operand must be type "bool".
|
|
*/
|
|
#ifdef __GCC_ASM_FLAG_OUTPUTS__
|
|
# define CC_SET(c) "\n\t/* output condition code " #c "*/\n"
|
|
# define CC_OUT(c) "=@cc" #c
|
|
#else
|
|
# define CC_SET(c) "\n\tset" #c " %[_cc_" #c "]\n"
|
|
# define CC_OUT(c) [_cc_ ## c] "=qm"
|
|
#endif
|
|
|
|
/* Exception table entry */
|
|
#ifdef __ASSEMBLY__
|
|
# define _ASM_EXTABLE_HANDLE(from, to, handler) \
|
|
ASM_EXTABLE_HANDLE from to handler
|
|
|
|
.macro ASM_EXTABLE_HANDLE from:req to:req handler:req
|
|
.pushsection "__ex_table","a"
|
|
.balign 4
|
|
.long (\from) - .
|
|
.long (\to) - .
|
|
.long (\handler) - .
|
|
.popsection
|
|
.endm
|
|
#else /* __ASSEMBLY__ */
|
|
|
|
# define _ASM_EXTABLE_HANDLE(from, to, handler) \
|
|
"ASM_EXTABLE_HANDLE from=" #from " to=" #to \
|
|
" handler=\"" #handler "\"\n\t"
|
|
|
|
/* For C file, we already have NOKPROBE_SYMBOL macro */
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
# define _ASM_EXTABLE(from, to) \
|
|
_ASM_EXTABLE_HANDLE(from, to, ex_handler_default)
|
|
|
|
# define _ASM_EXTABLE_UA(from, to) \
|
|
_ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess)
|
|
|
|
# define _ASM_EXTABLE_FAULT(from, to) \
|
|
_ASM_EXTABLE_HANDLE(from, to, ex_handler_fault)
|
|
|
|
# define _ASM_EXTABLE_EX(from, to) \
|
|
_ASM_EXTABLE_HANDLE(from, to, ex_handler_ext)
|
|
|
|
# define _ASM_EXTABLE_REFCOUNT(from, to) \
|
|
_ASM_EXTABLE_HANDLE(from, to, ex_handler_refcount)
|
|
|
|
# define _ASM_NOKPROBE(entry) \
|
|
.pushsection "_kprobe_blacklist","aw" ; \
|
|
_ASM_ALIGN ; \
|
|
_ASM_PTR (entry); \
|
|
.popsection
|
|
|
|
#ifdef __ASSEMBLY__
|
|
.macro ALIGN_DESTINATION
|
|
/* check for bad alignment of destination */
|
|
movl %edi,%ecx
|
|
andl $7,%ecx
|
|
jz 102f /* already aligned */
|
|
subl $8,%ecx
|
|
negl %ecx
|
|
subl %ecx,%edx
|
|
100: movb (%rsi),%al
|
|
101: movb %al,(%rdi)
|
|
incq %rsi
|
|
incq %rdi
|
|
decl %ecx
|
|
jnz 100b
|
|
102:
|
|
.section .fixup,"ax"
|
|
103: addl %ecx,%edx /* ecx is zerorest also */
|
|
jmp copy_user_handle_tail
|
|
.previous
|
|
|
|
_ASM_EXTABLE_UA(100b, 103b)
|
|
_ASM_EXTABLE_UA(101b, 103b)
|
|
.endm
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#ifndef __ASSEMBLY__
|
|
/*
|
|
* This output constraint should be used for any inline asm which has a "call"
|
|
* instruction. Otherwise the asm may be inserted before the frame pointer
|
|
* gets set up by the containing function. If you forget to do this, objtool
|
|
* may print a "call without frame pointer save/setup" warning.
|
|
*/
|
|
register unsigned long current_stack_pointer asm(_ASM_SP);
|
|
#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
|
|
#endif
|
|
|
|
#endif /* _ASM_X86_ASM_H */
|