mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-28 11:18:45 +07:00
77f48ec28e
As described in:
77b0bf55bc: ("kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs")
GCC's inlining heuristics are broken with common asm() patterns used in
kernel code, resulting in the effective disabling of inlining.
The workaround is to set an assembly macro and call it from the inline
assembly block - i.e. to macrify the affected block.
As a result GCC considers the inline assembly block as a single instruction.
This patch handles the LOCK prefix, allowing more aggresive inlining:
text data bss dec hex filename
18140140 10225284 2957312 31322736 1ddf270 ./vmlinux before
18146889 10225380 2957312 31329581 1de0d2d ./vmlinux after (+6845)
This is the reduction in non-inlined functions:
Before: 40286
After: 40218 (-68)
Tested-by: Kees Cook <keescook@chromium.org>
Signed-off-by: Nadav Amit <namit@vmware.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20181003213100.189959-6-namit@vmware.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
112 lines
2.5 KiB
C
112 lines
2.5 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_X86_ALTERNATIVE_ASM_H
|
|
#define _ASM_X86_ALTERNATIVE_ASM_H
|
|
|
|
#ifdef __ASSEMBLY__
|
|
|
|
#include <asm/asm.h>
|
|
|
|
#ifdef CONFIG_SMP
|
|
.macro LOCK_PREFIX_HERE
|
|
.pushsection .smp_locks,"a"
|
|
.balign 4
|
|
.long 671f - . # offset
|
|
.popsection
|
|
671:
|
|
.endm
|
|
|
|
.macro LOCK_PREFIX insn:vararg
|
|
LOCK_PREFIX_HERE
|
|
lock \insn
|
|
.endm
|
|
#else
|
|
.macro LOCK_PREFIX_HERE
|
|
.endm
|
|
|
|
.macro LOCK_PREFIX insn:vararg
|
|
.endm
|
|
#endif
|
|
|
|
/*
|
|
* Issue one struct alt_instr descriptor entry (need to put it into
|
|
* the section .altinstructions, see below). This entry contains
|
|
* enough information for the alternatives patching code to patch an
|
|
* instruction. See apply_alternatives().
|
|
*/
|
|
.macro altinstruction_entry orig alt feature orig_len alt_len pad_len
|
|
.long \orig - .
|
|
.long \alt - .
|
|
.word \feature
|
|
.byte \orig_len
|
|
.byte \alt_len
|
|
.byte \pad_len
|
|
.endm
|
|
|
|
/*
|
|
* Define an alternative between two instructions. If @feature is
|
|
* present, early code in apply_alternatives() replaces @oldinstr with
|
|
* @newinstr. ".skip" directive takes care of proper instruction padding
|
|
* in case @newinstr is longer than @oldinstr.
|
|
*/
|
|
.macro ALTERNATIVE oldinstr, newinstr, feature
|
|
140:
|
|
\oldinstr
|
|
141:
|
|
.skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
|
|
142:
|
|
|
|
.pushsection .altinstructions,"a"
|
|
altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f,142b-141b
|
|
.popsection
|
|
|
|
.pushsection .altinstr_replacement,"ax"
|
|
143:
|
|
\newinstr
|
|
144:
|
|
.popsection
|
|
.endm
|
|
|
|
#define old_len 141b-140b
|
|
#define new_len1 144f-143f
|
|
#define new_len2 145f-144f
|
|
|
|
/*
|
|
* gas compatible max based on the idea from:
|
|
* http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
|
|
*
|
|
* The additional "-" is needed because gas uses a "true" value of -1.
|
|
*/
|
|
#define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
|
|
|
|
|
|
/*
|
|
* Same as ALTERNATIVE macro above but for two alternatives. If CPU
|
|
* has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
|
|
* @feature2, it replaces @oldinstr with @feature2.
|
|
*/
|
|
.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
|
|
140:
|
|
\oldinstr
|
|
141:
|
|
.skip -((alt_max_short(new_len1, new_len2) - (old_len)) > 0) * \
|
|
(alt_max_short(new_len1, new_len2) - (old_len)),0x90
|
|
142:
|
|
|
|
.pushsection .altinstructions,"a"
|
|
altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f,142b-141b
|
|
altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f,142b-141b
|
|
.popsection
|
|
|
|
.pushsection .altinstr_replacement,"ax"
|
|
143:
|
|
\newinstr1
|
|
144:
|
|
\newinstr2
|
|
145:
|
|
.popsection
|
|
.endm
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* _ASM_X86_ALTERNATIVE_ASM_H */
|