2008-10-23 12:26:29 +07:00
|
|
|
#ifndef _ASM_X86_ALTERNATIVE_H
|
|
|
|
#define _ASM_X86_ALTERNATIVE_H
|
2008-01-30 19:30:30 +07:00
|
|
|
|
2016-04-27 02:23:25 +07:00
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
2008-01-30 19:30:30 +07:00
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/stddef.h>
|
2009-04-28 22:13:46 +07:00
|
|
|
#include <linux/stringify.h>
|
2008-01-30 19:30:30 +07:00
|
|
|
#include <asm/asm.h>
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Alternative inline assembly for SMP.
|
|
|
|
*
|
|
|
|
* The LOCK_PREFIX macro defined here replaces the LOCK and
|
|
|
|
* LOCK_PREFIX macros used everywhere in the source tree.
|
|
|
|
*
|
|
|
|
* SMP alternatives use the same data structures as the other
|
|
|
|
* alternatives and the X86_FEATURE_UP flag to indicate the case of a
|
|
|
|
* UP system running a SMP kernel. The existing apply_alternatives()
|
|
|
|
* works fine for patching a SMP kernel for UP.
|
|
|
|
*
|
|
|
|
* The SMP alternative tables can be kept after boot and contain both
|
|
|
|
* UP and SMP versions of the instructions to allow switching back to
|
|
|
|
* SMP at runtime, when hotplugging in a new CPU, which is especially
|
|
|
|
* useful in virtualized environments.
|
|
|
|
*
|
|
|
|
* The very common lock prefix is handled as special case in a
|
|
|
|
* separate table which is a pure address list without replacement ptr
|
|
|
|
* and size information. That keeps the table sizes small.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
2010-02-24 16:54:22 +07:00
|
|
|
#define LOCK_PREFIX_HERE \
|
2012-09-22 02:43:08 +07:00
|
|
|
".pushsection .smp_locks,\"a\"\n" \
|
|
|
|
".balign 4\n" \
|
|
|
|
".long 671f - .\n" /* offset */ \
|
|
|
|
".popsection\n" \
|
2010-02-24 16:54:22 +07:00
|
|
|
"671:"
|
|
|
|
|
|
|
|
#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
|
2008-01-30 19:30:30 +07:00
|
|
|
|
|
|
|
#else /* ! CONFIG_SMP */
|
2010-04-30 06:03:57 +07:00
|
|
|
#define LOCK_PREFIX_HERE ""
|
2008-01-30 19:30:30 +07:00
|
|
|
#define LOCK_PREFIX ""
|
|
|
|
#endif
|
|
|
|
|
|
|
|
struct alt_instr {
|
2011-07-13 20:24:10 +07:00
|
|
|
s32 instr_offset; /* original instruction */
|
|
|
|
s32 repl_offset; /* offset to replacement instruction */
|
2010-06-10 07:10:43 +07:00
|
|
|
u16 cpuid; /* cpuid bit set for replacement */
|
2008-01-30 19:30:30 +07:00
|
|
|
u8 instrlen; /* length of original instruction */
|
2014-12-27 16:41:52 +07:00
|
|
|
u8 replacementlen; /* length of new instruction */
|
|
|
|
u8 padlen; /* length of build-time padding */
|
|
|
|
} __packed;
|
2008-01-30 19:30:30 +07:00
|
|
|
|
2015-04-30 14:09:26 +07:00
|
|
|
/*
|
|
|
|
* Debug flag that can be tested to see whether alternative
|
|
|
|
* instructions were patched in already:
|
|
|
|
*/
|
|
|
|
extern int alternatives_patched;
|
|
|
|
|
2008-01-30 19:30:30 +07:00
|
|
|
extern void alternative_instructions(void);
|
|
|
|
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
|
|
|
|
|
|
|
|
struct module;
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
extern void alternatives_smp_module_add(struct module *mod, char *name,
|
|
|
|
void *locks, void *locks_end,
|
|
|
|
void *text, void *text_end);
|
|
|
|
extern void alternatives_smp_module_del(struct module *mod);
|
2012-08-06 14:59:49 +07:00
|
|
|
extern void alternatives_enable_smp(void);
|
2010-02-03 04:49:11 +07:00
|
|
|
extern int alternatives_text_reserved(void *start, void *end);
|
2010-11-24 07:11:40 +07:00
|
|
|
extern bool skip_smp_alternatives;
|
2008-01-30 19:30:30 +07:00
|
|
|
#else
|
|
|
|
static inline void alternatives_smp_module_add(struct module *mod, char *name,
|
2008-03-23 15:01:37 +07:00
|
|
|
void *locks, void *locks_end,
|
|
|
|
void *text, void *text_end) {}
|
2008-01-30 19:30:30 +07:00
|
|
|
static inline void alternatives_smp_module_del(struct module *mod) {}
|
2012-08-06 14:59:49 +07:00
|
|
|
static inline void alternatives_enable_smp(void) {}
|
2010-02-03 04:49:11 +07:00
|
|
|
static inline int alternatives_text_reserved(void *start, void *end)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2008-01-30 19:30:30 +07:00
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2014-12-27 16:41:52 +07:00
|
|
|
#define b_replacement(num) "664"#num
|
|
|
|
#define e_replacement(num) "665"#num
|
2012-05-25 08:19:45 +07:00
|
|
|
|
2014-12-27 16:41:52 +07:00
|
|
|
#define alt_end_marker "663"
|
|
|
|
#define alt_slen "662b-661b"
|
|
|
|
#define alt_pad_len alt_end_marker"b-662b"
|
|
|
|
#define alt_total_slen alt_end_marker"b-661b"
|
|
|
|
#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
|
2012-05-25 08:19:45 +07:00
|
|
|
|
2014-12-27 16:41:52 +07:00
|
|
|
#define __OLDINSTR(oldinstr, num) \
|
|
|
|
"661:\n\t" oldinstr "\n662:\n" \
|
|
|
|
".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
|
|
|
|
"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
|
2012-05-25 08:19:45 +07:00
|
|
|
|
2014-12-27 16:41:52 +07:00
|
|
|
#define OLDINSTR(oldinstr, num) \
|
|
|
|
__OLDINSTR(oldinstr, num) \
|
|
|
|
alt_end_marker ":\n"
|
|
|
|
|
x86/alternatives: Fix ALTERNATIVE_2 padding generation properly
Quentin caught a corner case with the generation of instruction
padding in the ALTERNATIVE_2 macro: if len(orig_insn) <
len(alt1) < len(alt2), then not enough padding gets added and
that is not good(tm) as we could overwrite the beginning of the
next instruction.
Luckily, at the time of this writing, we don't have
ALTERNATIVE_2() invocations which have that problem and even if
we did, a simple fix would be to prepend the instructions with
enough prefixes so that that corner case doesn't happen.
However, best it would be if we fixed it properly. See below for
a simple, abstracted example of what we're doing.
So what we ended up doing is, we compute the
max(len(alt1), len(alt2)) - len(orig_insn)
and feed that value to the .skip gas directive. The max() cannot
have conditionals due to gas limitations, thus the fancy integer
math.
With this patch, all ALTERNATIVE_2 sites get padded correctly;
generating obscure test cases pass too:
#define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
#define gen_skip(orig, alt1, alt2, marker) \
.skip -((alt_max_short(alt1, alt2) - (orig)) > 0) * \
(alt_max_short(alt1, alt2) - (orig)),marker
.pushsection .text, "ax"
.globl main
main:
gen_skip(1, 2, 4, 0x09)
gen_skip(4, 1, 2, 0x10)
...
.popsection
Thanks to Quentin for catching it and double-checking the fix!
Reported-by: Quentin Casasnovas <quentin.casasnovas@oracle.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20150404133443.GE21152@pd.tnic
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-04 20:34:43 +07:00
|
|
|
/*
|
|
|
|
* max without conditionals. Idea adapted from:
|
|
|
|
* http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
|
|
|
|
*
|
|
|
|
* The additional "-" is needed because gas works with s32s.
|
|
|
|
*/
|
|
|
|
#define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") - (" b ")))))"
|
|
|
|
|
2014-12-27 16:41:52 +07:00
|
|
|
/*
|
|
|
|
* Pad the second replacement alternative with additional NOPs if it is
|
|
|
|
* additionally longer than the first replacement alternative.
|
|
|
|
*/
|
x86/alternatives: Fix ALTERNATIVE_2 padding generation properly
Quentin caught a corner case with the generation of instruction
padding in the ALTERNATIVE_2 macro: if len(orig_insn) <
len(alt1) < len(alt2), then not enough padding gets added and
that is not good(tm) as we could overwrite the beginning of the
next instruction.
Luckily, at the time of this writing, we don't have
ALTERNATIVE_2() invocations which have that problem and even if
we did, a simple fix would be to prepend the instructions with
enough prefixes so that that corner case doesn't happen.
However, best it would be if we fixed it properly. See below for
a simple, abstracted example of what we're doing.
So what we ended up doing is, we compute the
max(len(alt1), len(alt2)) - len(orig_insn)
and feed that value to the .skip gas directive. The max() cannot
have conditionals due to gas limitations, thus the fancy integer
math.
With this patch, all ALTERNATIVE_2 sites get padded correctly;
generating obscure test cases pass too:
#define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
#define gen_skip(orig, alt1, alt2, marker) \
.skip -((alt_max_short(alt1, alt2) - (orig)) > 0) * \
(alt_max_short(alt1, alt2) - (orig)),marker
.pushsection .text, "ax"
.globl main
main:
gen_skip(1, 2, 4, 0x09)
gen_skip(4, 1, 2, 0x10)
...
.popsection
Thanks to Quentin for catching it and double-checking the fix!
Reported-by: Quentin Casasnovas <quentin.casasnovas@oracle.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20150404133443.GE21152@pd.tnic
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-04 20:34:43 +07:00
|
|
|
#define OLDINSTR_2(oldinstr, num1, num2) \
|
|
|
|
"661:\n\t" oldinstr "\n662:\n" \
|
|
|
|
".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \
|
|
|
|
"(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \
|
2014-12-27 16:41:52 +07:00
|
|
|
alt_end_marker ":\n"
|
|
|
|
|
|
|
|
#define ALTINSTR_ENTRY(feature, num) \
|
2012-05-25 08:19:45 +07:00
|
|
|
" .long 661b - .\n" /* label */ \
|
2014-12-27 16:41:52 +07:00
|
|
|
" .long " b_replacement(num)"f - .\n" /* new instruction */ \
|
2012-05-25 08:19:45 +07:00
|
|
|
" .word " __stringify(feature) "\n" /* feature bit */ \
|
2014-12-27 16:41:52 +07:00
|
|
|
" .byte " alt_total_slen "\n" /* source len */ \
|
|
|
|
" .byte " alt_rlen(num) "\n" /* replacement len */ \
|
|
|
|
" .byte " alt_pad_len "\n" /* pad len */
|
2012-05-25 08:19:45 +07:00
|
|
|
|
2014-12-27 16:41:52 +07:00
|
|
|
#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
|
|
|
|
b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
|
2012-05-25 08:19:45 +07:00
|
|
|
|
2009-04-28 22:13:46 +07:00
|
|
|
/* alternative assembly primitive: */
|
|
|
|
#define ALTERNATIVE(oldinstr, newinstr, feature) \
|
2014-12-27 16:41:52 +07:00
|
|
|
OLDINSTR(oldinstr, 1) \
|
2012-09-22 02:43:08 +07:00
|
|
|
".pushsection .altinstructions,\"a\"\n" \
|
2012-05-25 08:19:45 +07:00
|
|
|
ALTINSTR_ENTRY(feature, 1) \
|
2012-09-22 02:43:08 +07:00
|
|
|
".popsection\n" \
|
|
|
|
".pushsection .altinstr_replacement, \"ax\"\n" \
|
2012-05-25 08:19:45 +07:00
|
|
|
ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
|
2012-09-22 02:43:08 +07:00
|
|
|
".popsection"
|
2012-05-25 08:19:45 +07:00
|
|
|
|
|
|
|
#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
|
2014-12-27 16:41:52 +07:00
|
|
|
OLDINSTR_2(oldinstr, 1, 2) \
|
2012-09-22 02:43:08 +07:00
|
|
|
".pushsection .altinstructions,\"a\"\n" \
|
2012-05-25 08:19:45 +07:00
|
|
|
ALTINSTR_ENTRY(feature1, 1) \
|
|
|
|
ALTINSTR_ENTRY(feature2, 2) \
|
2012-09-22 02:43:08 +07:00
|
|
|
".popsection\n" \
|
|
|
|
".pushsection .altinstr_replacement, \"ax\"\n" \
|
2012-05-25 08:19:45 +07:00
|
|
|
ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
|
|
|
|
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
|
2012-09-22 02:43:08 +07:00
|
|
|
".popsection"
|
2009-04-28 22:13:46 +07:00
|
|
|
|
2008-01-30 19:30:30 +07:00
|
|
|
/*
|
|
|
|
* Alternative instructions for different CPU types or capabilities.
|
|
|
|
*
|
|
|
|
* This allows to use optimized instructions even on generic binary
|
|
|
|
* kernels.
|
|
|
|
*
|
|
|
|
* length of oldinstr must be longer or equal the length of newinstr
|
|
|
|
* It can be padded with nops as needed.
|
|
|
|
*
|
|
|
|
* For non barrier like inlines please define new variants
|
|
|
|
* without volatile and memory clobber.
|
|
|
|
*/
|
|
|
|
#define alternative(oldinstr, newinstr, feature) \
|
2009-04-28 22:13:46 +07:00
|
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
|
2008-01-30 19:30:30 +07:00
|
|
|
|
2014-12-27 16:41:52 +07:00
|
|
|
#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
|
|
|
|
asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
|
|
|
|
|
2008-01-30 19:30:30 +07:00
|
|
|
/*
|
|
|
|
* Alternative inline assembly with input.
|
|
|
|
*
|
|
|
|
* Pecularities:
|
|
|
|
* No memory clobber here.
|
|
|
|
* Argument numbers start with 1.
|
|
|
|
* Best is to use constraints that are fixed size (like (%1) ... "r")
|
|
|
|
* If you use variable sized constraints like "m" or "g" in the
|
|
|
|
* replacement make sure to pad to the worst case length.
|
2009-04-28 22:13:46 +07:00
|
|
|
* Leaving an unused argument 0 to keep API compatibility.
|
2008-01-30 19:30:30 +07:00
|
|
|
*/
|
|
|
|
#define alternative_input(oldinstr, newinstr, feature, input...) \
|
2009-04-28 22:13:46 +07:00
|
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
|
|
: : "i" (0), ## input)
|
2008-01-30 19:30:30 +07:00
|
|
|
|
x86/alternative: Add alternative_input_2 to support alternative with two features and input
alternative_input_2() replaces old instruction with new instructions with
input based on two features.
In alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, feature2,
input...),
feature2 has higher priority to replace oldinstr than feature1.
If CPU has feature2, newinstr2 replaces oldinstr and newinstr2 is
executed during run time.
If CPU doesn't have feature2, but it has feature1, newinstr1 replaces oldinstr
and newinstr1 is executed during run time.
If CPU doesn't have feature2 and feature1, oldinstr is executed during run
time.
Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Link: http://lkml.kernel.org/r/1401387164-43416-5-git-send-email-fenghua.yu@intel.com
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2014-05-30 01:12:32 +07:00
|
|
|
/*
|
|
|
|
* This is similar to alternative_input. But it has two features and
|
|
|
|
* respective instructions.
|
|
|
|
*
|
|
|
|
* If CPU has feature2, newinstr2 is used.
|
|
|
|
* Otherwise, if CPU has feature1, newinstr1 is used.
|
|
|
|
* Otherwise, oldinstr is used.
|
|
|
|
*/
|
|
|
|
#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \
|
|
|
|
feature2, input...) \
|
|
|
|
asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \
|
|
|
|
newinstr2, feature2) \
|
|
|
|
: : "i" (0), ## input)
|
|
|
|
|
2008-01-30 19:30:30 +07:00
|
|
|
/* Like alternative_input, but with a single output argument */
|
|
|
|
#define alternative_io(oldinstr, newinstr, feature, output, input...) \
|
2009-04-28 22:13:46 +07:00
|
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
|
|
: output : "i" (0), ## input)
|
2008-01-30 19:30:30 +07:00
|
|
|
|
2009-12-18 23:12:56 +07:00
|
|
|
/* Like alternative_io, but for replacing a direct call with another one. */
|
|
|
|
#define alternative_call(oldfunc, newfunc, feature, output, input...) \
|
|
|
|
asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
|
|
|
|
: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
|
|
|
|
|
2012-05-25 08:19:45 +07:00
|
|
|
/*
|
|
|
|
* Like alternative_call, but there are two features and respective functions.
|
|
|
|
* If CPU has feature2, function2 is used.
|
|
|
|
* Otherwise, if CPU has feature1, function1 is used.
|
|
|
|
* Otherwise, old function is used.
|
|
|
|
*/
|
|
|
|
#define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \
|
|
|
|
output, input...) \
|
2016-09-24 04:49:39 +07:00
|
|
|
{ \
|
|
|
|
register void *__sp asm(_ASM_SP); \
|
2012-05-25 08:19:45 +07:00
|
|
|
asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
|
|
|
|
"call %P[new2]", feature2) \
|
2016-09-24 04:49:39 +07:00
|
|
|
: output, "+r" (__sp) \
|
|
|
|
: [old] "i" (oldfunc), [new1] "i" (newfunc1), \
|
|
|
|
[new2] "i" (newfunc2), ## input); \
|
|
|
|
}
|
2012-05-25 08:19:45 +07:00
|
|
|
|
2008-01-30 19:30:30 +07:00
|
|
|
/*
|
|
|
|
* use this macro(s) if you need more than one output parameter
|
|
|
|
* in alternative_io
|
|
|
|
*/
|
2009-12-18 23:12:56 +07:00
|
|
|
#define ASM_OUTPUT2(a...) a
|
2008-01-30 19:30:30 +07:00
|
|
|
|
2012-01-20 23:21:41 +07:00
|
|
|
/*
|
|
|
|
* use this macro if you need clobbers but no inputs in
|
|
|
|
* alternative_{input,io,call}()
|
|
|
|
*/
|
|
|
|
#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
|
|
|
|
|
2016-04-27 02:23:25 +07:00
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
|
2008-10-23 12:26:29 +07:00
|
|
|
#endif /* _ASM_X86_ALTERNATIVE_H */
|