x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
|
|
2018-01-26 19:11:37 +07:00
|
|
|
#ifndef _ASM_X86_NOSPEC_BRANCH_H_
|
|
|
|
|
#define _ASM_X86_NOSPEC_BRANCH_H_
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
|
|
|
|
|
#include <asm/alternative.h>
|
|
|
|
|
#include <asm/alternative-asm.h>
|
|
|
|
|
#include <asm/cpufeatures.h>
|
2018-02-13 20:28:19 +07:00
|
|
|
#include <asm/msr-index.h>
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
|
2018-02-19 17:50:56 +07:00
|
|
|
/*
|
|
|
|
|
* Fill the CPU return stack buffer.
|
|
|
|
|
*
|
|
|
|
|
* Each entry in the RSB, if used for a speculative 'ret', contains an
|
|
|
|
|
* infinite 'pause; lfence; jmp' loop to capture speculative execution.
|
|
|
|
|
*
|
|
|
|
|
* This is required in various cases for retpoline and IBRS-based
|
|
|
|
|
* mitigations for the Spectre variant 2 vulnerability. Sometimes to
|
|
|
|
|
* eliminate potentially bogus entries from the RSB, and sometimes
|
|
|
|
|
* purely to ensure that it doesn't get empty, which on some CPUs would
|
|
|
|
|
* allow predictions from other (unwanted!) sources to be used.
|
|
|
|
|
*
|
|
|
|
|
* We define a CPP macro such that it can be used from both .S files and
|
|
|
|
|
* inline assembly. It's possible to do a .macro and then include that
|
|
|
|
|
* from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
|
|
|
|
|
#define RSB_FILL_LOOPS 16 /* To avoid underflow */
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Google experimented with loop-unrolling and this turned out to be
|
|
|
|
|
* the optimal version — two calls, each with their own speculation
|
|
|
|
|
* trap should their return address end up getting used, in a loop.
|
|
|
|
|
*/
|
|
|
|
|
#define __FILL_RETURN_BUFFER(reg, nr, sp) \
|
|
|
|
|
mov $(nr/2), reg; \
|
|
|
|
|
771: \
|
|
|
|
|
call 772f; \
|
|
|
|
|
773: /* speculation trap */ \
|
|
|
|
|
pause; \
|
|
|
|
|
lfence; \
|
|
|
|
|
jmp 773b; \
|
|
|
|
|
772: \
|
|
|
|
|
call 774f; \
|
|
|
|
|
775: /* speculation trap */ \
|
|
|
|
|
pause; \
|
|
|
|
|
lfence; \
|
|
|
|
|
jmp 775b; \
|
|
|
|
|
774: \
|
|
|
|
|
dec reg; \
|
|
|
|
|
jnz 771b; \
|
|
|
|
|
add $(BITS_PER_LONG/8) * nr, sp;
|
|
|
|
|
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
#ifdef __ASSEMBLY__
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This should be used immediately before a retpoline alternative. It tells
|
|
|
|
|
* objtool where the retpolines are so that it can make sense of the control
|
|
|
|
|
* flow by just reading the original instruction(s) and ignoring the
|
|
|
|
|
* alternatives.
|
|
|
|
|
*/
|
|
|
|
|
.macro ANNOTATE_NOSPEC_ALTERNATIVE
|
|
|
|
|
.Lannotate_\@:
|
|
|
|
|
.pushsection .discard.nospec
|
|
|
|
|
.long .Lannotate_\@ - .
|
|
|
|
|
.popsection
|
|
|
|
|
.endm
|
|
|
|
|
|
2018-01-18 04:34:34 +07:00
|
|
|
/*
|
|
|
|
|
* This should be used immediately before an indirect jump/call. It tells
|
|
|
|
|
* objtool the subsequent indirect jump/call is vouched safe for retpoline
|
|
|
|
|
* builds.
|
|
|
|
|
*/
|
|
|
|
|
.macro ANNOTATE_RETPOLINE_SAFE
|
|
|
|
|
.Lannotate_\@:
|
|
|
|
|
.pushsection .discard.retpoline_safe
|
|
|
|
|
_ASM_PTR .Lannotate_\@
|
|
|
|
|
.popsection
|
|
|
|
|
.endm
|
|
|
|
|
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
/*
|
|
|
|
|
* These are the bare retpoline primitives for indirect jmp and call.
|
|
|
|
|
* Do not use these directly; they only exist to make the ALTERNATIVE
|
|
|
|
|
* invocation below less ugly.
|
|
|
|
|
*/
|
|
|
|
|
.macro RETPOLINE_JMP reg:req
|
|
|
|
|
call .Ldo_rop_\@
|
|
|
|
|
.Lspec_trap_\@:
|
|
|
|
|
pause
|
2018-01-14 06:27:30 +07:00
|
|
|
lfence
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
jmp .Lspec_trap_\@
|
|
|
|
|
.Ldo_rop_\@:
|
|
|
|
|
mov \reg, (%_ASM_SP)
|
|
|
|
|
ret
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This is a wrapper around RETPOLINE_JMP so the called function in reg
|
|
|
|
|
* returns to the instruction after the macro.
|
|
|
|
|
*/
|
|
|
|
|
.macro RETPOLINE_CALL reg:req
|
|
|
|
|
jmp .Ldo_call_\@
|
|
|
|
|
.Ldo_retpoline_jmp_\@:
|
|
|
|
|
RETPOLINE_JMP \reg
|
|
|
|
|
.Ldo_call_\@:
|
|
|
|
|
call .Ldo_retpoline_jmp_\@
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
|
|
|
|
|
* indirect jmp/call which may be susceptible to the Spectre variant 2
|
|
|
|
|
* attack.
|
|
|
|
|
*/
|
|
|
|
|
.macro JMP_NOSPEC reg:req
|
|
|
|
|
#ifdef CONFIG_RETPOLINE
|
|
|
|
|
ANNOTATE_NOSPEC_ALTERNATIVE
|
2018-01-18 04:34:34 +07:00
|
|
|
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
|
2018-01-18 04:34:34 +07:00
|
|
|
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
#else
|
|
|
|
|
jmp *\reg
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro CALL_NOSPEC reg:req
|
|
|
|
|
#ifdef CONFIG_RETPOLINE
|
|
|
|
|
ANNOTATE_NOSPEC_ALTERNATIVE
|
2018-01-18 04:34:34 +07:00
|
|
|
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
|
2018-01-18 04:34:34 +07:00
|
|
|
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
#else
|
|
|
|
|
call *\reg
|
|
|
|
|
#endif
|
2018-01-12 18:11:27 +07:00
|
|
|
.endm
|
|
|
|
|
|
2018-02-19 17:50:56 +07:00
|
|
|
/*
|
|
|
|
|
* A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
|
|
|
|
|
* monstrosity above, manually.
|
|
|
|
|
*/
|
|
|
|
|
.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
|
2018-01-12 18:11:27 +07:00
|
|
|
#ifdef CONFIG_RETPOLINE
|
2018-02-19 17:50:56 +07:00
|
|
|
ANNOTATE_NOSPEC_ALTERNATIVE
|
|
|
|
|
ALTERNATIVE "jmp .Lskip_rsb_\@", \
|
|
|
|
|
__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
|
|
|
|
|
\ftr
|
|
|
|
|
.Lskip_rsb_\@:
|
2018-01-12 18:11:27 +07:00
|
|
|
#endif
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
#else /* __ASSEMBLY__ */
|
|
|
|
|
|
|
|
|
|
#define ANNOTATE_NOSPEC_ALTERNATIVE \
|
|
|
|
|
"999:\n\t" \
|
|
|
|
|
".pushsection .discard.nospec\n\t" \
|
|
|
|
|
".long 999b - .\n\t" \
|
|
|
|
|
".popsection\n\t"
|
|
|
|
|
|
2018-01-18 04:34:34 +07:00
|
|
|
#define ANNOTATE_RETPOLINE_SAFE \
|
|
|
|
|
"999:\n\t" \
|
|
|
|
|
".pushsection .discard.retpoline_safe\n\t" \
|
|
|
|
|
_ASM_PTR " 999b\n\t" \
|
|
|
|
|
".popsection\n\t"
|
|
|
|
|
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
#if defined(CONFIG_X86_64) && defined(RETPOLINE)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Since the inline asm uses the %V modifier which is only in newer GCC,
|
|
|
|
|
* the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
|
|
|
|
|
*/
|
|
|
|
|
# define CALL_NOSPEC \
|
|
|
|
|
ANNOTATE_NOSPEC_ALTERNATIVE \
|
|
|
|
|
ALTERNATIVE( \
|
2018-01-18 04:34:34 +07:00
|
|
|
ANNOTATE_RETPOLINE_SAFE \
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
"call *%[thunk_target]\n", \
|
|
|
|
|
"call __x86_indirect_thunk_%V[thunk_target]\n", \
|
|
|
|
|
X86_FEATURE_RETPOLINE)
|
|
|
|
|
# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
|
|
|
|
|
|
|
|
|
|
#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
|
|
|
|
|
/*
|
|
|
|
|
* For i386 we use the original ret-equivalent retpoline, because
|
|
|
|
|
* otherwise we'll run out of registers. We don't care about CET
|
|
|
|
|
* here, anyway.
|
|
|
|
|
*/
|
2018-03-14 18:24:27 +07:00
|
|
|
# define CALL_NOSPEC \
|
|
|
|
|
ALTERNATIVE( \
|
|
|
|
|
ANNOTATE_RETPOLINE_SAFE \
|
|
|
|
|
"call *%[thunk_target]\n", \
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
" jmp 904f;\n" \
|
|
|
|
|
" .align 16\n" \
|
|
|
|
|
"901: call 903f;\n" \
|
|
|
|
|
"902: pause;\n" \
|
2018-01-14 06:27:30 +07:00
|
|
|
" lfence;\n" \
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
" jmp 902b;\n" \
|
|
|
|
|
" .align 16\n" \
|
|
|
|
|
"903: addl $4, %%esp;\n" \
|
|
|
|
|
" pushl %[thunk_target];\n" \
|
|
|
|
|
" ret;\n" \
|
|
|
|
|
" .align 16\n" \
|
|
|
|
|
"904: call 901b;\n", \
|
|
|
|
|
X86_FEATURE_RETPOLINE)
|
|
|
|
|
|
|
|
|
|
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
|
2018-01-12 18:11:27 +07:00
|
|
|
#else /* No retpoline for C / inline asm */
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
# define CALL_NOSPEC "call *%[thunk_target]\n"
|
|
|
|
|
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
|
|
|
|
|
#endif
|
|
|
|
|
|
2018-01-12 04:46:26 +07:00
|
|
|
/* The Spectre V2 mitigation variants */
|
|
|
|
|
enum spectre_v2_mitigation {
|
|
|
|
|
SPECTRE_V2_NONE,
|
|
|
|
|
SPECTRE_V2_RETPOLINE_MINIMAL,
|
|
|
|
|
SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
|
|
|
|
|
SPECTRE_V2_RETPOLINE_GENERIC,
|
|
|
|
|
SPECTRE_V2_RETPOLINE_AMD,
|
|
|
|
|
SPECTRE_V2_IBRS,
|
|
|
|
|
};
|
|
|
|
|
|
2018-01-18 23:14:21 +07:00
|
|
|
extern char __indirect_thunk_start[];
|
|
|
|
|
extern char __indirect_thunk_end[];
|
|
|
|
|
|
2018-01-12 18:11:27 +07:00
|
|
|
/*
|
|
|
|
|
* On VMEXIT we must ensure that no RSB predictions learned in the guest
|
|
|
|
|
* can be followed in the host, by overwriting the RSB completely. Both
|
|
|
|
|
* retpoline and IBRS mitigations for Spectre v2 need this; only on future
|
2018-02-03 02:12:20 +07:00
|
|
|
* CPUs with IBRS_ALL *might* it be avoided.
|
2018-01-12 18:11:27 +07:00
|
|
|
*/
|
|
|
|
|
static inline void vmexit_fill_RSB(void)
|
|
|
|
|
{
|
|
|
|
|
#ifdef CONFIG_RETPOLINE
|
2018-02-19 17:50:56 +07:00
|
|
|
unsigned long loops;
|
|
|
|
|
|
|
|
|
|
asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
|
|
|
|
|
ALTERNATIVE("jmp 910f",
|
|
|
|
|
__stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
|
|
|
|
|
X86_FEATURE_RETPOLINE)
|
|
|
|
|
"910:"
|
|
|
|
|
: "=r" (loops), ASM_CALL_CONSTRAINT
|
|
|
|
|
: : "memory" );
|
2018-01-12 18:11:27 +07:00
|
|
|
#endif
|
|
|
|
|
}
|
2018-01-18 05:53:28 +07:00
|
|
|
|
2018-02-19 17:50:54 +07:00
|
|
|
#define alternative_msr_write(_msr, _val, _feature) \
|
|
|
|
|
asm volatile(ALTERNATIVE("", \
|
|
|
|
|
"movl %[msr], %%ecx\n\t" \
|
|
|
|
|
"movl %[val], %%eax\n\t" \
|
|
|
|
|
"movl $0, %%edx\n\t" \
|
|
|
|
|
"wrmsr", \
|
|
|
|
|
_feature) \
|
|
|
|
|
: : [msr] "i" (_msr), [val] "i" (_val) \
|
|
|
|
|
: "eax", "ecx", "edx", "memory")
|
|
|
|
|
|
2018-01-25 23:14:15 +07:00
|
|
|
static inline void indirect_branch_prediction_barrier(void)
|
|
|
|
|
{
|
2018-02-19 17:50:54 +07:00
|
|
|
alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
|
|
|
|
|
X86_FEATURE_USE_IBPB);
|
2018-01-25 23:14:15 +07:00
|
|
|
}
|
|
|
|
|
|
2018-02-19 17:50:54 +07:00
|
|
|
/*
|
|
|
|
|
* With retpoline, we must use IBRS to restrict branch prediction
|
|
|
|
|
* before calling into firmware.
|
2018-02-21 15:20:37 +07:00
|
|
|
*
|
|
|
|
|
* (Implemented as CPP macros due to header hell.)
|
2018-02-19 17:50:54 +07:00
|
|
|
*/
|
2018-02-21 15:20:37 +07:00
|
|
|
#define firmware_restrict_branch_speculation_start() \
|
|
|
|
|
do { \
|
|
|
|
|
preempt_disable(); \
|
|
|
|
|
alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS, \
|
|
|
|
|
X86_FEATURE_USE_IBRS_FW); \
|
|
|
|
|
} while (0)
|
2018-02-19 17:50:54 +07:00
|
|
|
|
2018-02-21 15:20:37 +07:00
|
|
|
#define firmware_restrict_branch_speculation_end() \
|
|
|
|
|
do { \
|
|
|
|
|
alternative_msr_write(MSR_IA32_SPEC_CTRL, 0, \
|
|
|
|
|
X86_FEATURE_USE_IBRS_FW); \
|
|
|
|
|
preempt_enable(); \
|
|
|
|
|
} while (0)
|
2018-01-25 23:14:15 +07:00
|
|
|
|
x86/retpoline: Add initial retpoline support
Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.
This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.
On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.
Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.
[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
2018-01-12 04:46:25 +07:00
|
|
|
#endif /* __ASSEMBLY__ */
|
bpf, x64: implement retpoline for tail call
Implement a retpoline [0] for the BPF tail call JIT'ing that converts
the indirect jump via jmp %rax that is used to make the long jump into
another JITed BPF image. Since this is subject to speculative execution,
we need to control the transient instruction sequence here as well
when CONFIG_RETPOLINE is set, and direct it into a pause + lfence loop.
The latter aligns also with what gcc / clang emits (e.g. [1]).
JIT dump after patch:
# bpftool p d x i 1
0: (18) r2 = map[id:1]
2: (b7) r3 = 0
3: (85) call bpf_tail_call#12
4: (b7) r0 = 2
5: (95) exit
With CONFIG_RETPOLINE:
# bpftool p d j i 1
[...]
33: cmp %edx,0x24(%rsi)
36: jbe 0x0000000000000072 |*
38: mov 0x24(%rbp),%eax
3e: cmp $0x20,%eax
41: ja 0x0000000000000072 |
43: add $0x1,%eax
46: mov %eax,0x24(%rbp)
4c: mov 0x90(%rsi,%rdx,8),%rax
54: test %rax,%rax
57: je 0x0000000000000072 |
59: mov 0x28(%rax),%rax
5d: add $0x25,%rax
61: callq 0x000000000000006d |+
66: pause |
68: lfence |
6b: jmp 0x0000000000000066 |
6d: mov %rax,(%rsp) |
71: retq |
72: mov $0x2,%eax
[...]
* relative fall-through jumps in error case
+ retpoline for indirect jump
Without CONFIG_RETPOLINE:
# bpftool p d j i 1
[...]
33: cmp %edx,0x24(%rsi)
36: jbe 0x0000000000000063 |*
38: mov 0x24(%rbp),%eax
3e: cmp $0x20,%eax
41: ja 0x0000000000000063 |
43: add $0x1,%eax
46: mov %eax,0x24(%rbp)
4c: mov 0x90(%rsi,%rdx,8),%rax
54: test %rax,%rax
57: je 0x0000000000000063 |
59: mov 0x28(%rax),%rax
5d: add $0x25,%rax
61: jmpq *%rax |-
63: mov $0x2,%eax
[...]
* relative fall-through jumps in error case
- plain indirect jump as before
[0] https://support.google.com/faqs/answer/7625886
[1] https://github.com/gcc-mirror/gcc/commit/a31e654fa107be968b802786d747e962c2fcdb2b
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2018-02-22 21:12:53 +07:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Below is used in the eBPF JIT compiler and emits the byte sequence
|
|
|
|
|
* for the following assembly:
|
|
|
|
|
*
|
|
|
|
|
* With retpolines configured:
|
|
|
|
|
*
|
|
|
|
|
* callq do_rop
|
|
|
|
|
* spec_trap:
|
|
|
|
|
* pause
|
|
|
|
|
* lfence
|
|
|
|
|
* jmp spec_trap
|
|
|
|
|
* do_rop:
|
|
|
|
|
* mov %rax,(%rsp)
|
|
|
|
|
* retq
|
|
|
|
|
*
|
|
|
|
|
* Without retpolines configured:
|
|
|
|
|
*
|
|
|
|
|
* jmp *%rax
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_RETPOLINE
|
|
|
|
|
# define RETPOLINE_RAX_BPF_JIT_SIZE 17
|
|
|
|
|
# define RETPOLINE_RAX_BPF_JIT() \
|
|
|
|
|
EMIT1_off32(0xE8, 7); /* callq do_rop */ \
|
|
|
|
|
/* spec_trap: */ \
|
|
|
|
|
EMIT2(0xF3, 0x90); /* pause */ \
|
|
|
|
|
EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
|
|
|
|
|
EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
|
|
|
|
|
/* do_rop: */ \
|
|
|
|
|
EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \
|
|
|
|
|
EMIT1(0xC3); /* retq */
|
|
|
|
|
#else
|
|
|
|
|
# define RETPOLINE_RAX_BPF_JIT_SIZE 2
|
|
|
|
|
# define RETPOLINE_RAX_BPF_JIT() \
|
|
|
|
|
EMIT2(0xFF, 0xE0); /* jmp *%rax */
|
|
|
|
|
#endif
|
|
|
|
|
|
2018-01-26 19:11:37 +07:00
|
|
|
#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
|