mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-28 11:18:45 +07:00
ff16701a29
Move the conditional invocation of __switch_to_xtra() into an inline function so the logic can be shared between 32 and 64 bit. Remove the handthrough of the TSS pointer and retrieve the pointer directly in the bitmap handling function. Use this_cpu_ptr() instead of the per_cpu() indirection. This is a preparatory change so integration of conditional indirect branch speculation optimization happens only in one place. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Ingo Molnar <mingo@kernel.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Andy Lutomirski <luto@kernel.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Jiri Kosina <jkosina@suse.cz> Cc: Tom Lendacky <thomas.lendacky@amd.com> Cc: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: David Woodhouse <dwmw@amazon.co.uk> Cc: Tim Chen <tim.c.chen@linux.intel.com> Cc: Andi Kleen <ak@linux.intel.com> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Casey Schaufler <casey.schaufler@intel.com> Cc: Asit Mallick <asit.k.mallick@intel.com> Cc: Arjan van de Ven <arjan@linux.intel.com> Cc: Jon Masters <jcm@redhat.com> Cc: Waiman Long <longman9394@gmail.com> Cc: Greg KH <gregkh@linuxfoundation.org> Cc: Dave Stewart <david.c.stewart@intel.com> Cc: Kees Cook <keescook@chromium.org> Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/20181125185005.280855518@linutronix.de
109 lines
3.0 KiB
C
109 lines
3.0 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_X86_SWITCH_TO_H
|
|
#define _ASM_X86_SWITCH_TO_H
|
|
|
|
#include <linux/sched/task_stack.h>
|
|
|
|
struct task_struct; /* one of the stranger aspects of C forward declarations */
|
|
|
|
struct task_struct *__switch_to_asm(struct task_struct *prev,
|
|
struct task_struct *next);
|
|
|
|
__visible struct task_struct *__switch_to(struct task_struct *prev,
|
|
struct task_struct *next);
|
|
|
|
/* This runs runs on the previous thread's stack. */
|
|
static inline void prepare_switch_to(struct task_struct *next)
|
|
{
|
|
#ifdef CONFIG_VMAP_STACK
|
|
/*
|
|
* If we switch to a stack that has a top-level paging entry
|
|
* that is not present in the current mm, the resulting #PF will
|
|
* will be promoted to a double-fault and we'll panic. Probe
|
|
* the new stack now so that vmalloc_fault can fix up the page
|
|
* tables if needed. This can only happen if we use a stack
|
|
* in vmap space.
|
|
*
|
|
* We assume that the stack is aligned so that it never spans
|
|
* more than one top-level paging entry.
|
|
*
|
|
* To minimize cache pollution, just follow the stack pointer.
|
|
*/
|
|
READ_ONCE(*(unsigned char *)next->thread.sp);
|
|
#endif
|
|
}
|
|
|
|
asmlinkage void ret_from_fork(void);
|
|
|
|
/*
|
|
* This is the structure pointed to by thread.sp for an inactive task. The
|
|
* order of the fields must match the code in __switch_to_asm().
|
|
*/
|
|
struct inactive_task_frame {
|
|
#ifdef CONFIG_X86_64
|
|
unsigned long r15;
|
|
unsigned long r14;
|
|
unsigned long r13;
|
|
unsigned long r12;
|
|
#else
|
|
unsigned long si;
|
|
unsigned long di;
|
|
#endif
|
|
unsigned long bx;
|
|
|
|
/*
|
|
* These two fields must be together. They form a stack frame header,
|
|
* needed by get_frame_pointer().
|
|
*/
|
|
unsigned long bp;
|
|
unsigned long ret_addr;
|
|
};
|
|
|
|
struct fork_frame {
|
|
struct inactive_task_frame frame;
|
|
struct pt_regs regs;
|
|
};
|
|
|
|
#define switch_to(prev, next, last) \
|
|
do { \
|
|
prepare_switch_to(next); \
|
|
\
|
|
((last) = __switch_to_asm((prev), (next))); \
|
|
} while (0)
|
|
|
|
#ifdef CONFIG_X86_32
|
|
static inline void refresh_sysenter_cs(struct thread_struct *thread)
|
|
{
|
|
/* Only happens when SEP is enabled, no need to test "SEP"arately: */
|
|
if (unlikely(this_cpu_read(cpu_tss_rw.x86_tss.ss1) == thread->sysenter_cs))
|
|
return;
|
|
|
|
this_cpu_write(cpu_tss_rw.x86_tss.ss1, thread->sysenter_cs);
|
|
wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
|
|
}
|
|
#endif
|
|
|
|
/* This is used when switching tasks or entering/exiting vm86 mode. */
|
|
static inline void update_task_stack(struct task_struct *task)
|
|
{
|
|
/* sp0 always points to the entry trampoline stack, which is constant: */
|
|
#ifdef CONFIG_X86_32
|
|
if (static_cpu_has(X86_FEATURE_XENPV))
|
|
load_sp0(task->thread.sp0);
|
|
else
|
|
this_cpu_write(cpu_tss_rw.x86_tss.sp1, task->thread.sp0);
|
|
#else
|
|
/*
|
|
* x86-64 updates x86_tss.sp1 via cpu_current_top_of_stack. That
|
|
* doesn't work on x86-32 because sp1 and
|
|
* cpu_current_top_of_stack have different values (because of
|
|
* the non-zero stack-padding on 32bit).
|
|
*/
|
|
if (static_cpu_has(X86_FEATURE_XENPV))
|
|
load_sp0(task_top_of_stack(task));
|
|
#endif
|
|
|
|
}
|
|
|
|
#endif /* _ASM_X86_SWITCH_TO_H */
|