mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-27 07:35:12 +07:00
07f07f55a2
Add a static key which controls the invocation of the CPU buffer clear mechanism on idle entry. This is independent of other MDS mitigations because the idle entry invocation to mitigate the potential leakage due to store buffer repartitioning is only necessary on SMT systems. Add the actual invocations to the different halt/mwait variants which covers all usage sites. mwaitx is not patched as it's not available on Intel CPUs. The buffer clear is only invoked before entering the C-State to prevent that stale data from the idling CPU is spilled to the Hyper-Thread sibling after the Store buffer got repartitioned and all entries are available to the non idle sibling. When coming out of idle the store buffer is partitioned again so each sibling has half of it available. Now CPU which returned from idle could be speculatively exposed to contents of the sibling, but the buffers are flushed either on exit to user space or on VMENTER. When later on conditional buffer clearing is implemented on top of this, then there is no action required either because before returning to user space the context switch will set the condition flag which causes a flush on the return to user path. Note, that the buffer clearing on idle is only sensible on CPUs which are solely affected by MSBDS and not any other variant of MDS because the other MDS variants cannot be mitigated when SMT is enabled, so the buffer clearing on idle would be a window dressing exercise. This intentionally does not handle the case in the acpi/processor_idle driver which uses the legacy IO port interface for C-State transitions for two reasons: - The acpi/processor_idle driver was replaced by the intel_idle driver almost a decade ago. Anything Nehalem upwards supports it and defaults to that new driver. - The legacy IO port interface is likely to be used on older and therefore unaffected CPUs or on systems which do not receive microcode updates anymore, so there is no point in adding that. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Borislav Petkov <bp@suse.de> Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Reviewed-by: Frederic Weisbecker <frederic@kernel.org> Reviewed-by: Jon Masters <jcm@redhat.com> Tested-by: Jon Masters <jcm@redhat.com>
210 lines
4.3 KiB
C
210 lines
4.3 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _X86_IRQFLAGS_H_
|
|
#define _X86_IRQFLAGS_H_
|
|
|
|
#include <asm/processor-flags.h>
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <asm/nospec-branch.h>
|
|
|
|
/* Provide __cpuidle; we can't safely include <linux/cpu.h> */
|
|
#define __cpuidle __attribute__((__section__(".cpuidle.text")))
|
|
|
|
/*
|
|
* Interrupt control:
|
|
*/
|
|
|
|
/* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */
|
|
extern inline unsigned long native_save_fl(void);
|
|
extern inline unsigned long native_save_fl(void)
|
|
{
|
|
unsigned long flags;
|
|
|
|
/*
|
|
* "=rm" is safe here, because "pop" adjusts the stack before
|
|
* it evaluates its effective address -- this is part of the
|
|
* documented behavior of the "pop" instruction.
|
|
*/
|
|
asm volatile("# __raw_save_flags\n\t"
|
|
"pushf ; pop %0"
|
|
: "=rm" (flags)
|
|
: /* no input */
|
|
: "memory");
|
|
|
|
return flags;
|
|
}
|
|
|
|
extern inline void native_restore_fl(unsigned long flags);
|
|
extern inline void native_restore_fl(unsigned long flags)
|
|
{
|
|
asm volatile("push %0 ; popf"
|
|
: /* no output */
|
|
:"g" (flags)
|
|
:"memory", "cc");
|
|
}
|
|
|
|
static inline void native_irq_disable(void)
|
|
{
|
|
asm volatile("cli": : :"memory");
|
|
}
|
|
|
|
static inline void native_irq_enable(void)
|
|
{
|
|
asm volatile("sti": : :"memory");
|
|
}
|
|
|
|
static inline __cpuidle void native_safe_halt(void)
|
|
{
|
|
mds_idle_clear_cpu_buffers();
|
|
asm volatile("sti; hlt": : :"memory");
|
|
}
|
|
|
|
static inline __cpuidle void native_halt(void)
|
|
{
|
|
mds_idle_clear_cpu_buffers();
|
|
asm volatile("hlt": : :"memory");
|
|
}
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_PARAVIRT_XXL
|
|
#include <asm/paravirt.h>
|
|
#else
|
|
#ifndef __ASSEMBLY__
|
|
#include <linux/types.h>
|
|
|
|
static inline notrace unsigned long arch_local_save_flags(void)
|
|
{
|
|
return native_save_fl();
|
|
}
|
|
|
|
static inline notrace void arch_local_irq_restore(unsigned long flags)
|
|
{
|
|
native_restore_fl(flags);
|
|
}
|
|
|
|
static inline notrace void arch_local_irq_disable(void)
|
|
{
|
|
native_irq_disable();
|
|
}
|
|
|
|
static inline notrace void arch_local_irq_enable(void)
|
|
{
|
|
native_irq_enable();
|
|
}
|
|
|
|
/*
|
|
* Used in the idle loop; sti takes one instruction cycle
|
|
* to complete:
|
|
*/
|
|
static inline __cpuidle void arch_safe_halt(void)
|
|
{
|
|
native_safe_halt();
|
|
}
|
|
|
|
/*
|
|
* Used when interrupts are already enabled or to
|
|
* shutdown the processor:
|
|
*/
|
|
static inline __cpuidle void halt(void)
|
|
{
|
|
native_halt();
|
|
}
|
|
|
|
/*
|
|
* For spinlocks, etc:
|
|
*/
|
|
static inline notrace unsigned long arch_local_irq_save(void)
|
|
{
|
|
unsigned long flags = arch_local_save_flags();
|
|
arch_local_irq_disable();
|
|
return flags;
|
|
}
|
|
#else
|
|
|
|
#define ENABLE_INTERRUPTS(x) sti
|
|
#define DISABLE_INTERRUPTS(x) cli
|
|
|
|
#ifdef CONFIG_X86_64
|
|
#ifdef CONFIG_DEBUG_ENTRY
|
|
#define SAVE_FLAGS(x) pushfq; popq %rax
|
|
#endif
|
|
|
|
#define SWAPGS swapgs
|
|
/*
|
|
* Currently paravirt can't handle swapgs nicely when we
|
|
* don't have a stack we can rely on (such as a user space
|
|
* stack). So we either find a way around these or just fault
|
|
* and emulate if a guest tries to call swapgs directly.
|
|
*
|
|
* Either way, this is a good way to document that we don't
|
|
* have a reliable stack. x86_64 only.
|
|
*/
|
|
#define SWAPGS_UNSAFE_STACK swapgs
|
|
|
|
#define INTERRUPT_RETURN jmp native_iret
|
|
#define USERGS_SYSRET64 \
|
|
swapgs; \
|
|
sysretq;
|
|
#define USERGS_SYSRET32 \
|
|
swapgs; \
|
|
sysretl
|
|
|
|
#else
|
|
#define INTERRUPT_RETURN iret
|
|
#endif
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
#endif /* CONFIG_PARAVIRT_XXL */
|
|
|
|
#ifndef __ASSEMBLY__
|
|
static inline int arch_irqs_disabled_flags(unsigned long flags)
|
|
{
|
|
return !(flags & X86_EFLAGS_IF);
|
|
}
|
|
|
|
static inline int arch_irqs_disabled(void)
|
|
{
|
|
unsigned long flags = arch_local_save_flags();
|
|
|
|
return arch_irqs_disabled_flags(flags);
|
|
}
|
|
#endif /* !__ASSEMBLY__ */
|
|
|
|
#ifdef __ASSEMBLY__
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
# define TRACE_IRQS_ON call trace_hardirqs_on_thunk;
|
|
# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk;
|
|
#else
|
|
# define TRACE_IRQS_ON
|
|
# define TRACE_IRQS_OFF
|
|
#endif
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
# ifdef CONFIG_X86_64
|
|
# define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk
|
|
# define LOCKDEP_SYS_EXIT_IRQ \
|
|
TRACE_IRQS_ON; \
|
|
sti; \
|
|
call lockdep_sys_exit_thunk; \
|
|
cli; \
|
|
TRACE_IRQS_OFF;
|
|
# else
|
|
# define LOCKDEP_SYS_EXIT \
|
|
pushl %eax; \
|
|
pushl %ecx; \
|
|
pushl %edx; \
|
|
call lockdep_sys_exit; \
|
|
popl %edx; \
|
|
popl %ecx; \
|
|
popl %eax;
|
|
# define LOCKDEP_SYS_EXIT_IRQ
|
|
# endif
|
|
#else
|
|
# define LOCKDEP_SYS_EXIT
|
|
# define LOCKDEP_SYS_EXIT_IRQ
|
|
#endif
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif
|