2015-04-22 15:53:34 +07:00
|
|
|
/*
|
|
|
|
* x86 FPU boot time init code
|
|
|
|
*/
|
2015-04-24 07:54:44 +07:00
|
|
|
#include <asm/fpu/internal.h>
|
2015-04-22 15:53:34 +07:00
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
2015-04-22 18:44:25 +07:00
|
|
|
/*
|
|
|
|
* Boot time CPU/FPU FDIV bug detection code:
|
|
|
|
*/
|
|
|
|
|
|
|
|
static double __initdata x = 4195835.0;
|
|
|
|
static double __initdata y = 3145727.0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This used to check for exceptions..
|
|
|
|
* However, it turns out that to support that,
|
|
|
|
* the XMM trap handlers basically had to
|
|
|
|
* be buggy. So let's have a correct XMM trap
|
|
|
|
* handler, and forget about printing out
|
|
|
|
* some status at boot.
|
|
|
|
*
|
|
|
|
* We should really only care about bugs here
|
|
|
|
* anyway. Not features.
|
|
|
|
*/
|
|
|
|
static void __init check_fpu(void)
|
|
|
|
{
|
|
|
|
s32 fdiv_bug;
|
|
|
|
|
|
|
|
kernel_fpu_begin();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* trap_init() enabled FXSR and company _before_ testing for FP
|
|
|
|
* problems here.
|
|
|
|
*
|
|
|
|
* Test for the divl bug: http://en.wikipedia.org/wiki/Fdiv_bug
|
|
|
|
*/
|
|
|
|
__asm__("fninit\n\t"
|
|
|
|
"fldl %1\n\t"
|
|
|
|
"fdivl %2\n\t"
|
|
|
|
"fmull %2\n\t"
|
|
|
|
"fldl %1\n\t"
|
|
|
|
"fsubp %%st,%%st(1)\n\t"
|
|
|
|
"fistpl %0\n\t"
|
|
|
|
"fwait\n\t"
|
|
|
|
"fninit"
|
|
|
|
: "=m" (*&fdiv_bug)
|
|
|
|
: "m" (*&x), "m" (*&y));
|
|
|
|
|
|
|
|
kernel_fpu_end();
|
|
|
|
|
|
|
|
if (fdiv_bug) {
|
|
|
|
set_cpu_bug(&boot_cpu_data, X86_BUG_FDIV);
|
|
|
|
pr_warn("Hmm, FPU with FDIV bug\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void fpu__init_check_bugs(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* kernel_fpu_begin/end() in check_fpu() relies on the patched
|
|
|
|
* alternative instructions.
|
|
|
|
*/
|
|
|
|
if (cpu_has_fpu)
|
|
|
|
check_fpu();
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Boot time FPU feature detection code:
|
|
|
|
*/
|
2015-04-22 15:53:34 +07:00
|
|
|
unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
|
2015-04-24 15:49:11 +07:00
|
|
|
|
2015-04-22 15:53:34 +07:00
|
|
|
unsigned int xstate_size;
|
|
|
|
EXPORT_SYMBOL_GPL(xstate_size);
|
|
|
|
|
|
|
|
static void mxcsr_feature_mask_init(void)
|
|
|
|
{
|
2015-04-24 15:49:11 +07:00
|
|
|
unsigned int mask = 0;
|
2015-04-22 15:53:34 +07:00
|
|
|
|
|
|
|
if (cpu_has_fxsr) {
|
2015-04-24 15:49:11 +07:00
|
|
|
struct i387_fxsave_struct fx_tmp __aligned(32) = { };
|
|
|
|
|
|
|
|
asm volatile("fxsave %0" : "+m" (fx_tmp));
|
|
|
|
|
|
|
|
mask = fx_tmp.mxcsr_mask;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If zero then use the default features mask,
|
|
|
|
* which has all features set, except the
|
|
|
|
* denormals-are-zero feature bit:
|
|
|
|
*/
|
2015-04-22 15:53:34 +07:00
|
|
|
if (mask == 0)
|
|
|
|
mask = 0x0000ffbf;
|
|
|
|
}
|
|
|
|
mxcsr_feature_mask &= mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void fpstate_xstate_init_size(void)
|
|
|
|
{
|
2015-04-26 15:35:57 +07:00
|
|
|
static bool on_boot_cpu = 1;
|
|
|
|
|
|
|
|
if (!on_boot_cpu)
|
|
|
|
return;
|
|
|
|
on_boot_cpu = 0;
|
|
|
|
|
2015-04-22 15:53:34 +07:00
|
|
|
/*
|
|
|
|
* Note that xstate_size might be overwriten later during
|
2015-04-25 11:52:53 +07:00
|
|
|
* fpu__init_system_xstate().
|
2015-04-22 15:53:34 +07:00
|
|
|
*/
|
|
|
|
|
|
|
|
if (!cpu_has_fpu) {
|
|
|
|
/*
|
|
|
|
* Disable xsave as we do not support it if i387
|
|
|
|
* emulation is enabled.
|
|
|
|
*/
|
|
|
|
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
|
|
|
|
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
|
|
|
|
xstate_size = sizeof(struct i387_soft_struct);
|
2015-04-25 09:29:26 +07:00
|
|
|
} else {
|
|
|
|
if (cpu_has_fxsr)
|
|
|
|
xstate_size = sizeof(struct i387_fxsave_struct);
|
|
|
|
else
|
|
|
|
xstate_size = sizeof(struct i387_fsave_struct);
|
2015-04-22 15:53:34 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-04-26 13:28:31 +07:00
|
|
|
/*
|
|
|
|
* Initialize the TS bit in CR0 according to the style of context-switches
|
|
|
|
* we are using:
|
|
|
|
*/
|
|
|
|
static void fpu__init_cpu_ctx_switch(void)
|
|
|
|
{
|
|
|
|
if (!cpu_has_eager_fpu)
|
|
|
|
stts();
|
|
|
|
else
|
|
|
|
clts();
|
|
|
|
}
|
|
|
|
|
2015-04-22 15:53:34 +07:00
|
|
|
/*
|
2015-04-26 19:32:34 +07:00
|
|
|
* Initialize the registers found in all CPUs, CR0 and CR4:
|
2015-04-22 15:53:34 +07:00
|
|
|
*/
|
2015-04-26 19:32:34 +07:00
|
|
|
static void fpu__init_cpu_generic(void)
|
2015-04-22 15:53:34 +07:00
|
|
|
{
|
|
|
|
unsigned long cr0;
|
|
|
|
unsigned long cr4_mask = 0;
|
|
|
|
|
|
|
|
#ifndef CONFIG_MATH_EMULATION
|
|
|
|
if (!cpu_has_fpu) {
|
|
|
|
pr_emerg("No FPU found and no math emulation present\n");
|
|
|
|
pr_emerg("Giving up\n");
|
|
|
|
for (;;)
|
|
|
|
asm volatile("hlt");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
if (cpu_has_fxsr)
|
|
|
|
cr4_mask |= X86_CR4_OSFXSR;
|
|
|
|
if (cpu_has_xmm)
|
|
|
|
cr4_mask |= X86_CR4_OSXMMEXCPT;
|
|
|
|
if (cr4_mask)
|
|
|
|
cr4_set_bits(cr4_mask);
|
|
|
|
|
|
|
|
cr0 = read_cr0();
|
|
|
|
cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
|
|
|
|
if (!cpu_has_fpu)
|
|
|
|
cr0 |= X86_CR0_EM;
|
|
|
|
write_cr0(cr0);
|
2015-04-26 19:32:34 +07:00
|
|
|
}
|
2015-04-22 15:53:34 +07:00
|
|
|
|
2015-04-26 19:32:34 +07:00
|
|
|
/*
|
|
|
|
* Enable all supported FPU features. Called when a CPU is brought online.
|
|
|
|
*/
|
|
|
|
void fpu__init_cpu(void)
|
|
|
|
{
|
|
|
|
fpu__init_cpu_generic();
|
2015-04-25 11:52:53 +07:00
|
|
|
fpu__init_cpu_xstate();
|
2015-04-25 13:27:44 +07:00
|
|
|
fpu__init_cpu_ctx_switch();
|
x86/fpu: Split fpu__cpu_init() into early-boot and cpu-boot parts
There are two kinds of FPU initialization sequences necessary to bring FPU
functionality up: once per system bootup activities, such as detection,
feature initialization, etc. of attributes that are shared by all CPUs
in the system - and per cpu initialization sequences run when a CPU is
brought online (either during bootup or during CPU hotplug onlining),
such as CR0/CR4 register setting, etc.
The FPU code is mixing these roles together, with no clear distinction.
Start sorting this out by splitting the main FPU detection routine
(fpu__cpu_init()) into two parts: fpu__init_system() for
one per system init activities, and fpu__init_cpu() for the
per CPU onlining init activities.
Note that xstate_init() is called from both variants for the time being,
because it has a dual nature as well. We'll fix that in upcoming patches.
Just do the split and call it as we used to before, don't introduce any
change in initialization behavior yet, beyond duplicate (and harmless)
fpu__init_cpu() and xstate_init() calls - which we'll fix in later
patches.
Reviewed-by: Borislav Petkov <bp@alien8.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-25 09:34:48 +07:00
|
|
|
}
|
|
|
|
|
2015-04-26 01:11:05 +07:00
|
|
|
static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO;
|
|
|
|
|
|
|
|
static int __init eager_fpu_setup(char *s)
|
|
|
|
{
|
|
|
|
if (!strcmp(s, "on"))
|
|
|
|
eagerfpu = ENABLE;
|
|
|
|
else if (!strcmp(s, "off"))
|
|
|
|
eagerfpu = DISABLE;
|
|
|
|
else if (!strcmp(s, "auto"))
|
|
|
|
eagerfpu = AUTO;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("eagerfpu=", eager_fpu_setup);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* setup_init_fpu_buf() is __init and it is OK to call it here because
|
|
|
|
* init_xstate_ctx will be unset only once during boot.
|
|
|
|
*/
|
2015-04-26 13:28:31 +07:00
|
|
|
static void fpu__init_system_ctx_switch(void)
|
2015-04-26 01:11:05 +07:00
|
|
|
{
|
|
|
|
WARN_ON(current->thread.fpu.fpstate_active);
|
|
|
|
current_thread_info()->status = 0;
|
|
|
|
|
|
|
|
/* Auto enable eagerfpu for xsaveopt */
|
|
|
|
if (cpu_has_xsaveopt && eagerfpu != DISABLE)
|
|
|
|
eagerfpu = ENABLE;
|
|
|
|
|
|
|
|
if (xfeatures_mask & XSTATE_EAGER) {
|
|
|
|
if (eagerfpu == DISABLE) {
|
|
|
|
pr_err("x86/fpu: eagerfpu switching disabled, disabling the following xstate features: 0x%llx.\n",
|
|
|
|
xfeatures_mask & XSTATE_EAGER);
|
|
|
|
xfeatures_mask &= ~XSTATE_EAGER;
|
|
|
|
} else {
|
|
|
|
eagerfpu = ENABLE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (eagerfpu == ENABLE)
|
|
|
|
setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
|
|
|
|
|
|
|
|
printk_once(KERN_INFO "x86/fpu: Using '%s' FPU context switches.\n", eagerfpu == ENABLE ? "eager" : "lazy");
|
|
|
|
}
|
|
|
|
|
x86/fpu: Split fpu__cpu_init() into early-boot and cpu-boot parts
There are two kinds of FPU initialization sequences necessary to bring FPU
functionality up: once per system bootup activities, such as detection,
feature initialization, etc. of attributes that are shared by all CPUs
in the system - and per cpu initialization sequences run when a CPU is
brought online (either during bootup or during CPU hotplug onlining),
such as CR0/CR4 register setting, etc.
The FPU code is mixing these roles together, with no clear distinction.
Start sorting this out by splitting the main FPU detection routine
(fpu__cpu_init()) into two parts: fpu__init_system() for
one per system init activities, and fpu__init_cpu() for the
per CPU onlining init activities.
Note that xstate_init() is called from both variants for the time being,
because it has a dual nature as well. We'll fix that in upcoming patches.
Just do the split and call it as we used to before, don't introduce any
change in initialization behavior yet, beyond duplicate (and harmless)
fpu__init_cpu() and xstate_init() calls - which we'll fix in later
patches.
Reviewed-by: Borislav Petkov <bp@alien8.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-25 09:34:48 +07:00
|
|
|
/*
|
|
|
|
* Called on the boot CPU once per system bootup, to set up the initial FPU state that
|
|
|
|
* is later cloned into all processes.
|
|
|
|
*/
|
|
|
|
void fpu__init_system(void)
|
|
|
|
{
|
|
|
|
/* The FPU has to be operational for some of the later FPU init activities: */
|
|
|
|
fpu__init_cpu();
|
2015-04-22 15:53:34 +07:00
|
|
|
|
2015-04-25 13:27:44 +07:00
|
|
|
/*
|
|
|
|
* But don't leave CR0::TS set yet, as some of the FPU setup methods depend
|
|
|
|
* on being able to execute FPU instructions that will fault on a set TS,
|
|
|
|
* such as the FXSAVE in mxcsr_feature_mask_init().
|
|
|
|
*/
|
|
|
|
clts();
|
|
|
|
|
2015-04-25 13:35:53 +07:00
|
|
|
/*
|
|
|
|
* Set up the legacy init FPU context. (xstate init might overwrite this
|
|
|
|
* with a more modern format, if the CPU supports it.)
|
|
|
|
*/
|
|
|
|
fx_finit(&init_xstate_ctx.i387);
|
|
|
|
|
2015-04-22 15:53:34 +07:00
|
|
|
mxcsr_feature_mask_init();
|
2015-04-26 15:35:57 +07:00
|
|
|
|
|
|
|
fpstate_xstate_init_size();
|
2015-04-25 11:52:53 +07:00
|
|
|
fpu__init_system_xstate();
|
2015-04-26 15:35:57 +07:00
|
|
|
|
2015-04-26 13:28:31 +07:00
|
|
|
fpu__init_system_ctx_switch();
|
2015-04-22 15:53:34 +07:00
|
|
|
}
|
2015-04-22 16:36:14 +07:00
|
|
|
|
|
|
|
static int __init no_387(char *s)
|
|
|
|
{
|
|
|
|
setup_clear_cpu_cap(X86_FEATURE_FPU);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
__setup("no387", no_387);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set the X86_FEATURE_FPU CPU-capability bit based on
|
|
|
|
* trying to execute an actual sequence of FPU instructions:
|
|
|
|
*/
|
|
|
|
void fpu__detect(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
unsigned long cr0;
|
|
|
|
u16 fsw, fcw;
|
|
|
|
|
|
|
|
fsw = fcw = 0xffff;
|
|
|
|
|
|
|
|
cr0 = read_cr0();
|
|
|
|
cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
|
|
|
|
write_cr0(cr0);
|
|
|
|
|
|
|
|
asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
|
|
|
|
: "+m" (fsw), "+m" (fcw));
|
|
|
|
|
|
|
|
if (fsw == 0 && (fcw & 0x103f) == 0x003f)
|
|
|
|
set_cpu_cap(c, X86_FEATURE_FPU);
|
|
|
|
else
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_FPU);
|
|
|
|
|
2015-04-25 13:27:44 +07:00
|
|
|
fpu__init_system();
|
2015-04-25 09:29:26 +07:00
|
|
|
/* The final cr0 value is set later, in fpu_init() */
|
2015-04-22 16:36:14 +07:00
|
|
|
}
|