mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-23 03:14:42 +07:00
1ae4de23ed
__vmcs_writel uses volatile asm, so there is no need to insert another one between the first and the second call to __vmcs_writel in order to prevent unwanted code moves for 32bit targets. Signed-off-by: Uros Bizjak <ubizjak@gmail.com> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
285 lines
7.7 KiB
C
285 lines
7.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __KVM_X86_VMX_INSN_H
|
|
#define __KVM_X86_VMX_INSN_H
|
|
|
|
#include <linux/nospec.h>
|
|
|
|
#include <asm/kvm_host.h>
|
|
#include <asm/vmx.h>
|
|
|
|
#include "evmcs.h"
|
|
#include "vmcs.h"
|
|
|
|
#define __ex(x) __kvm_handle_fault_on_reboot(x)
|
|
#define __ex_clear(x, reg) \
|
|
____kvm_handle_fault_on_reboot(x, "xor " reg ", " reg)
|
|
|
|
static __always_inline void vmcs_check16(unsigned long field)
|
|
{
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
|
|
"16-bit accessor invalid for 64-bit field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
|
|
"16-bit accessor invalid for 64-bit high field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
|
|
"16-bit accessor invalid for 32-bit high field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
|
|
"16-bit accessor invalid for natural width field");
|
|
}
|
|
|
|
static __always_inline void vmcs_check32(unsigned long field)
|
|
{
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
|
|
"32-bit accessor invalid for 16-bit field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
|
|
"32-bit accessor invalid for natural width field");
|
|
}
|
|
|
|
static __always_inline void vmcs_check64(unsigned long field)
|
|
{
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
|
|
"64-bit accessor invalid for 16-bit field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
|
|
"64-bit accessor invalid for 64-bit high field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
|
|
"64-bit accessor invalid for 32-bit field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
|
|
"64-bit accessor invalid for natural width field");
|
|
}
|
|
|
|
static __always_inline void vmcs_checkl(unsigned long field)
|
|
{
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
|
|
"Natural width accessor invalid for 16-bit field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
|
|
"Natural width accessor invalid for 64-bit field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
|
|
"Natural width accessor invalid for 64-bit high field");
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
|
|
"Natural width accessor invalid for 32-bit field");
|
|
}
|
|
|
|
static __always_inline unsigned long __vmcs_readl(unsigned long field)
|
|
{
|
|
unsigned long value;
|
|
|
|
asm volatile (__ex_clear("vmread %1, %0", "%k0")
|
|
: "=r"(value) : "r"(field));
|
|
return value;
|
|
}
|
|
|
|
static __always_inline u16 vmcs_read16(unsigned long field)
|
|
{
|
|
vmcs_check16(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_read16(field);
|
|
return __vmcs_readl(field);
|
|
}
|
|
|
|
static __always_inline u32 vmcs_read32(unsigned long field)
|
|
{
|
|
vmcs_check32(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_read32(field);
|
|
return __vmcs_readl(field);
|
|
}
|
|
|
|
static __always_inline u64 vmcs_read64(unsigned long field)
|
|
{
|
|
vmcs_check64(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_read64(field);
|
|
#ifdef CONFIG_X86_64
|
|
return __vmcs_readl(field);
|
|
#else
|
|
return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
|
|
#endif
|
|
}
|
|
|
|
static __always_inline unsigned long vmcs_readl(unsigned long field)
|
|
{
|
|
vmcs_checkl(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_read64(field);
|
|
return __vmcs_readl(field);
|
|
}
|
|
|
|
static noinline void vmwrite_error(unsigned long field, unsigned long value)
|
|
{
|
|
printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
|
|
field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
|
|
dump_stack();
|
|
}
|
|
|
|
static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
|
|
{
|
|
bool error;
|
|
|
|
asm volatile (__ex("vmwrite %2, %1") CC_SET(na)
|
|
: CC_OUT(na) (error) : "r"(field), "rm"(value));
|
|
if (unlikely(error))
|
|
vmwrite_error(field, value);
|
|
}
|
|
|
|
static __always_inline void vmcs_write16(unsigned long field, u16 value)
|
|
{
|
|
vmcs_check16(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_write16(field, value);
|
|
|
|
__vmcs_writel(field, value);
|
|
}
|
|
|
|
static __always_inline void vmcs_write32(unsigned long field, u32 value)
|
|
{
|
|
vmcs_check32(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_write32(field, value);
|
|
|
|
__vmcs_writel(field, value);
|
|
}
|
|
|
|
static __always_inline void vmcs_write64(unsigned long field, u64 value)
|
|
{
|
|
vmcs_check64(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_write64(field, value);
|
|
|
|
__vmcs_writel(field, value);
|
|
#ifndef CONFIG_X86_64
|
|
__vmcs_writel(field+1, value >> 32);
|
|
#endif
|
|
}
|
|
|
|
static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
|
|
{
|
|
vmcs_checkl(field);
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_write64(field, value);
|
|
|
|
__vmcs_writel(field, value);
|
|
}
|
|
|
|
static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
|
|
{
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
|
|
"vmcs_clear_bits does not support 64-bit fields");
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_write32(field, evmcs_read32(field) & ~mask);
|
|
|
|
__vmcs_writel(field, __vmcs_readl(field) & ~mask);
|
|
}
|
|
|
|
static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
|
|
{
|
|
BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
|
|
"vmcs_set_bits does not support 64-bit fields");
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_write32(field, evmcs_read32(field) | mask);
|
|
|
|
__vmcs_writel(field, __vmcs_readl(field) | mask);
|
|
}
|
|
|
|
static inline void vmcs_clear(struct vmcs *vmcs)
|
|
{
|
|
u64 phys_addr = __pa(vmcs);
|
|
bool error;
|
|
|
|
asm volatile (__ex("vmclear %1") CC_SET(na)
|
|
: CC_OUT(na) (error) : "m"(phys_addr));
|
|
if (unlikely(error))
|
|
printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
|
|
vmcs, phys_addr);
|
|
}
|
|
|
|
static inline void vmcs_load(struct vmcs *vmcs)
|
|
{
|
|
u64 phys_addr = __pa(vmcs);
|
|
bool error;
|
|
|
|
if (static_branch_unlikely(&enable_evmcs))
|
|
return evmcs_load(phys_addr);
|
|
|
|
asm volatile (__ex("vmptrld %1") CC_SET(na)
|
|
: CC_OUT(na) (error) : "m"(phys_addr));
|
|
if (unlikely(error))
|
|
printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
|
|
vmcs, phys_addr);
|
|
}
|
|
|
|
static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva)
|
|
{
|
|
struct {
|
|
u64 vpid : 16;
|
|
u64 rsvd : 48;
|
|
u64 gva;
|
|
} operand = { vpid, 0, gva };
|
|
bool error;
|
|
|
|
asm volatile (__ex("invvpid %2, %1") CC_SET(na)
|
|
: CC_OUT(na) (error) : "r"(ext), "m"(operand));
|
|
BUG_ON(error);
|
|
}
|
|
|
|
static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa)
|
|
{
|
|
struct {
|
|
u64 eptp, gpa;
|
|
} operand = {eptp, gpa};
|
|
bool error;
|
|
|
|
asm volatile (__ex("invept %2, %1") CC_SET(na)
|
|
: CC_OUT(na) (error) : "r"(ext), "m"(operand));
|
|
BUG_ON(error);
|
|
}
|
|
|
|
static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr)
|
|
{
|
|
if (vpid == 0)
|
|
return true;
|
|
|
|
if (cpu_has_vmx_invvpid_individual_addr()) {
|
|
__invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static inline void vpid_sync_vcpu_single(int vpid)
|
|
{
|
|
if (vpid == 0)
|
|
return;
|
|
|
|
if (cpu_has_vmx_invvpid_single())
|
|
__invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
|
|
}
|
|
|
|
static inline void vpid_sync_vcpu_global(void)
|
|
{
|
|
if (cpu_has_vmx_invvpid_global())
|
|
__invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
|
|
}
|
|
|
|
static inline void vpid_sync_context(int vpid)
|
|
{
|
|
if (cpu_has_vmx_invvpid_single())
|
|
vpid_sync_vcpu_single(vpid);
|
|
else
|
|
vpid_sync_vcpu_global();
|
|
}
|
|
|
|
static inline void ept_sync_global(void)
|
|
{
|
|
__invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
|
|
}
|
|
|
|
static inline void ept_sync_context(u64 eptp)
|
|
{
|
|
if (cpu_has_vmx_invept_context())
|
|
__invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
|
|
else
|
|
ept_sync_global();
|
|
}
|
|
|
|
#endif /* __KVM_X86_VMX_INSN_H */
|