2007-12-14 08:35:10 +07:00
|
|
|
#ifndef __KVM_X86_MMU_H
|
|
|
|
#define __KVM_X86_MMU_H
|
|
|
|
|
2007-12-16 16:02:48 +07:00
|
|
|
#include <linux/kvm_host.h>
|
2009-12-07 17:16:48 +07:00
|
|
|
#include "kvm_cache_regs.h"
|
2007-12-14 08:35:10 +07:00
|
|
|
|
2008-04-25 09:17:08 +07:00
|
|
|
#define PT64_PT_BITS 9
|
|
|
|
#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
|
|
|
|
#define PT32_PT_BITS 10
|
|
|
|
#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
|
|
|
|
|
|
|
|
#define PT_WRITABLE_SHIFT 1
|
2016-03-22 15:51:20 +07:00
|
|
|
#define PT_USER_SHIFT 2
|
2008-04-25 09:17:08 +07:00
|
|
|
|
|
|
|
#define PT_PRESENT_MASK (1ULL << 0)
|
|
|
|
#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
|
2016-03-22 15:51:20 +07:00
|
|
|
#define PT_USER_MASK (1ULL << PT_USER_SHIFT)
|
2008-04-25 09:17:08 +07:00
|
|
|
#define PT_PWT_MASK (1ULL << 3)
|
|
|
|
#define PT_PCD_MASK (1ULL << 4)
|
2008-05-15 17:51:35 +07:00
|
|
|
#define PT_ACCESSED_SHIFT 5
|
|
|
|
#define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
|
2012-09-12 17:44:53 +07:00
|
|
|
#define PT_DIRTY_SHIFT 6
|
|
|
|
#define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
|
2012-09-13 00:46:56 +07:00
|
|
|
#define PT_PAGE_SIZE_SHIFT 7
|
|
|
|
#define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
|
2008-04-25 09:17:08 +07:00
|
|
|
#define PT_PAT_MASK (1ULL << 7)
|
|
|
|
#define PT_GLOBAL_MASK (1ULL << 8)
|
|
|
|
#define PT64_NX_SHIFT 63
|
|
|
|
#define PT64_NX_MASK (1ULL << PT64_NX_SHIFT)
|
|
|
|
|
|
|
|
#define PT_PAT_SHIFT 7
|
|
|
|
#define PT_DIR_PAT_SHIFT 12
|
|
|
|
#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
|
|
|
|
|
|
|
|
#define PT32_DIR_PSE36_SIZE 4
|
|
|
|
#define PT32_DIR_PSE36_SHIFT 13
|
|
|
|
#define PT32_DIR_PSE36_MASK \
|
|
|
|
(((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
|
|
|
|
|
|
|
|
#define PT64_ROOT_LEVEL 4
|
|
|
|
#define PT32_ROOT_LEVEL 2
|
|
|
|
#define PT32E_ROOT_LEVEL 3
|
|
|
|
|
2010-01-05 18:02:26 +07:00
|
|
|
#define PT_PDPE_LEVEL 3
|
|
|
|
#define PT_DIRECTORY_LEVEL 2
|
|
|
|
#define PT_PAGE_TABLE_LEVEL 1
|
2015-05-13 13:42:21 +07:00
|
|
|
#define PT_MAX_HUGEPAGE_LEVEL (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES - 1)
|
2010-01-05 18:02:26 +07:00
|
|
|
|
2014-09-01 17:44:04 +07:00
|
|
|
static inline u64 rsvd_bits(int s, int e)
|
|
|
|
{
|
|
|
|
return ((1ULL << (e - s + 1)) - 1) << s;
|
|
|
|
}
|
|
|
|
|
2011-07-12 02:33:44 +07:00
|
|
|
void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask);
|
2013-06-07 15:51:25 +07:00
|
|
|
|
2015-08-05 11:04:24 +07:00
|
|
|
void
|
|
|
|
reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context);
|
|
|
|
|
2013-06-07 15:51:25 +07:00
|
|
|
/*
|
2015-11-04 19:41:21 +07:00
|
|
|
* Return values of handle_mmio_page_fault:
|
2013-06-07 15:51:25 +07:00
|
|
|
* RET_MMIO_PF_EMULATE: it is a real mmio page fault, emulate the instruction
|
2013-06-07 15:51:26 +07:00
|
|
|
* directly.
|
|
|
|
* RET_MMIO_PF_INVALID: invalid spte is detected then let the real page
|
|
|
|
* fault path update the mmio spte.
|
2013-06-07 15:51:25 +07:00
|
|
|
* RET_MMIO_PF_RETRY: let CPU fault again on the address.
|
2015-11-04 19:41:21 +07:00
|
|
|
* RET_MMIO_PF_BUG: a bug was detected (and a WARN was printed).
|
2013-06-07 15:51:25 +07:00
|
|
|
*/
|
|
|
|
enum {
|
|
|
|
RET_MMIO_PF_EMULATE = 1,
|
2013-06-07 15:51:26 +07:00
|
|
|
RET_MMIO_PF_INVALID = 2,
|
2013-06-07 15:51:25 +07:00
|
|
|
RET_MMIO_PF_RETRY = 0,
|
|
|
|
RET_MMIO_PF_BUG = -1
|
|
|
|
};
|
|
|
|
|
2015-11-04 19:41:21 +07:00
|
|
|
int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct);
|
2013-10-02 21:56:14 +07:00
|
|
|
void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly);
|
2009-06-11 22:07:42 +07:00
|
|
|
|
2010-08-20 08:11:05 +07:00
|
|
|
static inline unsigned int kvm_mmu_available_pages(struct kvm *kvm)
|
|
|
|
{
|
2013-03-13 08:36:43 +07:00
|
|
|
if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
|
|
|
|
return kvm->arch.n_max_mmu_pages -
|
|
|
|
kvm->arch.n_used_mmu_pages;
|
|
|
|
|
|
|
|
return 0;
|
2010-08-20 08:11:05 +07:00
|
|
|
}
|
|
|
|
|
2007-12-14 08:35:10 +07:00
|
|
|
static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return kvm_mmu_load(vcpu);
|
|
|
|
}
|
|
|
|
|
2014-04-17 16:06:16 +07:00
|
|
|
/*
|
|
|
|
* Currently, we have two sorts of write-protection, a) the first one
|
|
|
|
* write-protects guest page to sync the guest modification, b) another one is
|
|
|
|
* used to sync dirty bitmap when we do KVM_GET_DIRTY_LOG. The differences
|
|
|
|
* between these two sorts are:
|
|
|
|
* 1) the first case clears SPTE_MMU_WRITEABLE bit.
|
|
|
|
* 2) the first case requires flushing tlb immediately avoiding corrupting
|
|
|
|
* shadow page table between all vcpus so it should be in the protection of
|
|
|
|
* mmu-lock. And the another case does not need to flush tlb until returning
|
|
|
|
* the dirty bitmap to userspace since it only write-protects the page
|
|
|
|
* logged in the bitmap, that means the page in the dirty bitmap is not
|
|
|
|
* missed, so it can flush tlb out of mmu-lock.
|
|
|
|
*
|
|
|
|
* So, there is the problem: the first case can meet the corrupted tlb caused
|
|
|
|
* by another case which write-protects pages but without flush tlb
|
|
|
|
* immediately. In order to making the first case be aware this problem we let
|
|
|
|
* it flush tlb if we try to write-protect a spte whose SPTE_MMU_WRITEABLE bit
|
|
|
|
* is set, it works since another case never touches SPTE_MMU_WRITEABLE bit.
|
|
|
|
*
|
|
|
|
* Anyway, whenever a spte is updated (only permission and status bits are
|
|
|
|
* changed) we need to check whether the spte with SPTE_MMU_WRITEABLE becomes
|
|
|
|
* readonly, if that happens, we need to flush tlb. Fortunately,
|
|
|
|
* mmu_spte_update() has already handled it perfectly.
|
|
|
|
*
|
|
|
|
* The rules to use SPTE_MMU_WRITEABLE and PT_WRITABLE_MASK:
|
|
|
|
* - if we want to see if it has writable tlb entry or if the spte can be
|
|
|
|
* writable on the mmu mapping, check SPTE_MMU_WRITEABLE, this is the most
|
|
|
|
* case, otherwise
|
|
|
|
* - if we fix page fault on the spte or do write-protection by dirty logging,
|
|
|
|
* check PT_WRITABLE_MASK.
|
|
|
|
*
|
|
|
|
* TODO: introduce APIs to split these two cases.
|
|
|
|
*/
|
2011-07-12 02:23:20 +07:00
|
|
|
static inline int is_writable_pte(unsigned long pte)
|
|
|
|
{
|
|
|
|
return pte & PT_WRITABLE_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool is_write_protection(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
|
|
|
|
}
|
|
|
|
|
KVM: MMU: Optimize pte permission checks
walk_addr_generic() permission checks are a maze of branchy code, which is
performed four times per lookup. It depends on the type of access, efer.nxe,
cr0.wp, cr4.smep, and in the near future, cr4.smap.
Optimize this away by precalculating all variants and storing them in a
bitmap. The bitmap is recalculated when rarely-changing variables change
(cr0, cr4) and is indexed by the often-changing variables (page fault error
code, pte access permissions).
The permission check is moved to the end of the loop, otherwise an SMEP
fault could be reported as a false positive, when PDE.U=1 but PTE.U=0.
Noted by Xiao Guangrong.
The result is short, branch-free code.
Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>
2012-09-12 18:52:00 +07:00
|
|
|
/*
|
2016-03-08 16:08:16 +07:00
|
|
|
* Check if a given access (described through the I/D, W/R and U/S bits of a
|
|
|
|
* page fault error code pfec) causes a permission fault with the given PTE
|
|
|
|
* access rights (in ACC_* format).
|
|
|
|
*
|
|
|
|
* Return zero if the access does not fault; return the page fault error code
|
|
|
|
* if the access faults.
|
KVM: MMU: Optimize pte permission checks
walk_addr_generic() permission checks are a maze of branchy code, which is
performed four times per lookup. It depends on the type of access, efer.nxe,
cr0.wp, cr4.smep, and in the near future, cr4.smap.
Optimize this away by precalculating all variants and storing them in a
bitmap. The bitmap is recalculated when rarely-changing variables change
(cr0, cr4) and is indexed by the often-changing variables (page fault error
code, pte access permissions).
The permission check is moved to the end of the loop, otherwise an SMEP
fault could be reported as a false positive, when PDE.U=1 but PTE.U=0.
Noted by Xiao Guangrong.
The result is short, branch-free code.
Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>
2012-09-12 18:52:00 +07:00
|
|
|
*/
|
2016-03-08 16:08:16 +07:00
|
|
|
static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
2016-03-22 15:51:20 +07:00
|
|
|
unsigned pte_access, unsigned pte_pkey,
|
|
|
|
unsigned pfec)
|
2011-07-12 02:23:20 +07:00
|
|
|
{
|
2014-04-01 16:46:34 +07:00
|
|
|
int cpl = kvm_x86_ops->get_cpl(vcpu);
|
|
|
|
unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.
|
|
|
|
*
|
|
|
|
* If CPL = 3, SMAP applies to all supervisor-mode data accesses
|
|
|
|
* (these are implicit supervisor accesses) regardless of the value
|
|
|
|
* of EFLAGS.AC.
|
|
|
|
*
|
|
|
|
* This computes (cpl < 3) && (rflags & X86_EFLAGS_AC), leaving
|
|
|
|
* the result in X86_EFLAGS_AC. We then insert it in place of
|
|
|
|
* the PFERR_RSVD_MASK bit; this bit will always be zero in pfec,
|
|
|
|
* but it will be one in index if SMAP checks are being overridden.
|
|
|
|
* It is important to keep this branchless.
|
|
|
|
*/
|
|
|
|
unsigned long smap = (cpl - 3) & (rflags & X86_EFLAGS_AC);
|
|
|
|
int index = (pfec >> 1) +
|
|
|
|
(smap >> (X86_EFLAGS_AC_BIT - PFERR_RSVD_BIT + 1));
|
2016-03-22 15:51:20 +07:00
|
|
|
bool fault = (mmu->permissions[index] >> pte_access) & 1;
|
2016-03-25 20:19:35 +07:00
|
|
|
u32 errcode = PFERR_PRESENT_MASK;
|
2014-04-01 16:46:34 +07:00
|
|
|
|
2016-03-22 15:51:20 +07:00
|
|
|
WARN_ON(pfec & (PFERR_PK_MASK | PFERR_RSVD_MASK));
|
|
|
|
if (unlikely(mmu->pkru_mask)) {
|
|
|
|
u32 pkru_bits, offset;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* PKRU defines 32 bits, there are 16 domains and 2
|
|
|
|
* attribute bits per domain in pkru. pte_pkey is the
|
|
|
|
* index of the protection domain, so pte_pkey * 2 is
|
|
|
|
* is the index of the first bit for the domain.
|
|
|
|
*/
|
|
|
|
pkru_bits = (kvm_read_pkru(vcpu) >> (pte_pkey * 2)) & 3;
|
|
|
|
|
|
|
|
/* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
|
2016-03-25 20:19:35 +07:00
|
|
|
offset = (pfec & ~1) +
|
2016-03-22 15:51:20 +07:00
|
|
|
((pte_access & PT_USER_MASK) << (PFERR_RSVD_BIT - PT_USER_SHIFT));
|
|
|
|
|
|
|
|
pkru_bits &= mmu->pkru_mask >> offset;
|
2016-03-25 20:19:35 +07:00
|
|
|
errcode |= -pkru_bits & PFERR_PK_MASK;
|
2016-03-22 15:51:20 +07:00
|
|
|
fault |= (pkru_bits != 0);
|
|
|
|
}
|
|
|
|
|
2016-03-25 20:19:35 +07:00
|
|
|
return -(u32)fault & errcode;
|
2011-07-12 02:23:20 +07:00
|
|
|
}
|
KVM: MMU: Optimize pte permission checks
walk_addr_generic() permission checks are a maze of branchy code, which is
performed four times per lookup. It depends on the type of access, efer.nxe,
cr0.wp, cr4.smep, and in the near future, cr4.smap.
Optimize this away by precalculating all variants and storing them in a
bitmap. The bitmap is recalculated when rarely-changing variables change
(cr0, cr4) and is indexed by the often-changing variables (page fault error
code, pte access permissions).
The permission check is moved to the end of the loop, otherwise an SMEP
fault could be reported as a false positive, when PDE.U=1 but PTE.U=0.
Noted by Xiao Guangrong.
The result is short, branch-free code.
Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>
2012-09-12 18:52:00 +07:00
|
|
|
|
2013-05-31 07:36:22 +07:00
|
|
|
void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm);
|
2015-05-13 13:42:27 +07:00
|
|
|
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
|
2016-02-24 16:51:07 +07:00
|
|
|
|
|
|
|
void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn);
|
|
|
|
void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn);
|
2016-02-24 16:51:08 +07:00
|
|
|
bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
|
|
|
|
struct kvm_memory_slot *slot, u64 gfn);
|
2007-12-14 08:35:10 +07:00
|
|
|
#endif
|