2017-04-04 19:48:17 +07:00
|
|
|
#include <linux/pfn.h>
|
|
|
|
#include <asm/xen/page.h>
|
|
|
|
#include <asm/xen/hypercall.h>
|
|
|
|
#include <xen/interface/memory.h>
|
2015-07-17 11:51:35 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
#include "multicalls.h"
|
|
|
|
#include "mmu.h"
|
2015-07-17 11:51:35 +07:00
|
|
|
|
|
|
|
/*
|
2017-04-04 19:48:17 +07:00
|
|
|
* Protects atomic reservation decrease/increase against concurrent increases.
|
|
|
|
* Also protects non-atomic updates of current_pages and balloon lists.
|
2015-07-17 11:51:35 +07:00
|
|
|
*/
|
2017-04-04 19:48:17 +07:00
|
|
|
DEFINE_SPINLOCK(xen_reservation_lock);
|
2015-07-17 11:51:35 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
unsigned long arbitrary_virt_to_mfn(void *vaddr)
|
2015-07-17 11:51:35 +07:00
|
|
|
{
|
2017-04-04 19:48:17 +07:00
|
|
|
xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
|
2015-07-17 11:51:35 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
return PFN_DOWN(maddr.maddr);
|
2015-07-17 11:51:35 +07:00
|
|
|
}
|
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
xmaddr_t arbitrary_virt_to_machine(void *vaddr)
|
2010-11-24 19:09:41 +07:00
|
|
|
{
|
2017-04-04 19:48:17 +07:00
|
|
|
unsigned long address = (unsigned long)vaddr;
|
|
|
|
unsigned int level;
|
|
|
|
pte_t *pte;
|
|
|
|
unsigned offset;
|
2010-11-24 19:09:41 +07:00
|
|
|
|
|
|
|
/*
|
2017-04-04 19:48:17 +07:00
|
|
|
* if the PFN is in the linear mapped vaddr range, we can just use
|
|
|
|
* the (quick) virt_to_machine() p2m lookup
|
2010-11-24 19:09:41 +07:00
|
|
|
*/
|
2017-04-04 19:48:17 +07:00
|
|
|
if (virt_addr_valid(vaddr))
|
|
|
|
return virt_to_machine(vaddr);
|
2015-12-13 07:25:55 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
/* otherwise we have to do a (slower) full page-table walk */
|
2010-03-27 05:37:50 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
pte = lookup_address(address, &level);
|
|
|
|
BUG_ON(pte == NULL);
|
|
|
|
offset = address & ~PAGE_MASK;
|
|
|
|
return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
|
2009-08-20 19:30:02 +07:00
|
|
|
}
|
2017-04-04 19:48:17 +07:00
|
|
|
EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
|
2009-01-29 05:35:01 +07:00
|
|
|
|
2017-05-18 22:46:48 +07:00
|
|
|
static void xen_flush_tlb_all(void)
|
2009-02-10 03:05:46 +07:00
|
|
|
{
|
2017-04-04 19:48:17 +07:00
|
|
|
struct mmuext_op *op;
|
2009-02-10 03:05:46 +07:00
|
|
|
struct multicall_space mcs;
|
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
trace_xen_mmu_flush_tlb_all(0);
|
2009-02-10 03:05:46 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
preempt_disable();
|
2009-02-10 03:05:46 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
mcs = xen_mc_entry(sizeof(*op));
|
2009-02-10 03:05:46 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
op = mcs.args;
|
|
|
|
op->cmd = MMUEXT_TLB_FLUSH_ALL;
|
|
|
|
MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
|
2009-02-10 03:05:46 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
xen_mc_issue(PARAVIRT_LAZY_MMU);
|
2009-02-10 03:05:46 +07:00
|
|
|
|
2017-04-04 19:48:17 +07:00
|
|
|
preempt_enable();
|
2009-08-20 19:30:02 +07:00
|
|
|
}
|
2009-01-29 05:35:01 +07:00
|
|
|
|
2009-05-21 16:09:46 +07:00
|
|
|
#define REMAP_BATCH_SIZE 16
|
|
|
|
|
|
|
|
struct remap_data {
|
2015-03-11 21:49:57 +07:00
|
|
|
xen_pfn_t *mfn;
|
|
|
|
bool contiguous;
|
2009-05-21 16:09:46 +07:00
|
|
|
pgprot_t prot;
|
|
|
|
struct mmu_update *mmu_update;
|
|
|
|
};
|
|
|
|
|
|
|
|
static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
|
|
|
|
unsigned long addr, void *data)
|
|
|
|
{
|
|
|
|
struct remap_data *rmd = data;
|
2015-03-11 21:49:57 +07:00
|
|
|
pte_t pte = pte_mkspecial(mfn_pte(*rmd->mfn, rmd->prot));
|
|
|
|
|
2016-02-24 06:34:30 +07:00
|
|
|
/* If we have a contiguous range, just update the mfn itself,
|
2015-03-11 21:49:57 +07:00
|
|
|
else update pointer to be "next mfn". */
|
|
|
|
if (rmd->contiguous)
|
|
|
|
(*rmd->mfn)++;
|
|
|
|
else
|
|
|
|
rmd->mfn++;
|
2009-05-21 16:09:46 +07:00
|
|
|
|
2017-08-29 17:27:53 +07:00
|
|
|
rmd->mmu_update->ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
|
2009-05-21 16:09:46 +07:00
|
|
|
rmd->mmu_update->val = pte_val_ma(pte);
|
|
|
|
rmd->mmu_update++;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-08-07 23:34:41 +07:00
|
|
|
static int do_remap_gfn(struct vm_area_struct *vma,
|
2015-03-11 21:49:57 +07:00
|
|
|
unsigned long addr,
|
2015-08-07 23:34:41 +07:00
|
|
|
xen_pfn_t *gfn, int nr,
|
2015-03-11 21:49:57 +07:00
|
|
|
int *err_ptr, pgprot_t prot,
|
|
|
|
unsigned domid,
|
|
|
|
struct page **pages)
|
2009-05-21 16:09:46 +07:00
|
|
|
{
|
2015-03-11 21:49:57 +07:00
|
|
|
int err = 0;
|
2009-05-21 16:09:46 +07:00
|
|
|
struct remap_data rmd;
|
|
|
|
struct mmu_update mmu_update[REMAP_BATCH_SIZE];
|
|
|
|
unsigned long range;
|
2015-03-11 21:49:57 +07:00
|
|
|
int mapped = 0;
|
2009-05-21 16:09:46 +07:00
|
|
|
|
mm: kill vma flag VM_RESERVED and mm->reserved_vm counter
A long time ago, in v2.4, VM_RESERVED kept swapout process off VMA,
currently it lost original meaning but still has some effects:
| effect | alternative flags
-+------------------------+---------------------------------------------
1| account as reserved_vm | VM_IO
2| skip in core dump | VM_IO, VM_DONTDUMP
3| do not merge or expand | VM_IO, VM_DONTEXPAND, VM_HUGETLB, VM_PFNMAP
4| do not mlock | VM_IO, VM_DONTEXPAND, VM_HUGETLB, VM_PFNMAP
This patch removes reserved_vm counter from mm_struct. Seems like nobody
cares about it, it does not exported into userspace directly, it only
reduces total_vm showed in proc.
Thus VM_RESERVED can be replaced with VM_IO or pair VM_DONTEXPAND | VM_DONTDUMP.
remap_pfn_range() and io_remap_pfn_range() set VM_IO|VM_DONTEXPAND|VM_DONTDUMP.
remap_vmalloc_range() set VM_DONTEXPAND | VM_DONTDUMP.
[akpm@linux-foundation.org: drivers/vfio/pci/vfio_pci.c fixup]
Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Carsten Otte <cotte@de.ibm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Eric Paris <eparis@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: James Morris <james.l.morris@oracle.com>
Cc: Jason Baron <jbaron@redhat.com>
Cc: Kentaro Takeda <takedakn@nttdata.co.jp>
Cc: Matt Helsley <matthltc@us.ibm.com>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Robert Richter <robert.richter@amd.com>
Cc: Suresh Siddha <suresh.b.siddha@intel.com>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Venkatesh Pallipadi <venki@google.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-09 06:29:02 +07:00
|
|
|
BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
|
2009-05-21 16:09:46 +07:00
|
|
|
|
2015-08-07 23:34:41 +07:00
|
|
|
rmd.mfn = gfn;
|
2009-05-21 16:09:46 +07:00
|
|
|
rmd.prot = prot;
|
2016-02-24 06:34:30 +07:00
|
|
|
/* We use the err_ptr to indicate if there we are doing a contiguous
|
2015-03-11 21:49:57 +07:00
|
|
|
* mapping or a discontigious mapping. */
|
|
|
|
rmd.contiguous = !err_ptr;
|
2009-05-21 16:09:46 +07:00
|
|
|
|
|
|
|
while (nr) {
|
2015-03-11 21:49:57 +07:00
|
|
|
int index = 0;
|
|
|
|
int done = 0;
|
|
|
|
int batch = min(REMAP_BATCH_SIZE, nr);
|
|
|
|
int batch_left = batch;
|
2009-05-21 16:09:46 +07:00
|
|
|
range = (unsigned long)batch << PAGE_SHIFT;
|
|
|
|
|
|
|
|
rmd.mmu_update = mmu_update;
|
|
|
|
err = apply_to_page_range(vma->vm_mm, addr, range,
|
|
|
|
remap_area_mfn_pte_fn, &rmd);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
|
|
|
|
2015-03-11 21:49:57 +07:00
|
|
|
/* We record the error for each page that gives an error, but
|
|
|
|
* continue mapping until the whole set is done */
|
|
|
|
do {
|
|
|
|
int i;
|
|
|
|
|
|
|
|
err = HYPERVISOR_mmu_update(&mmu_update[index],
|
|
|
|
batch_left, &done, domid);
|
|
|
|
|
|
|
|
/*
|
2015-08-07 23:34:41 +07:00
|
|
|
* @err_ptr may be the same buffer as @gfn, so
|
|
|
|
* only clear it after each chunk of @gfn is
|
2015-03-11 21:49:57 +07:00
|
|
|
* used.
|
|
|
|
*/
|
|
|
|
if (err_ptr) {
|
|
|
|
for (i = index; i < index + done; i++)
|
|
|
|
err_ptr[i] = 0;
|
|
|
|
}
|
|
|
|
if (err < 0) {
|
|
|
|
if (!err_ptr)
|
|
|
|
goto out;
|
|
|
|
err_ptr[i] = err;
|
|
|
|
done++; /* Skip failed frame. */
|
|
|
|
} else
|
|
|
|
mapped += done;
|
|
|
|
batch_left -= done;
|
|
|
|
index += done;
|
|
|
|
} while (batch_left);
|
2009-05-21 16:09:46 +07:00
|
|
|
|
|
|
|
nr -= batch;
|
|
|
|
addr += range;
|
2015-03-11 21:49:57 +07:00
|
|
|
if (err_ptr)
|
|
|
|
err_ptr += batch;
|
2015-10-28 20:39:05 +07:00
|
|
|
cond_resched();
|
2009-05-21 16:09:46 +07:00
|
|
|
}
|
|
|
|
out:
|
|
|
|
|
2012-10-31 23:38:31 +07:00
|
|
|
xen_flush_tlb_all();
|
2009-05-21 16:09:46 +07:00
|
|
|
|
2015-03-11 21:49:57 +07:00
|
|
|
return err < 0 ? err : mapped;
|
|
|
|
}
|
|
|
|
|
2015-08-07 23:34:41 +07:00
|
|
|
int xen_remap_domain_gfn_range(struct vm_area_struct *vma,
|
2015-03-11 21:49:57 +07:00
|
|
|
unsigned long addr,
|
2015-08-07 23:34:41 +07:00
|
|
|
xen_pfn_t gfn, int nr,
|
2015-03-11 21:49:57 +07:00
|
|
|
pgprot_t prot, unsigned domid,
|
|
|
|
struct page **pages)
|
|
|
|
{
|
2017-11-04 00:04:11 +07:00
|
|
|
if (xen_feature(XENFEAT_auto_translated_physmap))
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2015-08-07 23:34:41 +07:00
|
|
|
return do_remap_gfn(vma, addr, &gfn, nr, NULL, prot, domid, pages);
|
2009-05-21 16:09:46 +07:00
|
|
|
}
|
2015-08-07 23:34:41 +07:00
|
|
|
EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range);
|
2012-10-18 03:37:49 +07:00
|
|
|
|
2015-08-07 23:34:41 +07:00
|
|
|
int xen_remap_domain_gfn_array(struct vm_area_struct *vma,
|
2015-03-11 21:49:57 +07:00
|
|
|
unsigned long addr,
|
2015-08-07 23:34:41 +07:00
|
|
|
xen_pfn_t *gfn, int nr,
|
2015-03-11 21:49:57 +07:00
|
|
|
int *err_ptr, pgprot_t prot,
|
|
|
|
unsigned domid, struct page **pages)
|
|
|
|
{
|
2017-11-04 00:04:11 +07:00
|
|
|
if (xen_feature(XENFEAT_auto_translated_physmap))
|
|
|
|
return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr,
|
|
|
|
prot, domid, pages);
|
|
|
|
|
2015-03-11 21:49:57 +07:00
|
|
|
/* We BUG_ON because it's a programmer error to pass a NULL err_ptr,
|
|
|
|
* and the consequences later is quite hard to detect what the actual
|
|
|
|
* cause of "wrong memory was mapped in".
|
|
|
|
*/
|
|
|
|
BUG_ON(err_ptr == NULL);
|
2015-08-07 23:34:41 +07:00
|
|
|
return do_remap_gfn(vma, addr, gfn, nr, err_ptr, prot, domid, pages);
|
2015-03-11 21:49:57 +07:00
|
|
|
}
|
2015-08-07 23:34:41 +07:00
|
|
|
EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array);
|
2015-03-11 21:49:57 +07:00
|
|
|
|
2012-10-18 03:37:49 +07:00
|
|
|
/* Returns: 0 success */
|
2015-08-07 23:34:41 +07:00
|
|
|
int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
|
2017-11-04 00:04:11 +07:00
|
|
|
int nr, struct page **pages)
|
2012-10-18 03:37:49 +07:00
|
|
|
{
|
2017-11-04 00:04:11 +07:00
|
|
|
if (xen_feature(XENFEAT_auto_translated_physmap))
|
|
|
|
return xen_xlate_unmap_gfn_range(vma, nr, pages);
|
|
|
|
|
|
|
|
if (!pages)
|
2012-10-18 03:37:49 +07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2015-08-07 23:34:41 +07:00
|
|
|
EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range);
|