2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* linux/mm/mincore.c
|
|
|
|
*
|
2006-12-17 00:44:32 +07:00
|
|
|
* Copyright (C) 1994-2006 Linus Torvalds
|
2005-04-17 05:20:36 +07:00
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The mincore() system call.
|
|
|
|
*/
|
|
|
|
#include <linux/pagemap.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 15:04:11 +07:00
|
|
|
#include <linux/gfp.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/mman.h>
|
|
|
|
#include <linux/syscalls.h>
|
2007-02-12 15:51:39 +07:00
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/swapops.h>
|
2009-12-15 08:59:58 +07:00
|
|
|
#include <linux/hugetlb.h>
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
#include <asm/uaccess.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
|
2010-05-25 04:32:10 +07:00
|
|
|
static void mincore_hugetlb_page_range(struct vm_area_struct *vma,
|
2010-05-25 04:32:11 +07:00
|
|
|
unsigned long addr, unsigned long end,
|
2010-05-25 04:32:10 +07:00
|
|
|
unsigned char *vec)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
struct hstate *h;
|
|
|
|
|
|
|
|
h = hstate_vma(vma);
|
|
|
|
while (1) {
|
|
|
|
unsigned char present;
|
|
|
|
pte_t *ptep;
|
|
|
|
/*
|
|
|
|
* Huge pages are always in RAM for now, but
|
|
|
|
* theoretically it needs to be checked.
|
|
|
|
*/
|
|
|
|
ptep = huge_pte_offset(current->mm,
|
|
|
|
addr & huge_page_mask(h));
|
|
|
|
present = ptep && !huge_pte_none(huge_ptep_get(ptep));
|
|
|
|
while (1) {
|
2010-05-25 04:32:11 +07:00
|
|
|
*vec = present;
|
|
|
|
vec++;
|
2010-05-25 04:32:10 +07:00
|
|
|
addr += PAGE_SIZE;
|
2010-05-25 04:32:11 +07:00
|
|
|
if (addr == end)
|
2010-05-25 04:32:10 +07:00
|
|
|
return;
|
|
|
|
/* check hugepage border */
|
|
|
|
if (!(addr & ~huge_page_mask(h)))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
BUG();
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
/*
|
|
|
|
* Later we can get more picky about what "in core" means precisely.
|
|
|
|
* For now, simply check to see if the page is in the page cache,
|
|
|
|
* and is up to date; i.e. that no page-in operation would be required
|
|
|
|
* at this time if an application were to map and access this page.
|
|
|
|
*/
|
2007-02-12 15:51:39 +07:00
|
|
|
static unsigned char mincore_page(struct address_space *mapping, pgoff_t pgoff)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
|
|
|
unsigned char present = 0;
|
2007-02-12 15:51:39 +07:00
|
|
|
struct page *page;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2007-02-12 15:51:39 +07:00
|
|
|
/*
|
|
|
|
* When tmpfs swaps out a page from a file, any process mapping that
|
|
|
|
* file will not get a swp_entry_t in its pte, but rather it is like
|
|
|
|
* any other file mapping (ie. marked !present and faulted in with
|
2008-04-28 16:12:10 +07:00
|
|
|
* tmpfs's .fault). So swapped out tmpfs mappings are tested here.
|
2007-02-12 15:51:39 +07:00
|
|
|
*/
|
2011-08-04 06:21:27 +07:00
|
|
|
#ifdef CONFIG_SWAP
|
2014-04-04 04:47:46 +07:00
|
|
|
if (shmem_mapping(mapping)) {
|
|
|
|
page = find_get_entry(mapping, pgoff);
|
|
|
|
/*
|
|
|
|
* shmem/tmpfs may return swap: account for swapcache
|
|
|
|
* page too.
|
|
|
|
*/
|
|
|
|
if (radix_tree_exceptional_entry(page)) {
|
|
|
|
swp_entry_t swp = radix_to_swp_entry(page);
|
|
|
|
page = find_get_page(swap_address_space(swp), swp.val);
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
page = find_get_page(mapping, pgoff);
|
|
|
|
#else
|
|
|
|
page = find_get_page(mapping, pgoff);
|
2011-08-04 06:21:27 +07:00
|
|
|
#endif
|
2005-04-17 05:20:36 +07:00
|
|
|
if (page) {
|
|
|
|
present = PageUptodate(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
}
|
|
|
|
|
|
|
|
return present;
|
|
|
|
}
|
|
|
|
|
2010-05-25 04:32:10 +07:00
|
|
|
static void mincore_unmapped_range(struct vm_area_struct *vma,
|
2010-05-25 04:32:11 +07:00
|
|
|
unsigned long addr, unsigned long end,
|
2010-05-25 04:32:10 +07:00
|
|
|
unsigned char *vec)
|
|
|
|
{
|
2010-05-25 04:32:11 +07:00
|
|
|
unsigned long nr = (end - addr) >> PAGE_SHIFT;
|
2010-05-25 04:32:10 +07:00
|
|
|
int i;
|
|
|
|
|
|
|
|
if (vma->vm_file) {
|
|
|
|
pgoff_t pgoff;
|
|
|
|
|
|
|
|
pgoff = linear_page_index(vma, addr);
|
|
|
|
for (i = 0; i < nr; i++, pgoff++)
|
|
|
|
vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff);
|
|
|
|
} else {
|
|
|
|
for (i = 0; i < nr; i++)
|
|
|
|
vec[i] = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mincore_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
|
2010-05-25 04:32:11 +07:00
|
|
|
unsigned long addr, unsigned long end,
|
2010-05-25 04:32:10 +07:00
|
|
|
unsigned char *vec)
|
|
|
|
{
|
2010-05-25 04:32:11 +07:00
|
|
|
unsigned long next;
|
2010-05-25 04:32:10 +07:00
|
|
|
spinlock_t *ptl;
|
|
|
|
pte_t *ptep;
|
|
|
|
|
|
|
|
ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
|
2010-05-25 04:32:11 +07:00
|
|
|
do {
|
2010-05-25 04:32:10 +07:00
|
|
|
pte_t pte = *ptep;
|
|
|
|
pgoff_t pgoff;
|
|
|
|
|
2010-05-25 04:32:11 +07:00
|
|
|
next = addr + PAGE_SIZE;
|
2010-05-25 04:32:10 +07:00
|
|
|
if (pte_none(pte))
|
2010-05-25 04:32:11 +07:00
|
|
|
mincore_unmapped_range(vma, addr, next, vec);
|
2010-05-25 04:32:10 +07:00
|
|
|
else if (pte_present(pte))
|
2010-05-25 04:32:11 +07:00
|
|
|
*vec = 1;
|
2010-05-25 04:32:10 +07:00
|
|
|
else if (pte_file(pte)) {
|
|
|
|
pgoff = pte_to_pgoff(pte);
|
2010-05-25 04:32:11 +07:00
|
|
|
*vec = mincore_page(vma->vm_file->f_mapping, pgoff);
|
2010-05-25 04:32:10 +07:00
|
|
|
} else { /* pte is a swap entry */
|
|
|
|
swp_entry_t entry = pte_to_swp_entry(pte);
|
|
|
|
|
|
|
|
if (is_migration_entry(entry)) {
|
|
|
|
/* migration entries are always uptodate */
|
2010-05-25 04:32:11 +07:00
|
|
|
*vec = 1;
|
2010-05-25 04:32:10 +07:00
|
|
|
} else {
|
|
|
|
#ifdef CONFIG_SWAP
|
|
|
|
pgoff = entry.val;
|
2013-02-23 07:34:37 +07:00
|
|
|
*vec = mincore_page(swap_address_space(entry),
|
|
|
|
pgoff);
|
2010-05-25 04:32:10 +07:00
|
|
|
#else
|
|
|
|
WARN_ON(1);
|
2010-05-25 04:32:11 +07:00
|
|
|
*vec = 1;
|
2010-05-25 04:32:10 +07:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
2010-05-25 04:32:11 +07:00
|
|
|
vec++;
|
|
|
|
} while (ptep++, addr = next, addr != end);
|
2010-05-25 04:32:10 +07:00
|
|
|
pte_unmap_unlock(ptep - 1, ptl);
|
|
|
|
}
|
|
|
|
|
2010-05-25 04:32:11 +07:00
|
|
|
static void mincore_pmd_range(struct vm_area_struct *vma, pud_t *pud,
|
|
|
|
unsigned long addr, unsigned long end,
|
|
|
|
unsigned char *vec)
|
|
|
|
{
|
|
|
|
unsigned long next;
|
|
|
|
pmd_t *pmd;
|
|
|
|
|
|
|
|
pmd = pmd_offset(pud, addr);
|
|
|
|
do {
|
|
|
|
next = pmd_addr_end(addr, end);
|
2011-01-14 06:47:02 +07:00
|
|
|
if (pmd_trans_huge(*pmd)) {
|
|
|
|
if (mincore_huge_pmd(vma, pmd, addr, next, vec)) {
|
|
|
|
vec += (next - addr) >> PAGE_SHIFT;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
/* fall through */
|
|
|
|
}
|
mm: thp: fix pmd_bad() triggering in code paths holding mmap_sem read mode
In some cases it may happen that pmd_none_or_clear_bad() is called with
the mmap_sem hold in read mode. In those cases the huge page faults can
allocate hugepmds under pmd_none_or_clear_bad() and that can trigger a
false positive from pmd_bad() that will not like to see a pmd
materializing as trans huge.
It's not khugepaged causing the problem, khugepaged holds the mmap_sem
in write mode (and all those sites must hold the mmap_sem in read mode
to prevent pagetables to go away from under them, during code review it
seems vm86 mode on 32bit kernels requires that too unless it's
restricted to 1 thread per process or UP builds). The race is only with
the huge pagefaults that can convert a pmd_none() into a
pmd_trans_huge().
Effectively all these pmd_none_or_clear_bad() sites running with
mmap_sem in read mode are somewhat speculative with the page faults, and
the result is always undefined when they run simultaneously. This is
probably why it wasn't common to run into this. For example if the
madvise(MADV_DONTNEED) runs zap_page_range() shortly before the page
fault, the hugepage will not be zapped, if the page fault runs first it
will be zapped.
Altering pmd_bad() not to error out if it finds hugepmds won't be enough
to fix this, because zap_pmd_range would then proceed to call
zap_pte_range (which would be incorrect if the pmd become a
pmd_trans_huge()).
The simplest way to fix this is to read the pmd in the local stack
(regardless of what we read, no need of actual CPU barriers, only
compiler barrier needed), and be sure it is not changing under the code
that computes its value. Even if the real pmd is changing under the
value we hold on the stack, we don't care. If we actually end up in
zap_pte_range it means the pmd was not none already and it was not huge,
and it can't become huge from under us (khugepaged locking explained
above).
All we need is to enforce that there is no way anymore that in a code
path like below, pmd_trans_huge can be false, but pmd_none_or_clear_bad
can run into a hugepmd. The overhead of a barrier() is just a compiler
tweak and should not be measurable (I only added it for THP builds). I
don't exclude different compiler versions may have prevented the race
too by caching the value of *pmd on the stack (that hasn't been
verified, but it wouldn't be impossible considering
pmd_none_or_clear_bad, pmd_bad, pmd_trans_huge, pmd_none are all inlines
and there's no external function called in between pmd_trans_huge and
pmd_none_or_clear_bad).
if (pmd_trans_huge(*pmd)) {
if (next-addr != HPAGE_PMD_SIZE) {
VM_BUG_ON(!rwsem_is_locked(&tlb->mm->mmap_sem));
split_huge_page_pmd(vma->vm_mm, pmd);
} else if (zap_huge_pmd(tlb, vma, pmd, addr))
continue;
/* fall through */
}
if (pmd_none_or_clear_bad(pmd))
Because this race condition could be exercised without special
privileges this was reported in CVE-2012-1179.
The race was identified and fully explained by Ulrich who debugged it.
I'm quoting his accurate explanation below, for reference.
====== start quote =======
mapcount 0 page_mapcount 1
kernel BUG at mm/huge_memory.c:1384!
At some point prior to the panic, a "bad pmd ..." message similar to the
following is logged on the console:
mm/memory.c:145: bad pmd ffff8800376e1f98(80000000314000e7).
The "bad pmd ..." message is logged by pmd_clear_bad() before it clears
the page's PMD table entry.
143 void pmd_clear_bad(pmd_t *pmd)
144 {
-> 145 pmd_ERROR(*pmd);
146 pmd_clear(pmd);
147 }
After the PMD table entry has been cleared, there is an inconsistency
between the actual number of PMD table entries that are mapping the page
and the page's map count (_mapcount field in struct page). When the page
is subsequently reclaimed, __split_huge_page() detects this inconsistency.
1381 if (mapcount != page_mapcount(page))
1382 printk(KERN_ERR "mapcount %d page_mapcount %d\n",
1383 mapcount, page_mapcount(page));
-> 1384 BUG_ON(mapcount != page_mapcount(page));
The root cause of the problem is a race of two threads in a multithreaded
process. Thread B incurs a page fault on a virtual address that has never
been accessed (PMD entry is zero) while Thread A is executing an madvise()
system call on a virtual address within the same 2 MB (huge page) range.
virtual address space
.---------------------.
| |
| |
.-|---------------------|
| | |
| | |<-- B(fault)
| | |
2 MB | |/////////////////////|-.
huge < |/////////////////////| > A(range)
page | |/////////////////////|-'
| | |
| | |
'-|---------------------|
| |
| |
'---------------------'
- Thread A is executing an madvise(..., MADV_DONTNEED) system call
on the virtual address range "A(range)" shown in the picture.
sys_madvise
// Acquire the semaphore in shared mode.
down_read(¤t->mm->mmap_sem)
...
madvise_vma
switch (behavior)
case MADV_DONTNEED:
madvise_dontneed
zap_page_range
unmap_vmas
unmap_page_range
zap_pud_range
zap_pmd_range
//
// Assume that this huge page has never been accessed.
// I.e. content of the PMD entry is zero (not mapped).
//
if (pmd_trans_huge(*pmd)) {
// We don't get here due to the above assumption.
}
//
// Assume that Thread B incurred a page fault and
.---------> // sneaks in here as shown below.
| //
| if (pmd_none_or_clear_bad(pmd))
| {
| if (unlikely(pmd_bad(*pmd)))
| pmd_clear_bad
| {
| pmd_ERROR
| // Log "bad pmd ..." message here.
| pmd_clear
| // Clear the page's PMD entry.
| // Thread B incremented the map count
| // in page_add_new_anon_rmap(), but
| // now the page is no longer mapped
| // by a PMD entry (-> inconsistency).
| }
| }
|
v
- Thread B is handling a page fault on virtual address "B(fault)" shown
in the picture.
...
do_page_fault
__do_page_fault
// Acquire the semaphore in shared mode.
down_read_trylock(&mm->mmap_sem)
...
handle_mm_fault
if (pmd_none(*pmd) && transparent_hugepage_enabled(vma))
// We get here due to the above assumption (PMD entry is zero).
do_huge_pmd_anonymous_page
alloc_hugepage_vma
// Allocate a new transparent huge page here.
...
__do_huge_pmd_anonymous_page
...
spin_lock(&mm->page_table_lock)
...
page_add_new_anon_rmap
// Here we increment the page's map count (starts at -1).
atomic_set(&page->_mapcount, 0)
set_pmd_at
// Here we set the page's PMD entry which will be cleared
// when Thread A calls pmd_clear_bad().
...
spin_unlock(&mm->page_table_lock)
The mmap_sem does not prevent the race because both threads are acquiring
it in shared mode (down_read). Thread B holds the page_table_lock while
the page's map count and PMD table entry are updated. However, Thread A
does not synchronize on that lock.
====== end quote =======
[akpm@linux-foundation.org: checkpatch fixes]
Reported-by: Ulrich Obergfell <uobergfe@redhat.com>
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Jones <davej@redhat.com>
Acked-by: Larry Woodman <lwoodman@redhat.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: <stable@vger.kernel.org> [2.6.38+]
Cc: Mark Salter <msalter@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-22 06:33:42 +07:00
|
|
|
if (pmd_none_or_trans_huge_or_clear_bad(pmd))
|
2010-05-25 04:32:11 +07:00
|
|
|
mincore_unmapped_range(vma, addr, next, vec);
|
|
|
|
else
|
|
|
|
mincore_pte_range(vma, pmd, addr, next, vec);
|
|
|
|
vec += (next - addr) >> PAGE_SHIFT;
|
|
|
|
} while (pmd++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mincore_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
|
|
|
|
unsigned long addr, unsigned long end,
|
|
|
|
unsigned char *vec)
|
|
|
|
{
|
|
|
|
unsigned long next;
|
|
|
|
pud_t *pud;
|
|
|
|
|
|
|
|
pud = pud_offset(pgd, addr);
|
|
|
|
do {
|
|
|
|
next = pud_addr_end(addr, end);
|
|
|
|
if (pud_none_or_clear_bad(pud))
|
|
|
|
mincore_unmapped_range(vma, addr, next, vec);
|
|
|
|
else
|
|
|
|
mincore_pmd_range(vma, pud, addr, next, vec);
|
|
|
|
vec += (next - addr) >> PAGE_SHIFT;
|
|
|
|
} while (pud++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mincore_page_range(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, unsigned long end,
|
|
|
|
unsigned char *vec)
|
|
|
|
{
|
|
|
|
unsigned long next;
|
|
|
|
pgd_t *pgd;
|
|
|
|
|
|
|
|
pgd = pgd_offset(vma->vm_mm, addr);
|
|
|
|
do {
|
|
|
|
next = pgd_addr_end(addr, end);
|
|
|
|
if (pgd_none_or_clear_bad(pgd))
|
|
|
|
mincore_unmapped_range(vma, addr, next, vec);
|
|
|
|
else
|
|
|
|
mincore_pud_range(vma, pgd, addr, next, vec);
|
|
|
|
vec += (next - addr) >> PAGE_SHIFT;
|
|
|
|
} while (pgd++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
2006-12-17 00:44:32 +07:00
|
|
|
/*
|
|
|
|
* Do a chunk of "sys_mincore()". We've already checked
|
|
|
|
* all the arguments, we hold the mmap semaphore: we should
|
|
|
|
* just return the amount of info we're asked for.
|
|
|
|
*/
|
2010-05-25 04:32:09 +07:00
|
|
|
static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2010-05-25 04:32:09 +07:00
|
|
|
struct vm_area_struct *vma;
|
2010-05-25 04:32:11 +07:00
|
|
|
unsigned long end;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2010-05-25 04:32:09 +07:00
|
|
|
vma = find_vma(current->mm, addr);
|
2006-12-17 07:01:50 +07:00
|
|
|
if (!vma || addr < vma->vm_start)
|
|
|
|
return -ENOMEM;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2010-05-25 04:32:11 +07:00
|
|
|
end = min(vma->vm_end, addr + (pages << PAGE_SHIFT));
|
2010-05-25 04:32:09 +07:00
|
|
|
|
2010-05-25 04:32:11 +07:00
|
|
|
if (is_vm_hugetlb_page(vma))
|
|
|
|
mincore_hugetlb_page_range(vma, addr, end, vec);
|
|
|
|
else
|
|
|
|
mincore_page_range(vma, addr, end, vec);
|
2007-02-12 15:51:39 +07:00
|
|
|
|
2010-05-25 04:32:11 +07:00
|
|
|
return (end - addr) >> PAGE_SHIFT;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The mincore(2) system call.
|
|
|
|
*
|
|
|
|
* mincore() returns the memory residency status of the pages in the
|
|
|
|
* current process's address space specified by [addr, addr + len).
|
|
|
|
* The status is returned in a vector of bytes. The least significant
|
|
|
|
* bit of each byte is 1 if the referenced page is in memory, otherwise
|
|
|
|
* it is zero.
|
|
|
|
*
|
|
|
|
* Because the status of a page can change after mincore() checks it
|
|
|
|
* but before it returns to the application, the returned vector may
|
|
|
|
* contain stale information. Only locked pages are guaranteed to
|
|
|
|
* remain in memory.
|
|
|
|
*
|
|
|
|
* return values:
|
|
|
|
* zero - success
|
|
|
|
* -EFAULT - vec points to an illegal address
|
|
|
|
* -EINVAL - addr is not a multiple of PAGE_CACHE_SIZE
|
|
|
|
* -ENOMEM - Addresses in the range [addr, addr + len] are
|
|
|
|
* invalid for the address space of this process, or
|
|
|
|
* specify one or more pages which are not currently
|
|
|
|
* mapped
|
|
|
|
* -EAGAIN - A kernel resource was temporarily unavailable.
|
|
|
|
*/
|
2009-01-14 20:14:16 +07:00
|
|
|
SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len,
|
|
|
|
unsigned char __user *, vec)
|
2005-04-17 05:20:36 +07:00
|
|
|
{
|
2006-12-17 00:44:32 +07:00
|
|
|
long retval;
|
|
|
|
unsigned long pages;
|
|
|
|
unsigned char *tmp;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2006-12-17 00:44:32 +07:00
|
|
|
/* Check the start address: needs to be page-aligned.. */
|
|
|
|
if (start & ~PAGE_CACHE_MASK)
|
|
|
|
return -EINVAL;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2006-12-17 00:44:32 +07:00
|
|
|
/* ..and we need to be passed a valid user-space range */
|
|
|
|
if (!access_ok(VERIFY_READ, (void __user *) start, len))
|
|
|
|
return -ENOMEM;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2006-12-17 00:44:32 +07:00
|
|
|
/* This also avoids any overflows on PAGE_CACHE_ALIGN */
|
|
|
|
pages = len >> PAGE_SHIFT;
|
|
|
|
pages += (len & ~PAGE_MASK) != 0;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2006-12-17 00:44:32 +07:00
|
|
|
if (!access_ok(VERIFY_WRITE, vec, pages))
|
|
|
|
return -EFAULT;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2006-12-17 00:44:32 +07:00
|
|
|
tmp = (void *) __get_free_page(GFP_USER);
|
|
|
|
if (!tmp)
|
2006-12-17 07:01:50 +07:00
|
|
|
return -EAGAIN;
|
2006-12-17 00:44:32 +07:00
|
|
|
|
|
|
|
retval = 0;
|
|
|
|
while (pages) {
|
|
|
|
/*
|
|
|
|
* Do at most PAGE_SIZE entries per iteration, due to
|
|
|
|
* the temporary buffer size.
|
|
|
|
*/
|
|
|
|
down_read(¤t->mm->mmap_sem);
|
2010-05-25 04:32:09 +07:00
|
|
|
retval = do_mincore(start, min(pages, PAGE_SIZE), tmp);
|
2006-12-17 00:44:32 +07:00
|
|
|
up_read(¤t->mm->mmap_sem);
|
|
|
|
|
|
|
|
if (retval <= 0)
|
|
|
|
break;
|
|
|
|
if (copy_to_user(vec, tmp, retval)) {
|
|
|
|
retval = -EFAULT;
|
|
|
|
break;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2006-12-17 00:44:32 +07:00
|
|
|
pages -= retval;
|
|
|
|
vec += retval;
|
|
|
|
start += retval << PAGE_SHIFT;
|
|
|
|
retval = 0;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|
2006-12-17 00:44:32 +07:00
|
|
|
free_page((unsigned long) tmp);
|
|
|
|
return retval;
|
2005-04-17 05:20:36 +07:00
|
|
|
}
|