mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-16 04:17:27 +07:00
dc0ef0df7b
This is a follow up work for oom_reaper [1]. As the async OOM killing depends on oom_sem for read we would really appreciate if a holder for write didn't stood in the way. This patchset is changing many of down_write calls to be killable to help those cases when the writer is blocked and waiting for readers to release the lock and so help __oom_reap_task to process the oom victim. Most of the patches are really trivial because the lock is help from a shallow syscall paths where we can return EINTR trivially and allow the current task to die (note that EINTR will never get to the userspace as the task has fatal signal pending). Others seem to be easy as well as the callers are already handling fatal errors and bail and return to userspace which should be sufficient to handle the failure gracefully. I am not familiar with all those code paths so a deeper review is really appreciated. As this work is touching more areas which are not directly connected I have tried to keep the CC list as small as possible and people who I believed would be familiar are CCed only to the specific patches (all should have received the cover though). This patchset is based on linux-next and it depends on down_write_killable for rw_semaphores which got merged into tip locking/rwsem branch and it is merged into this next tree. I guess it would be easiest to route these patches via mmotm because of the dependency on the tip tree but if respective maintainers prefer other way I have no objections. I haven't covered all the mmap_write(mm->mmap_sem) instances here $ git grep "down_write(.*\<mmap_sem\>)" next/master | wc -l 98 $ git grep "down_write(.*\<mmap_sem\>)" | wc -l 62 I have tried to cover those which should be relatively easy to review in this series because this alone should be a nice improvement. Other places can be changed on top. [0] http://lkml.kernel.org/r/1456752417-9626-1-git-send-email-mhocko@kernel.org [1] http://lkml.kernel.org/r/1452094975-551-1-git-send-email-mhocko@kernel.org [2] http://lkml.kernel.org/r/1456750705-7141-1-git-send-email-mhocko@kernel.org This patch (of 18): This is the first step in making mmap_sem write waiters killable. It focuses on the trivial ones which are taking the lock early after entering the syscall and they are not changing state before. Therefore it is very easy to change them to use down_write_killable and immediately return with -EINTR. This will allow the waiter to pass away without blocking the mmap_sem which might be required to make a forward progress. E.g. the oom reaper will need the lock for reading to dismantle the OOM victim address space. The only tricky function in this patch is vm_mmap_pgoff which has many call sites via vm_mmap. To reduce the risk keep vm_mmap with the original non-killable semantic for now. vm_munmap callers do not bother checking the return value so open code it into the munmap syscall path for now for simplicity. Signed-off-by: Michal Hocko <mhocko@suse.com> Acked-by: Vlastimil Babka <vbabka@suse.cz> Cc: Mel Gorman <mgorman@suse.de> Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> Cc: Konstantin Khlebnikov <koct9i@gmail.com> Cc: Hugh Dickins <hughd@google.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: David Rientjes <rientjes@google.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
457 lines
11 KiB
C
457 lines
11 KiB
C
/*
|
|
* mm/mprotect.c
|
|
*
|
|
* (C) Copyright 1994 Linus Torvalds
|
|
* (C) Copyright 2002 Christoph Hellwig
|
|
*
|
|
* Address space accounting code <alan@lxorguk.ukuu.org.uk>
|
|
* (C) Copyright 2002 Red Hat Inc, All Rights Reserved
|
|
*/
|
|
|
|
#include <linux/mm.h>
|
|
#include <linux/hugetlb.h>
|
|
#include <linux/shm.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/security.h>
|
|
#include <linux/mempolicy.h>
|
|
#include <linux/personality.h>
|
|
#include <linux/syscalls.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/swapops.h>
|
|
#include <linux/mmu_notifier.h>
|
|
#include <linux/migrate.h>
|
|
#include <linux/perf_event.h>
|
|
#include <linux/ksm.h>
|
|
#include <linux/pkeys.h>
|
|
#include <asm/uaccess.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include "internal.h"
|
|
|
|
/*
|
|
* For a prot_numa update we only hold mmap_sem for read so there is a
|
|
* potential race with faulting where a pmd was temporarily none. This
|
|
* function checks for a transhuge pmd under the appropriate lock. It
|
|
* returns a pte if it was successfully locked or NULL if it raced with
|
|
* a transhuge insertion.
|
|
*/
|
|
static pte_t *lock_pte_protection(struct vm_area_struct *vma, pmd_t *pmd,
|
|
unsigned long addr, int prot_numa, spinlock_t **ptl)
|
|
{
|
|
pte_t *pte;
|
|
spinlock_t *pmdl;
|
|
|
|
/* !prot_numa is protected by mmap_sem held for write */
|
|
if (!prot_numa)
|
|
return pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl);
|
|
|
|
pmdl = pmd_lock(vma->vm_mm, pmd);
|
|
if (unlikely(pmd_trans_huge(*pmd) || pmd_none(*pmd))) {
|
|
spin_unlock(pmdl);
|
|
return NULL;
|
|
}
|
|
|
|
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl);
|
|
spin_unlock(pmdl);
|
|
return pte;
|
|
}
|
|
|
|
static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
|
|
unsigned long addr, unsigned long end, pgprot_t newprot,
|
|
int dirty_accountable, int prot_numa)
|
|
{
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
pte_t *pte, oldpte;
|
|
spinlock_t *ptl;
|
|
unsigned long pages = 0;
|
|
|
|
pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl);
|
|
if (!pte)
|
|
return 0;
|
|
|
|
arch_enter_lazy_mmu_mode();
|
|
do {
|
|
oldpte = *pte;
|
|
if (pte_present(oldpte)) {
|
|
pte_t ptent;
|
|
bool preserve_write = prot_numa && pte_write(oldpte);
|
|
|
|
/*
|
|
* Avoid trapping faults against the zero or KSM
|
|
* pages. See similar comment in change_huge_pmd.
|
|
*/
|
|
if (prot_numa) {
|
|
struct page *page;
|
|
|
|
page = vm_normal_page(vma, addr, oldpte);
|
|
if (!page || PageKsm(page))
|
|
continue;
|
|
|
|
/* Avoid TLB flush if possible */
|
|
if (pte_protnone(oldpte))
|
|
continue;
|
|
}
|
|
|
|
ptent = ptep_modify_prot_start(mm, addr, pte);
|
|
ptent = pte_modify(ptent, newprot);
|
|
if (preserve_write)
|
|
ptent = pte_mkwrite(ptent);
|
|
|
|
/* Avoid taking write faults for known dirty pages */
|
|
if (dirty_accountable && pte_dirty(ptent) &&
|
|
(pte_soft_dirty(ptent) ||
|
|
!(vma->vm_flags & VM_SOFTDIRTY))) {
|
|
ptent = pte_mkwrite(ptent);
|
|
}
|
|
ptep_modify_prot_commit(mm, addr, pte, ptent);
|
|
pages++;
|
|
} else if (IS_ENABLED(CONFIG_MIGRATION)) {
|
|
swp_entry_t entry = pte_to_swp_entry(oldpte);
|
|
|
|
if (is_write_migration_entry(entry)) {
|
|
pte_t newpte;
|
|
/*
|
|
* A protection check is difficult so
|
|
* just be safe and disable write
|
|
*/
|
|
make_migration_entry_read(&entry);
|
|
newpte = swp_entry_to_pte(entry);
|
|
if (pte_swp_soft_dirty(oldpte))
|
|
newpte = pte_swp_mksoft_dirty(newpte);
|
|
set_pte_at(mm, addr, pte, newpte);
|
|
|
|
pages++;
|
|
}
|
|
}
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
arch_leave_lazy_mmu_mode();
|
|
pte_unmap_unlock(pte - 1, ptl);
|
|
|
|
return pages;
|
|
}
|
|
|
|
static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
|
|
pud_t *pud, unsigned long addr, unsigned long end,
|
|
pgprot_t newprot, int dirty_accountable, int prot_numa)
|
|
{
|
|
pmd_t *pmd;
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
unsigned long next;
|
|
unsigned long pages = 0;
|
|
unsigned long nr_huge_updates = 0;
|
|
unsigned long mni_start = 0;
|
|
|
|
pmd = pmd_offset(pud, addr);
|
|
do {
|
|
unsigned long this_pages;
|
|
|
|
next = pmd_addr_end(addr, end);
|
|
if (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)
|
|
&& pmd_none_or_clear_bad(pmd))
|
|
continue;
|
|
|
|
/* invoke the mmu notifier if the pmd is populated */
|
|
if (!mni_start) {
|
|
mni_start = addr;
|
|
mmu_notifier_invalidate_range_start(mm, mni_start, end);
|
|
}
|
|
|
|
if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
|
|
if (next - addr != HPAGE_PMD_SIZE) {
|
|
split_huge_pmd(vma, pmd, addr);
|
|
if (pmd_none(*pmd))
|
|
continue;
|
|
} else {
|
|
int nr_ptes = change_huge_pmd(vma, pmd, addr,
|
|
newprot, prot_numa);
|
|
|
|
if (nr_ptes) {
|
|
if (nr_ptes == HPAGE_PMD_NR) {
|
|
pages += HPAGE_PMD_NR;
|
|
nr_huge_updates++;
|
|
}
|
|
|
|
/* huge pmd was handled */
|
|
continue;
|
|
}
|
|
}
|
|
/* fall through, the trans huge pmd just split */
|
|
}
|
|
this_pages = change_pte_range(vma, pmd, addr, next, newprot,
|
|
dirty_accountable, prot_numa);
|
|
pages += this_pages;
|
|
} while (pmd++, addr = next, addr != end);
|
|
|
|
if (mni_start)
|
|
mmu_notifier_invalidate_range_end(mm, mni_start, end);
|
|
|
|
if (nr_huge_updates)
|
|
count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
|
|
return pages;
|
|
}
|
|
|
|
static inline unsigned long change_pud_range(struct vm_area_struct *vma,
|
|
pgd_t *pgd, unsigned long addr, unsigned long end,
|
|
pgprot_t newprot, int dirty_accountable, int prot_numa)
|
|
{
|
|
pud_t *pud;
|
|
unsigned long next;
|
|
unsigned long pages = 0;
|
|
|
|
pud = pud_offset(pgd, addr);
|
|
do {
|
|
next = pud_addr_end(addr, end);
|
|
if (pud_none_or_clear_bad(pud))
|
|
continue;
|
|
pages += change_pmd_range(vma, pud, addr, next, newprot,
|
|
dirty_accountable, prot_numa);
|
|
} while (pud++, addr = next, addr != end);
|
|
|
|
return pages;
|
|
}
|
|
|
|
static unsigned long change_protection_range(struct vm_area_struct *vma,
|
|
unsigned long addr, unsigned long end, pgprot_t newprot,
|
|
int dirty_accountable, int prot_numa)
|
|
{
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
pgd_t *pgd;
|
|
unsigned long next;
|
|
unsigned long start = addr;
|
|
unsigned long pages = 0;
|
|
|
|
BUG_ON(addr >= end);
|
|
pgd = pgd_offset(mm, addr);
|
|
flush_cache_range(vma, addr, end);
|
|
set_tlb_flush_pending(mm);
|
|
do {
|
|
next = pgd_addr_end(addr, end);
|
|
if (pgd_none_or_clear_bad(pgd))
|
|
continue;
|
|
pages += change_pud_range(vma, pgd, addr, next, newprot,
|
|
dirty_accountable, prot_numa);
|
|
} while (pgd++, addr = next, addr != end);
|
|
|
|
/* Only flush the TLB if we actually modified any entries: */
|
|
if (pages)
|
|
flush_tlb_range(vma, start, end);
|
|
clear_tlb_flush_pending(mm);
|
|
|
|
return pages;
|
|
}
|
|
|
|
unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
|
|
unsigned long end, pgprot_t newprot,
|
|
int dirty_accountable, int prot_numa)
|
|
{
|
|
unsigned long pages;
|
|
|
|
if (is_vm_hugetlb_page(vma))
|
|
pages = hugetlb_change_protection(vma, start, end, newprot);
|
|
else
|
|
pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
|
|
|
|
return pages;
|
|
}
|
|
|
|
int
|
|
mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
|
|
unsigned long start, unsigned long end, unsigned long newflags)
|
|
{
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
unsigned long oldflags = vma->vm_flags;
|
|
long nrpages = (end - start) >> PAGE_SHIFT;
|
|
unsigned long charged = 0;
|
|
pgoff_t pgoff;
|
|
int error;
|
|
int dirty_accountable = 0;
|
|
|
|
if (newflags == oldflags) {
|
|
*pprev = vma;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* If we make a private mapping writable we increase our commit;
|
|
* but (without finer accounting) cannot reduce our commit if we
|
|
* make it unwritable again. hugetlb mapping were accounted for
|
|
* even if read-only so there is no need to account for them here
|
|
*/
|
|
if (newflags & VM_WRITE) {
|
|
/* Check space limits when area turns into data. */
|
|
if (!may_expand_vm(mm, newflags, nrpages) &&
|
|
may_expand_vm(mm, oldflags, nrpages))
|
|
return -ENOMEM;
|
|
if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
|
|
VM_SHARED|VM_NORESERVE))) {
|
|
charged = nrpages;
|
|
if (security_vm_enough_memory_mm(mm, charged))
|
|
return -ENOMEM;
|
|
newflags |= VM_ACCOUNT;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* First try to merge with previous and/or next vma.
|
|
*/
|
|
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
|
|
*pprev = vma_merge(mm, *pprev, start, end, newflags,
|
|
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
|
|
vma->vm_userfaultfd_ctx);
|
|
if (*pprev) {
|
|
vma = *pprev;
|
|
goto success;
|
|
}
|
|
|
|
*pprev = vma;
|
|
|
|
if (start != vma->vm_start) {
|
|
error = split_vma(mm, vma, start, 1);
|
|
if (error)
|
|
goto fail;
|
|
}
|
|
|
|
if (end != vma->vm_end) {
|
|
error = split_vma(mm, vma, end, 0);
|
|
if (error)
|
|
goto fail;
|
|
}
|
|
|
|
success:
|
|
/*
|
|
* vm_flags and vm_page_prot are protected by the mmap_sem
|
|
* held in write mode.
|
|
*/
|
|
vma->vm_flags = newflags;
|
|
dirty_accountable = vma_wants_writenotify(vma);
|
|
vma_set_page_prot(vma);
|
|
|
|
change_protection(vma, start, end, vma->vm_page_prot,
|
|
dirty_accountable, 0);
|
|
|
|
/*
|
|
* Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
|
|
* fault on access.
|
|
*/
|
|
if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
|
|
(newflags & VM_WRITE)) {
|
|
populate_vma_page_range(vma, start, end, NULL);
|
|
}
|
|
|
|
vm_stat_account(mm, oldflags, -nrpages);
|
|
vm_stat_account(mm, newflags, nrpages);
|
|
perf_event_mmap(vma);
|
|
return 0;
|
|
|
|
fail:
|
|
vm_unacct_memory(charged);
|
|
return error;
|
|
}
|
|
|
|
SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
|
|
unsigned long, prot)
|
|
{
|
|
unsigned long nstart, end, tmp, reqprot;
|
|
struct vm_area_struct *vma, *prev;
|
|
int error = -EINVAL;
|
|
const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
|
|
const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
|
|
(prot & PROT_READ);
|
|
|
|
prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
|
|
if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
|
|
return -EINVAL;
|
|
|
|
if (start & ~PAGE_MASK)
|
|
return -EINVAL;
|
|
if (!len)
|
|
return 0;
|
|
len = PAGE_ALIGN(len);
|
|
end = start + len;
|
|
if (end <= start)
|
|
return -ENOMEM;
|
|
if (!arch_validate_prot(prot))
|
|
return -EINVAL;
|
|
|
|
reqprot = prot;
|
|
|
|
if (down_write_killable(¤t->mm->mmap_sem))
|
|
return -EINTR;
|
|
|
|
vma = find_vma(current->mm, start);
|
|
error = -ENOMEM;
|
|
if (!vma)
|
|
goto out;
|
|
prev = vma->vm_prev;
|
|
if (unlikely(grows & PROT_GROWSDOWN)) {
|
|
if (vma->vm_start >= end)
|
|
goto out;
|
|
start = vma->vm_start;
|
|
error = -EINVAL;
|
|
if (!(vma->vm_flags & VM_GROWSDOWN))
|
|
goto out;
|
|
} else {
|
|
if (vma->vm_start > start)
|
|
goto out;
|
|
if (unlikely(grows & PROT_GROWSUP)) {
|
|
end = vma->vm_end;
|
|
error = -EINVAL;
|
|
if (!(vma->vm_flags & VM_GROWSUP))
|
|
goto out;
|
|
}
|
|
}
|
|
if (start > vma->vm_start)
|
|
prev = vma;
|
|
|
|
for (nstart = start ; ; ) {
|
|
unsigned long newflags;
|
|
int pkey = arch_override_mprotect_pkey(vma, prot, -1);
|
|
|
|
/* Here we know that vma->vm_start <= nstart < vma->vm_end. */
|
|
|
|
/* Does the application expect PROT_READ to imply PROT_EXEC */
|
|
if (rier && (vma->vm_flags & VM_MAYEXEC))
|
|
prot |= PROT_EXEC;
|
|
|
|
newflags = calc_vm_prot_bits(prot, pkey);
|
|
newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC));
|
|
|
|
/* newflags >> 4 shift VM_MAY% in place of VM_% */
|
|
if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
|
|
error = -EACCES;
|
|
goto out;
|
|
}
|
|
|
|
error = security_file_mprotect(vma, reqprot, prot);
|
|
if (error)
|
|
goto out;
|
|
|
|
tmp = vma->vm_end;
|
|
if (tmp > end)
|
|
tmp = end;
|
|
error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
|
|
if (error)
|
|
goto out;
|
|
nstart = tmp;
|
|
|
|
if (nstart < prev->vm_end)
|
|
nstart = prev->vm_end;
|
|
if (nstart >= end)
|
|
goto out;
|
|
|
|
vma = prev->vm_next;
|
|
if (!vma || vma->vm_start != nstart) {
|
|
error = -ENOMEM;
|
|
goto out;
|
|
}
|
|
prot = reqprot;
|
|
}
|
|
out:
|
|
up_write(¤t->mm->mmap_sem);
|
|
return error;
|
|
}
|