AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2025-04-17 11:57:42 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

mm, thp: avoid unlikely branches for split_huge_pmd

Browse Source 
While doing MADV_DONTNEED on a large area of thp memory, I noticed we
encountered many unlikely() branches in profiles for each backing
hugepage.  This is because zap_pmd_range() would call split_huge_pmd(),
which rechecked the conditions that were already validated, but as part
of an unlikely() branch.

Avoid the unlikely() branch when in a context where pmd is known to be
good for __split_huge_pmd() directly.

Link: http://lkml.kernel.org/r/alpine.DEB.2.10.1610181600300.84525@chino.kir.corp.google.com
Signed-off-by: David Rientjes <rientjes@google.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
David Rientjes 2016-12-12 16:42:20 -08:00 committed by Linus Torvalds
parent 3f5000693f
commit fd60775aea
4 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/huge_mm.h
View File

@ -189,6 +189,8 @@ static inline void deferred_split_huge_page(struct page *page) {}
#define split_huge_pmd(__vma, __pmd, __address) \ #define split_huge_pmd(__vma, __pmd, __address) \
do { } while (0) do { } while (0)
static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long address, bool freeze, struct page *page) {}
static inline void split_huge_pmd_address(struct vm_area_struct *vma, static inline void split_huge_pmd_address(struct vm_area_struct *vma,
unsigned long address, bool freeze, struct page *page) {} unsigned long address, bool freeze, struct page *page) {}

4
mm/memory.c
View File

@ -1240,7 +1240,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
if (next - addr != HPAGE_PMD_SIZE) { if (next - addr != HPAGE_PMD_SIZE) {
VM_BUG_ON_VMA(vma_is_anonymous(vma) && VM_BUG_ON_VMA(vma_is_anonymous(vma) &&
!rwsem_is_locked(&tlb->mm->mmap_sem), vma); !rwsem_is_locked(&tlb->mm->mmap_sem), vma);
split_huge_pmd(vma, pmd, addr); __split_huge_pmd(vma, pmd, addr, false, NULL);
} else if (zap_huge_pmd(tlb, vma, pmd, addr)) } else if (zap_huge_pmd(tlb, vma, pmd, addr))
goto next; goto next;
/* fall through */ /* fall through */
@ -3454,7 +3454,7 @@ static int wp_huge_pmd(struct fault_env *fe, pmd_t orig_pmd)
/* COW handled on pte level: split pmd */ /* COW handled on pte level: split pmd */
VM_BUG_ON_VMA(fe->vma->vm_flags & VM_SHARED, fe->vma); VM_BUG_ON_VMA(fe->vma->vm_flags & VM_SHARED, fe->vma);
split_huge_pmd(fe->vma, fe->pmd, fe->address); __split_huge_pmd(fe->vma, fe->pmd, fe->address, false, NULL);
return VM_FAULT_FALLBACK; return VM_FAULT_FALLBACK;
} }

2
mm/mempolicy.c
View File

@ -496,7 +496,7 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
page = pmd_page(*pmd); page = pmd_page(*pmd);
if (is_huge_zero_page(page)) { if (is_huge_zero_page(page)) {
spin_unlock(ptl); spin_unlock(ptl);
split_huge_pmd(vma, pmd, addr); __split_huge_pmd(vma, pmd, addr, false, NULL);
} else { } else {
get_page(page); get_page(page);
spin_unlock(ptl); spin_unlock(ptl);

2
mm/mprotect.c
View File

@ -176,7 +176,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
if (next - addr != HPAGE_PMD_SIZE) { if (next - addr != HPAGE_PMD_SIZE) {
split_huge_pmd(vma, pmd, addr); __split_huge_pmd(vma, pmd, addr, false, NULL);
if (pmd_trans_unstable(pmd)) if (pmd_trans_unstable(pmd))
continue; continue;
} else { } else {