mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-11-25 07:20:53 +07:00
cbf86cfe04
Switching merge_across_nodes after running KSM is liable to oops on stale nodes still left over from the previous stable tree. It's not something that people will often want to do, but it would be lame to demand a reboot when they're trying to determine which merge_across_nodes setting is best. How can this happen? We only permit switching merge_across_nodes when pages_shared is 0, and usually set run 2 to force that beforehand, which ought to unmerge everything: yet oopses still occur when you then run 1. Three causes: 1. The old stable tree (built according to the inverse merge_across_nodes) has not been fully torn down. A stable node lingers until get_ksm_page() notices that the page it references no longer references it: but the page is not necessarily freed as soon as expected, particularly when swapcache. Fix this with a pass through the old stable tree, applying get_ksm_page() to each of the remaining nodes (most found stale and removed immediately), with forced removal of any left over. Unless the page is still mapped: I've not seen that case, it shouldn't occur, but better to WARN_ON_ONCE and EBUSY than BUG. 2. __ksm_enter() has a nice little optimization, to insert the new mm just behind ksmd's cursor, so there's a full pass for it to stabilize (or be removed) before ksmd addresses it. Nice when ksmd is running, but not so nice when we're trying to unmerge all mms: we were missing those mms forked and inserted behind the unmerge cursor. Easily fixed by inserting at the end when KSM_RUN_UNMERGE. 3. It is possible for a KSM page to be faulted back from swapcache into an mm, just after unmerge_and_remove_all_rmap_items() scanned past it. Fix this by copying on fault when KSM_RUN_UNMERGE: but that is private to ksm.c, so dissolve the distinction between ksm_might_need_to_copy() and ksm_does_need_to_copy(), doing it all in the one call into ksm.c. A long outstanding, unrelated bugfix sneaks in with that third fix: ksm_does_need_to_copy() would copy from a !PageUptodate page (implying I/O error when read in from swap) to a page which it then marks Uptodate. Fix this case by not copying, letting do_swap_page() discover the error. Signed-off-by: Hugh Dickins <hughd@google.com> Cc: Rik van Riel <riel@redhat.com> Cc: Petr Holasek <pholasek@redhat.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Izik Eidus <izik.eidus@ravellosystems.com> Cc: Gerald Schaefer <gerald.schaefer@de.ibm.com> Cc: KOSAKI Motohiro <kosaki.motohiro@gmail.com> Acked-by: Mel Gorman <mgorman@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
136 lines
3.7 KiB
C
136 lines
3.7 KiB
C
#ifndef __LINUX_KSM_H
|
|
#define __LINUX_KSM_H
|
|
/*
|
|
* Memory merging support.
|
|
*
|
|
* This code enables dynamic sharing of identical pages found in different
|
|
* memory areas, even if they are not shared by fork().
|
|
*/
|
|
|
|
#include <linux/bitops.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/rmap.h>
|
|
#include <linux/sched.h>
|
|
|
|
struct stable_node;
|
|
struct mem_cgroup;
|
|
|
|
#ifdef CONFIG_KSM
|
|
int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
|
|
unsigned long end, int advice, unsigned long *vm_flags);
|
|
int __ksm_enter(struct mm_struct *mm);
|
|
void __ksm_exit(struct mm_struct *mm);
|
|
|
|
static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
|
|
{
|
|
if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags))
|
|
return __ksm_enter(mm);
|
|
return 0;
|
|
}
|
|
|
|
static inline void ksm_exit(struct mm_struct *mm)
|
|
{
|
|
if (test_bit(MMF_VM_MERGEABLE, &mm->flags))
|
|
__ksm_exit(mm);
|
|
}
|
|
|
|
/*
|
|
* A KSM page is one of those write-protected "shared pages" or "merged pages"
|
|
* which KSM maps into multiple mms, wherever identical anonymous page content
|
|
* is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
|
|
* anon_vma, but to that page's node of the stable tree.
|
|
*/
|
|
static inline int PageKsm(struct page *page)
|
|
{
|
|
return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
|
|
(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
|
|
}
|
|
|
|
static inline struct stable_node *page_stable_node(struct page *page)
|
|
{
|
|
return PageKsm(page) ? page_rmapping(page) : NULL;
|
|
}
|
|
|
|
static inline void set_page_stable_node(struct page *page,
|
|
struct stable_node *stable_node)
|
|
{
|
|
page->mapping = (void *)stable_node +
|
|
(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
|
|
}
|
|
|
|
/*
|
|
* When do_swap_page() first faults in from swap what used to be a KSM page,
|
|
* no problem, it will be assigned to this vma's anon_vma; but thereafter,
|
|
* it might be faulted into a different anon_vma (or perhaps to a different
|
|
* offset in the same anon_vma). do_swap_page() cannot do all the locking
|
|
* needed to reconstitute a cross-anon_vma KSM page: for now it has to make
|
|
* a copy, and leave remerging the pages to a later pass of ksmd.
|
|
*
|
|
* We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE,
|
|
* but what if the vma was unmerged while the page was swapped out?
|
|
*/
|
|
struct page *ksm_might_need_to_copy(struct page *page,
|
|
struct vm_area_struct *vma, unsigned long address);
|
|
|
|
int page_referenced_ksm(struct page *page,
|
|
struct mem_cgroup *memcg, unsigned long *vm_flags);
|
|
int try_to_unmap_ksm(struct page *page, enum ttu_flags flags);
|
|
int rmap_walk_ksm(struct page *page, int (*rmap_one)(struct page *,
|
|
struct vm_area_struct *, unsigned long, void *), void *arg);
|
|
void ksm_migrate_page(struct page *newpage, struct page *oldpage);
|
|
|
|
#else /* !CONFIG_KSM */
|
|
|
|
static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void ksm_exit(struct mm_struct *mm)
|
|
{
|
|
}
|
|
|
|
static inline int PageKsm(struct page *page)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_MMU
|
|
static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
|
|
unsigned long end, int advice, unsigned long *vm_flags)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline struct page *ksm_might_need_to_copy(struct page *page,
|
|
struct vm_area_struct *vma, unsigned long address)
|
|
{
|
|
return page;
|
|
}
|
|
|
|
static inline int page_referenced_ksm(struct page *page,
|
|
struct mem_cgroup *memcg, unsigned long *vm_flags)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline int try_to_unmap_ksm(struct page *page, enum ttu_flags flags)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline int rmap_walk_ksm(struct page *page, int (*rmap_one)(struct page*,
|
|
struct vm_area_struct *, unsigned long, void *), void *arg)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
|
|
{
|
|
}
|
|
#endif /* CONFIG_MMU */
|
|
#endif /* !CONFIG_KSM */
|
|
|
|
#endif /* __LINUX_KSM_H */
|