2005-04-17 05:20:36 +07:00
|
|
|
#ifndef __ASM_SH_TLB_H
|
|
|
|
#define __ASM_SH_TLB_H
|
|
|
|
|
2007-11-10 18:35:53 +07:00
|
|
|
#ifdef CONFIG_SUPERH64
|
2012-10-03 00:01:25 +07:00
|
|
|
# include <asm/tlb_64.h>
|
2007-11-10 18:35:53 +07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
2009-03-17 19:19:49 +07:00
|
|
|
#include <linux/pagemap.h>
|
|
|
|
|
|
|
|
#ifdef CONFIG_MMU
|
2011-05-31 11:27:41 +07:00
|
|
|
#include <linux/swap.h>
|
2009-03-17 19:19:49 +07:00
|
|
|
#include <asm/pgalloc.h>
|
|
|
|
#include <asm/tlbflush.h>
|
2009-11-20 04:11:05 +07:00
|
|
|
#include <asm/mmu_context.h>
|
2009-03-17 19:19:49 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* TLB handling. This allows us to remove pages from the page
|
|
|
|
* tables, and efficiently handle the TLB issues.
|
|
|
|
*/
|
|
|
|
struct mmu_gather {
|
|
|
|
struct mm_struct *mm;
|
|
|
|
unsigned int fullmm;
|
|
|
|
unsigned long start, end;
|
|
|
|
};
|
2007-11-10 18:35:53 +07:00
|
|
|
|
2009-03-17 19:19:49 +07:00
|
|
|
static inline void init_tlb_gather(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
tlb->start = TASK_SIZE;
|
|
|
|
tlb->end = 0;
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2009-03-17 19:19:49 +07:00
|
|
|
if (tlb->fullmm) {
|
|
|
|
tlb->start = 0;
|
|
|
|
tlb->end = TASK_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-05-25 07:11:54 +07:00
|
|
|
static inline void
|
2017-08-11 05:24:05 +07:00
|
|
|
arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
|
|
|
|
unsigned long start, unsigned long end)
|
2009-03-17 19:19:49 +07:00
|
|
|
{
|
|
|
|
tlb->mm = mm;
|
Fix TLB gather virtual address range invalidation corner cases
Ben Tebulin reported:
"Since v3.7.2 on two independent machines a very specific Git
repository fails in 9/10 cases on git-fsck due to an SHA1/memory
failures. This only occurs on a very specific repository and can be
reproduced stably on two independent laptops. Git mailing list ran
out of ideas and for me this looks like some very exotic kernel issue"
and bisected the failure to the backport of commit 53a59fc67f97 ("mm:
limit mmu_gather batching to fix soft lockups on !CONFIG_PREEMPT").
That commit itself is not actually buggy, but what it does is to make it
much more likely to hit the partial TLB invalidation case, since it
introduces a new case in tlb_next_batch() that previously only ever
happened when running out of memory.
The real bug is that the TLB gather virtual memory range setup is subtly
buggered. It was introduced in commit 597e1c3580b7 ("mm/mmu_gather:
enable tlb flush range in generic mmu_gather"), and the range handling
was already fixed at least once in commit e6c495a96ce0 ("mm: fix the TLB
range flushed when __tlb_remove_page() runs out of slots"), but that fix
was not complete.
The problem with the TLB gather virtual address range is that it isn't
set up by the initial tlb_gather_mmu() initialization (which didn't get
the TLB range information), but it is set up ad-hoc later by the
functions that actually flush the TLB. And so any such case that forgot
to update the TLB range entries would potentially miss TLB invalidates.
Rather than try to figure out exactly which particular ad-hoc range
setup was missing (I personally suspect it's the hugetlb case in
zap_huge_pmd(), which didn't have the same logic as zap_pte_range()
did), this patch just gets rid of the problem at the source: make the
TLB range information available to tlb_gather_mmu(), and initialize it
when initializing all the other tlb gather fields.
This makes the patch larger, but conceptually much simpler. And the end
result is much more understandable; even if you want to play games with
partial ranges when invalidating the TLB contents in chunks, now the
range information is always there, and anybody who doesn't want to
bother with it won't introduce subtle bugs.
Ben verified that this fixes his problem.
Reported-bisected-and-tested-by: Ben Tebulin <tebulin@googlemail.com>
Build-testing-by: Stephen Rothwell <sfr@canb.auug.org.au>
Build-testing-by: Richard Weinberger <richard.weinberger@gmail.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: stable@vger.kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-08-16 01:42:25 +07:00
|
|
|
tlb->start = start;
|
|
|
|
tlb->end = end;
|
|
|
|
tlb->fullmm = !(start | (end+1));
|
2009-03-17 19:19:49 +07:00
|
|
|
|
|
|
|
init_tlb_gather(tlb);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
2017-08-11 05:24:05 +07:00
|
|
|
arch_tlb_finish_mmu(struct mmu_gather *tlb,
|
mm: fix MADV_[FREE|DONTNEED] TLB flush miss problem
Nadav reported parallel MADV_DONTNEED on same range has a stale TLB
problem and Mel fixed it[1] and found same problem on MADV_FREE[2].
Quote from Mel Gorman:
"The race in question is CPU 0 running madv_free and updating some PTEs
while CPU 1 is also running madv_free and looking at the same PTEs.
CPU 1 may have writable TLB entries for a page but fail the pte_dirty
check (because CPU 0 has updated it already) and potentially fail to
flush.
Hence, when madv_free on CPU 1 returns, there are still potentially
writable TLB entries and the underlying PTE is still present so that a
subsequent write does not necessarily propagate the dirty bit to the
underlying PTE any more. Reclaim at some unknown time at the future
may then see that the PTE is still clean and discard the page even
though a write has happened in the meantime. I think this is possible
but I could have missed some protection in madv_free that prevents it
happening."
This patch aims for solving both problems all at once and is ready for
other problem with KSM, MADV_FREE and soft-dirty story[3].
TLB batch API(tlb_[gather|finish]_mmu] uses [inc|dec]_tlb_flush_pending
and mmu_tlb_flush_pending so that when tlb_finish_mmu is called, we can
catch there are parallel threads going on. In that case, forcefully,
flush TLB to prevent for user to access memory via stale TLB entry
although it fail to gather page table entry.
I confirmed this patch works with [4] test program Nadav gave so this
patch supersedes "mm: Always flush VMA ranges affected by zap_page_range
v2" in current mmotm.
NOTE:
This patch modifies arch-specific TLB gathering interface(x86, ia64,
s390, sh, um). It seems most of architecture are straightforward but
s390 need to be careful because tlb_flush_mmu works only if
mm->context.flush_mm is set to non-zero which happens only a pte entry
really is cleared by ptep_get_and_clear and friends. However, this
problem never changes the pte entries but need to flush to prevent
memory access from stale tlb.
[1] http://lkml.kernel.org/r/20170725101230.5v7gvnjmcnkzzql3@techsingularity.net
[2] http://lkml.kernel.org/r/20170725100722.2dxnmgypmwnrfawp@suse.de
[3] http://lkml.kernel.org/r/BD3A0EBE-ECF4-41D4-87FA-C755EA9AB6BD@gmail.com
[4] https://patchwork.kernel.org/patch/9861621/
[minchan@kernel.org: decrease tlb flush pending count in tlb_finish_mmu]
Link: http://lkml.kernel.org/r/20170808080821.GA31730@bbox
Link: http://lkml.kernel.org/r/20170802000818.4760-7-namit@vmware.com
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Nadav Amit <namit@vmware.com>
Reported-by: Nadav Amit <namit@vmware.com>
Reported-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-08-11 05:24:12 +07:00
|
|
|
unsigned long start, unsigned long end, bool force)
|
2009-03-17 19:19:49 +07:00
|
|
|
{
|
mm: fix MADV_[FREE|DONTNEED] TLB flush miss problem
Nadav reported parallel MADV_DONTNEED on same range has a stale TLB
problem and Mel fixed it[1] and found same problem on MADV_FREE[2].
Quote from Mel Gorman:
"The race in question is CPU 0 running madv_free and updating some PTEs
while CPU 1 is also running madv_free and looking at the same PTEs.
CPU 1 may have writable TLB entries for a page but fail the pte_dirty
check (because CPU 0 has updated it already) and potentially fail to
flush.
Hence, when madv_free on CPU 1 returns, there are still potentially
writable TLB entries and the underlying PTE is still present so that a
subsequent write does not necessarily propagate the dirty bit to the
underlying PTE any more. Reclaim at some unknown time at the future
may then see that the PTE is still clean and discard the page even
though a write has happened in the meantime. I think this is possible
but I could have missed some protection in madv_free that prevents it
happening."
This patch aims for solving both problems all at once and is ready for
other problem with KSM, MADV_FREE and soft-dirty story[3].
TLB batch API(tlb_[gather|finish]_mmu] uses [inc|dec]_tlb_flush_pending
and mmu_tlb_flush_pending so that when tlb_finish_mmu is called, we can
catch there are parallel threads going on. In that case, forcefully,
flush TLB to prevent for user to access memory via stale TLB entry
although it fail to gather page table entry.
I confirmed this patch works with [4] test program Nadav gave so this
patch supersedes "mm: Always flush VMA ranges affected by zap_page_range
v2" in current mmotm.
NOTE:
This patch modifies arch-specific TLB gathering interface(x86, ia64,
s390, sh, um). It seems most of architecture are straightforward but
s390 need to be careful because tlb_flush_mmu works only if
mm->context.flush_mm is set to non-zero which happens only a pte entry
really is cleared by ptep_get_and_clear and friends. However, this
problem never changes the pte entries but need to flush to prevent
memory access from stale tlb.
[1] http://lkml.kernel.org/r/20170725101230.5v7gvnjmcnkzzql3@techsingularity.net
[2] http://lkml.kernel.org/r/20170725100722.2dxnmgypmwnrfawp@suse.de
[3] http://lkml.kernel.org/r/BD3A0EBE-ECF4-41D4-87FA-C755EA9AB6BD@gmail.com
[4] https://patchwork.kernel.org/patch/9861621/
[minchan@kernel.org: decrease tlb flush pending count in tlb_finish_mmu]
Link: http://lkml.kernel.org/r/20170808080821.GA31730@bbox
Link: http://lkml.kernel.org/r/20170802000818.4760-7-namit@vmware.com
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Nadav Amit <namit@vmware.com>
Reported-by: Nadav Amit <namit@vmware.com>
Reported-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-08-11 05:24:12 +07:00
|
|
|
if (tlb->fullmm || force)
|
2009-03-17 19:19:49 +07:00
|
|
|
flush_tlb_mm(tlb->mm);
|
|
|
|
|
|
|
|
/* keep the page table cache within bounds */
|
|
|
|
check_pgt_cache();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long address)
|
|
|
|
{
|
|
|
|
if (tlb->start > address)
|
|
|
|
tlb->start = address;
|
|
|
|
if (tlb->end < address + PAGE_SIZE)
|
|
|
|
tlb->end = address + PAGE_SIZE;
|
|
|
|
}
|
2005-04-17 05:20:36 +07:00
|
|
|
|
2016-12-13 07:42:37 +07:00
|
|
|
#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
|
|
|
|
tlb_remove_tlb_entry(tlb, ptep, address)
|
|
|
|
|
2005-04-17 05:20:36 +07:00
|
|
|
/*
|
2009-03-17 19:19:49 +07:00
|
|
|
* In the case of tlb vma handling, we can optimise these away in the
|
|
|
|
* case where we're doing a full MM flush. When we're doing a munmap,
|
|
|
|
* the vmas are adjusted to only cover the region to be torn down.
|
2005-04-17 05:20:36 +07:00
|
|
|
*/
|
2009-03-17 19:19:49 +07:00
|
|
|
static inline void
|
|
|
|
tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
if (!tlb->fullmm)
|
|
|
|
flush_cache_range(vma, vma->vm_start, vma->vm_end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
if (!tlb->fullmm && tlb->end) {
|
|
|
|
flush_tlb_range(vma, tlb->start, tlb->end);
|
|
|
|
init_tlb_gather(tlb);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-04-26 06:05:40 +07:00
|
|
|
static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2011-05-25 07:11:54 +07:00
|
|
|
static inline void tlb_flush_mmu(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
|
|
|
|
{
|
|
|
|
free_page_and_swap_cache(page);
|
2016-07-27 05:24:09 +07:00
|
|
|
return false; /* avoid calling tlb_flush_mmu */
|
2011-05-25 07:11:54 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
|
|
|
|
{
|
|
|
|
__tlb_remove_page(tlb, page);
|
|
|
|
}
|
|
|
|
|
2016-07-27 05:24:12 +07:00
|
|
|
static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
|
|
|
|
struct page *page, int page_size)
|
|
|
|
{
|
|
|
|
return __tlb_remove_page(tlb, page);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
|
|
|
|
struct page *page, int page_size)
|
|
|
|
{
|
|
|
|
return tlb_remove_page(tlb, page);
|
|
|
|
}
|
|
|
|
|
2016-12-13 07:42:40 +07:00
|
|
|
#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
|
|
|
|
static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
|
|
|
|
unsigned int page_size)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-22 12:44:28 +07:00
|
|
|
#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep)
|
|
|
|
#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
|
|
|
|
#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
|
2009-03-17 19:19:49 +07:00
|
|
|
|
|
|
|
#define tlb_migrate_finish(mm) do { } while (0)
|
|
|
|
|
2010-01-19 13:20:35 +07:00
|
|
|
#if defined(CONFIG_CPU_SH4) || defined(CONFIG_SUPERH64)
|
2009-11-18 04:05:31 +07:00
|
|
|
extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);
|
|
|
|
extern void tlb_unwire_entry(void);
|
|
|
|
#else
|
|
|
|
static inline void tlb_wire_entry(struct vm_area_struct *vma ,
|
|
|
|
unsigned long addr, pte_t pte)
|
|
|
|
{
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_unwire_entry(void)
|
|
|
|
{
|
|
|
|
BUG();
|
|
|
|
}
|
2010-01-19 13:20:35 +07:00
|
|
|
#endif
|
2009-11-18 04:05:31 +07:00
|
|
|
|
2009-03-17 19:19:49 +07:00
|
|
|
#else /* CONFIG_MMU */
|
|
|
|
|
|
|
|
#define tlb_start_vma(tlb, vma) do { } while (0)
|
|
|
|
#define tlb_end_vma(tlb, vma) do { } while (0)
|
|
|
|
#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
|
|
|
|
#define tlb_flush(tlb) do { } while (0)
|
2005-04-17 05:20:36 +07:00
|
|
|
|
|
|
|
#include <asm-generic/tlb.h>
|
2007-11-10 18:35:53 +07:00
|
|
|
|
2009-03-17 19:19:49 +07:00
|
|
|
#endif /* CONFIG_MMU */
|
2007-11-10 18:35:53 +07:00
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* __ASM_SH_TLB_H */
|