2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
#ifndef __ASM_PGTABLE_H
|
|
|
|
#define __ASM_PGTABLE_H
|
|
|
|
|
2015-07-10 23:24:28 +07:00
|
|
|
#include <asm/bug.h>
|
2012-03-05 18:49:27 +07:00
|
|
|
#include <asm/proc-fns.h>
|
|
|
|
|
|
|
|
#include <asm/memory.h>
|
|
|
|
#include <asm/pgtable-hwdef.h>
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Software defined PTE bits definition.
|
|
|
|
*/
|
2012-12-18 21:15:15 +07:00
|
|
|
#define PTE_VALID (_AT(pteval_t, 1) << 0)
|
2015-09-12 00:22:02 +07:00
|
|
|
#define PTE_WRITE (PTE_DBM) /* same as DBM (51) */
|
2012-03-05 18:49:27 +07:00
|
|
|
#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
|
|
|
|
#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)
|
2013-11-27 23:59:27 +07:00
|
|
|
#define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* VMALLOC and SPARSEMEM_VMEMMAP ranges.
|
2014-07-16 23:42:43 +07:00
|
|
|
*
|
|
|
|
* VMEMAP_SIZE: allows the whole VA space to be covered by a struct page array
|
|
|
|
* (rounded up to PUD_SIZE).
|
|
|
|
* VMALLOC_START: beginning of the kernel VA space
|
|
|
|
* VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space,
|
|
|
|
* fixed mappings and modules
|
2012-03-05 18:49:27 +07:00
|
|
|
*/
|
2014-07-16 23:42:43 +07:00
|
|
|
#define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE)
|
2015-10-12 22:52:58 +07:00
|
|
|
|
|
|
|
#ifndef CONFIG_KASAN
|
2015-09-17 16:38:07 +07:00
|
|
|
#define VMALLOC_START (VA_START)
|
2015-10-12 22:52:58 +07:00
|
|
|
#else
|
|
|
|
#include <asm/kasan.h>
|
|
|
|
#define VMALLOC_START (KASAN_SHADOW_END + SZ_64K)
|
|
|
|
#endif
|
|
|
|
|
2014-07-16 23:42:43 +07:00
|
|
|
#define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
|
|
|
|
|
2015-02-12 06:26:41 +07:00
|
|
|
#define FIRST_USER_ADDRESS 0UL
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
2015-07-10 23:24:28 +07:00
|
|
|
|
|
|
|
#include <linux/mmdebug.h>
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
extern void __pte_error(const char *file, int line, unsigned long val);
|
|
|
|
extern void __pmd_error(const char *file, int line, unsigned long val);
|
2014-05-12 16:40:51 +07:00
|
|
|
extern void __pud_error(const char *file, int line, unsigned long val);
|
2012-03-05 18:49:27 +07:00
|
|
|
extern void __pgd_error(const char *file, int line, unsigned long val);
|
|
|
|
|
2014-04-03 21:57:15 +07:00
|
|
|
#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
|
|
|
|
#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2016-01-07 23:07:20 +07:00
|
|
|
#define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE))
|
|
|
|
#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE))
|
|
|
|
#define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_NC))
|
|
|
|
#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_WT))
|
|
|
|
#define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL))
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2014-04-03 21:57:15 +07:00
|
|
|
#define PROT_SECT_DEVICE_nGnRE (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE))
|
|
|
|
#define PROT_SECT_NORMAL (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
|
|
|
|
#define PROT_SECT_NORMAL_EXEC (PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
|
2012-12-18 21:15:15 +07:00
|
|
|
|
2014-04-03 21:57:15 +07:00
|
|
|
#define _PAGE_DEFAULT (PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2014-04-03 21:57:15 +07:00
|
|
|
#define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE)
|
2015-11-09 15:55:46 +07:00
|
|
|
#define PAGE_KERNEL_RO __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
|
2016-01-07 23:07:20 +07:00
|
|
|
#define PAGE_KERNEL_ROX __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
|
2014-04-03 21:57:15 +07:00
|
|
|
#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE)
|
2015-10-08 00:00:22 +07:00
|
|
|
#define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT)
|
2012-11-16 00:21:16 +07:00
|
|
|
|
2014-04-03 21:57:15 +07:00
|
|
|
#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP)
|
2012-12-08 01:35:41 +07:00
|
|
|
#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
|
|
|
|
|
2014-04-03 21:57:15 +07:00
|
|
|
#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
|
2014-09-18 04:56:20 +07:00
|
|
|
#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
|
2012-12-08 01:35:41 +07:00
|
|
|
|
2015-10-01 19:06:07 +07:00
|
|
|
#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN)
|
2014-04-03 21:57:15 +07:00
|
|
|
#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
|
|
|
|
#define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
|
|
|
|
#define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
|
|
|
|
#define PAGE_COPY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
|
|
|
|
#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
|
|
|
|
#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
|
|
|
|
|
|
|
|
#define __P000 PAGE_NONE
|
|
|
|
#define __P001 PAGE_READONLY
|
|
|
|
#define __P010 PAGE_COPY
|
|
|
|
#define __P011 PAGE_COPY
|
2014-05-16 22:44:32 +07:00
|
|
|
#define __P100 PAGE_READONLY_EXEC
|
2014-04-03 21:57:15 +07:00
|
|
|
#define __P101 PAGE_READONLY_EXEC
|
|
|
|
#define __P110 PAGE_COPY_EXEC
|
|
|
|
#define __P111 PAGE_COPY_EXEC
|
|
|
|
|
|
|
|
#define __S000 PAGE_NONE
|
|
|
|
#define __S001 PAGE_READONLY
|
|
|
|
#define __S010 PAGE_SHARED
|
|
|
|
#define __S011 PAGE_SHARED
|
2014-05-16 22:44:32 +07:00
|
|
|
#define __S100 PAGE_READONLY_EXEC
|
2014-04-03 21:57:15 +07:00
|
|
|
#define __S101 PAGE_READONLY_EXEC
|
|
|
|
#define __S110 PAGE_SHARED_EXEC
|
|
|
|
#define __S111 PAGE_SHARED_EXEC
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* ZERO_PAGE is a global shared page that is always zero: used
|
|
|
|
* for zero-mapped memory areas etc..
|
|
|
|
*/
|
arm64: mm: place empty_zero_page in bss
Currently the zero page is set up in paging_init, and thus we cannot use
the zero page earlier. We use the zero page as a reserved TTBR value
from which no TLB entries may be allocated (e.g. when uninstalling the
idmap). To enable such usage earlier (as may be required for invasive
changes to the kernel page tables), and to minimise the time that the
idmap is active, we need to be able to use the zero page before
paging_init.
This patch follows the example set by x86, by allocating the zero page
at compile time, in .bss. This means that the zero page itself is
available immediately upon entry to start_kernel (as we zero .bss before
this), and also means that the zero page takes up no space in the raw
Image binary. The associated struct page is allocated in bootmem_init,
and remains unavailable until this time.
Outside of arch code, the only users of empty_zero_page assume that the
empty_zero_page symbol refers to the zeroed memory itself, and that
ZERO_PAGE(x) must be used to acquire the associated struct page,
following the example of x86. This patch also brings arm64 inline with
these assumptions.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Tested-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: Jeremy Linton <jeremy.linton@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-01-25 18:44:57 +07:00
|
|
|
extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
|
|
|
|
#define ZERO_PAGE(vaddr) virt_to_page(empty_zero_page)
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2014-07-21 20:52:49 +07:00
|
|
|
#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
#define pte_pfn(pte) ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT)
|
|
|
|
|
|
|
|
#define pfn_pte(pfn,prot) (__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
|
|
|
|
|
|
|
|
#define pte_none(pte) (!pte_val(pte))
|
|
|
|
#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
|
|
|
|
#define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
|
2014-07-21 20:52:49 +07:00
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* The following only work if pte_present(). Undefined behaviour otherwise.
|
|
|
|
*/
|
2014-02-25 18:38:53 +07:00
|
|
|
#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)))
|
|
|
|
#define pte_young(pte) (!!(pte_val(pte) & PTE_AF))
|
|
|
|
#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))
|
|
|
|
#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))
|
2012-11-16 00:21:16 +07:00
|
|
|
#define pte_exec(pte) (!(pte_val(pte) & PTE_UXN))
|
2015-10-08 00:00:21 +07:00
|
|
|
#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
|
2016-01-07 23:07:20 +07:00
|
|
|
#define pte_user(pte) (!!(pte_val(pte) & PTE_USER))
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2015-07-10 23:24:28 +07:00
|
|
|
#ifdef CONFIG_ARM64_HW_AFDBM
|
2015-09-12 00:22:00 +07:00
|
|
|
#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
|
2015-07-10 23:24:28 +07:00
|
|
|
#else
|
|
|
|
#define pte_hw_dirty(pte) (0)
|
|
|
|
#endif
|
|
|
|
#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
|
|
|
|
#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))
|
|
|
|
|
2015-07-28 22:14:03 +07:00
|
|
|
#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))
|
2014-06-09 17:55:03 +07:00
|
|
|
#define pte_valid_not_user(pte) \
|
|
|
|
((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID)
|
2015-10-31 01:56:19 +07:00
|
|
|
#define pte_valid_young(pte) \
|
|
|
|
((pte_val(pte) & (PTE_VALID | PTE_AF)) == (PTE_VALID | PTE_AF))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Could the pte be present in the TLB? We must check mm_tlb_flush_pending
|
|
|
|
* so that we don't erroneously return false for pages that have been
|
|
|
|
* remapped as PROT_NONE but are yet to be flushed from the TLB.
|
|
|
|
*/
|
|
|
|
#define pte_accessible(mm, pte) \
|
|
|
|
(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte))
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2014-08-20 02:41:42 +07:00
|
|
|
static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
|
2014-01-15 21:07:12 +07:00
|
|
|
{
|
2014-08-20 02:41:42 +07:00
|
|
|
pte_val(pte) &= ~pgprot_val(prot);
|
2014-01-15 21:07:12 +07:00
|
|
|
return pte;
|
|
|
|
}
|
|
|
|
|
2014-08-20 02:41:42 +07:00
|
|
|
static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
|
2014-01-15 21:07:12 +07:00
|
|
|
{
|
2014-08-20 02:41:42 +07:00
|
|
|
pte_val(pte) |= pgprot_val(prot);
|
2014-01-15 21:07:12 +07:00
|
|
|
return pte;
|
|
|
|
}
|
|
|
|
|
2014-08-20 02:41:42 +07:00
|
|
|
static inline pte_t pte_wrprotect(pte_t pte)
|
|
|
|
{
|
|
|
|
return clear_pte_bit(pte, __pgprot(PTE_WRITE));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline pte_t pte_mkwrite(pte_t pte)
|
|
|
|
{
|
|
|
|
return set_pte_bit(pte, __pgprot(PTE_WRITE));
|
|
|
|
}
|
|
|
|
|
2014-01-15 21:07:12 +07:00
|
|
|
static inline pte_t pte_mkclean(pte_t pte)
|
|
|
|
{
|
2014-08-20 02:41:42 +07:00
|
|
|
return clear_pte_bit(pte, __pgprot(PTE_DIRTY));
|
2014-01-15 21:07:12 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline pte_t pte_mkdirty(pte_t pte)
|
|
|
|
{
|
2014-08-20 02:41:42 +07:00
|
|
|
return set_pte_bit(pte, __pgprot(PTE_DIRTY));
|
2014-01-15 21:07:12 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline pte_t pte_mkold(pte_t pte)
|
|
|
|
{
|
2014-08-20 02:41:42 +07:00
|
|
|
return clear_pte_bit(pte, __pgprot(PTE_AF));
|
2014-01-15 21:07:12 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline pte_t pte_mkyoung(pte_t pte)
|
|
|
|
{
|
2014-08-20 02:41:42 +07:00
|
|
|
return set_pte_bit(pte, __pgprot(PTE_AF));
|
2014-01-15 21:07:12 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline pte_t pte_mkspecial(pte_t pte)
|
|
|
|
{
|
2014-08-20 02:41:42 +07:00
|
|
|
return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
|
2014-01-15 21:07:12 +07:00
|
|
|
}
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2015-10-08 00:00:21 +07:00
|
|
|
static inline pte_t pte_mkcont(pte_t pte)
|
|
|
|
{
|
2015-12-18 02:31:26 +07:00
|
|
|
pte = set_pte_bit(pte, __pgprot(PTE_CONT));
|
|
|
|
return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE));
|
2015-10-08 00:00:21 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline pte_t pte_mknoncont(pte_t pte)
|
|
|
|
{
|
|
|
|
return clear_pte_bit(pte, __pgprot(PTE_CONT));
|
|
|
|
}
|
|
|
|
|
2015-12-18 02:31:26 +07:00
|
|
|
static inline pmd_t pmd_mkcont(pmd_t pmd)
|
|
|
|
{
|
|
|
|
return __pmd(pmd_val(pmd) | PMD_SECT_CONT);
|
|
|
|
}
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
static inline void set_pte(pte_t *ptep, pte_t pte)
|
|
|
|
{
|
|
|
|
*ptep = pte;
|
2014-06-09 17:55:03 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Only if the new pte is valid and kernel, otherwise TLB maintenance
|
|
|
|
* or update_mmu_cache() have the necessary barriers.
|
|
|
|
*/
|
|
|
|
if (pte_valid_not_user(pte)) {
|
|
|
|
dsb(ishst);
|
|
|
|
isb();
|
|
|
|
}
|
2012-03-05 18:49:27 +07:00
|
|
|
}
|
|
|
|
|
2015-07-10 23:24:28 +07:00
|
|
|
struct mm_struct;
|
|
|
|
struct vm_area_struct;
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
extern void __sync_icache_dcache(pte_t pteval, unsigned long addr);
|
|
|
|
|
2015-07-10 23:24:28 +07:00
|
|
|
/*
|
|
|
|
* PTE bits configuration in the presence of hardware Dirty Bit Management
|
|
|
|
* (PTE_WRITE == PTE_DBM):
|
|
|
|
*
|
|
|
|
* Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw)
|
|
|
|
* 0 0 | 1 0 0
|
|
|
|
* 0 1 | 1 1 0
|
|
|
|
* 1 0 | 1 0 1
|
|
|
|
* 1 1 | 0 1 x
|
|
|
|
*
|
|
|
|
* When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
|
|
|
|
* the page fault mechanism. Checking the dirty status of a pte becomes:
|
|
|
|
*
|
2015-09-12 00:22:00 +07:00
|
|
|
* PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
|
2015-07-10 23:24:28 +07:00
|
|
|
*/
|
2012-03-05 18:49:27 +07:00
|
|
|
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, pte_t pte)
|
|
|
|
{
|
2016-01-07 23:07:20 +07:00
|
|
|
if (pte_valid(pte)) {
|
2015-07-10 23:24:28 +07:00
|
|
|
if (pte_sw_dirty(pte) && pte_write(pte))
|
2014-01-15 21:07:13 +07:00
|
|
|
pte_val(pte) &= ~PTE_RDONLY;
|
|
|
|
else
|
|
|
|
pte_val(pte) |= PTE_RDONLY;
|
2016-01-07 23:07:20 +07:00
|
|
|
if (pte_user(pte) && pte_exec(pte) && !pte_special(pte))
|
|
|
|
__sync_icache_dcache(pte, addr);
|
2013-01-09 18:08:10 +07:00
|
|
|
}
|
|
|
|
|
2015-07-10 23:24:28 +07:00
|
|
|
/*
|
|
|
|
* If the existing pte is valid, check for potential race with
|
|
|
|
* hardware updates of the pte (ptep_set_access_flags safely changes
|
|
|
|
* valid ptes without going through an invalid entry).
|
|
|
|
*/
|
2015-12-09 00:39:15 +07:00
|
|
|
if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) &&
|
|
|
|
pte_valid(*ptep) && pte_valid(pte)) {
|
|
|
|
VM_WARN_ONCE(!pte_young(pte),
|
|
|
|
"%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
|
|
|
|
__func__, pte_val(*ptep), pte_val(pte));
|
|
|
|
VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(pte),
|
|
|
|
"%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
|
|
|
|
__func__, pte_val(*ptep), pte_val(pte));
|
2015-07-10 23:24:28 +07:00
|
|
|
}
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
set_pte(ptep, pte);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Huge pte definitions.
|
|
|
|
*/
|
2013-04-10 19:48:00 +07:00
|
|
|
#define pte_huge(pte) (!(pte_val(pte) & PTE_TABLE_BIT))
|
|
|
|
#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Hugetlb definitions.
|
|
|
|
*/
|
2015-12-18 02:31:26 +07:00
|
|
|
#define HUGE_MAX_HSTATE 4
|
2013-04-10 19:48:00 +07:00
|
|
|
#define HPAGE_SHIFT PMD_SHIFT
|
|
|
|
#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
|
|
|
|
#define HPAGE_MASK (~(HPAGE_SIZE - 1))
|
|
|
|
#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
#define __HAVE_ARCH_PTE_SPECIAL
|
|
|
|
|
2014-10-10 05:29:25 +07:00
|
|
|
static inline pte_t pud_pte(pud_t pud)
|
|
|
|
{
|
|
|
|
return __pte(pud_val(pud));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline pmd_t pud_pmd(pud_t pud)
|
|
|
|
{
|
|
|
|
return __pmd(pud_val(pud));
|
|
|
|
}
|
|
|
|
|
2014-02-25 17:02:13 +07:00
|
|
|
static inline pte_t pmd_pte(pmd_t pmd)
|
|
|
|
{
|
|
|
|
return __pte(pmd_val(pmd));
|
|
|
|
}
|
2013-04-19 22:23:57 +07:00
|
|
|
|
2014-02-25 17:02:13 +07:00
|
|
|
static inline pmd_t pte_pmd(pte_t pte)
|
|
|
|
{
|
|
|
|
return __pmd(pte_val(pte));
|
|
|
|
}
|
2013-04-19 22:23:57 +07:00
|
|
|
|
2014-10-20 20:42:07 +07:00
|
|
|
static inline pgprot_t mk_sect_prot(pgprot_t prot)
|
|
|
|
{
|
|
|
|
return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT);
|
|
|
|
}
|
|
|
|
|
2013-04-19 22:23:57 +07:00
|
|
|
/*
|
|
|
|
* THP definitions.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
#define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
|
2014-10-10 05:29:25 +07:00
|
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
2013-04-19 22:23:57 +07:00
|
|
|
|
2014-12-11 06:44:36 +07:00
|
|
|
#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
|
2014-02-25 17:02:13 +07:00
|
|
|
#define pmd_young(pmd) pte_young(pmd_pte(pmd))
|
|
|
|
#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
|
|
|
|
#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
|
|
|
|
#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
|
2016-01-16 07:55:37 +07:00
|
|
|
#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
|
2014-02-25 17:02:13 +07:00
|
|
|
#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
|
|
|
|
#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
|
2014-06-18 20:06:27 +07:00
|
|
|
#define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_TYPE_MASK))
|
2013-04-19 22:23:57 +07:00
|
|
|
|
2014-02-25 17:02:13 +07:00
|
|
|
#define __HAVE_ARCH_PMD_WRITE
|
|
|
|
#define pmd_write(pmd) pte_write(pmd_pte(pmd))
|
2013-04-19 22:23:57 +07:00
|
|
|
|
|
|
|
#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
|
|
|
|
|
|
|
|
#define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
|
|
|
|
#define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
|
|
|
|
#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)
|
|
|
|
|
2014-10-10 05:29:25 +07:00
|
|
|
#define pud_write(pud) pte_write(pud_pte(pud))
|
2014-05-06 20:02:27 +07:00
|
|
|
#define pud_pfn(pud) (((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
|
2013-04-19 22:23:57 +07:00
|
|
|
|
arm64: mm: fix pmd_write CoW brokenness
Commit 9c7e535fcc17 ("arm64: mm: Route pmd thp functions through pte
equivalents") changed the pmd manipulator and accessor functions to
convert the target pmd to a pte, process it with the pte functions, then
convert it back. Along the way, we gained support for PTE_WRITE, however
this is completely ignored by set_pmd_at, and so we fail to set the
PMD_SECT_RDONLY for PMDs, resulting in all sorts of lovely failures (like
CoW not working).
Partially reverting the offending commit (by making use of
PMD_SECT_RDONLY explicitly for pmd_{write,wrprotect,mkwrite} functions)
leads to further issues because pmd_write can then return potentially
incorrect values for page table entries marked as RDONLY, leading to
BUG_ON(pmd_write(entry)) tripping under some THP workloads.
This patch fixes the issue by routing set_pmd_at through set_pte_at,
which correctly takes the PTE_WRITE flag into account. Given that
THP mappings are always anonymous, the additional cache-flushing code
in __sync_icache_dcache won't impose any significant overhead as the
flush will be skipped.
Cc: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Steve Capper <steve.capper@arm.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2014-05-28 01:11:58 +07:00
|
|
|
#define set_pmd_at(mm, addr, pmdp, pmd) set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))
|
2013-04-19 22:23:57 +07:00
|
|
|
|
|
|
|
static inline int has_transparent_hugepage(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2014-04-03 21:57:15 +07:00
|
|
|
#define __pgprot_modify(prot,mask,bits) \
|
|
|
|
__pgprot((pgprot_val(prot) & ~(mask)) | (bits))
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* Mark the prot value as uncacheable and unbufferable.
|
|
|
|
*/
|
|
|
|
#define pgprot_noncached(prot) \
|
2014-03-12 23:07:06 +07:00
|
|
|
__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN)
|
2012-03-05 18:49:27 +07:00
|
|
|
#define pgprot_writecombine(prot) \
|
2014-03-12 23:07:06 +07:00
|
|
|
__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
|
2014-09-29 21:29:31 +07:00
|
|
|
#define pgprot_device(prot) \
|
|
|
|
__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN)
|
2012-03-05 18:49:27 +07:00
|
|
|
#define __HAVE_PHYS_MEM_ACCESS_PROT
|
|
|
|
struct file;
|
|
|
|
extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
|
|
|
|
unsigned long size, pgprot_t vma_prot);
|
|
|
|
|
|
|
|
#define pmd_none(pmd) (!pmd_val(pmd))
|
|
|
|
#define pmd_present(pmd) (pmd_val(pmd))
|
|
|
|
|
|
|
|
#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
|
|
|
|
|
2012-12-08 01:35:41 +07:00
|
|
|
#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
|
|
|
PMD_TYPE_TABLE)
|
|
|
|
#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
|
|
|
PMD_TYPE_SECT)
|
|
|
|
|
2014-06-25 14:41:45 +07:00
|
|
|
#ifdef CONFIG_ARM64_64K_PAGES
|
2014-05-06 20:02:27 +07:00
|
|
|
#define pud_sect(pud) (0)
|
2014-12-09 14:26:47 +07:00
|
|
|
#define pud_table(pud) (1)
|
2014-05-06 20:02:27 +07:00
|
|
|
#else
|
|
|
|
#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
|
|
|
PUD_TYPE_SECT)
|
2014-12-09 14:26:47 +07:00
|
|
|
#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
|
|
|
PUD_TYPE_TABLE)
|
2014-05-06 20:02:27 +07:00
|
|
|
#endif
|
2012-12-08 01:35:41 +07:00
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
|
|
|
|
{
|
|
|
|
*pmdp = pmd;
|
2014-05-02 22:24:10 +07:00
|
|
|
dsb(ishst);
|
2014-06-09 17:55:03 +07:00
|
|
|
isb();
|
2012-03-05 18:49:27 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void pmd_clear(pmd_t *pmdp)
|
|
|
|
{
|
|
|
|
set_pmd(pmdp, __pmd(0));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline pte_t *pmd_page_vaddr(pmd_t pmd)
|
|
|
|
{
|
|
|
|
return __va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK);
|
|
|
|
}
|
|
|
|
|
2016-01-25 18:45:03 +07:00
|
|
|
/* Find an entry in the third-level page table. */
|
|
|
|
#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
|
|
|
|
|
|
|
|
#define pte_offset_kernel(dir,addr) (pmd_page_vaddr(*(dir)) + pte_index(addr))
|
|
|
|
|
|
|
|
#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
|
|
|
|
#define pte_offset_map_nested(dir,addr) pte_offset_kernel((dir), (addr))
|
|
|
|
#define pte_unmap(pte) do { } while (0)
|
|
|
|
#define pte_unmap_nested(pte) do { } while (0)
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Conversion functions: convert a page and protection to a page entry,
|
|
|
|
* and a page entry and page directory to the page they refer to.
|
|
|
|
*/
|
|
|
|
#define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot)
|
|
|
|
|
2015-04-15 05:45:39 +07:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 2
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2014-07-21 20:52:49 +07:00
|
|
|
#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
#define pud_none(pud) (!pud_val(pud))
|
|
|
|
#define pud_bad(pud) (!(pud_val(pud) & 2))
|
|
|
|
#define pud_present(pud) (pud_val(pud))
|
|
|
|
|
|
|
|
static inline void set_pud(pud_t *pudp, pud_t pud)
|
|
|
|
{
|
|
|
|
*pudp = pud;
|
2014-05-02 22:24:10 +07:00
|
|
|
dsb(ishst);
|
2014-06-09 17:55:03 +07:00
|
|
|
isb();
|
2012-03-05 18:49:27 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void pud_clear(pud_t *pudp)
|
|
|
|
{
|
|
|
|
set_pud(pudp, __pud(0));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline pmd_t *pud_page_vaddr(pud_t pud)
|
|
|
|
{
|
|
|
|
return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
|
|
|
|
}
|
|
|
|
|
2014-07-21 20:52:49 +07:00
|
|
|
/* Find an entry in the second-level page table. */
|
|
|
|
#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
|
|
|
|
|
|
|
|
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
|
|
|
|
{
|
|
|
|
return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
|
|
|
|
}
|
|
|
|
|
2014-12-20 07:49:40 +07:00
|
|
|
#define pud_page(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
|
2014-10-10 05:29:25 +07:00
|
|
|
|
2015-04-15 05:45:39 +07:00
|
|
|
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2015-04-15 05:45:39 +07:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 3
|
2014-05-12 16:40:51 +07:00
|
|
|
|
2014-07-21 20:52:49 +07:00
|
|
|
#define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud))
|
|
|
|
|
2014-05-12 16:40:51 +07:00
|
|
|
#define pgd_none(pgd) (!pgd_val(pgd))
|
|
|
|
#define pgd_bad(pgd) (!(pgd_val(pgd) & 2))
|
|
|
|
#define pgd_present(pgd) (pgd_val(pgd))
|
|
|
|
|
|
|
|
static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
|
|
|
|
{
|
|
|
|
*pgdp = pgd;
|
|
|
|
dsb(ishst);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void pgd_clear(pgd_t *pgdp)
|
|
|
|
{
|
|
|
|
set_pgd(pgdp, __pgd(0));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline pud_t *pgd_page_vaddr(pgd_t pgd)
|
|
|
|
{
|
|
|
|
return __va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
|
|
|
|
}
|
|
|
|
|
2014-07-21 20:52:49 +07:00
|
|
|
/* Find an entry in the frst-level page table. */
|
|
|
|
#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
|
|
|
|
|
|
|
|
static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr)
|
|
|
|
{
|
|
|
|
return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr);
|
|
|
|
}
|
|
|
|
|
2014-12-20 07:49:40 +07:00
|
|
|
#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK))
|
|
|
|
|
2015-04-15 05:45:39 +07:00
|
|
|
#endif /* CONFIG_PGTABLE_LEVELS > 3 */
|
2014-05-12 16:40:51 +07:00
|
|
|
|
2014-07-21 20:52:49 +07:00
|
|
|
#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd))
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/* to find an entry in a page-table-directory */
|
|
|
|
#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
|
|
|
|
|
|
|
|
#define pgd_offset(mm, addr) ((mm)->pgd+pgd_index(addr))
|
|
|
|
|
|
|
|
/* to find an entry in a kernel page-table-directory */
|
|
|
|
#define pgd_offset_k(addr) pgd_offset(&init_mm, addr)
|
|
|
|
|
|
|
|
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
|
|
|
|
{
|
2012-12-18 21:15:15 +07:00
|
|
|
const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
|
2015-10-01 19:06:07 +07:00
|
|
|
PTE_PROT_NONE | PTE_VALID | PTE_WRITE;
|
2015-07-10 23:24:28 +07:00
|
|
|
/* preserve the hardware dirty information */
|
|
|
|
if (pte_hw_dirty(pte))
|
2015-09-12 00:22:01 +07:00
|
|
|
pte = pte_mkdirty(pte);
|
2012-03-05 18:49:27 +07:00
|
|
|
pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
|
|
|
|
return pte;
|
|
|
|
}
|
|
|
|
|
2014-02-25 17:02:13 +07:00
|
|
|
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
|
|
|
{
|
|
|
|
return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
|
|
|
|
}
|
|
|
|
|
2015-07-10 23:24:28 +07:00
|
|
|
#ifdef CONFIG_ARM64_HW_AFDBM
|
|
|
|
/*
|
|
|
|
* Atomic pte/pmd modifications.
|
|
|
|
*/
|
|
|
|
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
|
|
|
|
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
|
|
|
|
unsigned long address,
|
|
|
|
pte_t *ptep)
|
|
|
|
{
|
|
|
|
pteval_t pteval;
|
|
|
|
unsigned int tmp, res;
|
|
|
|
|
|
|
|
asm volatile("// ptep_test_and_clear_young\n"
|
|
|
|
" prfm pstl1strm, %2\n"
|
|
|
|
"1: ldxr %0, %2\n"
|
|
|
|
" ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n"
|
|
|
|
" and %0, %0, %4 // clear PTE_AF\n"
|
|
|
|
" stxr %w1, %0, %2\n"
|
|
|
|
" cbnz %w1, 1b\n"
|
|
|
|
: "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res)
|
|
|
|
: "L" (~PTE_AF), "I" (ilog2(PTE_AF)));
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
|
|
|
|
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
|
|
|
unsigned long address,
|
|
|
|
pmd_t *pmdp)
|
|
|
|
{
|
|
|
|
return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
|
|
|
|
|
|
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
|
|
|
|
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
|
|
|
|
unsigned long address, pte_t *ptep)
|
|
|
|
{
|
|
|
|
pteval_t old_pteval;
|
|
|
|
unsigned int tmp;
|
|
|
|
|
|
|
|
asm volatile("// ptep_get_and_clear\n"
|
|
|
|
" prfm pstl1strm, %2\n"
|
|
|
|
"1: ldxr %0, %2\n"
|
|
|
|
" stxr %w1, xzr, %2\n"
|
|
|
|
" cbnz %w1, 1b\n"
|
|
|
|
: "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)));
|
|
|
|
|
|
|
|
return __pte(old_pteval);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
|
|
|
|
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
|
|
|
|
unsigned long address, pmd_t *pmdp)
|
|
|
|
{
|
|
|
|
return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp));
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ptep_set_wrprotect - mark read-only while trasferring potential hardware
|
|
|
|
* dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
|
|
|
|
*/
|
|
|
|
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
|
|
|
|
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
|
|
|
|
{
|
|
|
|
pteval_t pteval;
|
|
|
|
unsigned long tmp;
|
|
|
|
|
|
|
|
asm volatile("// ptep_set_wrprotect\n"
|
|
|
|
" prfm pstl1strm, %2\n"
|
|
|
|
"1: ldxr %0, %2\n"
|
|
|
|
" tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n"
|
|
|
|
" csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n"
|
|
|
|
" orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n"
|
|
|
|
" and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n"
|
|
|
|
" stxr %w1, %0, %2\n"
|
|
|
|
" cbnz %w1, 1b\n"
|
|
|
|
: "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
|
|
|
|
: "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE)
|
|
|
|
: "cc");
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
#define __HAVE_ARCH_PMDP_SET_WRPROTECT
|
|
|
|
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
|
|
|
|
unsigned long address, pmd_t *pmdp)
|
|
|
|
{
|
|
|
|
ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
#endif /* CONFIG_ARM64_HW_AFDBM */
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
|
|
|
|
extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Encode and decode a swap entry:
|
2013-11-27 23:59:27 +07:00
|
|
|
* bits 0-1: present (must be zero)
|
2015-02-11 05:10:15 +07:00
|
|
|
* bits 2-7: swap type
|
|
|
|
* bits 8-57: swap offset
|
2012-03-05 18:49:27 +07:00
|
|
|
*/
|
2015-02-11 05:10:15 +07:00
|
|
|
#define __SWP_TYPE_SHIFT 2
|
2012-03-05 18:49:27 +07:00
|
|
|
#define __SWP_TYPE_BITS 6
|
2015-02-11 05:10:15 +07:00
|
|
|
#define __SWP_OFFSET_BITS 50
|
2012-03-05 18:49:27 +07:00
|
|
|
#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1)
|
|
|
|
#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
|
2013-11-27 23:59:27 +07:00
|
|
|
#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1)
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
|
2013-11-27 23:59:27 +07:00
|
|
|
#define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK)
|
2012-03-05 18:49:27 +07:00
|
|
|
#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
|
|
|
|
|
|
|
|
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
|
|
|
|
#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val })
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ensure that there are not more swap files than can be encoded in the kernel
|
2014-03-11 17:23:39 +07:00
|
|
|
* PTEs.
|
2012-03-05 18:49:27 +07:00
|
|
|
*/
|
|
|
|
#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
|
|
|
|
|
|
|
|
extern int kern_addr_valid(unsigned long addr);
|
|
|
|
|
|
|
|
#include <asm-generic/pgtable.h>
|
|
|
|
|
2016-01-05 22:36:59 +07:00
|
|
|
void pgd_cache_init(void);
|
|
|
|
#define pgtable_cache_init pgd_cache_init
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2015-07-17 01:26:02 +07:00
|
|
|
/*
|
|
|
|
* On AArch64, the cache coherency is handled via the set_pte_at() function.
|
|
|
|
*/
|
|
|
|
static inline void update_mmu_cache(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
|
|
|
/*
|
2015-10-07 00:46:30 +07:00
|
|
|
* We don't do anything here, so there's a very small chance of
|
|
|
|
* us retaking a user fault which we just fixed up. The alternative
|
|
|
|
* is doing a dsb(ishst), but that penalises the fastpath.
|
2015-07-17 01:26:02 +07:00
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
|
|
|
#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
|
|
|
|
|
2015-10-16 20:34:50 +07:00
|
|
|
#define kc_vaddr_to_offset(v) ((v) & ~VA_START)
|
|
|
|
#define kc_offset_to_vaddr(o) ((o) | VA_START)
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
#endif /* !__ASSEMBLY__ */
|
|
|
|
|
|
|
|
#endif /* __ASM_PGTABLE_H */
|