2011-11-23 00:30:29 +07:00
|
|
|
/*
|
|
|
|
* arch/arm/include/asm/pgtable-3level.h
|
|
|
|
*
|
|
|
|
* Copyright (C) 2011 ARM Ltd.
|
|
|
|
* Author: Catalin Marinas <catalin.marinas@arm.com>
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
*/
|
|
|
|
#ifndef _ASM_PGTABLE_3LEVEL_H
|
|
|
|
#define _ASM_PGTABLE_3LEVEL_H
|
|
|
|
|
|
|
|
/*
|
|
|
|
* With LPAE, there are 3 levels of page tables. Each level has 512 entries of
|
|
|
|
* 8 bytes each, occupying a 4K page. The first level table covers a range of
|
|
|
|
* 512GB, each entry representing 1GB. Since we are limited to 4GB input
|
|
|
|
* address range, only 4 entries in the PGD are used.
|
|
|
|
*
|
|
|
|
* There are enough spare bits in a page table entry for the kernel specific
|
|
|
|
* state.
|
|
|
|
*/
|
|
|
|
#define PTRS_PER_PTE 512
|
|
|
|
#define PTRS_PER_PMD 512
|
|
|
|
#define PTRS_PER_PGD 4
|
|
|
|
|
2013-05-02 19:52:01 +07:00
|
|
|
#define PTE_HWTABLE_PTRS (0)
|
2011-11-23 00:30:29 +07:00
|
|
|
#define PTE_HWTABLE_OFF (0)
|
|
|
|
#define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u64))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* PGDIR_SHIFT determines the size a top-level page table entry can map.
|
|
|
|
*/
|
|
|
|
#define PGDIR_SHIFT 30
|
|
|
|
|
|
|
|
/*
|
|
|
|
* PMD_SHIFT determines the size a middle-level page table entry can map.
|
|
|
|
*/
|
|
|
|
#define PMD_SHIFT 21
|
|
|
|
|
|
|
|
#define PMD_SIZE (1UL << PMD_SHIFT)
|
2012-07-23 00:40:38 +07:00
|
|
|
#define PMD_MASK (~((1 << PMD_SHIFT) - 1))
|
2011-11-23 00:30:29 +07:00
|
|
|
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
|
2012-07-23 00:40:38 +07:00
|
|
|
#define PGDIR_MASK (~((1 << PGDIR_SHIFT) - 1))
|
2011-11-23 00:30:29 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* section address mask and size definitions.
|
|
|
|
*/
|
|
|
|
#define SECTION_SHIFT 21
|
|
|
|
#define SECTION_SIZE (1UL << SECTION_SHIFT)
|
2012-07-23 00:40:38 +07:00
|
|
|
#define SECTION_MASK (~((1 << SECTION_SHIFT) - 1))
|
2011-11-23 00:30:29 +07:00
|
|
|
|
|
|
|
#define USER_PTRS_PER_PGD (PAGE_OFFSET / PGDIR_SIZE)
|
|
|
|
|
2012-07-25 20:32:38 +07:00
|
|
|
/*
|
|
|
|
* Hugetlb definitions.
|
|
|
|
*/
|
|
|
|
#define HPAGE_SHIFT PMD_SHIFT
|
|
|
|
#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
|
|
|
|
#define HPAGE_MASK (~(HPAGE_SIZE - 1))
|
|
|
|
#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
|
|
|
|
|
2011-11-23 00:30:29 +07:00
|
|
|
/*
|
|
|
|
* "Linux" PTE definitions for LPAE.
|
|
|
|
*
|
|
|
|
* These bits overlap with the hardware bits but the naming is preserved for
|
|
|
|
* consistency with the classic page table format.
|
|
|
|
*/
|
2012-07-19 17:51:05 +07:00
|
|
|
#define L_PTE_VALID (_AT(pteval_t, 1) << 0) /* Valid */
|
|
|
|
#define L_PTE_PRESENT (_AT(pteval_t, 3) << 0) /* Present */
|
2011-11-23 00:30:29 +07:00
|
|
|
#define L_PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !PRESENT */
|
|
|
|
#define L_PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
|
|
|
|
#define L_PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
|
|
|
|
#define L_PTE_YOUNG (_AT(pteval_t, 1) << 10) /* AF */
|
|
|
|
#define L_PTE_XN (_AT(pteval_t, 1) << 54) /* XN */
|
2014-07-18 22:16:15 +07:00
|
|
|
#define L_PTE_DIRTY (_AT(pteval_t, 1) << 55)
|
|
|
|
#define L_PTE_SPECIAL (_AT(pteval_t, 1) << 56)
|
2012-09-01 11:22:12 +07:00
|
|
|
#define L_PTE_NONE (_AT(pteval_t, 1) << 57) /* PROT_NONE */
|
2014-07-18 22:16:15 +07:00
|
|
|
#define L_PTE_RDONLY (_AT(pteval_t, 1) << 58) /* READ ONLY */
|
2011-11-23 00:30:29 +07:00
|
|
|
|
2014-07-18 22:16:15 +07:00
|
|
|
#define L_PMD_SECT_VALID (_AT(pmdval_t, 1) << 0)
|
|
|
|
#define L_PMD_SECT_DIRTY (_AT(pmdval_t, 1) << 55)
|
|
|
|
#define L_PMD_SECT_SPLITTING (_AT(pmdval_t, 1) << 56)
|
|
|
|
#define L_PMD_SECT_NONE (_AT(pmdval_t, 1) << 57)
|
|
|
|
#define L_PMD_SECT_RDONLY (_AT(pteval_t, 1) << 58)
|
2012-07-25 20:39:26 +07:00
|
|
|
|
2011-11-23 00:30:29 +07:00
|
|
|
/*
|
|
|
|
* To be used in assembly code with the upper page attributes.
|
|
|
|
*/
|
|
|
|
#define L_PTE_XN_HIGH (1 << (54 - 32))
|
|
|
|
#define L_PTE_DIRTY_HIGH (1 << (55 - 32))
|
|
|
|
|
|
|
|
/*
|
|
|
|
* AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
|
|
|
|
*/
|
|
|
|
#define L_PTE_MT_UNCACHED (_AT(pteval_t, 0) << 2) /* strongly ordered */
|
|
|
|
#define L_PTE_MT_BUFFERABLE (_AT(pteval_t, 1) << 2) /* normal non-cacheable */
|
|
|
|
#define L_PTE_MT_WRITETHROUGH (_AT(pteval_t, 2) << 2) /* normal inner write-through */
|
|
|
|
#define L_PTE_MT_WRITEBACK (_AT(pteval_t, 3) << 2) /* normal inner write-back */
|
|
|
|
#define L_PTE_MT_WRITEALLOC (_AT(pteval_t, 7) << 2) /* normal inner write-alloc */
|
|
|
|
#define L_PTE_MT_DEV_SHARED (_AT(pteval_t, 4) << 2) /* device */
|
|
|
|
#define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 4) << 2) /* device */
|
|
|
|
#define L_PTE_MT_DEV_WC (_AT(pteval_t, 1) << 2) /* normal non-cacheable */
|
|
|
|
#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */
|
|
|
|
#define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2)
|
|
|
|
|
2011-11-23 00:30:29 +07:00
|
|
|
/*
|
|
|
|
* Software PGD flags.
|
|
|
|
*/
|
|
|
|
#define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */
|
|
|
|
|
2013-01-21 06:28:04 +07:00
|
|
|
/*
|
|
|
|
* 2nd stage PTE definitions for LPAE.
|
|
|
|
*/
|
2014-02-03 04:21:31 +07:00
|
|
|
#define L_PTE_S2_MT_UNCACHED (_AT(pteval_t, 0x0) << 2) /* strongly ordered */
|
|
|
|
#define L_PTE_S2_MT_WRITETHROUGH (_AT(pteval_t, 0xa) << 2) /* normal inner write-through */
|
|
|
|
#define L_PTE_S2_MT_WRITEBACK (_AT(pteval_t, 0xf) << 2) /* normal inner write-back */
|
|
|
|
#define L_PTE_S2_MT_DEV_SHARED (_AT(pteval_t, 0x1) << 2) /* device */
|
|
|
|
#define L_PTE_S2_MT_MASK (_AT(pteval_t, 0xf) << 2)
|
2013-01-21 06:28:04 +07:00
|
|
|
|
2014-02-03 04:21:31 +07:00
|
|
|
#define L_PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[1] */
|
|
|
|
#define L_PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */
|
|
|
|
|
|
|
|
#define L_PMD_S2_RDWR (_AT(pmdval_t, 3) << 6) /* HAP[2:1] */
|
2012-11-01 23:14:45 +07:00
|
|
|
|
2013-01-21 06:28:04 +07:00
|
|
|
/*
|
|
|
|
* Hyp-mode PL2 PTE definitions for LPAE.
|
|
|
|
*/
|
|
|
|
#define L_PTE_HYP L_PTE_USER
|
|
|
|
|
2011-11-23 00:30:29 +07:00
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
|
|
#define pud_none(pud) (!pud_val(pud))
|
|
|
|
#define pud_bad(pud) (!(pud_val(pud) & 2))
|
|
|
|
#define pud_present(pud) (pud_val(pud))
|
2013-01-21 06:28:04 +07:00
|
|
|
#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
|
|
|
PMD_TYPE_TABLE)
|
|
|
|
#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
|
|
|
PMD_TYPE_SECT)
|
2013-10-23 22:13:02 +07:00
|
|
|
#define pmd_large(pmd) pmd_sect(pmd)
|
2011-11-23 00:30:29 +07:00
|
|
|
|
|
|
|
#define pud_clear(pudp) \
|
|
|
|
do { \
|
|
|
|
*pudp = __pud(0); \
|
|
|
|
clean_pmd_entry(pudp); \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#define set_pud(pudp, pud) \
|
|
|
|
do { \
|
|
|
|
*pudp = pud; \
|
|
|
|
flush_pmd_entry(pudp); \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
static inline pmd_t *pud_page_vaddr(pud_t pud)
|
|
|
|
{
|
|
|
|
return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Find an entry in the second-level page table.. */
|
|
|
|
#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
|
|
|
|
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
|
|
|
|
{
|
|
|
|
return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
|
|
|
|
|
|
|
|
#define copy_pmd(pmdpd,pmdps) \
|
|
|
|
do { \
|
|
|
|
*pmdpd = *pmdps; \
|
|
|
|
flush_pmd_entry(pmdpd); \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#define pmd_clear(pmdp) \
|
|
|
|
do { \
|
|
|
|
*pmdp = __pmd(0); \
|
|
|
|
clean_pmd_entry(pmdp); \
|
|
|
|
} while (0)
|
|
|
|
|
2013-05-17 18:32:55 +07:00
|
|
|
/*
|
|
|
|
* For 3 levels of paging the PTE_EXT_NG bit will be set for user address ptes
|
|
|
|
* that are written to a page table but not for ptes created with mk_pte.
|
|
|
|
*
|
|
|
|
* In hugetlb_no_page, a new huge pte (new_pte) is generated and passed to
|
|
|
|
* hugetlb_cow, where it is compared with an entry in a page table.
|
|
|
|
* This comparison test fails erroneously leading ultimately to a memory leak.
|
|
|
|
*
|
|
|
|
* To correct this behaviour, we mask off PTE_EXT_NG for any pte that is
|
|
|
|
* present before running the comparison.
|
|
|
|
*/
|
|
|
|
#define __HAVE_ARCH_PTE_SAME
|
|
|
|
#define pte_same(pte_a,pte_b) ((pte_present(pte_a) ? pte_val(pte_a) & ~PTE_EXT_NG \
|
|
|
|
: pte_val(pte_a)) \
|
|
|
|
== (pte_present(pte_b) ? pte_val(pte_b) & ~PTE_EXT_NG \
|
|
|
|
: pte_val(pte_b)))
|
|
|
|
|
2011-11-23 00:30:29 +07:00
|
|
|
#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)|(ext)))
|
|
|
|
|
2012-07-25 20:32:38 +07:00
|
|
|
#define pte_huge(pte) (pte_val(pte) && !(pte_val(pte) & PTE_TABLE_BIT))
|
|
|
|
#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT))
|
|
|
|
|
2014-07-18 22:15:27 +07:00
|
|
|
#define pmd_isset(pmd, val) ((u32)(val) == (val) ? pmd_val(pmd) & (val) \
|
|
|
|
: !!(pmd_val(pmd) & (val)))
|
|
|
|
#define pmd_isclear(pmd, val) (!(pmd_val(pmd) & (val)))
|
|
|
|
|
|
|
|
#define pmd_young(pmd) (pmd_isset((pmd), PMD_SECT_AF))
|
2014-10-10 05:29:16 +07:00
|
|
|
#define pte_special(pte) (pte_isset((pte), L_PTE_SPECIAL))
|
|
|
|
static inline pte_t pte_mkspecial(pte_t pte)
|
|
|
|
{
|
|
|
|
pte_val(pte) |= L_PTE_SPECIAL;
|
|
|
|
return pte;
|
|
|
|
}
|
|
|
|
#define __HAVE_ARCH_PTE_SPECIAL
|
2012-07-25 20:39:26 +07:00
|
|
|
|
|
|
|
#define __HAVE_ARCH_PMD_WRITE
|
2014-07-18 22:16:15 +07:00
|
|
|
#define pmd_write(pmd) (pmd_isclear((pmd), L_PMD_SECT_RDONLY))
|
|
|
|
#define pmd_dirty(pmd) (pmd_isset((pmd), L_PMD_SECT_DIRTY))
|
2014-10-10 05:29:20 +07:00
|
|
|
#define pud_page(pud) pmd_page(__pmd(pud_val(pud)))
|
|
|
|
#define pud_write(pud) pmd_write(__pmd(pud_val(pud)))
|
2012-07-25 20:39:26 +07:00
|
|
|
|
ARM: 7858/1: mm: make UACCESS_WITH_MEMCPY huge page aware
The memory pinning code in uaccess_with_memcpy.c does not check
for HugeTLB or THP pmds, and will enter an infinite loop should
a __copy_to_user or __clear_user occur against a huge page.
This patch adds detection code for huge pages to pin_page_for_write.
As this code can be executed in a fast path it refers to the actual
pmds rather than the vma. If a HugeTLB or THP is found (they have
the same pmd representation on ARM), the page table spinlock is
taken to prevent modification whilst the page is pinned.
On ARM, huge pages are only represented as pmds, thus no huge pud
checks are performed. (For huge puds one would lock the page table
in a similar manner as in the pmd case).
Two helper functions are introduced; pmd_thp_or_huge will check
whether or not a page is huge or transparent huge (which have the
same pmd layout on ARM), and pmd_hugewillfault will detect whether
or not a page fault will occur on write to the page.
Running the following test (with the chunking from read_zero
removed):
$ dd if=/dev/zero of=/dev/null bs=10M count=1024
Gave: 2.3 GB/s backed by normal pages,
2.9 GB/s backed by huge pages,
5.1 GB/s backed by huge pages, with page mask=HPAGE_MASK.
After some discussion, it was decided not to adopt the HPAGE_MASK,
as this would have a significant detrimental effect on the overall
system latency due to page_table_lock being held for too long.
This could be revisited if split huge page locks are adopted.
Signed-off-by: Steve Capper <steve.capper@linaro.org>
Reviewed-by: Nicolas Pitre <nico@linaro.org>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2013-10-14 15:49:10 +07:00
|
|
|
#define pmd_hugewillfault(pmd) (!pmd_young(pmd) || !pmd_write(pmd))
|
|
|
|
#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd))
|
|
|
|
|
2012-07-25 20:39:26 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2014-07-18 22:15:27 +07:00
|
|
|
#define pmd_trans_huge(pmd) (pmd_val(pmd) && !pmd_table(pmd))
|
2014-07-18 22:16:15 +07:00
|
|
|
#define pmd_trans_splitting(pmd) (pmd_isset((pmd), L_PMD_SECT_SPLITTING))
|
2014-10-10 05:29:20 +07:00
|
|
|
|
|
|
|
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
|
|
|
|
#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
|
|
|
|
void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
|
|
|
|
pmd_t *pmdp);
|
|
|
|
#endif
|
2012-07-25 20:39:26 +07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#define PMD_BIT_FUNC(fn,op) \
|
|
|
|
static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; }
|
|
|
|
|
2014-07-18 22:16:15 +07:00
|
|
|
PMD_BIT_FUNC(wrprotect, |= L_PMD_SECT_RDONLY);
|
2012-07-25 20:39:26 +07:00
|
|
|
PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF);
|
2014-07-18 22:16:15 +07:00
|
|
|
PMD_BIT_FUNC(mksplitting, |= L_PMD_SECT_SPLITTING);
|
|
|
|
PMD_BIT_FUNC(mkwrite, &= ~L_PMD_SECT_RDONLY);
|
|
|
|
PMD_BIT_FUNC(mkdirty, |= L_PMD_SECT_DIRTY);
|
2012-07-25 20:39:26 +07:00
|
|
|
PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF);
|
|
|
|
|
|
|
|
#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
|
|
|
|
|
|
|
|
#define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
|
|
|
|
#define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
|
|
|
|
#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)
|
|
|
|
|
|
|
|
/* represent a notpresent pmd by zero, this is used by pmdp_invalidate */
|
|
|
|
#define pmd_mknotpresent(pmd) (__pmd(0))
|
|
|
|
|
|
|
|
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
|
|
|
{
|
2014-07-18 22:16:15 +07:00
|
|
|
const pmdval_t mask = PMD_SECT_USER | PMD_SECT_XN | L_PMD_SECT_RDONLY |
|
|
|
|
L_PMD_SECT_VALID | L_PMD_SECT_NONE;
|
2012-07-25 20:39:26 +07:00
|
|
|
pmd_val(pmd) = (pmd_val(pmd) & ~mask) | (pgprot_val(newprot) & mask);
|
|
|
|
return pmd;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pmd_t *pmdp, pmd_t pmd)
|
|
|
|
{
|
|
|
|
BUG_ON(addr >= TASK_SIZE);
|
|
|
|
|
|
|
|
/* create a faulting entry if PROT_NONE protected */
|
2014-07-18 22:16:15 +07:00
|
|
|
if (pmd_val(pmd) & L_PMD_SECT_NONE)
|
|
|
|
pmd_val(pmd) &= ~L_PMD_SECT_VALID;
|
|
|
|
|
|
|
|
if (pmd_write(pmd) && pmd_dirty(pmd))
|
|
|
|
pmd_val(pmd) &= ~PMD_SECT_AP2;
|
|
|
|
else
|
|
|
|
pmd_val(pmd) |= PMD_SECT_AP2;
|
2012-07-25 20:39:26 +07:00
|
|
|
|
|
|
|
*pmdp = __pmd(pmd_val(pmd) | PMD_SECT_nG);
|
|
|
|
flush_pmd_entry(pmdp);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int has_transparent_hugepage(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2011-11-23 00:30:29 +07:00
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
|
2011-11-23 00:30:29 +07:00
|
|
|
#endif /* _ASM_PGTABLE_3LEVEL_H */
|