mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-21 03:40:25 +07:00
e77fafe9af
- 52-bit virtual addressing in the kernel
- New ABI to allow tagged user pointers to be dereferenced by syscalls
- Early RNG seeding by the bootloader
- Improve robustness of SMP boot
- Fix TLB invalidation in light of recent architectural clarifications
- Support for i.MX8 DDR PMU
- Remove direct LSE instruction patching in favour of static keys
- Function error injection using kprobes
- Support for the PPTT "thread" flag introduced by ACPI 6.3
- Move PSCI idle code into proper cpuidle driver
- Relaxation of implicit I/O memory barriers
- Build with RELR relocations when toolchain supports them
- Numerous cleanups and non-critical fixes
-----BEGIN PGP SIGNATURE-----
iQFEBAABCgAuFiEEPxTL6PPUbjXGY88ct6xw3ITBYzQFAl1yYREQHHdpbGxAa2Vy
bmVsLm9yZwAKCRC3rHDchMFjNAM3CAChqDFQkryXoHwdeEcaukMRVNxtxOi4pM4g
5xqkb7PoqRJssIblsuhaXjrSD97yWCgaqCmFe6rKoes++lP4bFcTe22KXPPyPBED
A+tK4nTuKKcZfVbEanUjI+ihXaHJmKZ/kwAxWsEBYZ4WCOe3voCiJVNO2fHxqg1M
8TskZ2BoayTbWMXih0eJg2MCy/xApBq4b3nZG4bKI7Z9UpXiKN1NYtDh98ZEBK4V
d/oNoHsJ2ZvIQsztoBJMsvr09DTCazCijWZiECadm6l41WEPFizngrACiSJLLtYo
0qu4qxgg9zgFlvBCRQmIYSggTuv35RgXSfcOwChmW5DUjHG+f9GK
=Ru4B
-----END PGP SIGNATURE-----
Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 updates from Will Deacon:
"Although there isn't tonnes of code in terms of line count, there are
a fair few headline features which I've noted both in the tag and also
in the merge commits when I pulled everything together.
The part I'm most pleased with is that we had 35 contributors this
time around, which feels like a big jump from the usual small group of
core arm64 arch developers. Hopefully they all enjoyed it so much that
they'll continue to contribute, but we'll see.
It's probably worth highlighting that we've pulled in a branch from
the risc-v folks which moves our CPU topology code out to where it can
be shared with others.
Summary:
- 52-bit virtual addressing in the kernel
- New ABI to allow tagged user pointers to be dereferenced by
syscalls
- Early RNG seeding by the bootloader
- Improve robustness of SMP boot
- Fix TLB invalidation in light of recent architectural
clarifications
- Support for i.MX8 DDR PMU
- Remove direct LSE instruction patching in favour of static keys
- Function error injection using kprobes
- Support for the PPTT "thread" flag introduced by ACPI 6.3
- Move PSCI idle code into proper cpuidle driver
- Relaxation of implicit I/O memory barriers
- Build with RELR relocations when toolchain supports them
- Numerous cleanups and non-critical fixes"
* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (114 commits)
arm64: remove __iounmap
arm64: atomics: Use K constraint when toolchain appears to support it
arm64: atomics: Undefine internal macros after use
arm64: lse: Make ARM64_LSE_ATOMICS depend on JUMP_LABEL
arm64: asm: Kill 'asm/atomic_arch.h'
arm64: lse: Remove unused 'alt_lse' assembly macro
arm64: atomics: Remove atomic_ll_sc compilation unit
arm64: avoid using hard-coded registers for LSE atomics
arm64: atomics: avoid out-of-line ll/sc atomics
arm64: Use correct ll/sc atomic constraints
jump_label: Don't warn on __exit jump entries
docs/perf: Add documentation for the i.MX8 DDR PMU
perf/imx_ddr: Add support for AXI ID filtering
arm64: kpti: ensure patched kernel text is fetched from PoU
arm64: fix fixmap copy for 16K pages and 48-bit VA
perf/smmuv3: Validate groups for global filtering
perf/smmuv3: Validate group size
arm64: Relax Documentation/arm64/tagged-pointers.rst
arm64: kvm: Replace hardcoded '1' with SYS_PAR_EL1_F
arm64: mm: Ignore spurious translation faults taken from the kernel
...
881 lines
25 KiB
C
881 lines
25 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*/
|
|
#ifndef __ASM_PGTABLE_H
|
|
#define __ASM_PGTABLE_H
|
|
|
|
#include <asm/bug.h>
|
|
#include <asm/proc-fns.h>
|
|
|
|
#include <asm/memory.h>
|
|
#include <asm/pgtable-hwdef.h>
|
|
#include <asm/pgtable-prot.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
/*
|
|
* VMALLOC range.
|
|
*
|
|
* VMALLOC_START: beginning of the kernel vmalloc space
|
|
* VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space
|
|
* and fixed mappings
|
|
*/
|
|
#define VMALLOC_START (MODULES_END)
|
|
#define VMALLOC_END (- PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
|
|
|
|
#define FIRST_USER_ADDRESS 0UL
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <asm/cmpxchg.h>
|
|
#include <asm/fixmap.h>
|
|
#include <linux/mmdebug.h>
|
|
#include <linux/mm_types.h>
|
|
#include <linux/sched.h>
|
|
|
|
extern struct page *vmemmap;
|
|
|
|
extern void __pte_error(const char *file, int line, unsigned long val);
|
|
extern void __pmd_error(const char *file, int line, unsigned long val);
|
|
extern void __pud_error(const char *file, int line, unsigned long val);
|
|
extern void __pgd_error(const char *file, int line, unsigned long val);
|
|
|
|
/*
|
|
* ZERO_PAGE is a global shared page that is always zero: used
|
|
* for zero-mapped memory areas etc..
|
|
*/
|
|
extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
|
|
#define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page))
|
|
|
|
#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
|
|
|
|
/*
|
|
* Macros to convert between a physical address and its placement in a
|
|
* page table entry, taking care of 52-bit addresses.
|
|
*/
|
|
#ifdef CONFIG_ARM64_PA_BITS_52
|
|
#define __pte_to_phys(pte) \
|
|
((pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << 36))
|
|
#define __phys_to_pte_val(phys) (((phys) | ((phys) >> 36)) & PTE_ADDR_MASK)
|
|
#else
|
|
#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK)
|
|
#define __phys_to_pte_val(phys) (phys)
|
|
#endif
|
|
|
|
#define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT)
|
|
#define pfn_pte(pfn,prot) \
|
|
__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
|
|
|
|
#define pte_none(pte) (!pte_val(pte))
|
|
#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
|
|
#define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
|
|
|
|
/*
|
|
* The following only work if pte_present(). Undefined behaviour otherwise.
|
|
*/
|
|
#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)))
|
|
#define pte_young(pte) (!!(pte_val(pte) & PTE_AF))
|
|
#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))
|
|
#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))
|
|
#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN))
|
|
#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
|
|
#define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP))
|
|
|
|
#define pte_cont_addr_end(addr, end) \
|
|
({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \
|
|
(__boundary - 1 < (end) - 1) ? __boundary : (end); \
|
|
})
|
|
|
|
#define pmd_cont_addr_end(addr, end) \
|
|
({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \
|
|
(__boundary - 1 < (end) - 1) ? __boundary : (end); \
|
|
})
|
|
|
|
#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
|
|
#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
|
|
#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))
|
|
|
|
#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))
|
|
/*
|
|
* Execute-only user mappings do not have the PTE_USER bit set. All valid
|
|
* kernel mappings have the PTE_UXN bit set.
|
|
*/
|
|
#define pte_valid_not_user(pte) \
|
|
((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))
|
|
#define pte_valid_young(pte) \
|
|
((pte_val(pte) & (PTE_VALID | PTE_AF)) == (PTE_VALID | PTE_AF))
|
|
#define pte_valid_user(pte) \
|
|
((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER))
|
|
|
|
/*
|
|
* Could the pte be present in the TLB? We must check mm_tlb_flush_pending
|
|
* so that we don't erroneously return false for pages that have been
|
|
* remapped as PROT_NONE but are yet to be flushed from the TLB.
|
|
*/
|
|
#define pte_accessible(mm, pte) \
|
|
(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte))
|
|
|
|
/*
|
|
* p??_access_permitted() is true for valid user mappings (subject to the
|
|
* write permission check) other than user execute-only which do not have the
|
|
* PTE_USER bit set. PROT_NONE mappings do not have the PTE_VALID bit set.
|
|
*/
|
|
#define pte_access_permitted(pte, write) \
|
|
(pte_valid_user(pte) && (!(write) || pte_write(pte)))
|
|
#define pmd_access_permitted(pmd, write) \
|
|
(pte_access_permitted(pmd_pte(pmd), (write)))
|
|
#define pud_access_permitted(pud, write) \
|
|
(pte_access_permitted(pud_pte(pud), (write)))
|
|
|
|
static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
|
|
{
|
|
pte_val(pte) &= ~pgprot_val(prot);
|
|
return pte;
|
|
}
|
|
|
|
static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
|
|
{
|
|
pte_val(pte) |= pgprot_val(prot);
|
|
return pte;
|
|
}
|
|
|
|
static inline pte_t pte_wrprotect(pte_t pte)
|
|
{
|
|
pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
|
|
pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
|
|
return pte;
|
|
}
|
|
|
|
static inline pte_t pte_mkwrite(pte_t pte)
|
|
{
|
|
pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
|
|
pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
|
|
return pte;
|
|
}
|
|
|
|
static inline pte_t pte_mkclean(pte_t pte)
|
|
{
|
|
pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY));
|
|
pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
|
|
|
|
return pte;
|
|
}
|
|
|
|
static inline pte_t pte_mkdirty(pte_t pte)
|
|
{
|
|
pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
|
|
|
|
if (pte_write(pte))
|
|
pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
|
|
|
|
return pte;
|
|
}
|
|
|
|
static inline pte_t pte_mkold(pte_t pte)
|
|
{
|
|
return clear_pte_bit(pte, __pgprot(PTE_AF));
|
|
}
|
|
|
|
static inline pte_t pte_mkyoung(pte_t pte)
|
|
{
|
|
return set_pte_bit(pte, __pgprot(PTE_AF));
|
|
}
|
|
|
|
static inline pte_t pte_mkspecial(pte_t pte)
|
|
{
|
|
return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
|
|
}
|
|
|
|
static inline pte_t pte_mkcont(pte_t pte)
|
|
{
|
|
pte = set_pte_bit(pte, __pgprot(PTE_CONT));
|
|
return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE));
|
|
}
|
|
|
|
static inline pte_t pte_mknoncont(pte_t pte)
|
|
{
|
|
return clear_pte_bit(pte, __pgprot(PTE_CONT));
|
|
}
|
|
|
|
static inline pte_t pte_mkpresent(pte_t pte)
|
|
{
|
|
return set_pte_bit(pte, __pgprot(PTE_VALID));
|
|
}
|
|
|
|
static inline pmd_t pmd_mkcont(pmd_t pmd)
|
|
{
|
|
return __pmd(pmd_val(pmd) | PMD_SECT_CONT);
|
|
}
|
|
|
|
static inline pte_t pte_mkdevmap(pte_t pte)
|
|
{
|
|
return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));
|
|
}
|
|
|
|
static inline void set_pte(pte_t *ptep, pte_t pte)
|
|
{
|
|
WRITE_ONCE(*ptep, pte);
|
|
|
|
/*
|
|
* Only if the new pte is valid and kernel, otherwise TLB maintenance
|
|
* or update_mmu_cache() have the necessary barriers.
|
|
*/
|
|
if (pte_valid_not_user(pte)) {
|
|
dsb(ishst);
|
|
isb();
|
|
}
|
|
}
|
|
|
|
extern void __sync_icache_dcache(pte_t pteval);
|
|
|
|
/*
|
|
* PTE bits configuration in the presence of hardware Dirty Bit Management
|
|
* (PTE_WRITE == PTE_DBM):
|
|
*
|
|
* Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw)
|
|
* 0 0 | 1 0 0
|
|
* 0 1 | 1 1 0
|
|
* 1 0 | 1 0 1
|
|
* 1 1 | 0 1 x
|
|
*
|
|
* When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
|
|
* the page fault mechanism. Checking the dirty status of a pte becomes:
|
|
*
|
|
* PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
|
|
*/
|
|
|
|
static inline void __check_racy_pte_update(struct mm_struct *mm, pte_t *ptep,
|
|
pte_t pte)
|
|
{
|
|
pte_t old_pte;
|
|
|
|
if (!IS_ENABLED(CONFIG_DEBUG_VM))
|
|
return;
|
|
|
|
old_pte = READ_ONCE(*ptep);
|
|
|
|
if (!pte_valid(old_pte) || !pte_valid(pte))
|
|
return;
|
|
if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
|
|
return;
|
|
|
|
/*
|
|
* Check for potential race with hardware updates of the pte
|
|
* (ptep_set_access_flags safely changes valid ptes without going
|
|
* through an invalid entry).
|
|
*/
|
|
VM_WARN_ONCE(!pte_young(pte),
|
|
"%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
|
|
__func__, pte_val(old_pte), pte_val(pte));
|
|
VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
|
|
"%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
|
|
__func__, pte_val(old_pte), pte_val(pte));
|
|
}
|
|
|
|
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
pte_t *ptep, pte_t pte)
|
|
{
|
|
if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
|
|
__sync_icache_dcache(pte);
|
|
|
|
__check_racy_pte_update(mm, ptep, pte);
|
|
|
|
set_pte(ptep, pte);
|
|
}
|
|
|
|
#define __HAVE_ARCH_PTE_SAME
|
|
static inline int pte_same(pte_t pte_a, pte_t pte_b)
|
|
{
|
|
pteval_t lhs, rhs;
|
|
|
|
lhs = pte_val(pte_a);
|
|
rhs = pte_val(pte_b);
|
|
|
|
if (pte_present(pte_a))
|
|
lhs &= ~PTE_RDONLY;
|
|
|
|
if (pte_present(pte_b))
|
|
rhs &= ~PTE_RDONLY;
|
|
|
|
return (lhs == rhs);
|
|
}
|
|
|
|
/*
|
|
* Huge pte definitions.
|
|
*/
|
|
#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT))
|
|
|
|
/*
|
|
* Hugetlb definitions.
|
|
*/
|
|
#define HUGE_MAX_HSTATE 4
|
|
#define HPAGE_SHIFT PMD_SHIFT
|
|
#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
|
|
#define HPAGE_MASK (~(HPAGE_SIZE - 1))
|
|
#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
|
|
|
|
static inline pte_t pgd_pte(pgd_t pgd)
|
|
{
|
|
return __pte(pgd_val(pgd));
|
|
}
|
|
|
|
static inline pte_t pud_pte(pud_t pud)
|
|
{
|
|
return __pte(pud_val(pud));
|
|
}
|
|
|
|
static inline pud_t pte_pud(pte_t pte)
|
|
{
|
|
return __pud(pte_val(pte));
|
|
}
|
|
|
|
static inline pmd_t pud_pmd(pud_t pud)
|
|
{
|
|
return __pmd(pud_val(pud));
|
|
}
|
|
|
|
static inline pte_t pmd_pte(pmd_t pmd)
|
|
{
|
|
return __pte(pmd_val(pmd));
|
|
}
|
|
|
|
static inline pmd_t pte_pmd(pte_t pte)
|
|
{
|
|
return __pmd(pte_val(pte));
|
|
}
|
|
|
|
static inline pgprot_t mk_pud_sect_prot(pgprot_t prot)
|
|
{
|
|
return __pgprot((pgprot_val(prot) & ~PUD_TABLE_BIT) | PUD_TYPE_SECT);
|
|
}
|
|
|
|
static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot)
|
|
{
|
|
return __pgprot((pgprot_val(prot) & ~PMD_TABLE_BIT) | PMD_TYPE_SECT);
|
|
}
|
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
/*
|
|
* See the comment in include/asm-generic/pgtable.h
|
|
*/
|
|
static inline int pte_protnone(pte_t pte)
|
|
{
|
|
return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE;
|
|
}
|
|
|
|
static inline int pmd_protnone(pmd_t pmd)
|
|
{
|
|
return pte_protnone(pmd_pte(pmd));
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* THP definitions.
|
|
*/
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
#define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
|
|
#define pmd_present(pmd) pte_present(pmd_pte(pmd))
|
|
#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
|
|
#define pmd_young(pmd) pte_young(pmd_pte(pmd))
|
|
#define pmd_valid(pmd) pte_valid(pmd_pte(pmd))
|
|
#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
|
|
#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
|
|
#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
|
|
#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
|
|
#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
|
|
#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
|
|
#define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID))
|
|
|
|
#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd))
|
|
|
|
#define pmd_write(pmd) pte_write(pmd_pte(pmd))
|
|
|
|
#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
#define pmd_devmap(pmd) pte_devmap(pmd_pte(pmd))
|
|
#endif
|
|
static inline pmd_t pmd_mkdevmap(pmd_t pmd)
|
|
{
|
|
return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP)));
|
|
}
|
|
|
|
#define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd))
|
|
#define __phys_to_pmd_val(phys) __phys_to_pte_val(phys)
|
|
#define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
|
|
#define pfn_pmd(pfn,prot) __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
|
|
#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)
|
|
|
|
#define pud_young(pud) pte_young(pud_pte(pud))
|
|
#define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud)))
|
|
#define pud_write(pud) pte_write(pud_pte(pud))
|
|
|
|
#define pud_mkhuge(pud) (__pud(pud_val(pud) & ~PUD_TABLE_BIT))
|
|
|
|
#define __pud_to_phys(pud) __pte_to_phys(pud_pte(pud))
|
|
#define __phys_to_pud_val(phys) __phys_to_pte_val(phys)
|
|
#define pud_pfn(pud) ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
|
|
#define pfn_pud(pfn,prot) __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
|
|
|
|
#define set_pmd_at(mm, addr, pmdp, pmd) set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))
|
|
|
|
#define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd))
|
|
#define __phys_to_pgd_val(phys) __phys_to_pte_val(phys)
|
|
|
|
#define __pgprot_modify(prot,mask,bits) \
|
|
__pgprot((pgprot_val(prot) & ~(mask)) | (bits))
|
|
|
|
/*
|
|
* Mark the prot value as uncacheable and unbufferable.
|
|
*/
|
|
#define pgprot_noncached(prot) \
|
|
__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN)
|
|
#define pgprot_writecombine(prot) \
|
|
__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
|
|
#define pgprot_device(prot) \
|
|
__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN)
|
|
#define __HAVE_PHYS_MEM_ACCESS_PROT
|
|
struct file;
|
|
extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
|
|
unsigned long size, pgprot_t vma_prot);
|
|
|
|
#define pmd_none(pmd) (!pmd_val(pmd))
|
|
|
|
#define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT))
|
|
|
|
#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
|
PMD_TYPE_TABLE)
|
|
#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
|
PMD_TYPE_SECT)
|
|
|
|
#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
|
|
static inline bool pud_sect(pud_t pud) { return false; }
|
|
static inline bool pud_table(pud_t pud) { return true; }
|
|
#else
|
|
#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
|
PUD_TYPE_SECT)
|
|
#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
|
PUD_TYPE_TABLE)
|
|
#endif
|
|
|
|
extern pgd_t init_pg_dir[PTRS_PER_PGD];
|
|
extern pgd_t init_pg_end[];
|
|
extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
|
|
extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
|
|
extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
|
|
|
|
extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
|
|
|
|
static inline bool in_swapper_pgdir(void *addr)
|
|
{
|
|
return ((unsigned long)addr & PAGE_MASK) ==
|
|
((unsigned long)swapper_pg_dir & PAGE_MASK);
|
|
}
|
|
|
|
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
|
|
{
|
|
#ifdef __PAGETABLE_PMD_FOLDED
|
|
if (in_swapper_pgdir(pmdp)) {
|
|
set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
|
|
return;
|
|
}
|
|
#endif /* __PAGETABLE_PMD_FOLDED */
|
|
|
|
WRITE_ONCE(*pmdp, pmd);
|
|
|
|
if (pmd_valid(pmd)) {
|
|
dsb(ishst);
|
|
isb();
|
|
}
|
|
}
|
|
|
|
static inline void pmd_clear(pmd_t *pmdp)
|
|
{
|
|
set_pmd(pmdp, __pmd(0));
|
|
}
|
|
|
|
static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
|
|
{
|
|
return __pmd_to_phys(pmd);
|
|
}
|
|
|
|
static inline void pte_unmap(pte_t *pte) { }
|
|
|
|
/* Find an entry in the third-level page table. */
|
|
#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
|
|
|
|
#define pte_offset_phys(dir,addr) (pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t))
|
|
#define pte_offset_kernel(dir,addr) ((pte_t *)__va(pte_offset_phys((dir), (addr))))
|
|
|
|
#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
|
|
|
|
#define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr))
|
|
#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr))
|
|
#define pte_clear_fixmap() clear_fixmap(FIX_PTE)
|
|
|
|
#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(__pmd_to_phys(pmd)))
|
|
|
|
/* use ONLY for statically allocated translation tables */
|
|
#define pte_offset_kimg(dir,addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
|
|
|
|
/*
|
|
* Conversion functions: convert a page and protection to a page entry,
|
|
* and a page entry and page directory to the page they refer to.
|
|
*/
|
|
#define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot)
|
|
|
|
#if CONFIG_PGTABLE_LEVELS > 2
|
|
|
|
#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
|
|
|
|
#define pud_none(pud) (!pud_val(pud))
|
|
#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT))
|
|
#define pud_present(pud) pte_present(pud_pte(pud))
|
|
#define pud_valid(pud) pte_valid(pud_pte(pud))
|
|
|
|
static inline void set_pud(pud_t *pudp, pud_t pud)
|
|
{
|
|
#ifdef __PAGETABLE_PUD_FOLDED
|
|
if (in_swapper_pgdir(pudp)) {
|
|
set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
|
|
return;
|
|
}
|
|
#endif /* __PAGETABLE_PUD_FOLDED */
|
|
|
|
WRITE_ONCE(*pudp, pud);
|
|
|
|
if (pud_valid(pud)) {
|
|
dsb(ishst);
|
|
isb();
|
|
}
|
|
}
|
|
|
|
static inline void pud_clear(pud_t *pudp)
|
|
{
|
|
set_pud(pudp, __pud(0));
|
|
}
|
|
|
|
static inline phys_addr_t pud_page_paddr(pud_t pud)
|
|
{
|
|
return __pud_to_phys(pud);
|
|
}
|
|
|
|
/* Find an entry in the second-level page table. */
|
|
#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
|
|
|
|
#define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
|
|
#define pmd_offset(dir, addr) ((pmd_t *)__va(pmd_offset_phys((dir), (addr))))
|
|
|
|
#define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
|
|
#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr))
|
|
#define pmd_clear_fixmap() clear_fixmap(FIX_PMD)
|
|
|
|
#define pud_page(pud) pfn_to_page(__phys_to_pfn(__pud_to_phys(pud)))
|
|
|
|
/* use ONLY for statically allocated translation tables */
|
|
#define pmd_offset_kimg(dir,addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
|
|
|
|
#else
|
|
|
|
#define pud_page_paddr(pud) ({ BUILD_BUG(); 0; })
|
|
|
|
/* Match pmd_offset folding in <asm/generic/pgtable-nopmd.h> */
|
|
#define pmd_set_fixmap(addr) NULL
|
|
#define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp)
|
|
#define pmd_clear_fixmap()
|
|
|
|
#define pmd_offset_kimg(dir,addr) ((pmd_t *)dir)
|
|
|
|
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
|
|
|
|
#if CONFIG_PGTABLE_LEVELS > 3
|
|
|
|
#define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud))
|
|
|
|
#define pgd_none(pgd) (!pgd_val(pgd))
|
|
#define pgd_bad(pgd) (!(pgd_val(pgd) & 2))
|
|
#define pgd_present(pgd) (pgd_val(pgd))
|
|
|
|
static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
|
|
{
|
|
if (in_swapper_pgdir(pgdp)) {
|
|
set_swapper_pgd(pgdp, pgd);
|
|
return;
|
|
}
|
|
|
|
WRITE_ONCE(*pgdp, pgd);
|
|
dsb(ishst);
|
|
isb();
|
|
}
|
|
|
|
static inline void pgd_clear(pgd_t *pgdp)
|
|
{
|
|
set_pgd(pgdp, __pgd(0));
|
|
}
|
|
|
|
static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
|
|
{
|
|
return __pgd_to_phys(pgd);
|
|
}
|
|
|
|
/* Find an entry in the frst-level page table. */
|
|
#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
|
|
|
|
#define pud_offset_phys(dir, addr) (pgd_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
|
|
#define pud_offset(dir, addr) ((pud_t *)__va(pud_offset_phys((dir), (addr))))
|
|
|
|
#define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr))
|
|
#define pud_set_fixmap_offset(pgd, addr) pud_set_fixmap(pud_offset_phys(pgd, addr))
|
|
#define pud_clear_fixmap() clear_fixmap(FIX_PUD)
|
|
|
|
#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
|
|
|
|
/* use ONLY for statically allocated translation tables */
|
|
#define pud_offset_kimg(dir,addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))
|
|
|
|
#else
|
|
|
|
#define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0;})
|
|
|
|
/* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
|
|
#define pud_set_fixmap(addr) NULL
|
|
#define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp)
|
|
#define pud_clear_fixmap()
|
|
|
|
#define pud_offset_kimg(dir,addr) ((pud_t *)dir)
|
|
|
|
#endif /* CONFIG_PGTABLE_LEVELS > 3 */
|
|
|
|
#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd))
|
|
|
|
/* to find an entry in a page-table-directory */
|
|
#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
|
|
|
|
#define pgd_offset_raw(pgd, addr) ((pgd) + pgd_index(addr))
|
|
|
|
#define pgd_offset(mm, addr) (pgd_offset_raw((mm)->pgd, (addr)))
|
|
|
|
/* to find an entry in a kernel page-table-directory */
|
|
#define pgd_offset_k(addr) pgd_offset(&init_mm, addr)
|
|
|
|
#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
|
|
#define pgd_clear_fixmap() clear_fixmap(FIX_PGD)
|
|
|
|
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
|
|
{
|
|
const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
|
|
PTE_PROT_NONE | PTE_VALID | PTE_WRITE;
|
|
/* preserve the hardware dirty information */
|
|
if (pte_hw_dirty(pte))
|
|
pte = pte_mkdirty(pte);
|
|
pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
|
|
return pte;
|
|
}
|
|
|
|
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
|
{
|
|
return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
|
|
}
|
|
|
|
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
|
|
extern int ptep_set_access_flags(struct vm_area_struct *vma,
|
|
unsigned long address, pte_t *ptep,
|
|
pte_t entry, int dirty);
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
|
|
static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
|
|
unsigned long address, pmd_t *pmdp,
|
|
pmd_t entry, int dirty)
|
|
{
|
|
return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty);
|
|
}
|
|
|
|
static inline int pud_devmap(pud_t pud)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline int pgd_devmap(pgd_t pgd)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Atomic pte/pmd modifications.
|
|
*/
|
|
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
|
|
static inline int __ptep_test_and_clear_young(pte_t *ptep)
|
|
{
|
|
pte_t old_pte, pte;
|
|
|
|
pte = READ_ONCE(*ptep);
|
|
do {
|
|
old_pte = pte;
|
|
pte = pte_mkold(pte);
|
|
pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
|
|
pte_val(old_pte), pte_val(pte));
|
|
} while (pte_val(pte) != pte_val(old_pte));
|
|
|
|
return pte_young(pte);
|
|
}
|
|
|
|
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
|
|
unsigned long address,
|
|
pte_t *ptep)
|
|
{
|
|
return __ptep_test_and_clear_young(ptep);
|
|
}
|
|
|
|
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
|
|
static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
|
|
unsigned long address, pte_t *ptep)
|
|
{
|
|
int young = ptep_test_and_clear_young(vma, address, ptep);
|
|
|
|
if (young) {
|
|
/*
|
|
* We can elide the trailing DSB here since the worst that can
|
|
* happen is that a CPU continues to use the young entry in its
|
|
* TLB and we mistakenly reclaim the associated page. The
|
|
* window for such an event is bounded by the next
|
|
* context-switch, which provides a DSB to complete the TLB
|
|
* invalidation.
|
|
*/
|
|
flush_tlb_page_nosync(vma, address);
|
|
}
|
|
|
|
return young;
|
|
}
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
|
|
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
|
unsigned long address,
|
|
pmd_t *pmdp)
|
|
{
|
|
return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
|
|
}
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
|
|
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
|
|
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
|
|
unsigned long address, pte_t *ptep)
|
|
{
|
|
return __pte(xchg_relaxed(&pte_val(*ptep), 0));
|
|
}
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
|
|
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
|
|
unsigned long address, pmd_t *pmdp)
|
|
{
|
|
return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp));
|
|
}
|
|
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
|
|
/*
|
|
* ptep_set_wrprotect - mark read-only while trasferring potential hardware
|
|
* dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
|
|
*/
|
|
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
|
|
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
|
|
{
|
|
pte_t old_pte, pte;
|
|
|
|
pte = READ_ONCE(*ptep);
|
|
do {
|
|
old_pte = pte;
|
|
/*
|
|
* If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY
|
|
* clear), set the PTE_DIRTY bit.
|
|
*/
|
|
if (pte_hw_dirty(pte))
|
|
pte = pte_mkdirty(pte);
|
|
pte = pte_wrprotect(pte);
|
|
pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
|
|
pte_val(old_pte), pte_val(pte));
|
|
} while (pte_val(pte) != pte_val(old_pte));
|
|
}
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
#define __HAVE_ARCH_PMDP_SET_WRPROTECT
|
|
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
|
|
unsigned long address, pmd_t *pmdp)
|
|
{
|
|
ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
|
|
}
|
|
|
|
#define pmdp_establish pmdp_establish
|
|
static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
|
|
unsigned long address, pmd_t *pmdp, pmd_t pmd)
|
|
{
|
|
return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Encode and decode a swap entry:
|
|
* bits 0-1: present (must be zero)
|
|
* bits 2-7: swap type
|
|
* bits 8-57: swap offset
|
|
* bit 58: PTE_PROT_NONE (must be zero)
|
|
*/
|
|
#define __SWP_TYPE_SHIFT 2
|
|
#define __SWP_TYPE_BITS 6
|
|
#define __SWP_OFFSET_BITS 50
|
|
#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1)
|
|
#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
|
|
#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1)
|
|
|
|
#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
|
|
#define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK)
|
|
#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
|
|
|
|
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
|
|
#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val })
|
|
|
|
/*
|
|
* Ensure that there are not more swap files than can be encoded in the kernel
|
|
* PTEs.
|
|
*/
|
|
#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
|
|
|
|
extern int kern_addr_valid(unsigned long addr);
|
|
|
|
#include <asm-generic/pgtable.h>
|
|
|
|
static inline void pgtable_cache_init(void) { }
|
|
|
|
/*
|
|
* On AArch64, the cache coherency is handled via the set_pte_at() function.
|
|
*/
|
|
static inline void update_mmu_cache(struct vm_area_struct *vma,
|
|
unsigned long addr, pte_t *ptep)
|
|
{
|
|
/*
|
|
* We don't do anything here, so there's a very small chance of
|
|
* us retaking a user fault which we just fixed up. The alternative
|
|
* is doing a dsb(ishst), but that penalises the fastpath.
|
|
*/
|
|
}
|
|
|
|
#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
|
|
|
|
#define kc_vaddr_to_offset(v) ((v) & ~PAGE_END)
|
|
#define kc_offset_to_vaddr(o) ((o) | PAGE_END)
|
|
|
|
#ifdef CONFIG_ARM64_PA_BITS_52
|
|
#define phys_to_ttbr(addr) (((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
|
|
#else
|
|
#define phys_to_ttbr(addr) (addr)
|
|
#endif
|
|
|
|
#endif /* !__ASSEMBLY__ */
|
|
|
|
#endif /* __ASM_PGTABLE_H */
|