2008-07-18 11:55:51 +07:00
|
|
|
/*
|
|
|
|
|
* pgtable.h: SpitFire page table operations.
|
|
|
|
|
*
|
|
|
|
|
* Copyright 1996,1997 David S. Miller (davem@caip.rutgers.edu)
|
|
|
|
|
* Copyright 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef _SPARC64_PGTABLE_H
|
|
|
|
|
#define _SPARC64_PGTABLE_H
|
|
|
|
|
|
|
|
|
|
/* This file contains the functions and defines necessary to modify and use
|
|
|
|
|
* the SpitFire page tables.
|
|
|
|
|
*/
|
|
|
|
|
|
2017-03-09 21:24:05 +07:00
|
|
|
#include <asm-generic/5level-fixup.h>
|
2008-07-18 11:55:51 +07:00
|
|
|
#include <linux/compiler.h>
|
|
|
|
|
#include <linux/const.h>
|
|
|
|
|
#include <asm/types.h>
|
|
|
|
|
#include <asm/spitfire.h>
|
|
|
|
|
#include <asm/asi.h>
|
|
|
|
|
#include <asm/page.h>
|
|
|
|
|
#include <asm/processor.h>
|
|
|
|
|
|
|
|
|
|
/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
|
|
|
|
|
* The page copy blockops can use 0x6000000 to 0x8000000.
|
2014-05-08 04:07:32 +07:00
|
|
|
* The 8K TSB is mapped in the 0x8000000 to 0x8400000 range.
|
|
|
|
|
* The 4M TSB is mapped in the 0x8400000 to 0x8800000 range.
|
2008-07-18 11:55:51 +07:00
|
|
|
* The PROM resides in an area spanning 0xf0000000 to 0x100000000.
|
|
|
|
|
* The vmalloc area spans 0x100000000 to 0x200000000.
|
|
|
|
|
* Since modules need to be in the lowest 32-bits of the address space,
|
|
|
|
|
* we place them right before the OBP area from 0x10000000 to 0xf0000000.
|
|
|
|
|
* There is a single static kernel PMD which maps from 0x0 to address
|
|
|
|
|
* 0x400000000.
|
|
|
|
|
*/
|
|
|
|
|
#define TLBTEMP_BASE _AC(0x0000000006000000,UL)
|
2014-05-08 04:07:32 +07:00
|
|
|
#define TSBMAP_8K_BASE _AC(0x0000000008000000,UL)
|
|
|
|
|
#define TSBMAP_4M_BASE _AC(0x0000000008400000,UL)
|
2008-07-18 11:55:51 +07:00
|
|
|
#define MODULES_VADDR _AC(0x0000000010000000,UL)
|
|
|
|
|
#define MODULES_LEN _AC(0x00000000e0000000,UL)
|
|
|
|
|
#define MODULES_END _AC(0x00000000f0000000,UL)
|
|
|
|
|
#define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL)
|
|
|
|
|
#define HI_OBP_ADDRESS _AC(0x0000000100000000,UL)
|
|
|
|
|
#define VMALLOC_START _AC(0x0000000100000000,UL)
|
2014-09-28 01:05:21 +07:00
|
|
|
#define VMEMMAP_BASE VMALLOC_END
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* PMD_SHIFT determines the size of the area a second-level page
|
|
|
|
|
* table can map
|
|
|
|
|
*/
|
sparc64: Move from 4MB to 8MB huge pages.
The impetus for this is that we would like to move to 64-bit PMDs and
PGDs, but that would result in only supporting a 42-bit address space
with the current page table layout. It'd be nice to support at least
43-bits.
The reason we'd end up with only 42-bits after making PMDs and PGDs
64-bit is that we only use half-page sized PTE tables in order to make
PMDs line up to 4MB, the hardware huge page size we use.
So what we do here is we make huge pages 8MB, and fabricate them using
4MB hw TLB entries.
Facilitate this by providing a "REAL_HPAGE_SHIFT" which is used in
places that really need to operate on hardware 4MB pages.
Use full pages (512 entries) for PTE tables, and adjust PMD_SHIFT,
PGD_SHIFT, and the build time CPP test as needed. Use a CPP test to
make sure REAL_HPAGE_SHIFT and the _PAGE_SZHUGE_* we use match up.
This makes the pgtable cache completely unused, so remove the code
managing it and the state used in mm_context_t. Now we have less
spinlocks taken in the page table allocation path.
The technique we use to fabricate the 8MB pages is to transfer bit 22
from the missing virtual address into the PTEs physical address field.
That takes care of the transparent huge pages case.
For hugetlb, we fill things in at the PTE level and that code already
puts the sub huge page physical bits into the PTEs, based upon the
offset, so there is nothing special we need to do. It all just works
out.
So, a small amount of complexity in the THP case, but this code is
about to get much simpler when we move the 64-bit PMDs as we can move
away from the fancy 32-bit huge PMD encoding and just put a real PTE
value in there.
With bug fixes and help from Bob Picco.
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-09-26 03:48:49 +07:00
|
|
|
#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3))
|
2008-07-18 11:55:51 +07:00
|
|
|
#define PMD_SIZE (_AC(1,UL) << PMD_SHIFT)
|
|
|
|
|
#define PMD_MASK (~(PMD_SIZE-1))
|
2013-09-26 04:33:16 +07:00
|
|
|
#define PMD_BITS (PAGE_SHIFT - 3)
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2014-09-27 11:19:46 +07:00
|
|
|
/* PUD_SHIFT determines the size of the area a third-level page
|
|
|
|
|
* table can map
|
|
|
|
|
*/
|
|
|
|
|
#define PUD_SHIFT (PMD_SHIFT + PMD_BITS)
|
|
|
|
|
#define PUD_SIZE (_AC(1,UL) << PUD_SHIFT)
|
|
|
|
|
#define PUD_MASK (~(PUD_SIZE-1))
|
|
|
|
|
#define PUD_BITS (PAGE_SHIFT - 3)
|
|
|
|
|
|
|
|
|
|
/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
|
|
|
|
|
#define PGDIR_SHIFT (PUD_SHIFT + PUD_BITS)
|
2008-07-18 11:55:51 +07:00
|
|
|
#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT)
|
|
|
|
|
#define PGDIR_MASK (~(PGDIR_SIZE-1))
|
2013-09-26 04:33:16 +07:00
|
|
|
#define PGDIR_BITS (PAGE_SHIFT - 3)
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2014-09-25 11:49:29 +07:00
|
|
|
#if (MAX_PHYS_ADDRESS_BITS > PGDIR_SHIFT + PGDIR_BITS)
|
|
|
|
|
#error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support
|
|
|
|
|
#endif
|
|
|
|
|
|
2014-09-27 11:19:46 +07:00
|
|
|
#if (PGDIR_SHIFT + PGDIR_BITS) != 53
|
2012-10-09 06:34:20 +07:00
|
|
|
#error Page table parameters do not cover virtual address space properly.
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#if (PMD_SHIFT != HPAGE_SHIFT)
|
|
|
|
|
#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages.
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
2014-09-28 01:05:21 +07:00
|
|
|
extern unsigned long VMALLOC_END;
|
|
|
|
|
|
|
|
|
|
#define vmemmap ((struct page *)VMEMMAP_BASE)
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#include <linux/sched.h>
|
|
|
|
|
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
bool kern_addr_valid(unsigned long addr);
|
2014-04-30 03:03:27 +07:00
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* Entries per page directory level. */
|
sparc64: Move from 4MB to 8MB huge pages.
The impetus for this is that we would like to move to 64-bit PMDs and
PGDs, but that would result in only supporting a 42-bit address space
with the current page table layout. It'd be nice to support at least
43-bits.
The reason we'd end up with only 42-bits after making PMDs and PGDs
64-bit is that we only use half-page sized PTE tables in order to make
PMDs line up to 4MB, the hardware huge page size we use.
So what we do here is we make huge pages 8MB, and fabricate them using
4MB hw TLB entries.
Facilitate this by providing a "REAL_HPAGE_SHIFT" which is used in
places that really need to operate on hardware 4MB pages.
Use full pages (512 entries) for PTE tables, and adjust PMD_SHIFT,
PGD_SHIFT, and the build time CPP test as needed. Use a CPP test to
make sure REAL_HPAGE_SHIFT and the _PAGE_SZHUGE_* we use match up.
This makes the pgtable cache completely unused, so remove the code
managing it and the state used in mm_context_t. Now we have less
spinlocks taken in the page table allocation path.
The technique we use to fabricate the 8MB pages is to transfer bit 22
from the missing virtual address into the PTEs physical address field.
That takes care of the transparent huge pages case.
For hugetlb, we fill things in at the PTE level and that code already
puts the sub huge page physical bits into the PTEs, based upon the
offset, so there is nothing special we need to do. It all just works
out.
So, a small amount of complexity in the THP case, but this code is
about to get much simpler when we move the 64-bit PMDs as we can move
away from the fancy 32-bit huge PMD encoding and just put a real PTE
value in there.
With bug fixes and help from Bob Picco.
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-09-26 03:48:49 +07:00
|
|
|
#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3))
|
2008-07-18 11:55:51 +07:00
|
|
|
#define PTRS_PER_PMD (1UL << PMD_BITS)
|
2014-09-27 11:19:46 +07:00
|
|
|
#define PTRS_PER_PUD (1UL << PUD_BITS)
|
2008-07-18 11:55:51 +07:00
|
|
|
#define PTRS_PER_PGD (1UL << PGDIR_BITS)
|
|
|
|
|
|
|
|
|
|
/* Kernel has a separate 44bit address space. */
|
2015-02-12 06:26:41 +07:00
|
|
|
#define FIRST_USER_ADDRESS 0UL
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2014-04-30 03:28:23 +07:00
|
|
|
#define pmd_ERROR(e) \
|
|
|
|
|
pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \
|
|
|
|
|
__FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))
|
2014-09-27 11:19:46 +07:00
|
|
|
#define pud_ERROR(e) \
|
|
|
|
|
pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n", \
|
|
|
|
|
__FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
|
2014-04-30 03:28:23 +07:00
|
|
|
#define pgd_ERROR(e) \
|
|
|
|
|
pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \
|
|
|
|
|
__FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
#endif /* !(__ASSEMBLY__) */
|
|
|
|
|
|
|
|
|
|
/* PTE bits which are the same in SUN4U and SUN4V format. */
|
|
|
|
|
#define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */
|
|
|
|
|
#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
|
2011-07-26 07:12:21 +07:00
|
|
|
#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */
|
2013-09-27 03:45:15 +07:00
|
|
|
#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
#define _PAGE_PUD_HUGE _PAGE_PMD_HUGE
|
2011-07-26 07:12:21 +07:00
|
|
|
|
|
|
|
|
/* Advertise support for _PAGE_SPECIAL */
|
|
|
|
|
#define __HAVE_ARCH_PTE_SPECIAL
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* SUN4U pte bits... */
|
|
|
|
|
#define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */
|
|
|
|
|
#define _PAGE_SZ512K_4U _AC(0x4000000000000000,UL) /* 512K Page */
|
|
|
|
|
#define _PAGE_SZ64K_4U _AC(0x2000000000000000,UL) /* 64K Page */
|
|
|
|
|
#define _PAGE_SZ8K_4U _AC(0x0000000000000000,UL) /* 8K Page */
|
|
|
|
|
#define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */
|
|
|
|
|
#define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */
|
|
|
|
|
#define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */
|
2011-07-26 07:12:21 +07:00
|
|
|
#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */
|
2013-09-27 03:45:15 +07:00
|
|
|
#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page */
|
2008-07-18 11:55:51 +07:00
|
|
|
#define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */
|
|
|
|
|
#define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */
|
|
|
|
|
#define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */
|
|
|
|
|
#define _PAGE_SZALL_4U _AC(0x6001000000000000,UL) /* All pgsz bits */
|
|
|
|
|
#define _PAGE_SN_4U _AC(0x0000800000000000,UL) /* (Cheetah) Snoop */
|
|
|
|
|
#define _PAGE_RES2_4U _AC(0x0000780000000000,UL) /* Reserved */
|
|
|
|
|
#define _PAGE_PADDR_4U _AC(0x000007FFFFFFE000,UL) /* (Cheetah) pa[42:13] */
|
|
|
|
|
#define _PAGE_SOFT_4U _AC(0x0000000000001F80,UL) /* Software bits: */
|
|
|
|
|
#define _PAGE_EXEC_4U _AC(0x0000000000001000,UL) /* Executable SW bit */
|
|
|
|
|
#define _PAGE_MODIFIED_4U _AC(0x0000000000000800,UL) /* Modified (dirty) */
|
|
|
|
|
#define _PAGE_ACCESSED_4U _AC(0x0000000000000400,UL) /* Accessed (ref'd) */
|
|
|
|
|
#define _PAGE_READ_4U _AC(0x0000000000000200,UL) /* Readable SW Bit */
|
|
|
|
|
#define _PAGE_WRITE_4U _AC(0x0000000000000100,UL) /* Writable SW Bit */
|
|
|
|
|
#define _PAGE_PRESENT_4U _AC(0x0000000000000080,UL) /* Present */
|
|
|
|
|
#define _PAGE_L_4U _AC(0x0000000000000040,UL) /* Locked TTE */
|
|
|
|
|
#define _PAGE_CP_4U _AC(0x0000000000000020,UL) /* Cacheable in P-Cache */
|
|
|
|
|
#define _PAGE_CV_4U _AC(0x0000000000000010,UL) /* Cacheable in V-Cache */
|
|
|
|
|
#define _PAGE_E_4U _AC(0x0000000000000008,UL) /* side-Effect */
|
|
|
|
|
#define _PAGE_P_4U _AC(0x0000000000000004,UL) /* Privileged Page */
|
|
|
|
|
#define _PAGE_W_4U _AC(0x0000000000000002,UL) /* Writable */
|
|
|
|
|
|
|
|
|
|
/* SUN4V pte bits... */
|
|
|
|
|
#define _PAGE_NFO_4V _AC(0x4000000000000000,UL) /* No Fault Only */
|
|
|
|
|
#define _PAGE_SOFT2_4V _AC(0x3F00000000000000,UL) /* Software bits, set 2 */
|
|
|
|
|
#define _PAGE_MODIFIED_4V _AC(0x2000000000000000,UL) /* Modified (dirty) */
|
|
|
|
|
#define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */
|
|
|
|
|
#define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */
|
|
|
|
|
#define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */
|
2011-07-26 07:12:21 +07:00
|
|
|
#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */
|
2013-09-27 03:45:15 +07:00
|
|
|
#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page */
|
2008-07-18 11:55:51 +07:00
|
|
|
#define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */
|
|
|
|
|
#define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */
|
|
|
|
|
#define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */
|
|
|
|
|
#define _PAGE_CP_4V _AC(0x0000000000000400,UL) /* Cacheable in P-Cache */
|
|
|
|
|
#define _PAGE_CV_4V _AC(0x0000000000000200,UL) /* Cacheable in V-Cache */
|
|
|
|
|
#define _PAGE_P_4V _AC(0x0000000000000100,UL) /* Privileged Page */
|
|
|
|
|
#define _PAGE_EXEC_4V _AC(0x0000000000000080,UL) /* Executable Page */
|
|
|
|
|
#define _PAGE_W_4V _AC(0x0000000000000040,UL) /* Writable */
|
|
|
|
|
#define _PAGE_SOFT_4V _AC(0x0000000000000030,UL) /* Software bits */
|
|
|
|
|
#define _PAGE_PRESENT_4V _AC(0x0000000000000010,UL) /* Present */
|
|
|
|
|
#define _PAGE_RESV_4V _AC(0x0000000000000008,UL) /* Reserved */
|
|
|
|
|
#define _PAGE_SZ16GB_4V _AC(0x0000000000000007,UL) /* 16GB Page */
|
|
|
|
|
#define _PAGE_SZ2GB_4V _AC(0x0000000000000006,UL) /* 2GB Page */
|
|
|
|
|
#define _PAGE_SZ256MB_4V _AC(0x0000000000000005,UL) /* 256MB Page */
|
|
|
|
|
#define _PAGE_SZ32MB_4V _AC(0x0000000000000004,UL) /* 32MB Page */
|
|
|
|
|
#define _PAGE_SZ4MB_4V _AC(0x0000000000000003,UL) /* 4MB Page */
|
|
|
|
|
#define _PAGE_SZ512K_4V _AC(0x0000000000000002,UL) /* 512K Page */
|
|
|
|
|
#define _PAGE_SZ64K_4V _AC(0x0000000000000001,UL) /* 64K Page */
|
|
|
|
|
#define _PAGE_SZ8K_4V _AC(0x0000000000000000,UL) /* 8K Page */
|
|
|
|
|
#define _PAGE_SZALL_4V _AC(0x0000000000000007,UL) /* All pgsz bits */
|
|
|
|
|
|
|
|
|
|
#define _PAGE_SZBITS_4U _PAGE_SZ8K_4U
|
|
|
|
|
#define _PAGE_SZBITS_4V _PAGE_SZ8K_4V
|
2012-10-09 06:34:19 +07:00
|
|
|
|
sparc64: Move from 4MB to 8MB huge pages.
The impetus for this is that we would like to move to 64-bit PMDs and
PGDs, but that would result in only supporting a 42-bit address space
with the current page table layout. It'd be nice to support at least
43-bits.
The reason we'd end up with only 42-bits after making PMDs and PGDs
64-bit is that we only use half-page sized PTE tables in order to make
PMDs line up to 4MB, the hardware huge page size we use.
So what we do here is we make huge pages 8MB, and fabricate them using
4MB hw TLB entries.
Facilitate this by providing a "REAL_HPAGE_SHIFT" which is used in
places that really need to operate on hardware 4MB pages.
Use full pages (512 entries) for PTE tables, and adjust PMD_SHIFT,
PGD_SHIFT, and the build time CPP test as needed. Use a CPP test to
make sure REAL_HPAGE_SHIFT and the _PAGE_SZHUGE_* we use match up.
This makes the pgtable cache completely unused, so remove the code
managing it and the state used in mm_context_t. Now we have less
spinlocks taken in the page table allocation path.
The technique we use to fabricate the 8MB pages is to transfer bit 22
from the missing virtual address into the PTEs physical address field.
That takes care of the transparent huge pages case.
For hugetlb, we fill things in at the PTE level and that code already
puts the sub huge page physical bits into the PTEs, based upon the
offset, so there is nothing special we need to do. It all just works
out.
So, a small amount of complexity in the THP case, but this code is
about to get much simpler when we move the 64-bit PMDs as we can move
away from the fancy 32-bit huge PMD encoding and just put a real PTE
value in there.
With bug fixes and help from Bob Picco.
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-09-26 03:48:49 +07:00
|
|
|
#if REAL_HPAGE_SHIFT != 22
|
|
|
|
|
#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U
|
|
|
|
|
#define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V
|
|
|
|
|
|
|
|
|
|
/* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */
|
|
|
|
|
#define __P000 __pgprot(0)
|
|
|
|
|
#define __P001 __pgprot(0)
|
|
|
|
|
#define __P010 __pgprot(0)
|
|
|
|
|
#define __P011 __pgprot(0)
|
|
|
|
|
#define __P100 __pgprot(0)
|
|
|
|
|
#define __P101 __pgprot(0)
|
|
|
|
|
#define __P110 __pgprot(0)
|
|
|
|
|
#define __P111 __pgprot(0)
|
|
|
|
|
|
|
|
|
|
#define __S000 __pgprot(0)
|
|
|
|
|
#define __S001 __pgprot(0)
|
|
|
|
|
#define __S010 __pgprot(0)
|
|
|
|
|
#define __S011 __pgprot(0)
|
|
|
|
|
#define __S100 __pgprot(0)
|
|
|
|
|
#define __S101 __pgprot(0)
|
|
|
|
|
#define __S110 __pgprot(0)
|
|
|
|
|
#define __S111 __pgprot(0)
|
|
|
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
2014-05-17 04:25:50 +07:00
|
|
|
pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long);
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2014-05-17 04:25:50 +07:00
|
|
|
unsigned long pte_sz_bits(unsigned long size);
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
extern pgprot_t PAGE_KERNEL;
|
|
|
|
|
extern pgprot_t PAGE_KERNEL_LOCKED;
|
|
|
|
|
extern pgprot_t PAGE_COPY;
|
|
|
|
|
extern pgprot_t PAGE_SHARED;
|
|
|
|
|
|
2016-03-05 02:21:18 +07:00
|
|
|
/* XXX This ugliness is for the atyfb driver's sparc mmap() support. XXX */
|
2008-07-18 11:55:51 +07:00
|
|
|
extern unsigned long _PAGE_IE;
|
|
|
|
|
extern unsigned long _PAGE_E;
|
|
|
|
|
extern unsigned long _PAGE_CACHE;
|
|
|
|
|
|
|
|
|
|
extern unsigned long pg_iobits;
|
|
|
|
|
extern unsigned long _PAGE_ALL_SZ_BITS;
|
|
|
|
|
|
|
|
|
|
extern struct page *mem_map_zero;
|
|
|
|
|
#define ZERO_PAGE(vaddr) (mem_map_zero)
|
|
|
|
|
|
|
|
|
|
/* PFNs are real physical page numbers. However, mem_map only begins to record
|
|
|
|
|
* per-page information starting at pfn_base. This is to handle systems where
|
|
|
|
|
* the first physical page in the machine is at some huge physical address,
|
|
|
|
|
* such as 4GB. This is common on a partitioned E10000, for example.
|
|
|
|
|
*/
|
|
|
|
|
static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long paddr = pfn << PAGE_SHIFT;
|
2012-10-09 06:34:19 +07:00
|
|
|
|
|
|
|
|
BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
|
|
|
|
|
return __pte(paddr | pgprot_val(prot));
|
2008-07-18 11:55:51 +07:00
|
|
|
}
|
|
|
|
|
#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = pfn_pte(page_nr, pgprot);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
2013-09-27 03:45:15 +07:00
|
|
|
#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
|
2012-10-09 06:34:29 +07:00
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* This one can be done with two shifts. */
|
|
|
|
|
static inline unsigned long pte_pfn(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long ret;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sllx %1, %2, %0\n"
|
|
|
|
|
" srlx %0, %3, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sllx %1, %4, %0\n"
|
|
|
|
|
" srlx %0, %5, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (ret)
|
|
|
|
|
: "r" (pte_val(pte)),
|
|
|
|
|
"i" (21), "i" (21 + PAGE_SHIFT),
|
|
|
|
|
"i" (8), "i" (8 + PAGE_SHIFT));
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
#define pte_page(x) pfn_to_page(pte_pfn(x))
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_modify(pte_t pte, pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask, tmp;
|
|
|
|
|
|
2014-04-29 09:11:27 +07:00
|
|
|
/* SUN4U: 0x630107ffffffec38 (negated == 0x9cfef800000013c7)
|
|
|
|
|
* SUN4V: 0x33ffffffffffee07 (negated == 0xcc000000000011f8)
|
2008-07-18 11:55:51 +07:00
|
|
|
*
|
|
|
|
|
* Even if we use negation tricks the result is still a 6
|
|
|
|
|
* instruction sequence, so don't try to play fancy and just
|
|
|
|
|
* do the most straightforward implementation.
|
|
|
|
|
*
|
|
|
|
|
* Note: We encode this into 3 sun4v 2-insn patch sequences.
|
|
|
|
|
*/
|
|
|
|
|
|
2012-10-09 06:34:19 +07:00
|
|
|
BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
|
2008-07-18 11:55:51 +07:00
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sethi %%uhi(%2), %1\n"
|
|
|
|
|
" sethi %%hi(%2), %0\n"
|
|
|
|
|
"\n662: or %1, %%ulo(%2), %1\n"
|
|
|
|
|
" or %0, %%lo(%2), %0\n"
|
|
|
|
|
"\n663: sllx %1, 32, %1\n"
|
|
|
|
|
" or %0, %1, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%3), %1\n"
|
|
|
|
|
" sethi %%hi(%3), %0\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%ulo(%3), %1\n"
|
|
|
|
|
" or %0, %%lo(%3), %0\n"
|
|
|
|
|
" .word 663b\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" or %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
2015-05-27 23:00:46 +07:00
|
|
|
" .section .sun_m7_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%4), %1\n"
|
|
|
|
|
" sethi %%hi(%4), %0\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%ulo(%4), %1\n"
|
|
|
|
|
" or %0, %%lo(%4), %0\n"
|
|
|
|
|
" .word 663b\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" or %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
2008-07-18 11:55:51 +07:00
|
|
|
: "=r" (mask), "=r" (tmp)
|
|
|
|
|
: "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U |
|
2014-04-29 09:11:27 +07:00
|
|
|
_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U |
|
2013-09-27 03:45:15 +07:00
|
|
|
_PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U),
|
2008-07-18 11:55:51 +07:00
|
|
|
"i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
|
2014-04-29 09:11:27 +07:00
|
|
|
_PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V |
|
2015-05-27 23:00:46 +07:00
|
|
|
_PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V),
|
|
|
|
|
"i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
|
|
|
|
|
_PAGE_CP_4V | _PAGE_E_4V |
|
2013-09-27 03:45:15 +07:00
|
|
|
_PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V));
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask));
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
|
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
|
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_modify(pte, newprot);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
static inline pgprot_t pgprot_noncached(pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pgprot_val(prot);
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: andn %0, %2, %0\n"
|
|
|
|
|
" or %0, %3, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" andn %0, %4, %0\n"
|
|
|
|
|
" or %0, %5, %0\n"
|
|
|
|
|
" .previous\n"
|
2015-05-27 23:00:46 +07:00
|
|
|
" .section .sun_m7_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" andn %0, %6, %0\n"
|
|
|
|
|
" or %0, %5, %0\n"
|
|
|
|
|
" .previous\n"
|
2008-07-18 11:55:51 +07:00
|
|
|
: "=r" (val)
|
|
|
|
|
: "0" (val), "i" (_PAGE_CP_4U | _PAGE_CV_4U), "i" (_PAGE_E_4U),
|
2015-05-27 23:00:46 +07:00
|
|
|
"i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V),
|
|
|
|
|
"i" (_PAGE_CP_4V));
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
return __pgprot(val);
|
|
|
|
|
}
|
|
|
|
|
/* Various pieces of code check for platform support by ifdef testing
|
|
|
|
|
* on "pgprot_noncached". That's broken and should be fixed, but for
|
|
|
|
|
* now...
|
|
|
|
|
*/
|
|
|
|
|
#define pgprot_noncached pgprot_noncached
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
|
2017-02-02 07:16:36 +07:00
|
|
|
extern pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
|
|
|
|
|
struct page *page, int writable);
|
|
|
|
|
#define arch_make_huge_pte arch_make_huge_pte
|
|
|
|
|
static inline unsigned long __pte_default_huge_mask(void)
|
2008-07-18 11:55:51 +07:00
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sethi %%uhi(%1), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" mov %2, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V));
|
|
|
|
|
|
2016-03-31 01:17:13 +07:00
|
|
|
return mask;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkhuge(pte_t pte)
|
|
|
|
|
{
|
2017-02-02 07:16:36 +07:00
|
|
|
return __pte(pte_val(pte) | __pte_default_huge_mask());
|
2016-03-31 01:17:13 +07:00
|
|
|
}
|
|
|
|
|
|
2017-02-02 07:16:36 +07:00
|
|
|
static inline bool is_default_hugetlb_pte(pte_t pte)
|
2016-03-31 01:17:13 +07:00
|
|
|
{
|
2017-02-02 07:16:36 +07:00
|
|
|
unsigned long mask = __pte_default_huge_mask();
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask) == mask;
|
2008-07-18 11:55:51 +07:00
|
|
|
}
|
2016-03-31 01:17:13 +07:00
|
|
|
|
2016-07-29 14:54:21 +07:00
|
|
|
static inline bool is_hugetlb_pmd(pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
return !!(pmd_val(pmd) & _PAGE_PMD_HUGE);
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
|
static inline pmd_t pmd_mkhuge(pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkhuge(pte);
|
|
|
|
|
pte_val(pte) |= _PAGE_PMD_HUGE;
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2016-03-31 01:17:13 +07:00
|
|
|
#else
|
|
|
|
|
static inline bool is_hugetlb_pte(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
return false;
|
|
|
|
|
}
|
2008-07-18 11:55:51 +07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkdirty(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), tmp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: or %0, %3, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
"\n662: nop\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%4), %1\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%lo(%4), %1\n"
|
|
|
|
|
" or %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val), "=r" (tmp)
|
|
|
|
|
: "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U),
|
|
|
|
|
"i" (_PAGE_MODIFIED_4V | _PAGE_W_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkclean(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), tmp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: andn %0, %3, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
"\n662: nop\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%4), %1\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%lo(%4), %1\n"
|
|
|
|
|
" andn %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val), "=r" (tmp)
|
|
|
|
|
: "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U),
|
|
|
|
|
"i" (_PAGE_MODIFIED_4V | _PAGE_W_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkwrite(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val | mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_wrprotect(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), tmp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: andn %0, %3, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
"\n662: nop\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%4), %1\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%lo(%4), %1\n"
|
|
|
|
|
" andn %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val), "=r" (tmp)
|
|
|
|
|
: "0" (val), "i" (_PAGE_WRITE_4U | _PAGE_W_4U),
|
|
|
|
|
"i" (_PAGE_WRITE_4V | _PAGE_W_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkold(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
|
|
|
|
|
|
|
|
|
|
mask |= _PAGE_R;
|
|
|
|
|
|
|
|
|
|
return __pte(pte_val(pte) & ~mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkyoung(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
|
|
|
|
|
|
|
|
|
|
mask |= _PAGE_R;
|
|
|
|
|
|
|
|
|
|
return __pte(pte_val(pte) | mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkspecial(pte_t pte)
|
|
|
|
|
{
|
2011-07-26 07:12:21 +07:00
|
|
|
pte_val(pte) |= _PAGE_SPECIAL;
|
2008-07-18 11:55:51 +07:00
|
|
|
return pte;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_young(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_dirty(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_write(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_exec(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sethi %%hi(%1), %0\n"
|
|
|
|
|
" .section .sun4v_1insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" mov %2, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_EXEC_4U), "i" (_PAGE_EXEC_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_present(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte);
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: and %0, %2, %0\n"
|
|
|
|
|
" .section .sun4v_1insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" and %0, %3, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val)
|
|
|
|
|
: "0" (val), "i" (_PAGE_PRESENT_4U), "i" (_PAGE_PRESENT_4V));
|
|
|
|
|
|
|
|
|
|
return val;
|
|
|
|
|
}
|
|
|
|
|
|
2012-12-19 07:06:16 +07:00
|
|
|
#define pte_accessible pte_accessible
|
mm: fix TLB flush race between migration, and change_protection_range
There are a few subtle races, between change_protection_range (used by
mprotect and change_prot_numa) on one side, and NUMA page migration and
compaction on the other side.
The basic race is that there is a time window between when the PTE gets
made non-present (PROT_NONE or NUMA), and the TLB is flushed.
During that time, a CPU may continue writing to the page.
This is fine most of the time, however compaction or the NUMA migration
code may come in, and migrate the page away.
When that happens, the CPU may continue writing, through the cached
translation, to what is no longer the current memory location of the
process.
This only affects x86, which has a somewhat optimistic pte_accessible.
All other architectures appear to be safe, and will either always flush,
or flush whenever there is a valid mapping, even with no permissions
(SPARC).
The basic race looks like this:
CPU A CPU B CPU C
load TLB entry
make entry PTE/PMD_NUMA
fault on entry
read/write old page
start migrating page
change PTE/PMD to new page
read/write old page [*]
flush TLB
reload TLB from new entry
read/write new page
lose data
[*] the old page may belong to a new user at this point!
The obvious fix is to flush remote TLB entries, by making sure that
pte_accessible aware of the fact that PROT_NONE and PROT_NUMA memory may
still be accessible if there is a TLB flush pending for the mm.
This should fix both NUMA migration and compaction.
[mgorman@suse.de: fix build]
Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Alex Thorlton <athorlton@sgi.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-12-19 08:08:44 +07:00
|
|
|
static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
|
2012-12-19 07:06:16 +07:00
|
|
|
{
|
|
|
|
|
return pte_val(a) & _PAGE_VALID;
|
|
|
|
|
}
|
|
|
|
|
|
2011-07-26 07:12:21 +07:00
|
|
|
static inline unsigned long pte_special(pte_t pte)
|
2008-07-18 11:55:51 +07:00
|
|
|
{
|
2011-07-26 07:12:21 +07:00
|
|
|
return pte_val(pte) & _PAGE_SPECIAL;
|
2008-07-18 11:55:51 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline unsigned long pmd_large(pmd_t pmd)
|
2013-02-14 03:21:06 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
2014-04-26 00:21:12 +07:00
|
|
|
return pte_val(pte) & _PAGE_PMD_HUGE;
|
2013-02-14 03:21:06 +07:00
|
|
|
}
|
|
|
|
|
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
static inline unsigned long pmd_pfn(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
return pte_pfn(pte);
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2017-04-01 05:31:42 +07:00
|
|
|
#define __HAVE_ARCH_PMD_WRITE
|
|
|
|
|
static inline unsigned long pmd_write(pmd_t pmd)
|
2014-12-11 06:44:36 +07:00
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
2017-04-01 05:31:42 +07:00
|
|
|
return pte_write(pte);
|
2014-12-11 06:44:36 +07:00
|
|
|
}
|
|
|
|
|
|
2017-04-01 05:31:42 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
|
static inline unsigned long pmd_dirty(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
2017-04-01 05:31:42 +07:00
|
|
|
return pte_dirty(pte);
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2017-04-01 05:31:42 +07:00
|
|
|
static inline unsigned long pmd_young(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
2012-10-09 06:34:29 +07:00
|
|
|
|
2017-04-01 05:31:42 +07:00
|
|
|
return pte_young(pte);
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline unsigned long pmd_trans_huge(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
return pte_val(pte) & _PAGE_PMD_HUGE;
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mkold(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkold(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_wrprotect(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_wrprotect(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mkdirty(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkdirty(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2016-01-16 07:55:24 +07:00
|
|
|
static inline pmd_t pmd_mkclean(pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkclean(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
|
|
|
|
}
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
static inline pmd_t pmd_mkyoung(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkyoung(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mkwrite(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkwrite(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline pgprot_t pmd_pgprot(pmd_t entry)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pmd_val(entry);
|
|
|
|
|
|
|
|
|
|
return __pgprot(val);
|
|
|
|
|
}
|
2012-10-09 06:34:29 +07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
static inline int pmd_present(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-26 04:33:16 +07:00
|
|
|
return pmd_val(pmd) != 0UL;
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define pmd_none(pmd) (!pmd_val(pmd))
|
|
|
|
|
|
2014-04-30 03:03:27 +07:00
|
|
|
/* pmd_bad() is only called on non-trans-huge PMDs. Our encoding is
|
|
|
|
|
* very simple, it's just the physical address. PTE tables are of
|
|
|
|
|
* size PAGE_SIZE so make sure the sub-PAGE_SIZE bits are clear and
|
|
|
|
|
* the top bits outside of the range of any physical address size we
|
|
|
|
|
* support are clear as well. We also validate the physical itself.
|
|
|
|
|
*/
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
#define pmd_bad(pmd) (pmd_val(pmd) & ~PAGE_MASK)
|
2014-04-30 03:03:27 +07:00
|
|
|
|
|
|
|
|
#define pud_none(pud) (!pud_val(pud))
|
|
|
|
|
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
#define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK)
|
2014-04-30 03:03:27 +07:00
|
|
|
|
2014-09-27 11:19:46 +07:00
|
|
|
#define pgd_none(pgd) (!pgd_val(pgd))
|
|
|
|
|
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
#define pgd_bad(pgd) (pgd_val(pgd) & ~PAGE_MASK)
|
2014-09-27 11:19:46 +07:00
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2014-05-17 04:25:50 +07:00
|
|
|
void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pmd_t *pmdp, pmd_t pmd);
|
2012-10-09 06:34:29 +07:00
|
|
|
#else
|
|
|
|
|
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pmd_t *pmdp, pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
*pmdp = pmd;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
unsigned long val = __pa((unsigned long) (ptep));
|
2012-10-09 06:34:29 +07:00
|
|
|
|
|
|
|
|
pmd_val(*pmdp) = val;
|
|
|
|
|
}
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pud_set(pudp, pmdp) \
|
2013-09-27 03:45:15 +07:00
|
|
|
(pud_val(*(pudp)) = (__pa((unsigned long) (pmdp))))
|
2012-10-09 06:34:29 +07:00
|
|
|
static inline unsigned long __pmd_page(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
unsigned long pfn;
|
|
|
|
|
|
|
|
|
|
pfn = pte_pfn(pte);
|
|
|
|
|
|
|
|
|
|
return ((unsigned long) __va(pfn << PAGE_SHIFT));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd))
|
|
|
|
|
#define pud_page_vaddr(pud) \
|
2013-09-27 03:45:15 +07:00
|
|
|
((unsigned long) __va(pud_val(pud)))
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud))
|
2013-09-26 04:33:16 +07:00
|
|
|
#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pud_present(pud) (pud_val(pud) != 0U)
|
2013-09-26 04:33:16 +07:00
|
|
|
#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL)
|
2014-09-27 11:19:46 +07:00
|
|
|
#define pgd_page_vaddr(pgd) \
|
|
|
|
|
((unsigned long) __va(pgd_val(pgd)))
|
|
|
|
|
#define pgd_present(pgd) (pgd_val(pgd) != 0U)
|
2016-12-09 18:24:00 +07:00
|
|
|
#define pgd_clear(pgdp) (pgd_val(*(pgdp)) = 0UL)
|
2008-07-18 11:55:51 +07:00
|
|
|
|
sparc64: Fix physical memory management regressions with large max_phys_bits.
If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.
Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.
Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.
The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.
The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.
The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.
These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image. Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.
So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.
We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.
On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled. Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>
2014-09-25 10:56:11 +07:00
|
|
|
static inline unsigned long pud_large(pud_t pud)
|
|
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pud_val(pud));
|
|
|
|
|
|
|
|
|
|
return pte_val(pte) & _PAGE_PMD_HUGE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pud_pfn(pud_t pud)
|
|
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pud_val(pud));
|
|
|
|
|
|
|
|
|
|
return pte_pfn(pte);
|
|
|
|
|
}
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* Same in both SUN4V and SUN4U. */
|
|
|
|
|
#define pte_none(pte) (!pte_val(pte))
|
|
|
|
|
|
2014-09-27 11:19:46 +07:00
|
|
|
#define pgd_set(pgdp, pudp) \
|
|
|
|
|
(pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* to find an entry in a page-table-directory. */
|
|
|
|
|
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
|
|
|
|
|
#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
|
|
|
|
|
|
|
|
|
|
/* to find an entry in a kernel page-table-directory */
|
|
|
|
|
#define pgd_offset_k(address) pgd_offset(&init_mm, address)
|
|
|
|
|
|
2014-09-27 11:19:46 +07:00
|
|
|
/* Find an entry in the third-level page table.. */
|
|
|
|
|
#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
|
|
|
|
|
#define pud_offset(pgdp, address) \
|
|
|
|
|
((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* Find an entry in the second-level page table.. */
|
|
|
|
|
#define pmd_offset(pudp, address) \
|
|
|
|
|
((pmd_t *) pud_page_vaddr(*(pudp)) + \
|
|
|
|
|
(((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)))
|
|
|
|
|
|
|
|
|
|
/* Find an entry in the third-level page table.. */
|
|
|
|
|
#define pte_index(dir, address) \
|
|
|
|
|
((pte_t *) __pmd_page(*(dir)) + \
|
|
|
|
|
((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)))
|
|
|
|
|
#define pte_offset_kernel pte_index
|
|
|
|
|
#define pte_offset_map pte_index
|
|
|
|
|
#define pte_unmap(pte) do { } while (0)
|
|
|
|
|
|
2017-02-04 06:16:44 +07:00
|
|
|
/* We cannot include <linux/mm_types.h> at this point yet: */
|
|
|
|
|
extern struct mm_struct init_mm;
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* Actual page table PTE updates. */
|
2014-05-17 04:25:50 +07:00
|
|
|
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
|
2017-02-02 07:16:36 +07:00
|
|
|
pte_t *ptep, pte_t orig, int fullmm,
|
|
|
|
|
unsigned int hugepage_shift);
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2016-03-31 01:17:13 +07:00
|
|
|
static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
|
2017-02-02 07:16:36 +07:00
|
|
|
pte_t *ptep, pte_t orig, int fullmm,
|
|
|
|
|
unsigned int hugepage_shift)
|
2016-03-31 01:17:13 +07:00
|
|
|
{
|
|
|
|
|
/* It is more efficient to let flush_tlb_kernel_range()
|
|
|
|
|
* handle init_mm tlb flushes.
|
|
|
|
|
*
|
|
|
|
|
* SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
|
|
|
|
|
* and SUN4V pte layout, so this inline test is fine.
|
|
|
|
|
*/
|
|
|
|
|
if (likely(mm != &init_mm) && pte_accessible(mm, orig))
|
2017-02-02 07:16:36 +07:00
|
|
|
tlb_batch_add(mm, vaddr, ptep, orig, fullmm, hugepage_shift);
|
2016-03-31 01:17:13 +07:00
|
|
|
}
|
|
|
|
|
|
2015-06-25 06:57:44 +07:00
|
|
|
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
|
|
|
|
|
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
|
|
|
|
|
unsigned long addr,
|
|
|
|
|
pmd_t *pmdp)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
|
|
|
|
pmd_t pmd = *pmdp;
|
2013-09-26 04:33:16 +07:00
|
|
|
set_pmd_at(mm, addr, pmdp, __pmd(0UL));
|
2012-10-09 06:34:29 +07:00
|
|
|
return pmd;
|
|
|
|
|
}
|
|
|
|
|
|
2011-05-25 07:11:50 +07:00
|
|
|
static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pte_t *ptep, pte_t pte, int fullmm)
|
2008-07-18 11:55:51 +07:00
|
|
|
{
|
|
|
|
|
pte_t orig = *ptep;
|
|
|
|
|
|
|
|
|
|
*ptep = pte;
|
2017-02-02 07:16:36 +07:00
|
|
|
maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT);
|
2008-07-18 11:55:51 +07:00
|
|
|
}
|
|
|
|
|
|
2011-05-25 07:11:50 +07:00
|
|
|
#define set_pte_at(mm,addr,ptep,pte) \
|
|
|
|
|
__set_pte_at((mm), (addr), (ptep), (pte), 0)
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pte_clear(mm,addr,ptep) \
|
|
|
|
|
set_pte_at((mm), (addr), (ptep), __pte(0UL))
|
|
|
|
|
|
2011-05-25 07:11:50 +07:00
|
|
|
#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
|
|
|
|
|
#define pte_clear_not_present_full(mm,addr,ptep,fullmm) \
|
|
|
|
|
__set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm))
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#ifdef DCACHE_ALIASING_POSSIBLE
|
|
|
|
|
#define __HAVE_ARCH_MOVE_PTE
|
|
|
|
|
#define move_pte(pte, prot, old_addr, new_addr) \
|
|
|
|
|
({ \
|
|
|
|
|
pte_t newpte = (pte); \
|
|
|
|
|
if (tlb_type != hypervisor && pte_present(pte)) { \
|
|
|
|
|
unsigned long this_pfn = pte_pfn(pte); \
|
|
|
|
|
\
|
|
|
|
|
if (pfn_valid(this_pfn) && \
|
|
|
|
|
(((old_addr) ^ (new_addr)) & (1 << 13))) \
|
|
|
|
|
flush_dcache_page_all(current->mm, \
|
|
|
|
|
pfn_to_page(this_pfn)); \
|
|
|
|
|
} \
|
|
|
|
|
newpte; \
|
|
|
|
|
})
|
|
|
|
|
#endif
|
|
|
|
|
|
2013-09-26 04:33:16 +07:00
|
|
|
extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2014-05-17 04:25:50 +07:00
|
|
|
void paging_init(void);
|
|
|
|
|
unsigned long find_ecache_flush_span(unsigned long size);
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2011-04-22 05:45:45 +07:00
|
|
|
struct seq_file;
|
2014-05-17 04:25:50 +07:00
|
|
|
void mmu_info(struct seq_file *);
|
2011-04-22 05:45:45 +07:00
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
struct vm_area_struct;
|
2014-05-17 04:25:50 +07:00
|
|
|
void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *);
|
2012-10-09 06:34:29 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2014-05-17 04:25:50 +07:00
|
|
|
void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
|
|
|
|
|
pmd_t *pmd);
|
2012-10-09 06:34:29 +07:00
|
|
|
|
2014-04-25 03:58:02 +07:00
|
|
|
#define __HAVE_ARCH_PMDP_INVALIDATE
|
|
|
|
|
extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
|
|
|
|
|
pmd_t *pmdp);
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#define __HAVE_ARCH_PGTABLE_DEPOSIT
|
2014-05-17 04:25:50 +07:00
|
|
|
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
|
|
|
|
pgtable_t pgtable);
|
2012-10-09 06:34:29 +07:00
|
|
|
|
|
|
|
|
#define __HAVE_ARCH_PGTABLE_WITHDRAW
|
2014-05-17 04:25:50 +07:00
|
|
|
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
|
2012-10-09 06:34:29 +07:00
|
|
|
#endif
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* Encode and de-code a swap entry */
|
|
|
|
|
#define __swp_type(entry) (((entry).val >> PAGE_SHIFT) & 0xffUL)
|
|
|
|
|
#define __swp_offset(entry) ((entry).val >> (PAGE_SHIFT + 8UL))
|
|
|
|
|
#define __swp_entry(type, offset) \
|
|
|
|
|
( (swp_entry_t) \
|
|
|
|
|
{ \
|
|
|
|
|
(((long)(type) << PAGE_SHIFT) | \
|
|
|
|
|
((long)(offset) << (PAGE_SHIFT + 8UL))) \
|
|
|
|
|
} )
|
|
|
|
|
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
|
|
|
|
|
#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
|
|
|
|
|
|
2014-05-17 04:25:50 +07:00
|
|
|
int page_in_phys_avail(unsigned long paddr);
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in
|
|
|
|
|
* its high 4 bits. These macros/functions put it there or get it from there.
|
|
|
|
|
*/
|
|
|
|
|
#define MK_IOSPACE_PFN(space, pfn) (pfn | (space << (BITS_PER_LONG - 4)))
|
|
|
|
|
#define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4))
|
|
|
|
|
#define GET_PFN(pfn) (pfn & 0x0fffffffffffffffUL)
|
|
|
|
|
|
2014-05-17 04:25:50 +07:00
|
|
|
int remap_pfn_range(struct vm_area_struct *, unsigned long, unsigned long,
|
|
|
|
|
unsigned long, pgprot_t);
|
2011-11-18 09:17:59 +07:00
|
|
|
|
|
|
|
|
static inline int io_remap_pfn_range(struct vm_area_struct *vma,
|
|
|
|
|
unsigned long from, unsigned long pfn,
|
|
|
|
|
unsigned long size, pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT;
|
|
|
|
|
int space = GET_IOSPACE(pfn);
|
|
|
|
|
unsigned long phys_base;
|
|
|
|
|
|
|
|
|
|
phys_base = offset | (((unsigned long) space) << 32UL);
|
|
|
|
|
|
|
|
|
|
return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot);
|
|
|
|
|
}
|
2013-05-11 23:13:10 +07:00
|
|
|
#define io_remap_pfn_range io_remap_pfn_range
|
2011-11-18 09:17:59 +07:00
|
|
|
|
sparc64: Fix race in TLB batch processing.
As reported by Dave Kleikamp, when we emit cross calls to do batched
TLB flush processing we have a race because we do not synchronize on
the sibling cpus completing the cross call.
So meanwhile the TLB batch can be reset (tb->tlb_nr set to zero, etc.)
and either flushes are missed or flushes will flush the wrong
addresses.
Fix this by using generic infrastructure to synchonize on the
completion of the cross call.
This first required getting the flush_tlb_pending() call out from
switch_to() which operates with locks held and interrupts disabled.
The problem is that smp_call_function_many() cannot be invoked with
IRQs disabled and this is explicitly checked for with WARN_ON_ONCE().
We get the batch processing outside of locked IRQ disabled sections by
using some ideas from the powerpc port. Namely, we only batch inside
of arch_{enter,leave}_lazy_mmu_mode() calls. If we're not in such a
region, we flush TLBs synchronously.
1) Get rid of xcall_flush_tlb_pending and per-cpu type
implementations.
2) Do TLB batch cross calls instead via:
smp_call_function_many()
tlb_pending_func()
__flush_tlb_pending()
3) Batch only in lazy mmu sequences:
a) Add 'active' member to struct tlb_batch
b) Define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
c) Set 'active' in arch_enter_lazy_mmu_mode()
d) Run batch and clear 'active' in arch_leave_lazy_mmu_mode()
e) Check 'active' in tlb_batch_add_one() and do a synchronous
flush if it's clear.
4) Add infrastructure for synchronous TLB page flushes.
a) Implement __flush_tlb_page and per-cpu variants, patch
as needed.
b) Likewise for xcall_flush_tlb_page.
c) Implement smp_flush_tlb_page() to invoke the cross-call.
d) Wire up global_flush_tlb_page() to the right routine based
upon CONFIG_SMP
5) It turns out that singleton batches are very common, 2 out of every
3 batch flushes have only a single entry in them.
The batch flush waiting is very expensive, both because of the poll
on sibling cpu completeion, as well as because passing the tlb batch
pointer to the sibling cpus invokes a shared memory dereference.
Therefore, in flush_tlb_pending(), if there is only one entry in
the batch perform a completely asynchronous global_flush_tlb_page()
instead.
Reported-by: Dave Kleikamp <dave.kleikamp@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Dave Kleikamp <dave.kleikamp@oracle.com>
2013-04-20 04:26:26 +07:00
|
|
|
#include <asm/tlbflush.h>
|
2008-07-18 11:55:51 +07:00
|
|
|
#include <asm-generic/pgtable.h>
|
|
|
|
|
|
|
|
|
|
/* We provide our own get_unmapped_area to cope with VA holes and
|
|
|
|
|
* SHM area cache aliasing for userland.
|
|
|
|
|
*/
|
|
|
|
|
#define HAVE_ARCH_UNMAPPED_AREA
|
|
|
|
|
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
|
|
|
|
|
|
|
|
|
|
/* We provide a special get_unmapped_area for framebuffer mmaps to try and use
|
|
|
|
|
* the largest alignment possible such that larget PTEs can be used.
|
|
|
|
|
*/
|
2014-05-17 04:25:50 +07:00
|
|
|
unsigned long get_fb_unmapped_area(struct file *filp, unsigned long,
|
|
|
|
|
unsigned long, unsigned long,
|
|
|
|
|
unsigned long);
|
2008-07-18 11:55:51 +07:00
|
|
|
#define HAVE_ARCH_FB_UNMAPPED_AREA
|
|
|
|
|
|
2014-05-17 04:25:50 +07:00
|
|
|
void pgtable_cache_init(void);
|
|
|
|
|
void sun4v_register_fault_status(void);
|
|
|
|
|
void sun4v_ktsb_register(void);
|
|
|
|
|
void __init cheetah_ecache_flush_init(void);
|
|
|
|
|
void sun4v_patch_tlb_handlers(void);
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
extern unsigned long cmdline_memory_size;
|
|
|
|
|
|
2014-05-17 04:25:50 +07:00
|
|
|
asmlinkage void do_sparc64_fault(struct pt_regs *regs);
|
2008-09-12 14:10:32 +07:00
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#endif /* !(__ASSEMBLY__) */
|
|
|
|
|
|
|
|
|
|
#endif /* !(_SPARC64_PGTABLE_H) */
|