2008-07-18 11:55:51 +07:00
|
|
|
/*
|
|
|
|
|
* pgtable.h: SpitFire page table operations.
|
|
|
|
|
*
|
|
|
|
|
* Copyright 1996,1997 David S. Miller (davem@caip.rutgers.edu)
|
|
|
|
|
* Copyright 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef _SPARC64_PGTABLE_H
|
|
|
|
|
#define _SPARC64_PGTABLE_H
|
|
|
|
|
|
|
|
|
|
/* This file contains the functions and defines necessary to modify and use
|
|
|
|
|
* the SpitFire page tables.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/compiler.h>
|
|
|
|
|
#include <linux/const.h>
|
|
|
|
|
#include <asm/types.h>
|
|
|
|
|
#include <asm/spitfire.h>
|
|
|
|
|
#include <asm/asi.h>
|
|
|
|
|
#include <asm/page.h>
|
|
|
|
|
#include <asm/processor.h>
|
|
|
|
|
|
2012-04-01 15:54:38 +07:00
|
|
|
#include <asm-generic/pgtable-nopud.h>
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
|
|
|
|
|
* The page copy blockops can use 0x6000000 to 0x8000000.
|
|
|
|
|
* The TSB is mapped in the 0x8000000 to 0xa000000 range.
|
|
|
|
|
* The PROM resides in an area spanning 0xf0000000 to 0x100000000.
|
|
|
|
|
* The vmalloc area spans 0x100000000 to 0x200000000.
|
|
|
|
|
* Since modules need to be in the lowest 32-bits of the address space,
|
|
|
|
|
* we place them right before the OBP area from 0x10000000 to 0xf0000000.
|
|
|
|
|
* There is a single static kernel PMD which maps from 0x0 to address
|
|
|
|
|
* 0x400000000.
|
|
|
|
|
*/
|
|
|
|
|
#define TLBTEMP_BASE _AC(0x0000000006000000,UL)
|
|
|
|
|
#define TSBMAP_BASE _AC(0x0000000008000000,UL)
|
|
|
|
|
#define MODULES_VADDR _AC(0x0000000010000000,UL)
|
|
|
|
|
#define MODULES_LEN _AC(0x00000000e0000000,UL)
|
|
|
|
|
#define MODULES_END _AC(0x00000000f0000000,UL)
|
|
|
|
|
#define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL)
|
|
|
|
|
#define HI_OBP_ADDRESS _AC(0x0000000100000000,UL)
|
|
|
|
|
#define VMALLOC_START _AC(0x0000000100000000,UL)
|
2009-09-29 04:39:58 +07:00
|
|
|
#define VMALLOC_END _AC(0x0000010000000000,UL)
|
|
|
|
|
#define VMEMMAP_BASE _AC(0x0000010000000000,UL)
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
#define vmemmap ((struct page *)VMEMMAP_BASE)
|
|
|
|
|
|
|
|
|
|
/* PMD_SHIFT determines the size of the area a second-level page
|
|
|
|
|
* table can map
|
|
|
|
|
*/
|
sparc64: Move from 4MB to 8MB huge pages.
The impetus for this is that we would like to move to 64-bit PMDs and
PGDs, but that would result in only supporting a 42-bit address space
with the current page table layout. It'd be nice to support at least
43-bits.
The reason we'd end up with only 42-bits after making PMDs and PGDs
64-bit is that we only use half-page sized PTE tables in order to make
PMDs line up to 4MB, the hardware huge page size we use.
So what we do here is we make huge pages 8MB, and fabricate them using
4MB hw TLB entries.
Facilitate this by providing a "REAL_HPAGE_SHIFT" which is used in
places that really need to operate on hardware 4MB pages.
Use full pages (512 entries) for PTE tables, and adjust PMD_SHIFT,
PGD_SHIFT, and the build time CPP test as needed. Use a CPP test to
make sure REAL_HPAGE_SHIFT and the _PAGE_SZHUGE_* we use match up.
This makes the pgtable cache completely unused, so remove the code
managing it and the state used in mm_context_t. Now we have less
spinlocks taken in the page table allocation path.
The technique we use to fabricate the 8MB pages is to transfer bit 22
from the missing virtual address into the PTEs physical address field.
That takes care of the transparent huge pages case.
For hugetlb, we fill things in at the PTE level and that code already
puts the sub huge page physical bits into the PTEs, based upon the
offset, so there is nothing special we need to do. It all just works
out.
So, a small amount of complexity in the THP case, but this code is
about to get much simpler when we move the 64-bit PMDs as we can move
away from the fancy 32-bit huge PMD encoding and just put a real PTE
value in there.
With bug fixes and help from Bob Picco.
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-09-26 03:48:49 +07:00
|
|
|
#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3))
|
2008-07-18 11:55:51 +07:00
|
|
|
#define PMD_SIZE (_AC(1,UL) << PMD_SHIFT)
|
|
|
|
|
#define PMD_MASK (~(PMD_SIZE-1))
|
2013-09-26 04:33:16 +07:00
|
|
|
#define PMD_BITS (PAGE_SHIFT - 3)
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* PGDIR_SHIFT determines what a third-level page table entry can map */
|
sparc64: Move from 4MB to 8MB huge pages.
The impetus for this is that we would like to move to 64-bit PMDs and
PGDs, but that would result in only supporting a 42-bit address space
with the current page table layout. It'd be nice to support at least
43-bits.
The reason we'd end up with only 42-bits after making PMDs and PGDs
64-bit is that we only use half-page sized PTE tables in order to make
PMDs line up to 4MB, the hardware huge page size we use.
So what we do here is we make huge pages 8MB, and fabricate them using
4MB hw TLB entries.
Facilitate this by providing a "REAL_HPAGE_SHIFT" which is used in
places that really need to operate on hardware 4MB pages.
Use full pages (512 entries) for PTE tables, and adjust PMD_SHIFT,
PGD_SHIFT, and the build time CPP test as needed. Use a CPP test to
make sure REAL_HPAGE_SHIFT and the _PAGE_SZHUGE_* we use match up.
This makes the pgtable cache completely unused, so remove the code
managing it and the state used in mm_context_t. Now we have less
spinlocks taken in the page table allocation path.
The technique we use to fabricate the 8MB pages is to transfer bit 22
from the missing virtual address into the PTEs physical address field.
That takes care of the transparent huge pages case.
For hugetlb, we fill things in at the PTE level and that code already
puts the sub huge page physical bits into the PTEs, based upon the
offset, so there is nothing special we need to do. It all just works
out.
So, a small amount of complexity in the THP case, but this code is
about to get much simpler when we move the 64-bit PMDs as we can move
away from the fancy 32-bit huge PMD encoding and just put a real PTE
value in there.
With bug fixes and help from Bob Picco.
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-09-26 03:48:49 +07:00
|
|
|
#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS)
|
2008-07-18 11:55:51 +07:00
|
|
|
#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT)
|
|
|
|
|
#define PGDIR_MASK (~(PGDIR_SIZE-1))
|
2013-09-26 04:33:16 +07:00
|
|
|
#define PGDIR_BITS (PAGE_SHIFT - 3)
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2013-09-26 04:33:16 +07:00
|
|
|
#if (PGDIR_SHIFT + PGDIR_BITS) != 43
|
2012-10-09 06:34:20 +07:00
|
|
|
#error Page table parameters do not cover virtual address space properly.
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#if (PMD_SHIFT != HPAGE_SHIFT)
|
|
|
|
|
#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages.
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
|
|
|
|
#include <linux/sched.h>
|
|
|
|
|
|
|
|
|
|
/* Entries per page directory level. */
|
sparc64: Move from 4MB to 8MB huge pages.
The impetus for this is that we would like to move to 64-bit PMDs and
PGDs, but that would result in only supporting a 42-bit address space
with the current page table layout. It'd be nice to support at least
43-bits.
The reason we'd end up with only 42-bits after making PMDs and PGDs
64-bit is that we only use half-page sized PTE tables in order to make
PMDs line up to 4MB, the hardware huge page size we use.
So what we do here is we make huge pages 8MB, and fabricate them using
4MB hw TLB entries.
Facilitate this by providing a "REAL_HPAGE_SHIFT" which is used in
places that really need to operate on hardware 4MB pages.
Use full pages (512 entries) for PTE tables, and adjust PMD_SHIFT,
PGD_SHIFT, and the build time CPP test as needed. Use a CPP test to
make sure REAL_HPAGE_SHIFT and the _PAGE_SZHUGE_* we use match up.
This makes the pgtable cache completely unused, so remove the code
managing it and the state used in mm_context_t. Now we have less
spinlocks taken in the page table allocation path.
The technique we use to fabricate the 8MB pages is to transfer bit 22
from the missing virtual address into the PTEs physical address field.
That takes care of the transparent huge pages case.
For hugetlb, we fill things in at the PTE level and that code already
puts the sub huge page physical bits into the PTEs, based upon the
offset, so there is nothing special we need to do. It all just works
out.
So, a small amount of complexity in the THP case, but this code is
about to get much simpler when we move the 64-bit PMDs as we can move
away from the fancy 32-bit huge PMD encoding and just put a real PTE
value in there.
With bug fixes and help from Bob Picco.
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-09-26 03:48:49 +07:00
|
|
|
#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3))
|
2008-07-18 11:55:51 +07:00
|
|
|
#define PTRS_PER_PMD (1UL << PMD_BITS)
|
|
|
|
|
#define PTRS_PER_PGD (1UL << PGDIR_BITS)
|
|
|
|
|
|
|
|
|
|
/* Kernel has a separate 44bit address space. */
|
|
|
|
|
#define FIRST_USER_ADDRESS 0
|
|
|
|
|
|
|
|
|
|
#define pte_ERROR(e) __builtin_trap()
|
|
|
|
|
#define pmd_ERROR(e) __builtin_trap()
|
|
|
|
|
#define pgd_ERROR(e) __builtin_trap()
|
|
|
|
|
|
|
|
|
|
#endif /* !(__ASSEMBLY__) */
|
|
|
|
|
|
|
|
|
|
/* PTE bits which are the same in SUN4U and SUN4V format. */
|
|
|
|
|
#define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */
|
|
|
|
|
#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
|
2011-07-26 07:12:21 +07:00
|
|
|
#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */
|
2013-09-27 03:45:15 +07:00
|
|
|
#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */
|
2011-07-26 07:12:21 +07:00
|
|
|
|
|
|
|
|
/* Advertise support for _PAGE_SPECIAL */
|
|
|
|
|
#define __HAVE_ARCH_PTE_SPECIAL
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* SUN4U pte bits... */
|
|
|
|
|
#define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */
|
|
|
|
|
#define _PAGE_SZ512K_4U _AC(0x4000000000000000,UL) /* 512K Page */
|
|
|
|
|
#define _PAGE_SZ64K_4U _AC(0x2000000000000000,UL) /* 64K Page */
|
|
|
|
|
#define _PAGE_SZ8K_4U _AC(0x0000000000000000,UL) /* 8K Page */
|
|
|
|
|
#define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */
|
|
|
|
|
#define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */
|
|
|
|
|
#define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */
|
2011-07-26 07:12:21 +07:00
|
|
|
#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */
|
2013-09-27 03:45:15 +07:00
|
|
|
#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page */
|
2008-07-18 11:55:51 +07:00
|
|
|
#define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */
|
|
|
|
|
#define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */
|
|
|
|
|
#define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */
|
|
|
|
|
#define _PAGE_SZALL_4U _AC(0x6001000000000000,UL) /* All pgsz bits */
|
|
|
|
|
#define _PAGE_SN_4U _AC(0x0000800000000000,UL) /* (Cheetah) Snoop */
|
|
|
|
|
#define _PAGE_RES2_4U _AC(0x0000780000000000,UL) /* Reserved */
|
|
|
|
|
#define _PAGE_PADDR_4U _AC(0x000007FFFFFFE000,UL) /* (Cheetah) pa[42:13] */
|
|
|
|
|
#define _PAGE_SOFT_4U _AC(0x0000000000001F80,UL) /* Software bits: */
|
|
|
|
|
#define _PAGE_EXEC_4U _AC(0x0000000000001000,UL) /* Executable SW bit */
|
|
|
|
|
#define _PAGE_MODIFIED_4U _AC(0x0000000000000800,UL) /* Modified (dirty) */
|
|
|
|
|
#define _PAGE_FILE_4U _AC(0x0000000000000800,UL) /* Pagecache page */
|
|
|
|
|
#define _PAGE_ACCESSED_4U _AC(0x0000000000000400,UL) /* Accessed (ref'd) */
|
|
|
|
|
#define _PAGE_READ_4U _AC(0x0000000000000200,UL) /* Readable SW Bit */
|
|
|
|
|
#define _PAGE_WRITE_4U _AC(0x0000000000000100,UL) /* Writable SW Bit */
|
|
|
|
|
#define _PAGE_PRESENT_4U _AC(0x0000000000000080,UL) /* Present */
|
|
|
|
|
#define _PAGE_L_4U _AC(0x0000000000000040,UL) /* Locked TTE */
|
|
|
|
|
#define _PAGE_CP_4U _AC(0x0000000000000020,UL) /* Cacheable in P-Cache */
|
|
|
|
|
#define _PAGE_CV_4U _AC(0x0000000000000010,UL) /* Cacheable in V-Cache */
|
|
|
|
|
#define _PAGE_E_4U _AC(0x0000000000000008,UL) /* side-Effect */
|
|
|
|
|
#define _PAGE_P_4U _AC(0x0000000000000004,UL) /* Privileged Page */
|
|
|
|
|
#define _PAGE_W_4U _AC(0x0000000000000002,UL) /* Writable */
|
|
|
|
|
|
|
|
|
|
/* SUN4V pte bits... */
|
|
|
|
|
#define _PAGE_NFO_4V _AC(0x4000000000000000,UL) /* No Fault Only */
|
|
|
|
|
#define _PAGE_SOFT2_4V _AC(0x3F00000000000000,UL) /* Software bits, set 2 */
|
|
|
|
|
#define _PAGE_MODIFIED_4V _AC(0x2000000000000000,UL) /* Modified (dirty) */
|
|
|
|
|
#define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */
|
|
|
|
|
#define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */
|
|
|
|
|
#define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */
|
2011-07-26 07:12:21 +07:00
|
|
|
#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */
|
2013-09-27 03:45:15 +07:00
|
|
|
#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page */
|
2008-07-18 11:55:51 +07:00
|
|
|
#define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */
|
|
|
|
|
#define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */
|
|
|
|
|
#define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */
|
|
|
|
|
#define _PAGE_CP_4V _AC(0x0000000000000400,UL) /* Cacheable in P-Cache */
|
|
|
|
|
#define _PAGE_CV_4V _AC(0x0000000000000200,UL) /* Cacheable in V-Cache */
|
|
|
|
|
#define _PAGE_P_4V _AC(0x0000000000000100,UL) /* Privileged Page */
|
|
|
|
|
#define _PAGE_EXEC_4V _AC(0x0000000000000080,UL) /* Executable Page */
|
|
|
|
|
#define _PAGE_W_4V _AC(0x0000000000000040,UL) /* Writable */
|
|
|
|
|
#define _PAGE_SOFT_4V _AC(0x0000000000000030,UL) /* Software bits */
|
|
|
|
|
#define _PAGE_FILE_4V _AC(0x0000000000000020,UL) /* Pagecache page */
|
|
|
|
|
#define _PAGE_PRESENT_4V _AC(0x0000000000000010,UL) /* Present */
|
|
|
|
|
#define _PAGE_RESV_4V _AC(0x0000000000000008,UL) /* Reserved */
|
|
|
|
|
#define _PAGE_SZ16GB_4V _AC(0x0000000000000007,UL) /* 16GB Page */
|
|
|
|
|
#define _PAGE_SZ2GB_4V _AC(0x0000000000000006,UL) /* 2GB Page */
|
|
|
|
|
#define _PAGE_SZ256MB_4V _AC(0x0000000000000005,UL) /* 256MB Page */
|
|
|
|
|
#define _PAGE_SZ32MB_4V _AC(0x0000000000000004,UL) /* 32MB Page */
|
|
|
|
|
#define _PAGE_SZ4MB_4V _AC(0x0000000000000003,UL) /* 4MB Page */
|
|
|
|
|
#define _PAGE_SZ512K_4V _AC(0x0000000000000002,UL) /* 512K Page */
|
|
|
|
|
#define _PAGE_SZ64K_4V _AC(0x0000000000000001,UL) /* 64K Page */
|
|
|
|
|
#define _PAGE_SZ8K_4V _AC(0x0000000000000000,UL) /* 8K Page */
|
|
|
|
|
#define _PAGE_SZALL_4V _AC(0x0000000000000007,UL) /* All pgsz bits */
|
|
|
|
|
|
|
|
|
|
#define _PAGE_SZBITS_4U _PAGE_SZ8K_4U
|
|
|
|
|
#define _PAGE_SZBITS_4V _PAGE_SZ8K_4V
|
2012-10-09 06:34:19 +07:00
|
|
|
|
sparc64: Move from 4MB to 8MB huge pages.
The impetus for this is that we would like to move to 64-bit PMDs and
PGDs, but that would result in only supporting a 42-bit address space
with the current page table layout. It'd be nice to support at least
43-bits.
The reason we'd end up with only 42-bits after making PMDs and PGDs
64-bit is that we only use half-page sized PTE tables in order to make
PMDs line up to 4MB, the hardware huge page size we use.
So what we do here is we make huge pages 8MB, and fabricate them using
4MB hw TLB entries.
Facilitate this by providing a "REAL_HPAGE_SHIFT" which is used in
places that really need to operate on hardware 4MB pages.
Use full pages (512 entries) for PTE tables, and adjust PMD_SHIFT,
PGD_SHIFT, and the build time CPP test as needed. Use a CPP test to
make sure REAL_HPAGE_SHIFT and the _PAGE_SZHUGE_* we use match up.
This makes the pgtable cache completely unused, so remove the code
managing it and the state used in mm_context_t. Now we have less
spinlocks taken in the page table allocation path.
The technique we use to fabricate the 8MB pages is to transfer bit 22
from the missing virtual address into the PTEs physical address field.
That takes care of the transparent huge pages case.
For hugetlb, we fill things in at the PTE level and that code already
puts the sub huge page physical bits into the PTEs, based upon the
offset, so there is nothing special we need to do. It all just works
out.
So, a small amount of complexity in the THP case, but this code is
about to get much simpler when we move the 64-bit PMDs as we can move
away from the fancy 32-bit huge PMD encoding and just put a real PTE
value in there.
With bug fixes and help from Bob Picco.
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-09-26 03:48:49 +07:00
|
|
|
#if REAL_HPAGE_SHIFT != 22
|
|
|
|
|
#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U
|
|
|
|
|
#define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V
|
|
|
|
|
|
|
|
|
|
/* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */
|
|
|
|
|
#define __P000 __pgprot(0)
|
|
|
|
|
#define __P001 __pgprot(0)
|
|
|
|
|
#define __P010 __pgprot(0)
|
|
|
|
|
#define __P011 __pgprot(0)
|
|
|
|
|
#define __P100 __pgprot(0)
|
|
|
|
|
#define __P101 __pgprot(0)
|
|
|
|
|
#define __P110 __pgprot(0)
|
|
|
|
|
#define __P111 __pgprot(0)
|
|
|
|
|
|
|
|
|
|
#define __S000 __pgprot(0)
|
|
|
|
|
#define __S001 __pgprot(0)
|
|
|
|
|
#define __S010 __pgprot(0)
|
|
|
|
|
#define __S011 __pgprot(0)
|
|
|
|
|
#define __S100 __pgprot(0)
|
|
|
|
|
#define __S101 __pgprot(0)
|
|
|
|
|
#define __S110 __pgprot(0)
|
|
|
|
|
#define __S111 __pgprot(0)
|
|
|
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
|
|
|
|
extern pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long);
|
|
|
|
|
|
|
|
|
|
extern unsigned long pte_sz_bits(unsigned long size);
|
|
|
|
|
|
|
|
|
|
extern pgprot_t PAGE_KERNEL;
|
|
|
|
|
extern pgprot_t PAGE_KERNEL_LOCKED;
|
|
|
|
|
extern pgprot_t PAGE_COPY;
|
|
|
|
|
extern pgprot_t PAGE_SHARED;
|
|
|
|
|
|
|
|
|
|
/* XXX This uglyness is for the atyfb driver's sparc mmap() support. XXX */
|
|
|
|
|
extern unsigned long _PAGE_IE;
|
|
|
|
|
extern unsigned long _PAGE_E;
|
|
|
|
|
extern unsigned long _PAGE_CACHE;
|
|
|
|
|
|
|
|
|
|
extern unsigned long pg_iobits;
|
|
|
|
|
extern unsigned long _PAGE_ALL_SZ_BITS;
|
|
|
|
|
|
|
|
|
|
extern struct page *mem_map_zero;
|
|
|
|
|
#define ZERO_PAGE(vaddr) (mem_map_zero)
|
|
|
|
|
|
|
|
|
|
/* PFNs are real physical page numbers. However, mem_map only begins to record
|
|
|
|
|
* per-page information starting at pfn_base. This is to handle systems where
|
|
|
|
|
* the first physical page in the machine is at some huge physical address,
|
|
|
|
|
* such as 4GB. This is common on a partitioned E10000, for example.
|
|
|
|
|
*/
|
|
|
|
|
static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long paddr = pfn << PAGE_SHIFT;
|
2012-10-09 06:34:19 +07:00
|
|
|
|
|
|
|
|
BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
|
|
|
|
|
return __pte(paddr | pgprot_val(prot));
|
2008-07-18 11:55:51 +07:00
|
|
|
}
|
|
|
|
|
#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = pfn_pte(page_nr, pgprot);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
2013-09-27 03:45:15 +07:00
|
|
|
#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
|
2012-10-09 06:34:29 +07:00
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
/* This one can be done with two shifts. */
|
|
|
|
|
static inline unsigned long pte_pfn(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long ret;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sllx %1, %2, %0\n"
|
|
|
|
|
" srlx %0, %3, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sllx %1, %4, %0\n"
|
|
|
|
|
" srlx %0, %5, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (ret)
|
|
|
|
|
: "r" (pte_val(pte)),
|
|
|
|
|
"i" (21), "i" (21 + PAGE_SHIFT),
|
|
|
|
|
"i" (8), "i" (8 + PAGE_SHIFT));
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
#define pte_page(x) pfn_to_page(pte_pfn(x))
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_modify(pte_t pte, pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask, tmp;
|
|
|
|
|
|
|
|
|
|
/* SUN4U: 0x600307ffffffecb8 (negated == 0x9ffcf80000001347)
|
|
|
|
|
* SUN4V: 0x30ffffffffffee17 (negated == 0xcf000000000011e8)
|
|
|
|
|
*
|
|
|
|
|
* Even if we use negation tricks the result is still a 6
|
|
|
|
|
* instruction sequence, so don't try to play fancy and just
|
|
|
|
|
* do the most straightforward implementation.
|
|
|
|
|
*
|
|
|
|
|
* Note: We encode this into 3 sun4v 2-insn patch sequences.
|
|
|
|
|
*/
|
|
|
|
|
|
2012-10-09 06:34:19 +07:00
|
|
|
BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
|
2008-07-18 11:55:51 +07:00
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sethi %%uhi(%2), %1\n"
|
|
|
|
|
" sethi %%hi(%2), %0\n"
|
|
|
|
|
"\n662: or %1, %%ulo(%2), %1\n"
|
|
|
|
|
" or %0, %%lo(%2), %0\n"
|
|
|
|
|
"\n663: sllx %1, 32, %1\n"
|
|
|
|
|
" or %0, %1, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%3), %1\n"
|
|
|
|
|
" sethi %%hi(%3), %0\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%ulo(%3), %1\n"
|
|
|
|
|
" or %0, %%lo(%3), %0\n"
|
|
|
|
|
" .word 663b\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" or %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask), "=r" (tmp)
|
|
|
|
|
: "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U |
|
|
|
|
|
_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U |
|
2013-09-27 03:45:15 +07:00
|
|
|
_PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U),
|
2008-07-18 11:55:51 +07:00
|
|
|
"i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
|
|
|
|
|
_PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V |
|
2013-09-27 03:45:15 +07:00
|
|
|
_PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V));
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask));
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
|
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
|
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_modify(pte, newprot);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
static inline pte_t pgoff_to_pte(unsigned long off)
|
|
|
|
|
{
|
|
|
|
|
off <<= PAGE_SHIFT;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: or %0, %2, %0\n"
|
|
|
|
|
" .section .sun4v_1insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" or %0, %3, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (off)
|
|
|
|
|
: "0" (off), "i" (_PAGE_FILE_4U), "i" (_PAGE_FILE_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(off);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pgprot_t pgprot_noncached(pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pgprot_val(prot);
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: andn %0, %2, %0\n"
|
|
|
|
|
" or %0, %3, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" andn %0, %4, %0\n"
|
|
|
|
|
" or %0, %5, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val)
|
|
|
|
|
: "0" (val), "i" (_PAGE_CP_4U | _PAGE_CV_4U), "i" (_PAGE_E_4U),
|
|
|
|
|
"i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V));
|
|
|
|
|
|
|
|
|
|
return __pgprot(val);
|
|
|
|
|
}
|
|
|
|
|
/* Various pieces of code check for platform support by ifdef testing
|
|
|
|
|
* on "pgprot_noncached". That's broken and should be fixed, but for
|
|
|
|
|
* now...
|
|
|
|
|
*/
|
|
|
|
|
#define pgprot_noncached pgprot_noncached
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
|
2008-07-18 11:55:51 +07:00
|
|
|
static inline pte_t pte_mkhuge(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sethi %%uhi(%1), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" mov %2, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(pte_val(pte) | mask);
|
|
|
|
|
}
|
2013-09-27 03:45:15 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
|
static inline pmd_t pmd_mkhuge(pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkhuge(pte);
|
|
|
|
|
pte_val(pte) |= _PAGE_PMD_HUGE;
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2008-07-18 11:55:51 +07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkdirty(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), tmp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: or %0, %3, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
"\n662: nop\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%4), %1\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%lo(%4), %1\n"
|
|
|
|
|
" or %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val), "=r" (tmp)
|
|
|
|
|
: "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U),
|
|
|
|
|
"i" (_PAGE_MODIFIED_4V | _PAGE_W_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkclean(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), tmp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: andn %0, %3, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
"\n662: nop\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%4), %1\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%lo(%4), %1\n"
|
|
|
|
|
" andn %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val), "=r" (tmp)
|
|
|
|
|
: "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U),
|
|
|
|
|
"i" (_PAGE_MODIFIED_4V | _PAGE_W_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkwrite(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val | mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_wrprotect(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte), tmp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: andn %0, %3, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
"\n662: nop\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%4), %1\n"
|
|
|
|
|
" sllx %1, 32, %1\n"
|
|
|
|
|
" .word 662b\n"
|
|
|
|
|
" or %1, %%lo(%4), %1\n"
|
|
|
|
|
" andn %0, %1, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val), "=r" (tmp)
|
|
|
|
|
: "0" (val), "i" (_PAGE_WRITE_4U | _PAGE_W_4U),
|
|
|
|
|
"i" (_PAGE_WRITE_4V | _PAGE_W_4V));
|
|
|
|
|
|
|
|
|
|
return __pte(val);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkold(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
|
|
|
|
|
|
|
|
|
|
mask |= _PAGE_R;
|
|
|
|
|
|
|
|
|
|
return __pte(pte_val(pte) & ~mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkyoung(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
|
|
|
|
|
|
|
|
|
|
mask |= _PAGE_R;
|
|
|
|
|
|
|
|
|
|
return __pte(pte_val(pte) | mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pte_t pte_mkspecial(pte_t pte)
|
|
|
|
|
{
|
2011-07-26 07:12:21 +07:00
|
|
|
pte_val(pte) |= _PAGE_SPECIAL;
|
2008-07-18 11:55:51 +07:00
|
|
|
return pte;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_young(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_dirty(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_write(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: mov %1, %0\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" .section .sun4v_2insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" sethi %%uhi(%2), %0\n"
|
|
|
|
|
" sllx %0, 32, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_exec(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: sethi %%hi(%1), %0\n"
|
|
|
|
|
" .section .sun4v_1insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" mov %2, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (mask)
|
|
|
|
|
: "i" (_PAGE_EXEC_4U), "i" (_PAGE_EXEC_4V));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & mask);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_file(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte);
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: and %0, %2, %0\n"
|
|
|
|
|
" .section .sun4v_1insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" and %0, %3, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val)
|
|
|
|
|
: "0" (val), "i" (_PAGE_FILE_4U), "i" (_PAGE_FILE_4V));
|
|
|
|
|
|
|
|
|
|
return val;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pte_present(pte_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pte_val(pte);
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
"\n661: and %0, %2, %0\n"
|
|
|
|
|
" .section .sun4v_1insn_patch, \"ax\"\n"
|
|
|
|
|
" .word 661b\n"
|
|
|
|
|
" and %0, %3, %0\n"
|
|
|
|
|
" .previous\n"
|
|
|
|
|
: "=r" (val)
|
|
|
|
|
: "0" (val), "i" (_PAGE_PRESENT_4U), "i" (_PAGE_PRESENT_4V));
|
|
|
|
|
|
|
|
|
|
return val;
|
|
|
|
|
}
|
|
|
|
|
|
2012-12-19 07:06:16 +07:00
|
|
|
#define pte_accessible pte_accessible
|
mm: fix TLB flush race between migration, and change_protection_range
There are a few subtle races, between change_protection_range (used by
mprotect and change_prot_numa) on one side, and NUMA page migration and
compaction on the other side.
The basic race is that there is a time window between when the PTE gets
made non-present (PROT_NONE or NUMA), and the TLB is flushed.
During that time, a CPU may continue writing to the page.
This is fine most of the time, however compaction or the NUMA migration
code may come in, and migrate the page away.
When that happens, the CPU may continue writing, through the cached
translation, to what is no longer the current memory location of the
process.
This only affects x86, which has a somewhat optimistic pte_accessible.
All other architectures appear to be safe, and will either always flush,
or flush whenever there is a valid mapping, even with no permissions
(SPARC).
The basic race looks like this:
CPU A CPU B CPU C
load TLB entry
make entry PTE/PMD_NUMA
fault on entry
read/write old page
start migrating page
change PTE/PMD to new page
read/write old page [*]
flush TLB
reload TLB from new entry
read/write new page
lose data
[*] the old page may belong to a new user at this point!
The obvious fix is to flush remote TLB entries, by making sure that
pte_accessible aware of the fact that PROT_NONE and PROT_NUMA memory may
still be accessible if there is a TLB flush pending for the mm.
This should fix both NUMA migration and compaction.
[mgorman@suse.de: fix build]
Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Alex Thorlton <athorlton@sgi.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-12-19 08:08:44 +07:00
|
|
|
static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
|
2012-12-19 07:06:16 +07:00
|
|
|
{
|
|
|
|
|
return pte_val(a) & _PAGE_VALID;
|
|
|
|
|
}
|
|
|
|
|
|
2011-07-26 07:12:21 +07:00
|
|
|
static inline unsigned long pte_special(pte_t pte)
|
2008-07-18 11:55:51 +07:00
|
|
|
{
|
2011-07-26 07:12:21 +07:00
|
|
|
return pte_val(pte) & _PAGE_SPECIAL;
|
2008-07-18 11:55:51 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline unsigned long pmd_large(pmd_t pmd)
|
2013-02-14 03:21:06 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
return (pte_val(pte) & _PAGE_PMD_HUGE) && pte_present(pte);
|
2013-02-14 03:21:06 +07:00
|
|
|
}
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline unsigned long pmd_young(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
return pte_young(pte);
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline unsigned long pmd_write(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
return pte_write(pte);
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long pmd_pfn(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
2012-10-09 06:34:29 +07:00
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
return pte_pfn(pte);
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline unsigned long pmd_trans_huge(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
return pte_val(pte) & _PAGE_PMD_HUGE;
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline unsigned long pmd_trans_splitting(pmd_t pmd)
|
2012-10-09 06:34:29 +07:00
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
return pmd_trans_huge(pmd) && pte_special(pte);
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define has_transparent_hugepage() 1
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mkold(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkold(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_wrprotect(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_wrprotect(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mkdirty(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkdirty(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mkyoung(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkyoung(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mkwrite(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkwrite(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pmd_t pmd_mksplitting(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
|
|
|
|
|
pte = pte_mkspecial(pte);
|
|
|
|
|
|
|
|
|
|
return __pmd(pte_val(pte));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
2013-09-27 03:45:15 +07:00
|
|
|
static inline pgprot_t pmd_pgprot(pmd_t entry)
|
|
|
|
|
{
|
|
|
|
|
unsigned long val = pmd_val(entry);
|
|
|
|
|
|
|
|
|
|
return __pgprot(val);
|
|
|
|
|
}
|
2012-10-09 06:34:29 +07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
static inline int pmd_present(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-26 04:33:16 +07:00
|
|
|
return pmd_val(pmd) != 0UL;
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define pmd_none(pmd) (!pmd_val(pmd))
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
|
extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pmd_t *pmdp, pmd_t pmd);
|
|
|
|
|
#else
|
|
|
|
|
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pmd_t *pmdp, pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
*pmdp = pmd;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
unsigned long val = __pa((unsigned long) (ptep));
|
2012-10-09 06:34:29 +07:00
|
|
|
|
|
|
|
|
pmd_val(*pmdp) = val;
|
|
|
|
|
}
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pud_set(pudp, pmdp) \
|
2013-09-27 03:45:15 +07:00
|
|
|
(pud_val(*(pudp)) = (__pa((unsigned long) (pmdp))))
|
2012-10-09 06:34:29 +07:00
|
|
|
static inline unsigned long __pmd_page(pmd_t pmd)
|
|
|
|
|
{
|
2013-09-27 03:45:15 +07:00
|
|
|
pte_t pte = __pte(pmd_val(pmd));
|
|
|
|
|
unsigned long pfn;
|
|
|
|
|
|
|
|
|
|
pfn = pte_pfn(pte);
|
|
|
|
|
|
|
|
|
|
return ((unsigned long) __va(pfn << PAGE_SHIFT));
|
2012-10-09 06:34:29 +07:00
|
|
|
}
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd))
|
|
|
|
|
#define pud_page_vaddr(pud) \
|
2013-09-27 03:45:15 +07:00
|
|
|
((unsigned long) __va(pud_val(pud)))
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud))
|
|
|
|
|
#define pmd_bad(pmd) (0)
|
2013-09-26 04:33:16 +07:00
|
|
|
#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pud_none(pud) (!pud_val(pud))
|
|
|
|
|
#define pud_bad(pud) (0)
|
|
|
|
|
#define pud_present(pud) (pud_val(pud) != 0U)
|
2013-09-26 04:33:16 +07:00
|
|
|
#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL)
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* Same in both SUN4V and SUN4U. */
|
|
|
|
|
#define pte_none(pte) (!pte_val(pte))
|
|
|
|
|
|
|
|
|
|
/* to find an entry in a page-table-directory. */
|
|
|
|
|
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
|
|
|
|
|
#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
|
|
|
|
|
|
|
|
|
|
/* to find an entry in a kernel page-table-directory */
|
|
|
|
|
#define pgd_offset_k(address) pgd_offset(&init_mm, address)
|
|
|
|
|
|
|
|
|
|
/* Find an entry in the second-level page table.. */
|
|
|
|
|
#define pmd_offset(pudp, address) \
|
|
|
|
|
((pmd_t *) pud_page_vaddr(*(pudp)) + \
|
|
|
|
|
(((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)))
|
|
|
|
|
|
|
|
|
|
/* Find an entry in the third-level page table.. */
|
|
|
|
|
#define pte_index(dir, address) \
|
|
|
|
|
((pte_t *) __pmd_page(*(dir)) + \
|
|
|
|
|
((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)))
|
|
|
|
|
#define pte_offset_kernel pte_index
|
|
|
|
|
#define pte_offset_map pte_index
|
|
|
|
|
#define pte_unmap(pte) do { } while (0)
|
|
|
|
|
|
|
|
|
|
/* Actual page table PTE updates. */
|
2011-05-25 07:11:50 +07:00
|
|
|
extern void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
|
|
|
|
|
pte_t *ptep, pte_t orig, int fullmm);
|
2008-07-18 11:55:51 +07:00
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
|
|
|
|
|
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
|
|
|
|
|
unsigned long addr,
|
|
|
|
|
pmd_t *pmdp)
|
|
|
|
|
{
|
|
|
|
|
pmd_t pmd = *pmdp;
|
2013-09-26 04:33:16 +07:00
|
|
|
set_pmd_at(mm, addr, pmdp, __pmd(0UL));
|
2012-10-09 06:34:29 +07:00
|
|
|
return pmd;
|
|
|
|
|
}
|
|
|
|
|
|
2011-05-25 07:11:50 +07:00
|
|
|
static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
pte_t *ptep, pte_t pte, int fullmm)
|
2008-07-18 11:55:51 +07:00
|
|
|
{
|
|
|
|
|
pte_t orig = *ptep;
|
|
|
|
|
|
|
|
|
|
*ptep = pte;
|
|
|
|
|
|
|
|
|
|
/* It is more efficient to let flush_tlb_kernel_range()
|
|
|
|
|
* handle init_mm tlb flushes.
|
|
|
|
|
*
|
|
|
|
|
* SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
|
|
|
|
|
* and SUN4V pte layout, so this inline test is fine.
|
|
|
|
|
*/
|
mm: fix TLB flush race between migration, and change_protection_range
There are a few subtle races, between change_protection_range (used by
mprotect and change_prot_numa) on one side, and NUMA page migration and
compaction on the other side.
The basic race is that there is a time window between when the PTE gets
made non-present (PROT_NONE or NUMA), and the TLB is flushed.
During that time, a CPU may continue writing to the page.
This is fine most of the time, however compaction or the NUMA migration
code may come in, and migrate the page away.
When that happens, the CPU may continue writing, through the cached
translation, to what is no longer the current memory location of the
process.
This only affects x86, which has a somewhat optimistic pte_accessible.
All other architectures appear to be safe, and will either always flush,
or flush whenever there is a valid mapping, even with no permissions
(SPARC).
The basic race looks like this:
CPU A CPU B CPU C
load TLB entry
make entry PTE/PMD_NUMA
fault on entry
read/write old page
start migrating page
change PTE/PMD to new page
read/write old page [*]
flush TLB
reload TLB from new entry
read/write new page
lose data
[*] the old page may belong to a new user at this point!
The obvious fix is to flush remote TLB entries, by making sure that
pte_accessible aware of the fact that PROT_NONE and PROT_NUMA memory may
still be accessible if there is a TLB flush pending for the mm.
This should fix both NUMA migration and compaction.
[mgorman@suse.de: fix build]
Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Cc: Alex Thorlton <athorlton@sgi.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-12-19 08:08:44 +07:00
|
|
|
if (likely(mm != &init_mm) && pte_accessible(mm, orig))
|
2011-05-25 07:11:50 +07:00
|
|
|
tlb_batch_add(mm, addr, ptep, orig, fullmm);
|
2008-07-18 11:55:51 +07:00
|
|
|
}
|
|
|
|
|
|
2011-05-25 07:11:50 +07:00
|
|
|
#define set_pte_at(mm,addr,ptep,pte) \
|
|
|
|
|
__set_pte_at((mm), (addr), (ptep), (pte), 0)
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#define pte_clear(mm,addr,ptep) \
|
|
|
|
|
set_pte_at((mm), (addr), (ptep), __pte(0UL))
|
|
|
|
|
|
2011-05-25 07:11:50 +07:00
|
|
|
#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
|
|
|
|
|
#define pte_clear_not_present_full(mm,addr,ptep,fullmm) \
|
|
|
|
|
__set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm))
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#ifdef DCACHE_ALIASING_POSSIBLE
|
|
|
|
|
#define __HAVE_ARCH_MOVE_PTE
|
|
|
|
|
#define move_pte(pte, prot, old_addr, new_addr) \
|
|
|
|
|
({ \
|
|
|
|
|
pte_t newpte = (pte); \
|
|
|
|
|
if (tlb_type != hypervisor && pte_present(pte)) { \
|
|
|
|
|
unsigned long this_pfn = pte_pfn(pte); \
|
|
|
|
|
\
|
|
|
|
|
if (pfn_valid(this_pfn) && \
|
|
|
|
|
(((old_addr) ^ (new_addr)) & (1 << 13))) \
|
|
|
|
|
flush_dcache_page_all(current->mm, \
|
|
|
|
|
pfn_to_page(this_pfn)); \
|
|
|
|
|
} \
|
|
|
|
|
newpte; \
|
|
|
|
|
})
|
|
|
|
|
#endif
|
|
|
|
|
|
2013-09-26 04:33:16 +07:00
|
|
|
extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
|
|
|
|
|
extern pmd_t swapper_low_pmd_dir[PTRS_PER_PMD];
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
extern void paging_init(void);
|
|
|
|
|
extern unsigned long find_ecache_flush_span(unsigned long size);
|
|
|
|
|
|
2011-04-22 05:45:45 +07:00
|
|
|
struct seq_file;
|
|
|
|
|
extern void mmu_info(struct seq_file *);
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
struct vm_area_struct;
|
MM: Pass a PTE pointer to update_mmu_cache() rather than the PTE itself
On VIVT ARM, when we have multiple shared mappings of the same file
in the same MM, we need to ensure that we have coherency across all
copies. We do this via make_coherent() by making the pages
uncacheable.
This used to work fine, until we allowed highmem with highpte - we
now have a page table which is mapped as required, and is not available
for modification via update_mmu_cache().
Ralf Beache suggested getting rid of the PTE value passed to
update_mmu_cache():
On MIPS update_mmu_cache() calls __update_tlb() which walks pagetables
to construct a pointer to the pte again. Passing a pte_t * is much
more elegant. Maybe we might even replace the pte argument with the
pte_t?
Ben Herrenschmidt would also like the pte pointer for PowerPC:
Passing the ptep in there is exactly what I want. I want that
-instead- of the PTE value, because I have issue on some ppc cases,
for I$/D$ coherency, where set_pte_at() may decide to mask out the
_PAGE_EXEC.
So, pass in the mapped page table pointer into update_mmu_cache(), and
remove the PTE value, updating all implementations and call sites to
suit.
Includes a fix from Stephen Rothwell:
sparc: fix fallout from update_mmu_cache API change
Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>
Acked-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2009-12-18 23:40:18 +07:00
|
|
|
extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *);
|
2012-10-09 06:34:29 +07:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
|
|
extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
|
|
|
|
|
pmd_t *pmd);
|
|
|
|
|
|
2014-04-25 03:58:02 +07:00
|
|
|
#define __HAVE_ARCH_PMDP_INVALIDATE
|
|
|
|
|
extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
|
|
|
|
|
pmd_t *pmdp);
|
|
|
|
|
|
2012-10-09 06:34:29 +07:00
|
|
|
#define __HAVE_ARCH_PGTABLE_DEPOSIT
|
2013-06-06 07:14:02 +07:00
|
|
|
extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
|
|
|
|
pgtable_t pgtable);
|
2012-10-09 06:34:29 +07:00
|
|
|
|
|
|
|
|
#define __HAVE_ARCH_PGTABLE_WITHDRAW
|
2013-06-06 07:14:02 +07:00
|
|
|
extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
|
2012-10-09 06:34:29 +07:00
|
|
|
#endif
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* Encode and de-code a swap entry */
|
|
|
|
|
#define __swp_type(entry) (((entry).val >> PAGE_SHIFT) & 0xffUL)
|
|
|
|
|
#define __swp_offset(entry) ((entry).val >> (PAGE_SHIFT + 8UL))
|
|
|
|
|
#define __swp_entry(type, offset) \
|
|
|
|
|
( (swp_entry_t) \
|
|
|
|
|
{ \
|
|
|
|
|
(((long)(type) << PAGE_SHIFT) | \
|
|
|
|
|
((long)(offset) << (PAGE_SHIFT + 8UL))) \
|
|
|
|
|
} )
|
|
|
|
|
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
|
|
|
|
|
#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
|
|
|
|
|
|
|
|
|
|
/* File offset in PTE support. */
|
|
|
|
|
extern unsigned long pte_file(pte_t);
|
|
|
|
|
#define pte_to_pgoff(pte) (pte_val(pte) >> PAGE_SHIFT)
|
|
|
|
|
extern pte_t pgoff_to_pte(unsigned long);
|
|
|
|
|
#define PTE_FILE_MAX_BITS (64UL - PAGE_SHIFT - 1UL)
|
|
|
|
|
|
sparc64: Validate linear D-TLB misses.
When page alloc debugging is not enabled, we essentially accept any
virtual address for linear kernel TLB misses. But with kgdb, kernel
address probing, and other facilities we can try to access arbitrary
crap.
So, make sure the address we miss on will translate to physical memory
that actually exists.
In order to make this work we have to embed the valid address bitmap
into the kernel image. And in order to make that less expensive we
make an adjustment, in that the max physical memory address is
decreased to "1 << 41", even on the chips that support a 42-bit
physical address space. We can do this because bit 41 indicates
"I/O space" and thus covers non-memory ranges.
The result of this is that:
1) kpte_linear_bitmap shrinks from 2K to 1K in size
2) we need 64K more for the valid address bitmap
We can't let the valid address bitmap be dynamically allocated
once we start using it to validate TLB misses, otherwise we have
crazy issues to deal with wrt. recursive TLB misses and such.
If we're in a TLB miss it could be the deepest trap level that's legal
inside of the cpu. So if we TLB miss referencing the bitmap, the cpu
will be out of trap levels and enter RED state.
To guard against out-of-range accesses to the bitmap, we have to check
to make sure no bits in the physical address above bit 40 are set. We
could export and use last_valid_pfn for this check, but that's just an
unnecessary extra memory reference.
On the plus side of all this, since we load all of these translations
into the special 4MB mapping TSB, and we check the TSB first for TLB
misses, there should be absolutely no real cost for these new checks
in the TLB miss path.
Reported-by: heyongli@gmail.com
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-08-26 06:47:46 +07:00
|
|
|
extern unsigned long sparc64_valid_addr_bitmap[];
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
|
sparc64: Validate linear D-TLB misses.
When page alloc debugging is not enabled, we essentially accept any
virtual address for linear kernel TLB misses. But with kgdb, kernel
address probing, and other facilities we can try to access arbitrary
crap.
So, make sure the address we miss on will translate to physical memory
that actually exists.
In order to make this work we have to embed the valid address bitmap
into the kernel image. And in order to make that less expensive we
make an adjustment, in that the max physical memory address is
decreased to "1 << 41", even on the chips that support a 42-bit
physical address space. We can do this because bit 41 indicates
"I/O space" and thus covers non-memory ranges.
The result of this is that:
1) kpte_linear_bitmap shrinks from 2K to 1K in size
2) we need 64K more for the valid address bitmap
We can't let the valid address bitmap be dynamically allocated
once we start using it to validate TLB misses, otherwise we have
crazy issues to deal with wrt. recursive TLB misses and such.
If we're in a TLB miss it could be the deepest trap level that's legal
inside of the cpu. So if we TLB miss referencing the bitmap, the cpu
will be out of trap levels and enter RED state.
To guard against out-of-range accesses to the bitmap, we have to check
to make sure no bits in the physical address above bit 40 are set. We
could export and use last_valid_pfn for this check, but that's just an
unnecessary extra memory reference.
On the plus side of all this, since we load all of these translations
into the special 4MB mapping TSB, and we check the TSB first for TLB
misses, there should be absolutely no real cost for these new checks
in the TLB miss path.
Reported-by: heyongli@gmail.com
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-08-26 06:47:46 +07:00
|
|
|
static inline bool kern_addr_valid(unsigned long addr)
|
|
|
|
|
{
|
|
|
|
|
unsigned long paddr = __pa(addr);
|
|
|
|
|
|
|
|
|
|
if ((paddr >> 41UL) != 0UL)
|
|
|
|
|
return false;
|
|
|
|
|
return test_bit(paddr >> 22, sparc64_valid_addr_bitmap);
|
|
|
|
|
}
|
2008-07-18 11:55:51 +07:00
|
|
|
|
|
|
|
|
extern int page_in_phys_avail(unsigned long paddr);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in
|
|
|
|
|
* its high 4 bits. These macros/functions put it there or get it from there.
|
|
|
|
|
*/
|
|
|
|
|
#define MK_IOSPACE_PFN(space, pfn) (pfn | (space << (BITS_PER_LONG - 4)))
|
|
|
|
|
#define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4))
|
|
|
|
|
#define GET_PFN(pfn) (pfn & 0x0fffffffffffffffUL)
|
|
|
|
|
|
2011-11-18 09:17:59 +07:00
|
|
|
extern int remap_pfn_range(struct vm_area_struct *, unsigned long, unsigned long,
|
|
|
|
|
unsigned long, pgprot_t);
|
|
|
|
|
|
|
|
|
|
static inline int io_remap_pfn_range(struct vm_area_struct *vma,
|
|
|
|
|
unsigned long from, unsigned long pfn,
|
|
|
|
|
unsigned long size, pgprot_t prot)
|
|
|
|
|
{
|
|
|
|
|
unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT;
|
|
|
|
|
int space = GET_IOSPACE(pfn);
|
|
|
|
|
unsigned long phys_base;
|
|
|
|
|
|
|
|
|
|
phys_base = offset | (((unsigned long) space) << 32UL);
|
|
|
|
|
|
|
|
|
|
return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot);
|
|
|
|
|
}
|
2013-05-11 23:13:10 +07:00
|
|
|
#define io_remap_pfn_range io_remap_pfn_range
|
2011-11-18 09:17:59 +07:00
|
|
|
|
sparc64: Fix race in TLB batch processing.
As reported by Dave Kleikamp, when we emit cross calls to do batched
TLB flush processing we have a race because we do not synchronize on
the sibling cpus completing the cross call.
So meanwhile the TLB batch can be reset (tb->tlb_nr set to zero, etc.)
and either flushes are missed or flushes will flush the wrong
addresses.
Fix this by using generic infrastructure to synchonize on the
completion of the cross call.
This first required getting the flush_tlb_pending() call out from
switch_to() which operates with locks held and interrupts disabled.
The problem is that smp_call_function_many() cannot be invoked with
IRQs disabled and this is explicitly checked for with WARN_ON_ONCE().
We get the batch processing outside of locked IRQ disabled sections by
using some ideas from the powerpc port. Namely, we only batch inside
of arch_{enter,leave}_lazy_mmu_mode() calls. If we're not in such a
region, we flush TLBs synchronously.
1) Get rid of xcall_flush_tlb_pending and per-cpu type
implementations.
2) Do TLB batch cross calls instead via:
smp_call_function_many()
tlb_pending_func()
__flush_tlb_pending()
3) Batch only in lazy mmu sequences:
a) Add 'active' member to struct tlb_batch
b) Define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
c) Set 'active' in arch_enter_lazy_mmu_mode()
d) Run batch and clear 'active' in arch_leave_lazy_mmu_mode()
e) Check 'active' in tlb_batch_add_one() and do a synchronous
flush if it's clear.
4) Add infrastructure for synchronous TLB page flushes.
a) Implement __flush_tlb_page and per-cpu variants, patch
as needed.
b) Likewise for xcall_flush_tlb_page.
c) Implement smp_flush_tlb_page() to invoke the cross-call.
d) Wire up global_flush_tlb_page() to the right routine based
upon CONFIG_SMP
5) It turns out that singleton batches are very common, 2 out of every
3 batch flushes have only a single entry in them.
The batch flush waiting is very expensive, both because of the poll
on sibling cpu completeion, as well as because passing the tlb batch
pointer to the sibling cpus invokes a shared memory dereference.
Therefore, in flush_tlb_pending(), if there is only one entry in
the batch perform a completely asynchronous global_flush_tlb_page()
instead.
Reported-by: Dave Kleikamp <dave.kleikamp@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Dave Kleikamp <dave.kleikamp@oracle.com>
2013-04-20 04:26:26 +07:00
|
|
|
#include <asm/tlbflush.h>
|
2008-07-18 11:55:51 +07:00
|
|
|
#include <asm-generic/pgtable.h>
|
|
|
|
|
|
|
|
|
|
/* We provide our own get_unmapped_area to cope with VA holes and
|
|
|
|
|
* SHM area cache aliasing for userland.
|
|
|
|
|
*/
|
|
|
|
|
#define HAVE_ARCH_UNMAPPED_AREA
|
|
|
|
|
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
|
|
|
|
|
|
|
|
|
|
/* We provide a special get_unmapped_area for framebuffer mmaps to try and use
|
|
|
|
|
* the largest alignment possible such that larget PTEs can be used.
|
|
|
|
|
*/
|
|
|
|
|
extern unsigned long get_fb_unmapped_area(struct file *filp, unsigned long,
|
|
|
|
|
unsigned long, unsigned long,
|
|
|
|
|
unsigned long);
|
|
|
|
|
#define HAVE_ARCH_FB_UNMAPPED_AREA
|
|
|
|
|
|
|
|
|
|
extern void pgtable_cache_init(void);
|
|
|
|
|
extern void sun4v_register_fault_status(void);
|
|
|
|
|
extern void sun4v_ktsb_register(void);
|
|
|
|
|
extern void __init cheetah_ecache_flush_init(void);
|
|
|
|
|
extern void sun4v_patch_tlb_handlers(void);
|
|
|
|
|
|
|
|
|
|
extern unsigned long cmdline_memory_size;
|
|
|
|
|
|
2008-09-12 14:10:32 +07:00
|
|
|
extern asmlinkage void do_sparc64_fault(struct pt_regs *regs);
|
|
|
|
|
|
2008-07-18 11:55:51 +07:00
|
|
|
#endif /* !(__ASSEMBLY__) */
|
|
|
|
|
|
|
|
|
|
#endif /* !(_SPARC64_PGTABLE_H) */
|