mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-02 10:46:50 +07:00
powerpc/8xx: Make pinning of ITLBs optional
As stated in a comment in head_8xx.S, today we "Always pin the first 8 MB ITLB to prevent ITLB misses while mucking around with SRR0/SRR1 in asm". This issue has just been cleared by the preceding patch, therefore we can make this pinning optional (on by default) and independent of DATA pinning. This patch also makes pinning of IMMR independent of pinning of DATA. Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
This commit is contained in:
Christophe Leroy
Michael Ellerman
parent
0eb0d2e77d
commit
a3059b0ca0
@ -1178,10 +1178,20 @@ config PIN_TLB
|
||||
bool "Pinned Kernel TLBs (860 ONLY)"
|
||||
depends on ADVANCED_OPTIONS && PPC_8xx
|
||||
|
||||
config PIN_TLB_DATA
|
||||
bool "Pinned TLB for DATA"
|
||||
depends on PIN_TLB
|
||||
default y
|
||||
|
||||
config PIN_TLB_IMMR
|
||||
bool "Pinned TLB for IMMR"
|
||||
depends on PIN_TLB
|
||||
default y
|
||||
|
||||
config PIN_TLB_TEXT
|
||||
bool "Pinned TLB for TEXT"
|
||||
depends on PIN_TLB
|
||||
default y
|
||||
endmenu
|
||||
|
||||
if PPC64
|
||||
|
||||
@ -55,6 +55,15 @@
|
||||
#define SIMPLE_KERNEL_ADDRESS 1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* We need an ITLB miss handler for kernel addresses if:
|
||||
* - Either we have modules
|
||||
* - Or we have not pinned the first 8M
|
||||
*/
|
||||
#if defined(CONFIG_MODULES) || !defined(CONFIG_PIN_TLB_TEXT) || \
|
||||
defined(CONFIG_DEBUG_PAGEALLOC)
|
||||
#define ITLB_MISS_KERNEL 1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Value for the bits that have fixed value in RPN entries.
|
||||
@ -317,7 +326,7 @@ SystemCall:
|
||||
#endif
|
||||
|
||||
InstructionTLBMiss:
|
||||
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
|
||||
#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
|
||||
mtspr SPRN_SPRG_SCRATCH2, r3
|
||||
#endif
|
||||
EXCEPTION_PROLOG_0
|
||||
@ -335,23 +344,31 @@ InstructionTLBMiss:
|
||||
INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
|
||||
/* Only modules will cause ITLB Misses as we always
|
||||
* pin the first 8MB of kernel memory */
|
||||
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
|
||||
#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
|
||||
mfcr r3
|
||||
#endif
|
||||
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
|
||||
#ifdef SIMPLE_KERNEL_ADDRESS
|
||||
#ifdef ITLB_MISS_KERNEL
|
||||
#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
|
||||
andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
|
||||
#else
|
||||
rlwinm r11, r10, 16, 0xfff8
|
||||
cmpli cr0, r11, PAGE_OFFSET@h
|
||||
#ifndef CONFIG_PIN_TLB_TEXT
|
||||
/* It is assumed that kernel code fits into the first 8M page */
|
||||
_ENTRY(ITLBMiss_cmp)
|
||||
cmpli cr7, r11, (PAGE_OFFSET + 0x0800000)@h
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
||||
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
|
||||
#ifdef SIMPLE_KERNEL_ADDRESS
|
||||
#ifdef ITLB_MISS_KERNEL
|
||||
#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
|
||||
beq+ 3f
|
||||
#else
|
||||
blt+ 3f
|
||||
#endif
|
||||
#ifndef CONFIG_PIN_TLB_TEXT
|
||||
blt cr7, ITLBMissLinear
|
||||
#endif
|
||||
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
||||
3:
|
||||
@ -370,7 +387,7 @@ InstructionTLBMiss:
|
||||
rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
|
||||
lwz r10, 0(r10) /* Get the pte */
|
||||
4:
|
||||
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
|
||||
#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
|
||||
mtcr r3
|
||||
#endif
|
||||
/* Insert the APG into the TWC from the Linux PTE. */
|
||||
@ -401,7 +418,7 @@ InstructionTLBMiss:
|
||||
MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */
|
||||
|
||||
/* Restore registers */
|
||||
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
|
||||
#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
|
||||
mfspr r3, SPRN_SPRG_SCRATCH2
|
||||
#endif
|
||||
EXCEPTION_EPILOG_0
|
||||
@ -696,6 +713,22 @@ DTLBMissLinear:
|
||||
EXCEPTION_EPILOG_0
|
||||
rfi
|
||||
|
||||
#ifndef CONFIG_PIN_TLB_TEXT
|
||||
ITLBMissLinear:
|
||||
mtcr r3
|
||||
/* Set 8M byte page and mark it valid */
|
||||
li r11, MI_PS8MEG | MI_SVALID | _PAGE_EXEC
|
||||
MTSPR_CPU6(SPRN_MI_TWC, r11, r3)
|
||||
rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */
|
||||
ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \
|
||||
_PAGE_PRESENT
|
||||
MTSPR_CPU6(SPRN_MI_RPN, r10, r11) /* Update TLB entry */
|
||||
|
||||
mfspr r3, SPRN_SPRG_SCRATCH2
|
||||
EXCEPTION_EPILOG_0
|
||||
rfi
|
||||
#endif
|
||||
|
||||
/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
|
||||
* by decoding the registers used by the dcbx instruction and adding them.
|
||||
* DAR is set to the calculated address.
|
||||
@ -955,15 +988,14 @@ initial_mmu:
|
||||
mtspr SPRN_MD_CTR, r10 /* remove PINNED DTLB entries */
|
||||
|
||||
tlbia /* Invalidate all TLB entries */
|
||||
/* Always pin the first 8 MB ITLB to prevent ITLB
|
||||
misses while mucking around with SRR0/SRR1 in asm
|
||||
*/
|
||||
#ifdef CONFIG_PIN_TLB_TEXT
|
||||
lis r8, MI_RSV4I@h
|
||||
ori r8, r8, 0x1c00
|
||||
|
||||
mtspr SPRN_MI_CTR, r8 /* Set instruction MMU control */
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_PIN_TLB
|
||||
#ifdef CONFIG_PIN_TLB_DATA
|
||||
oris r10, r10, MD_RSV4I@h
|
||||
mtspr SPRN_MD_CTR, r10 /* Set data TLB control */
|
||||
#endif
|
||||
@ -989,6 +1021,7 @@ initial_mmu:
|
||||
* internal registers (among other things).
|
||||
*/
|
||||
#ifdef CONFIG_PIN_TLB_IMMR
|
||||
oris r10, r10, MD_RSV4I@h
|
||||
ori r10, r10, 0x1c00
|
||||
mtspr SPRN_MD_CTR, r10
|
||||
|
||||
|
||||
@ -65,7 +65,7 @@ unsigned long p_block_mapped(phys_addr_t pa)
|
||||
void __init MMU_init_hw(void)
|
||||
{
|
||||
/* PIN up to the 3 first 8Mb after IMMR in DTLB table */
|
||||
#ifdef CONFIG_PIN_TLB
|
||||
#ifdef CONFIG_PIN_TLB_DATA
|
||||
unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000;
|
||||
unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY;
|
||||
#ifdef CONFIG_PIN_TLB_IMMR
|
||||
@ -103,6 +103,9 @@ static void mmu_mapin_immr(void)
|
||||
extern unsigned int DTLBMiss_jmp;
|
||||
#endif
|
||||
extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;
|
||||
#ifndef CONFIG_PIN_TLB_TEXT
|
||||
extern unsigned int ITLBMiss_cmp;
|
||||
#endif
|
||||
|
||||
void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped)
|
||||
{
|
||||
@ -122,6 +125,9 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
|
||||
mmu_mapin_immr();
|
||||
#ifndef CONFIG_PIN_TLB_IMMR
|
||||
patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);
|
||||
#endif
|
||||
#ifndef CONFIG_PIN_TLB_TEXT
|
||||
mmu_patch_cmp_limit(&ITLBMiss_cmp, 0);
|
||||
#endif
|
||||
} else {
|
||||
mapped = top & ~(LARGE_PAGE_SIZE_8M - 1);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user