mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-16 19:57:38 +07:00
3c835670ab
Arrange the sections in the FRV arch so that a BRA instruction with a 16-bit displacement can always reach from the trap table to entry.S, tlb-miss.S and break.S. The problem otherwise is that the linker can insert sufficient code between the slots in the trap table and the targets of the branch instructions in those slots that the displacement field in the instruction isn't sufficiently large. This is because the branch targets were in the .text section along with most of the other code in the kernel. Signed-off-by: David Howells <dhowells@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
631 lines
18 KiB
ArmAsm
631 lines
18 KiB
ArmAsm
/* tlb-miss.S: TLB miss handlers
|
|
*
|
|
* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
|
|
* Written by David Howells (dhowells@redhat.com)
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*/
|
|
|
|
#include <linux/sys.h>
|
|
#include <linux/linkage.h>
|
|
#include <asm/page.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/highmem.h>
|
|
#include <asm/spr-regs.h>
|
|
|
|
.section .text.tlbmiss
|
|
.balign 4
|
|
|
|
.globl __entry_insn_mmu_miss
|
|
__entry_insn_mmu_miss:
|
|
break
|
|
nop
|
|
|
|
.globl __entry_insn_mmu_exception
|
|
__entry_insn_mmu_exception:
|
|
break
|
|
nop
|
|
|
|
.globl __entry_data_mmu_miss
|
|
__entry_data_mmu_miss:
|
|
break
|
|
nop
|
|
|
|
.globl __entry_data_mmu_exception
|
|
__entry_data_mmu_exception:
|
|
break
|
|
nop
|
|
|
|
###############################################################################
|
|
#
|
|
# handle a lookup failure of one sort or another in a kernel TLB handler
|
|
# On entry:
|
|
# GR29 - faulting address
|
|
# SCR2 - saved CCR
|
|
#
|
|
###############################################################################
|
|
.type __tlb_kernel_fault,@function
|
|
__tlb_kernel_fault:
|
|
# see if we're supposed to re-enable single-step mode upon return
|
|
sethi.p %hi(__break_tlb_miss_return_break),gr30
|
|
setlo %lo(__break_tlb_miss_return_break),gr30
|
|
movsg pcsr,gr31
|
|
|
|
subcc gr31,gr30,gr0,icc0
|
|
beq icc0,#0,__tlb_kernel_fault_sstep
|
|
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
movgs gr29,scr2 /* save EAR0 value */
|
|
sethi.p %hi(__kernel_current_task),gr29
|
|
setlo %lo(__kernel_current_task),gr29
|
|
ldi.p @(gr29,#0),gr29 /* restore GR29 */
|
|
|
|
bra __entry_kernel_handle_mmu_fault
|
|
|
|
# we've got to re-enable single-stepping
|
|
__tlb_kernel_fault_sstep:
|
|
sethi.p %hi(__break_tlb_miss_real_return_info),gr30
|
|
setlo %lo(__break_tlb_miss_real_return_info),gr30
|
|
lddi @(gr30,0),gr30
|
|
movgs gr30,pcsr
|
|
movgs gr31,psr
|
|
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
movgs gr29,scr2 /* save EAR0 value */
|
|
sethi.p %hi(__kernel_current_task),gr29
|
|
setlo %lo(__kernel_current_task),gr29
|
|
ldi.p @(gr29,#0),gr29 /* restore GR29 */
|
|
bra __entry_kernel_handle_mmu_fault_sstep
|
|
|
|
.size __tlb_kernel_fault, .-__tlb_kernel_fault
|
|
|
|
###############################################################################
|
|
#
|
|
# handle a lookup failure of one sort or another in a user TLB handler
|
|
# On entry:
|
|
# GR28 - faulting address
|
|
# SCR2 - saved CCR
|
|
#
|
|
###############################################################################
|
|
.type __tlb_user_fault,@function
|
|
__tlb_user_fault:
|
|
# see if we're supposed to re-enable single-step mode upon return
|
|
sethi.p %hi(__break_tlb_miss_return_break),gr30
|
|
setlo %lo(__break_tlb_miss_return_break),gr30
|
|
movsg pcsr,gr31
|
|
subcc gr31,gr30,gr0,icc0
|
|
beq icc0,#0,__tlb_user_fault_sstep
|
|
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
bra __entry_uspace_handle_mmu_fault
|
|
|
|
# we've got to re-enable single-stepping
|
|
__tlb_user_fault_sstep:
|
|
sethi.p %hi(__break_tlb_miss_real_return_info),gr30
|
|
setlo %lo(__break_tlb_miss_real_return_info),gr30
|
|
lddi @(gr30,0),gr30
|
|
movgs gr30,pcsr
|
|
movgs gr31,psr
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
bra __entry_uspace_handle_mmu_fault_sstep
|
|
|
|
.size __tlb_user_fault, .-__tlb_user_fault
|
|
|
|
###############################################################################
|
|
#
|
|
# Kernel instruction TLB miss handler
|
|
# On entry:
|
|
# GR1 - kernel stack pointer
|
|
# GR28 - saved exception frame pointer
|
|
# GR29 - faulting address
|
|
# GR31 - EAR0 ^ SCR0
|
|
# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
|
|
# DAMR3 - mapped page directory
|
|
# DAMR4 - mapped page table as matched by SCR0
|
|
#
|
|
###############################################################################
|
|
.globl __entry_kernel_insn_tlb_miss
|
|
.type __entry_kernel_insn_tlb_miss,@function
|
|
__entry_kernel_insn_tlb_miss:
|
|
#if 0
|
|
sethi.p %hi(0xe1200004),gr30
|
|
setlo %lo(0xe1200004),gr30
|
|
st gr0,@(gr30,gr0)
|
|
sethi.p %hi(0xffc00100),gr30
|
|
setlo %lo(0xffc00100),gr30
|
|
sth gr30,@(gr30,gr0)
|
|
membar
|
|
#endif
|
|
|
|
movsg ccr,gr30 /* save CCR */
|
|
movgs gr30,scr2
|
|
|
|
# see if the cached page table mapping is appropriate
|
|
srlicc.p gr31,#26,gr0,icc0
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bne icc0,#0,__itlb_k_PTD_miss
|
|
|
|
__itlb_k_PTD_mapped:
|
|
# access the PTD with EAR0[25:14]
|
|
# - DAMLR4 points to the virtual address of the appropriate page table
|
|
# - the PTD holds 4096 PTEs
|
|
# - the PTD must be accessed uncached
|
|
# - the PTE must be marked accessed if it was valid
|
|
#
|
|
and gr31,gr30,gr31
|
|
movsg damlr4,gr30
|
|
add gr30,gr31,gr31
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
|
|
|
# we're using IAMR1 as an extra TLB entry
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
|
# - need to check DAMR1 lest we cause an multiple-DAT-hit exception
|
|
# - IAMPR1 has no WP bit, and we mustn't lose WP information
|
|
movsg iampr1,gr31
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
|
setlos.p 0xfffff000,gr31
|
|
beq icc0,#0,__itlb_k_nopunt /* punt not required */
|
|
|
|
movsg iamlr1,gr31
|
|
movgs gr31,tplr /* set TPLR.CXN */
|
|
tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */
|
|
|
|
movsg dampr1,gr31
|
|
ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */
|
|
movgs gr31,tppr
|
|
movsg iamlr1,gr31 /* set TPLR.CXN */
|
|
movgs gr31,tplr
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
|
setlos #0xfffff000,gr31
|
|
bne icc0,#0,__tlb_kernel_fault
|
|
|
|
__itlb_k_nopunt:
|
|
|
|
# assemble the new TLB entry
|
|
and gr29,gr31,gr29
|
|
movsg cxnr,gr31
|
|
or gr29,gr31,gr29
|
|
movgs gr29,iamlr1 /* xAMLR = address | context number */
|
|
movgs gr30,iampr1
|
|
movgs gr29,damlr1
|
|
movgs gr30,dampr1
|
|
|
|
# return, restoring registers
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
sethi.p %hi(__kernel_current_task),gr29
|
|
setlo %lo(__kernel_current_task),gr29
|
|
ldi @(gr29,#0),gr29
|
|
rett #0
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
|
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
|
# appropriate page table and map that instead
|
|
# - access the PGD with EAR0[31:26]
|
|
# - DAMLR3 points to the virtual address of the page directory
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
|
__itlb_k_PTD_miss:
|
|
srli gr29,#26,gr31 /* calculate PGE offset */
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
|
|
|
movsg damlr3,gr30
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
|
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
|
|
|
# map this PTD instead and record coverage address
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */
|
|
slli.p gr31,#18,gr31
|
|
bne icc1,#0,__itlb_k_bigpage
|
|
movgs gr30,dampr4
|
|
movgs gr31,scr0
|
|
|
|
# we can now resume normal service
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bra __itlb_k_PTD_mapped
|
|
|
|
__itlb_k_bigpage:
|
|
break
|
|
nop
|
|
|
|
.size __entry_kernel_insn_tlb_miss, .-__entry_kernel_insn_tlb_miss
|
|
|
|
###############################################################################
|
|
#
|
|
# Kernel data TLB miss handler
|
|
# On entry:
|
|
# GR1 - kernel stack pointer
|
|
# GR28 - saved exception frame pointer
|
|
# GR29 - faulting address
|
|
# GR31 - EAR0 ^ SCR1
|
|
# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
|
|
# DAMR3 - mapped page directory
|
|
# DAMR5 - mapped page table as matched by SCR1
|
|
#
|
|
###############################################################################
|
|
.globl __entry_kernel_data_tlb_miss
|
|
.type __entry_kernel_data_tlb_miss,@function
|
|
__entry_kernel_data_tlb_miss:
|
|
#if 0
|
|
sethi.p %hi(0xe1200004),gr30
|
|
setlo %lo(0xe1200004),gr30
|
|
st gr0,@(gr30,gr0)
|
|
sethi.p %hi(0xffc00100),gr30
|
|
setlo %lo(0xffc00100),gr30
|
|
sth gr30,@(gr30,gr0)
|
|
membar
|
|
#endif
|
|
|
|
movsg ccr,gr30 /* save CCR */
|
|
movgs gr30,scr2
|
|
|
|
# see if the cached page table mapping is appropriate
|
|
srlicc.p gr31,#26,gr0,icc0
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bne icc0,#0,__dtlb_k_PTD_miss
|
|
|
|
__dtlb_k_PTD_mapped:
|
|
# access the PTD with EAR0[25:14]
|
|
# - DAMLR5 points to the virtual address of the appropriate page table
|
|
# - the PTD holds 4096 PTEs
|
|
# - the PTD must be accessed uncached
|
|
# - the PTE must be marked accessed if it was valid
|
|
#
|
|
and gr31,gr30,gr31
|
|
movsg damlr5,gr30
|
|
add gr30,gr31,gr31
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
|
|
|
# we're using DAMR1 as an extra TLB entry
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
|
# - need to check IAMR1 lest we cause an multiple-DAT-hit exception
|
|
movsg dampr1,gr31
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
|
setlos.p 0xfffff000,gr31
|
|
beq icc0,#0,__dtlb_k_nopunt /* punt not required */
|
|
|
|
movsg damlr1,gr31
|
|
movgs gr31,tplr /* set TPLR.CXN */
|
|
tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */
|
|
|
|
movsg dampr1,gr31
|
|
ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */
|
|
movgs gr31,tppr
|
|
movsg damlr1,gr31 /* set TPLR.CXN */
|
|
movgs gr31,tplr
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
|
setlos #0xfffff000,gr31
|
|
bne icc0,#0,__tlb_kernel_fault
|
|
|
|
__dtlb_k_nopunt:
|
|
|
|
# assemble the new TLB entry
|
|
and gr29,gr31,gr29
|
|
movsg cxnr,gr31
|
|
or gr29,gr31,gr29
|
|
movgs gr29,iamlr1 /* xAMLR = address | context number */
|
|
movgs gr30,iampr1
|
|
movgs gr29,damlr1
|
|
movgs gr30,dampr1
|
|
|
|
# return, restoring registers
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
sethi.p %hi(__kernel_current_task),gr29
|
|
setlo %lo(__kernel_current_task),gr29
|
|
ldi @(gr29,#0),gr29
|
|
rett #0
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
|
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
|
# appropriate page table and map that instead
|
|
# - access the PGD with EAR0[31:26]
|
|
# - DAMLR3 points to the virtual address of the page directory
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
|
__dtlb_k_PTD_miss:
|
|
srli gr29,#26,gr31 /* calculate PGE offset */
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
|
|
|
movsg damlr3,gr30
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
|
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
|
|
|
# map this PTD instead and record coverage address
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */
|
|
slli.p gr31,#18,gr31
|
|
bne icc1,#0,__dtlb_k_bigpage
|
|
movgs gr30,dampr5
|
|
movgs gr31,scr1
|
|
|
|
# we can now resume normal service
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bra __dtlb_k_PTD_mapped
|
|
|
|
__dtlb_k_bigpage:
|
|
break
|
|
nop
|
|
|
|
.size __entry_kernel_data_tlb_miss, .-__entry_kernel_data_tlb_miss
|
|
|
|
###############################################################################
|
|
#
|
|
# Userspace instruction TLB miss handler (with PGE prediction)
|
|
# On entry:
|
|
# GR28 - faulting address
|
|
# GR31 - EAR0 ^ SCR0
|
|
# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
|
|
# DAMR3 - mapped page directory
|
|
# DAMR4 - mapped page table as matched by SCR0
|
|
#
|
|
###############################################################################
|
|
.globl __entry_user_insn_tlb_miss
|
|
.type __entry_user_insn_tlb_miss,@function
|
|
__entry_user_insn_tlb_miss:
|
|
#if 0
|
|
sethi.p %hi(0xe1200004),gr30
|
|
setlo %lo(0xe1200004),gr30
|
|
st gr0,@(gr30,gr0)
|
|
sethi.p %hi(0xffc00100),gr30
|
|
setlo %lo(0xffc00100),gr30
|
|
sth gr30,@(gr30,gr0)
|
|
membar
|
|
#endif
|
|
|
|
movsg ccr,gr30 /* save CCR */
|
|
movgs gr30,scr2
|
|
|
|
# see if the cached page table mapping is appropriate
|
|
srlicc.p gr31,#26,gr0,icc0
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bne icc0,#0,__itlb_u_PTD_miss
|
|
|
|
__itlb_u_PTD_mapped:
|
|
# access the PTD with EAR0[25:14]
|
|
# - DAMLR4 points to the virtual address of the appropriate page table
|
|
# - the PTD holds 4096 PTEs
|
|
# - the PTD must be accessed uncached
|
|
# - the PTE must be marked accessed if it was valid
|
|
#
|
|
and gr31,gr30,gr31
|
|
movsg damlr4,gr30
|
|
add gr30,gr31,gr31
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
|
beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
|
|
|
# we're using IAMR1/DAMR1 as an extra TLB entry
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
|
movsg dampr1,gr31
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
|
setlos.p 0xfffff000,gr31
|
|
beq icc0,#0,__itlb_u_nopunt /* punt not required */
|
|
|
|
movsg dampr1,gr31
|
|
movgs gr31,tppr
|
|
movsg damlr1,gr31 /* set TPLR.CXN */
|
|
movgs gr31,tplr
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
|
setlos #0xfffff000,gr31
|
|
bne icc0,#0,__tlb_user_fault
|
|
|
|
__itlb_u_nopunt:
|
|
|
|
# assemble the new TLB entry
|
|
and gr28,gr31,gr28
|
|
movsg cxnr,gr31
|
|
or gr28,gr31,gr28
|
|
movgs gr28,iamlr1 /* xAMLR = address | context number */
|
|
movgs gr30,iampr1
|
|
movgs gr28,damlr1
|
|
movgs gr30,dampr1
|
|
|
|
# return, restoring registers
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
rett #0
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
|
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
|
# appropriate page table and map that instead
|
|
# - access the PGD with EAR0[31:26]
|
|
# - DAMLR3 points to the virtual address of the page directory
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
|
__itlb_u_PTD_miss:
|
|
srli gr28,#26,gr31 /* calculate PGE offset */
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
|
|
|
movsg damlr3,gr30
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
|
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
|
|
|
# map this PTD instead and record coverage address
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
|
beq icc0,#0,__tlb_user_fault /* jump if PGE not present */
|
|
slli.p gr31,#18,gr31
|
|
bne icc1,#0,__itlb_u_bigpage
|
|
movgs gr30,dampr4
|
|
movgs gr31,scr0
|
|
|
|
# we can now resume normal service
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bra __itlb_u_PTD_mapped
|
|
|
|
__itlb_u_bigpage:
|
|
break
|
|
nop
|
|
|
|
.size __entry_user_insn_tlb_miss, .-__entry_user_insn_tlb_miss
|
|
|
|
###############################################################################
|
|
#
|
|
# Userspace data TLB miss handler
|
|
# On entry:
|
|
# GR28 - faulting address
|
|
# GR31 - EAR0 ^ SCR1
|
|
# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
|
|
# DAMR3 - mapped page directory
|
|
# DAMR5 - mapped page table as matched by SCR1
|
|
#
|
|
###############################################################################
|
|
.globl __entry_user_data_tlb_miss
|
|
.type __entry_user_data_tlb_miss,@function
|
|
__entry_user_data_tlb_miss:
|
|
#if 0
|
|
sethi.p %hi(0xe1200004),gr30
|
|
setlo %lo(0xe1200004),gr30
|
|
st gr0,@(gr30,gr0)
|
|
sethi.p %hi(0xffc00100),gr30
|
|
setlo %lo(0xffc00100),gr30
|
|
sth gr30,@(gr30,gr0)
|
|
membar
|
|
#endif
|
|
|
|
movsg ccr,gr30 /* save CCR */
|
|
movgs gr30,scr2
|
|
|
|
# see if the cached page table mapping is appropriate
|
|
srlicc.p gr31,#26,gr0,icc0
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bne icc0,#0,__dtlb_u_PTD_miss
|
|
|
|
__dtlb_u_PTD_mapped:
|
|
# access the PTD with EAR0[25:14]
|
|
# - DAMLR5 points to the virtual address of the appropriate page table
|
|
# - the PTD holds 4096 PTEs
|
|
# - the PTD must be accessed uncached
|
|
# - the PTE must be marked accessed if it was valid
|
|
#
|
|
and gr31,gr30,gr31
|
|
movsg damlr5,gr30
|
|
|
|
__dtlb_u_using_iPTD:
|
|
add gr30,gr31,gr31
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
|
beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
|
|
|
# we're using DAMR1 as an extra TLB entry
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
|
movsg dampr1,gr31
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
|
setlos.p 0xfffff000,gr31
|
|
beq icc0,#0,__dtlb_u_nopunt /* punt not required */
|
|
|
|
movsg dampr1,gr31
|
|
movgs gr31,tppr
|
|
movsg damlr1,gr31 /* set TPLR.CXN */
|
|
movgs gr31,tplr
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
|
setlos #0xfffff000,gr31
|
|
bne icc0,#0,__tlb_user_fault
|
|
|
|
__dtlb_u_nopunt:
|
|
|
|
# assemble the new TLB entry
|
|
and gr28,gr31,gr28
|
|
movsg cxnr,gr31
|
|
or gr28,gr31,gr28
|
|
movgs gr28,iamlr1 /* xAMLR = address | context number */
|
|
movgs gr30,iampr1
|
|
movgs gr28,damlr1
|
|
movgs gr30,dampr1
|
|
|
|
# return, restoring registers
|
|
movsg scr2,gr30
|
|
movgs gr30,ccr
|
|
rett #0
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
|
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
|
# appropriate page table and map that instead
|
|
# - first of all, check the insn PGE cache - we may well get a hit there
|
|
# - access the PGD with EAR0[31:26]
|
|
# - DAMLR3 points to the virtual address of the page directory
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
|
__dtlb_u_PTD_miss:
|
|
movsg scr0,gr31 /* consult the insn-PGE-cache key */
|
|
xor gr28,gr31,gr31
|
|
srlicc gr31,#26,gr0,icc0
|
|
srli gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bne icc0,#0,__dtlb_u_iPGE_miss
|
|
|
|
# what we're looking for is covered by the insn-PGE-cache
|
|
setlos 0x3ffc,gr30
|
|
and gr31,gr30,gr31
|
|
movsg damlr4,gr30
|
|
bra __dtlb_u_using_iPTD
|
|
|
|
__dtlb_u_iPGE_miss:
|
|
srli gr28,#26,gr31 /* calculate PGE offset */
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
|
|
|
movsg damlr3,gr30
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
|
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
|
|
|
# map this PTD instead and record coverage address
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
|
beq icc0,#0,__tlb_user_fault /* jump if PGE not present */
|
|
slli.p gr31,#18,gr31
|
|
bne icc1,#0,__dtlb_u_bigpage
|
|
movgs gr30,dampr5
|
|
movgs gr31,scr1
|
|
|
|
# we can now resume normal service
|
|
setlos 0x3ffc,gr30
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
|
bra __dtlb_u_PTD_mapped
|
|
|
|
__dtlb_u_bigpage:
|
|
break
|
|
nop
|
|
|
|
.size __entry_user_data_tlb_miss, .-__entry_user_data_tlb_miss
|