mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-17 22:06:41 +07:00
cc6c98485f
The existing mechanism for handling IRQs on RISC-V is pretty ugly: the irq entry code selects the handler via Kconfig dependencies. Use the new generic IRQ handling infastructure, which allows boot time registration of the low level entry handler. This does add an additional load to the interrupt latency, but there's a lot of tuning left to be done there on RISC-V so it's OK for now. Signed-off-by: Palmer Dabbelt <palmer@sifive.com> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Christoph Hellwig <hch@lst.de> Acked-by: Stafford Horne <shorne@gmail.com> Cc: jonas@southpole.se Cc: catalin.marinas@arm.com Cc: Will Deacon <will.deacon@arm.com> Cc: linux@armlinux.org.uk Cc: stefan.kristiansson@saunalahti.fi Cc: openrisc@lists.librecores.org Cc: linux-riscv@lists.infradead.org Cc: linux-arm-kernel@lists.infradead.org Link: https://lkml.kernel.org/r/20180307235731.22627-3-palmer@sifive.com
468 lines
12 KiB
ArmAsm
468 lines
12 KiB
ArmAsm
/*
|
|
* Copyright (C) 2012 Regents of the University of California
|
|
* Copyright (C) 2017 SiFive
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation, version 2.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/linkage.h>
|
|
|
|
#include <asm/asm.h>
|
|
#include <asm/csr.h>
|
|
#include <asm/unistd.h>
|
|
#include <asm/thread_info.h>
|
|
#include <asm/asm-offsets.h>
|
|
|
|
.text
|
|
.altmacro
|
|
|
|
/*
|
|
* Prepares to enter a system call or exception by saving all registers to the
|
|
* stack.
|
|
*/
|
|
.macro SAVE_ALL
|
|
LOCAL _restore_kernel_tpsp
|
|
LOCAL _save_context
|
|
|
|
/*
|
|
* If coming from userspace, preserve the user thread pointer and load
|
|
* the kernel thread pointer. If we came from the kernel, sscratch
|
|
* will contain 0, and we should continue on the current TP.
|
|
*/
|
|
csrrw tp, sscratch, tp
|
|
bnez tp, _save_context
|
|
|
|
_restore_kernel_tpsp:
|
|
csrr tp, sscratch
|
|
REG_S sp, TASK_TI_KERNEL_SP(tp)
|
|
_save_context:
|
|
REG_S sp, TASK_TI_USER_SP(tp)
|
|
REG_L sp, TASK_TI_KERNEL_SP(tp)
|
|
addi sp, sp, -(PT_SIZE_ON_STACK)
|
|
REG_S x1, PT_RA(sp)
|
|
REG_S x3, PT_GP(sp)
|
|
REG_S x5, PT_T0(sp)
|
|
REG_S x6, PT_T1(sp)
|
|
REG_S x7, PT_T2(sp)
|
|
REG_S x8, PT_S0(sp)
|
|
REG_S x9, PT_S1(sp)
|
|
REG_S x10, PT_A0(sp)
|
|
REG_S x11, PT_A1(sp)
|
|
REG_S x12, PT_A2(sp)
|
|
REG_S x13, PT_A3(sp)
|
|
REG_S x14, PT_A4(sp)
|
|
REG_S x15, PT_A5(sp)
|
|
REG_S x16, PT_A6(sp)
|
|
REG_S x17, PT_A7(sp)
|
|
REG_S x18, PT_S2(sp)
|
|
REG_S x19, PT_S3(sp)
|
|
REG_S x20, PT_S4(sp)
|
|
REG_S x21, PT_S5(sp)
|
|
REG_S x22, PT_S6(sp)
|
|
REG_S x23, PT_S7(sp)
|
|
REG_S x24, PT_S8(sp)
|
|
REG_S x25, PT_S9(sp)
|
|
REG_S x26, PT_S10(sp)
|
|
REG_S x27, PT_S11(sp)
|
|
REG_S x28, PT_T3(sp)
|
|
REG_S x29, PT_T4(sp)
|
|
REG_S x30, PT_T5(sp)
|
|
REG_S x31, PT_T6(sp)
|
|
|
|
/*
|
|
* Disable user-mode memory access as it should only be set in the
|
|
* actual user copy routines.
|
|
*
|
|
* Disable the FPU to detect illegal usage of floating point in kernel
|
|
* space.
|
|
*/
|
|
li t0, SR_SUM | SR_FS
|
|
|
|
REG_L s0, TASK_TI_USER_SP(tp)
|
|
csrrc s1, sstatus, t0
|
|
csrr s2, sepc
|
|
csrr s3, sbadaddr
|
|
csrr s4, scause
|
|
csrr s5, sscratch
|
|
REG_S s0, PT_SP(sp)
|
|
REG_S s1, PT_SSTATUS(sp)
|
|
REG_S s2, PT_SEPC(sp)
|
|
REG_S s3, PT_SBADADDR(sp)
|
|
REG_S s4, PT_SCAUSE(sp)
|
|
REG_S s5, PT_TP(sp)
|
|
.endm
|
|
|
|
/*
|
|
* Prepares to return from a system call or exception by restoring all
|
|
* registers from the stack.
|
|
*/
|
|
.macro RESTORE_ALL
|
|
REG_L a0, PT_SSTATUS(sp)
|
|
REG_L a2, PT_SEPC(sp)
|
|
csrw sstatus, a0
|
|
csrw sepc, a2
|
|
|
|
REG_L x1, PT_RA(sp)
|
|
REG_L x3, PT_GP(sp)
|
|
REG_L x4, PT_TP(sp)
|
|
REG_L x5, PT_T0(sp)
|
|
REG_L x6, PT_T1(sp)
|
|
REG_L x7, PT_T2(sp)
|
|
REG_L x8, PT_S0(sp)
|
|
REG_L x9, PT_S1(sp)
|
|
REG_L x10, PT_A0(sp)
|
|
REG_L x11, PT_A1(sp)
|
|
REG_L x12, PT_A2(sp)
|
|
REG_L x13, PT_A3(sp)
|
|
REG_L x14, PT_A4(sp)
|
|
REG_L x15, PT_A5(sp)
|
|
REG_L x16, PT_A6(sp)
|
|
REG_L x17, PT_A7(sp)
|
|
REG_L x18, PT_S2(sp)
|
|
REG_L x19, PT_S3(sp)
|
|
REG_L x20, PT_S4(sp)
|
|
REG_L x21, PT_S5(sp)
|
|
REG_L x22, PT_S6(sp)
|
|
REG_L x23, PT_S7(sp)
|
|
REG_L x24, PT_S8(sp)
|
|
REG_L x25, PT_S9(sp)
|
|
REG_L x26, PT_S10(sp)
|
|
REG_L x27, PT_S11(sp)
|
|
REG_L x28, PT_T3(sp)
|
|
REG_L x29, PT_T4(sp)
|
|
REG_L x30, PT_T5(sp)
|
|
REG_L x31, PT_T6(sp)
|
|
|
|
REG_L x2, PT_SP(sp)
|
|
.endm
|
|
|
|
ENTRY(handle_exception)
|
|
SAVE_ALL
|
|
|
|
/*
|
|
* Set sscratch register to 0, so that if a recursive exception
|
|
* occurs, the exception vector knows it came from the kernel
|
|
*/
|
|
csrw sscratch, x0
|
|
|
|
/* Load the global pointer */
|
|
.option push
|
|
.option norelax
|
|
la gp, __global_pointer$
|
|
.option pop
|
|
|
|
la ra, ret_from_exception
|
|
/*
|
|
* MSB of cause differentiates between
|
|
* interrupts and exceptions
|
|
*/
|
|
bge s4, zero, 1f
|
|
|
|
/* Handle interrupts */
|
|
move a0, sp /* pt_regs */
|
|
REG_L a1, handle_arch_irq
|
|
jr a1
|
|
1:
|
|
/* Exceptions run with interrupts enabled */
|
|
csrs sstatus, SR_SIE
|
|
|
|
/* Handle syscalls */
|
|
li t0, EXC_SYSCALL
|
|
beq s4, t0, handle_syscall
|
|
|
|
/* Handle other exceptions */
|
|
slli t0, s4, RISCV_LGPTR
|
|
la t1, excp_vect_table
|
|
la t2, excp_vect_table_end
|
|
move a0, sp /* pt_regs */
|
|
add t0, t1, t0
|
|
/* Check if exception code lies within bounds */
|
|
bgeu t0, t2, 1f
|
|
REG_L t0, 0(t0)
|
|
jr t0
|
|
1:
|
|
tail do_trap_unknown
|
|
|
|
handle_syscall:
|
|
/* save the initial A0 value (needed in signal handlers) */
|
|
REG_S a0, PT_ORIG_A0(sp)
|
|
/*
|
|
* Advance SEPC to avoid executing the original
|
|
* scall instruction on sret
|
|
*/
|
|
addi s2, s2, 0x4
|
|
REG_S s2, PT_SEPC(sp)
|
|
/* Trace syscalls, but only if requested by the user. */
|
|
REG_L t0, TASK_TI_FLAGS(tp)
|
|
andi t0, t0, _TIF_SYSCALL_TRACE
|
|
bnez t0, handle_syscall_trace_enter
|
|
check_syscall_nr:
|
|
/* Check to make sure we don't jump to a bogus syscall number. */
|
|
li t0, __NR_syscalls
|
|
la s0, sys_ni_syscall
|
|
/* Syscall number held in a7 */
|
|
bgeu a7, t0, 1f
|
|
la s0, sys_call_table
|
|
slli t0, a7, RISCV_LGPTR
|
|
add s0, s0, t0
|
|
REG_L s0, 0(s0)
|
|
1:
|
|
jalr s0
|
|
|
|
ret_from_syscall:
|
|
/* Set user a0 to kernel a0 */
|
|
REG_S a0, PT_A0(sp)
|
|
/* Trace syscalls, but only if requested by the user. */
|
|
REG_L t0, TASK_TI_FLAGS(tp)
|
|
andi t0, t0, _TIF_SYSCALL_TRACE
|
|
bnez t0, handle_syscall_trace_exit
|
|
|
|
ret_from_exception:
|
|
REG_L s0, PT_SSTATUS(sp)
|
|
csrc sstatus, SR_SIE
|
|
andi s0, s0, SR_SPP
|
|
bnez s0, restore_all
|
|
|
|
resume_userspace:
|
|
/* Interrupts must be disabled here so flags are checked atomically */
|
|
REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */
|
|
andi s1, s0, _TIF_WORK_MASK
|
|
bnez s1, work_pending
|
|
|
|
/* Save unwound kernel stack pointer in thread_info */
|
|
addi s0, sp, PT_SIZE_ON_STACK
|
|
REG_S s0, TASK_TI_KERNEL_SP(tp)
|
|
|
|
/*
|
|
* Save TP into sscratch, so we can find the kernel data structures
|
|
* again.
|
|
*/
|
|
csrw sscratch, tp
|
|
|
|
restore_all:
|
|
RESTORE_ALL
|
|
sret
|
|
|
|
work_pending:
|
|
/* Enter slow path for supplementary processing */
|
|
la ra, ret_from_exception
|
|
andi s1, s0, _TIF_NEED_RESCHED
|
|
bnez s1, work_resched
|
|
work_notifysig:
|
|
/* Handle pending signals and notify-resume requests */
|
|
csrs sstatus, SR_SIE /* Enable interrupts for do_notify_resume() */
|
|
move a0, sp /* pt_regs */
|
|
move a1, s0 /* current_thread_info->flags */
|
|
tail do_notify_resume
|
|
work_resched:
|
|
tail schedule
|
|
|
|
/* Slow paths for ptrace. */
|
|
handle_syscall_trace_enter:
|
|
move a0, sp
|
|
call do_syscall_trace_enter
|
|
REG_L a0, PT_A0(sp)
|
|
REG_L a1, PT_A1(sp)
|
|
REG_L a2, PT_A2(sp)
|
|
REG_L a3, PT_A3(sp)
|
|
REG_L a4, PT_A4(sp)
|
|
REG_L a5, PT_A5(sp)
|
|
REG_L a6, PT_A6(sp)
|
|
REG_L a7, PT_A7(sp)
|
|
j check_syscall_nr
|
|
handle_syscall_trace_exit:
|
|
move a0, sp
|
|
call do_syscall_trace_exit
|
|
j ret_from_exception
|
|
|
|
END(handle_exception)
|
|
|
|
ENTRY(ret_from_fork)
|
|
la ra, ret_from_exception
|
|
tail schedule_tail
|
|
ENDPROC(ret_from_fork)
|
|
|
|
ENTRY(ret_from_kernel_thread)
|
|
call schedule_tail
|
|
/* Call fn(arg) */
|
|
la ra, ret_from_exception
|
|
move a0, s1
|
|
jr s0
|
|
ENDPROC(ret_from_kernel_thread)
|
|
|
|
|
|
/*
|
|
* Integer register context switch
|
|
* The callee-saved registers must be saved and restored.
|
|
*
|
|
* a0: previous task_struct (must be preserved across the switch)
|
|
* a1: next task_struct
|
|
*
|
|
* The value of a0 and a1 must be preserved by this function, as that's how
|
|
* arguments are passed to schedule_tail.
|
|
*/
|
|
ENTRY(__switch_to)
|
|
/* Save context into prev->thread */
|
|
li a4, TASK_THREAD_RA
|
|
add a3, a0, a4
|
|
add a4, a1, a4
|
|
REG_S ra, TASK_THREAD_RA_RA(a3)
|
|
REG_S sp, TASK_THREAD_SP_RA(a3)
|
|
REG_S s0, TASK_THREAD_S0_RA(a3)
|
|
REG_S s1, TASK_THREAD_S1_RA(a3)
|
|
REG_S s2, TASK_THREAD_S2_RA(a3)
|
|
REG_S s3, TASK_THREAD_S3_RA(a3)
|
|
REG_S s4, TASK_THREAD_S4_RA(a3)
|
|
REG_S s5, TASK_THREAD_S5_RA(a3)
|
|
REG_S s6, TASK_THREAD_S6_RA(a3)
|
|
REG_S s7, TASK_THREAD_S7_RA(a3)
|
|
REG_S s8, TASK_THREAD_S8_RA(a3)
|
|
REG_S s9, TASK_THREAD_S9_RA(a3)
|
|
REG_S s10, TASK_THREAD_S10_RA(a3)
|
|
REG_S s11, TASK_THREAD_S11_RA(a3)
|
|
/* Restore context from next->thread */
|
|
REG_L ra, TASK_THREAD_RA_RA(a4)
|
|
REG_L sp, TASK_THREAD_SP_RA(a4)
|
|
REG_L s0, TASK_THREAD_S0_RA(a4)
|
|
REG_L s1, TASK_THREAD_S1_RA(a4)
|
|
REG_L s2, TASK_THREAD_S2_RA(a4)
|
|
REG_L s3, TASK_THREAD_S3_RA(a4)
|
|
REG_L s4, TASK_THREAD_S4_RA(a4)
|
|
REG_L s5, TASK_THREAD_S5_RA(a4)
|
|
REG_L s6, TASK_THREAD_S6_RA(a4)
|
|
REG_L s7, TASK_THREAD_S7_RA(a4)
|
|
REG_L s8, TASK_THREAD_S8_RA(a4)
|
|
REG_L s9, TASK_THREAD_S9_RA(a4)
|
|
REG_L s10, TASK_THREAD_S10_RA(a4)
|
|
REG_L s11, TASK_THREAD_S11_RA(a4)
|
|
/* Swap the CPU entry around. */
|
|
lw a3, TASK_TI_CPU(a0)
|
|
lw a4, TASK_TI_CPU(a1)
|
|
sw a3, TASK_TI_CPU(a1)
|
|
sw a4, TASK_TI_CPU(a0)
|
|
#if TASK_TI != 0
|
|
#error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work."
|
|
addi tp, a1, TASK_TI
|
|
#else
|
|
move tp, a1
|
|
#endif
|
|
ret
|
|
ENDPROC(__switch_to)
|
|
|
|
ENTRY(__fstate_save)
|
|
li a2, TASK_THREAD_F0
|
|
add a0, a0, a2
|
|
li t1, SR_FS
|
|
csrs sstatus, t1
|
|
frcsr t0
|
|
fsd f0, TASK_THREAD_F0_F0(a0)
|
|
fsd f1, TASK_THREAD_F1_F0(a0)
|
|
fsd f2, TASK_THREAD_F2_F0(a0)
|
|
fsd f3, TASK_THREAD_F3_F0(a0)
|
|
fsd f4, TASK_THREAD_F4_F0(a0)
|
|
fsd f5, TASK_THREAD_F5_F0(a0)
|
|
fsd f6, TASK_THREAD_F6_F0(a0)
|
|
fsd f7, TASK_THREAD_F7_F0(a0)
|
|
fsd f8, TASK_THREAD_F8_F0(a0)
|
|
fsd f9, TASK_THREAD_F9_F0(a0)
|
|
fsd f10, TASK_THREAD_F10_F0(a0)
|
|
fsd f11, TASK_THREAD_F11_F0(a0)
|
|
fsd f12, TASK_THREAD_F12_F0(a0)
|
|
fsd f13, TASK_THREAD_F13_F0(a0)
|
|
fsd f14, TASK_THREAD_F14_F0(a0)
|
|
fsd f15, TASK_THREAD_F15_F0(a0)
|
|
fsd f16, TASK_THREAD_F16_F0(a0)
|
|
fsd f17, TASK_THREAD_F17_F0(a0)
|
|
fsd f18, TASK_THREAD_F18_F0(a0)
|
|
fsd f19, TASK_THREAD_F19_F0(a0)
|
|
fsd f20, TASK_THREAD_F20_F0(a0)
|
|
fsd f21, TASK_THREAD_F21_F0(a0)
|
|
fsd f22, TASK_THREAD_F22_F0(a0)
|
|
fsd f23, TASK_THREAD_F23_F0(a0)
|
|
fsd f24, TASK_THREAD_F24_F0(a0)
|
|
fsd f25, TASK_THREAD_F25_F0(a0)
|
|
fsd f26, TASK_THREAD_F26_F0(a0)
|
|
fsd f27, TASK_THREAD_F27_F0(a0)
|
|
fsd f28, TASK_THREAD_F28_F0(a0)
|
|
fsd f29, TASK_THREAD_F29_F0(a0)
|
|
fsd f30, TASK_THREAD_F30_F0(a0)
|
|
fsd f31, TASK_THREAD_F31_F0(a0)
|
|
sw t0, TASK_THREAD_FCSR_F0(a0)
|
|
csrc sstatus, t1
|
|
ret
|
|
ENDPROC(__fstate_save)
|
|
|
|
ENTRY(__fstate_restore)
|
|
li a2, TASK_THREAD_F0
|
|
add a0, a0, a2
|
|
li t1, SR_FS
|
|
lw t0, TASK_THREAD_FCSR_F0(a0)
|
|
csrs sstatus, t1
|
|
fld f0, TASK_THREAD_F0_F0(a0)
|
|
fld f1, TASK_THREAD_F1_F0(a0)
|
|
fld f2, TASK_THREAD_F2_F0(a0)
|
|
fld f3, TASK_THREAD_F3_F0(a0)
|
|
fld f4, TASK_THREAD_F4_F0(a0)
|
|
fld f5, TASK_THREAD_F5_F0(a0)
|
|
fld f6, TASK_THREAD_F6_F0(a0)
|
|
fld f7, TASK_THREAD_F7_F0(a0)
|
|
fld f8, TASK_THREAD_F8_F0(a0)
|
|
fld f9, TASK_THREAD_F9_F0(a0)
|
|
fld f10, TASK_THREAD_F10_F0(a0)
|
|
fld f11, TASK_THREAD_F11_F0(a0)
|
|
fld f12, TASK_THREAD_F12_F0(a0)
|
|
fld f13, TASK_THREAD_F13_F0(a0)
|
|
fld f14, TASK_THREAD_F14_F0(a0)
|
|
fld f15, TASK_THREAD_F15_F0(a0)
|
|
fld f16, TASK_THREAD_F16_F0(a0)
|
|
fld f17, TASK_THREAD_F17_F0(a0)
|
|
fld f18, TASK_THREAD_F18_F0(a0)
|
|
fld f19, TASK_THREAD_F19_F0(a0)
|
|
fld f20, TASK_THREAD_F20_F0(a0)
|
|
fld f21, TASK_THREAD_F21_F0(a0)
|
|
fld f22, TASK_THREAD_F22_F0(a0)
|
|
fld f23, TASK_THREAD_F23_F0(a0)
|
|
fld f24, TASK_THREAD_F24_F0(a0)
|
|
fld f25, TASK_THREAD_F25_F0(a0)
|
|
fld f26, TASK_THREAD_F26_F0(a0)
|
|
fld f27, TASK_THREAD_F27_F0(a0)
|
|
fld f28, TASK_THREAD_F28_F0(a0)
|
|
fld f29, TASK_THREAD_F29_F0(a0)
|
|
fld f30, TASK_THREAD_F30_F0(a0)
|
|
fld f31, TASK_THREAD_F31_F0(a0)
|
|
fscsr t0
|
|
csrc sstatus, t1
|
|
ret
|
|
ENDPROC(__fstate_restore)
|
|
|
|
|
|
.section ".rodata"
|
|
/* Exception vector table */
|
|
ENTRY(excp_vect_table)
|
|
RISCV_PTR do_trap_insn_misaligned
|
|
RISCV_PTR do_trap_insn_fault
|
|
RISCV_PTR do_trap_insn_illegal
|
|
RISCV_PTR do_trap_break
|
|
RISCV_PTR do_trap_load_misaligned
|
|
RISCV_PTR do_trap_load_fault
|
|
RISCV_PTR do_trap_store_misaligned
|
|
RISCV_PTR do_trap_store_fault
|
|
RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */
|
|
RISCV_PTR do_trap_ecall_s
|
|
RISCV_PTR do_trap_unknown
|
|
RISCV_PTR do_trap_ecall_m
|
|
RISCV_PTR do_page_fault /* instruction page fault */
|
|
RISCV_PTR do_page_fault /* load page fault */
|
|
RISCV_PTR do_trap_unknown
|
|
RISCV_PTR do_page_fault /* store page fault */
|
|
excp_vect_table_end:
|
|
END(excp_vect_table)
|