mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-19 22:58:02 +07:00
0aaba41b58
The vdso code for the getcpu() and the clock_gettime() call use the access register mode to access the per-CPU vdso data page with the current code. An alternative to the complicated AR mode is to use the secondary space mode. This makes the vdso faster and quite a bit simpler. The downside is that the uaccess code has to be changed quite a bit. Which instructions are used depends on the machine and what kind of uaccess operation is requested. The instruction dictates which ASCE value needs to be loaded into %cr1 and %cr7. The different cases: * User copy with MVCOS for z10 and newer machines The MVCOS instruction can copy between the primary space (aka user) and the home space (aka kernel) directly. For set_fs(KERNEL_DS) the kernel ASCE is loaded into %cr1. For set_fs(USER_DS) the user space is already loaded in %cr1. * User copy with MVCP/MVCS for older machines To be able to execute the MVCP/MVCS instructions the kernel needs to switch to primary mode. The control register %cr1 has to be set to the kernel ASCE and %cr7 to either the kernel ASCE or the user ASCE dependent on set_fs(KERNEL_DS) vs set_fs(USER_DS). * Data access in the user address space for strnlen / futex To use "normal" instruction with data from the user address space the secondary space mode is used. The kernel needs to switch to primary mode, %cr1 has to contain the kernel ASCE and %cr7 either the user ASCE or the kernel ASCE, dependent on set_fs. To load a new value into %cr1 or %cr7 is an expensive operation, the kernel tries to be lazy about it. E.g. for multiple user copies in a row with MVCP/MVCS the replacement of the vdso ASCE in %cr7 with the user ASCE is done only once. On return to user space a CPU bit is checked that loads the vdso ASCE again. To enable and disable the data access via the secondary space two new functions are added, enable_sacf_uaccess and disable_sacf_uaccess. The fact that a context is in secondary space uaccess mode is stored in the mm_segment_t value for the task. The code of an interrupt may use set_fs as long as it returns to the previous state it got with get_fs with another call to set_fs. The code in finish_arch_post_lock_switch simply has to do a set_fs with the current mm_segment_t value for the task. For CPUs with MVCOS: CPU running in | %cr1 ASCE | %cr7 ASCE | --------------------------------------|-----------|-----------| user space | user | vdso | kernel, USER_DS, normal-mode | user | vdso | kernel, USER_DS, normal-mode, lazy | user | user | kernel, USER_DS, sacf-mode | kernel | user | kernel, KERNEL_DS, normal-mode | kernel | vdso | kernel, KERNEL_DS, normal-mode, lazy | kernel | kernel | kernel, KERNEL_DS, sacf-mode | kernel | kernel | For CPUs without MVCOS: CPU running in | %cr1 ASCE | %cr7 ASCE | --------------------------------------|-----------|-----------| user space | user | vdso | kernel, USER_DS, normal-mode | user | vdso | kernel, USER_DS, normal-mode lazy | kernel | user | kernel, USER_DS, sacf-mode | kernel | user | kernel, KERNEL_DS, normal-mode | kernel | vdso | kernel, KERNEL_DS, normal-mode, lazy | kernel | kernel | kernel, KERNEL_DS, sacf-mode | kernel | kernel | The lines with "lazy" refer to the state after a copy via the secondary space with a delayed reload of %cr1 and %cr7. There are three hardware address spaces that can cause a DAT exception, primary, secondary and home space. The exception can be related to four different fault types: user space fault, vdso fault, kernel fault, and the gmap faults. Dependent on the set_fs state and normal vs. sacf mode there are a number of fault combinations: 1) user address space fault via the primary ASCE 2) gmap address space fault via the primary ASCE 3) kernel address space fault via the primary ASCE for machines with MVCOS and set_fs(KERNEL_DS) 4) vdso address space faults via the secondary ASCE with an invalid address while running in secondary space in problem state 5) user address space fault via the secondary ASCE for user-copy based on the secondary space mode, e.g. futex_ops or strnlen_user 6) kernel address space fault via the secondary ASCE for user-copy with secondary space mode with set_fs(KERNEL_DS) 7) kernel address space fault via the primary ASCE for user-copy with secondary space mode with set_fs(USER_DS) on machines without MVCOS. 8) kernel address space fault via the home space ASCE Replace user_space_fault() with a new function get_fault_type() that can distinguish all four different fault types. With these changes the futex atomic ops from the kernel and the strnlen_user will get a little bit slower, as well as the old style uaccess with MVCP/MVCS. All user accesses based on MVCOS will be as fast as before. On the positive side, the user space vdso code is a lot faster and Linux ceases to use the complicated AR mode. Reviewed-by: Heiko Carstens <heiko.carstens@de.ibm.com> Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com> Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
111 lines
3.2 KiB
ArmAsm
111 lines
3.2 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Copyright IBM Corp. 1999, 2010
|
|
*
|
|
* Author(s): Hartmut Penner <hp@de.ibm.com>
|
|
* Martin Schwidefsky <schwidefsky@de.ibm.com>
|
|
* Rob van der Heij <rvdhei@iae.nl>
|
|
* Heiko Carstens <heiko.carstens@de.ibm.com>
|
|
*
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/linkage.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/thread_info.h>
|
|
#include <asm/page.h>
|
|
|
|
__HEAD
|
|
ENTRY(startup_continue)
|
|
tm __LC_STFLE_FAC_LIST+5,0x80 # LPP available ?
|
|
jz 0f
|
|
xc __LC_LPP+1(7,0),__LC_LPP+1 # clear lpp and current_pid
|
|
mvi __LC_LPP,0x80 # and set LPP_MAGIC
|
|
.insn s,0xb2800000,__LC_LPP # load program parameter
|
|
0: larl %r1,tod_clock_base
|
|
mvc 0(16,%r1),__LC_BOOT_CLOCK
|
|
larl %r13,.LPG1 # get base
|
|
lctlg %c0,%c15,.Lctl-.LPG1(%r13) # load control registers
|
|
lg %r12,.Lparmaddr-.LPG1(%r13) # pointer to parameter area
|
|
# move IPL device to lowcore
|
|
larl %r0,boot_vdso_data
|
|
stg %r0,__LC_VDSO_PER_CPU
|
|
#
|
|
# Setup stack
|
|
#
|
|
larl %r14,init_task
|
|
stg %r14,__LC_CURRENT
|
|
larl %r15,init_thread_union
|
|
aghi %r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER) # init_task_union + THREAD_SIZE
|
|
stg %r15,__LC_KERNEL_STACK # set end of kernel stack
|
|
aghi %r15,-160
|
|
#
|
|
# Save ipl parameters, clear bss memory, initialize storage key for kernel pages,
|
|
# and create a kernel NSS if the SAVESYS= parm is defined
|
|
#
|
|
brasl %r14,startup_init
|
|
lpswe .Lentry-.LPG1(13) # jump to _stext in primary-space,
|
|
# virtual and never return ...
|
|
.align 16
|
|
.LPG1:
|
|
.Lentry:.quad 0x0000000180000000,_stext
|
|
.Lctl: .quad 0x04040000 # cr0: AFP registers & secondary space
|
|
.quad 0 # cr1: primary space segment table
|
|
.quad .Lduct # cr2: dispatchable unit control table
|
|
.quad 0 # cr3: instruction authorization
|
|
.quad 0xffff # cr4: instruction authorization
|
|
.quad .Lduct # cr5: primary-aste origin
|
|
.quad 0 # cr6: I/O interrupts
|
|
.quad 0 # cr7: secondary space segment table
|
|
.quad 0 # cr8: access registers translation
|
|
.quad 0 # cr9: tracing off
|
|
.quad 0 # cr10: tracing off
|
|
.quad 0 # cr11: tracing off
|
|
.quad 0 # cr12: tracing off
|
|
.quad 0 # cr13: home space segment table
|
|
.quad 0xc0000000 # cr14: machine check handling off
|
|
.quad .Llinkage_stack # cr15: linkage stack operations
|
|
.Lpcmsk:.quad 0x0000000180000000
|
|
.L4malign:.quad 0xffffffffffc00000
|
|
.Lscan2g:.quad 0x80000000 + 0x20000 - 8 # 2GB + 128K - 8
|
|
.Lnop: .long 0x07000700
|
|
.Lparmaddr:
|
|
.quad PARMAREA
|
|
.align 64
|
|
.Lduct: .long 0,.Laste,.Laste,0,.Lduald,0,0,0
|
|
.long 0,0,0,0,0,0,0,0
|
|
.Laste: .quad 0,0xffffffffffffffff,0,0,0,0,0,0
|
|
.align 128
|
|
.Lduald:.rept 8
|
|
.long 0x80000000,0,0,0 # invalid access-list entries
|
|
.endr
|
|
.Llinkage_stack:
|
|
.long 0,0,0x89000000,0,0,0,0x8a000000,0
|
|
|
|
ENTRY(_ehead)
|
|
|
|
.org 0x100000 - 0x11000 # head.o ends at 0x11000
|
|
#
|
|
# startup-code, running in absolute addressing mode
|
|
#
|
|
ENTRY(_stext)
|
|
basr %r13,0 # get base
|
|
.LPG3:
|
|
# check control registers
|
|
stctg %c0,%c15,0(%r15)
|
|
oi 6(%r15),0x60 # enable sigp emergency & external call
|
|
oi 4(%r15),0x10 # switch on low address proctection
|
|
lctlg %c0,%c15,0(%r15)
|
|
|
|
lam 0,15,.Laregs-.LPG3(%r13) # load acrs needed by uaccess
|
|
brasl %r14,start_kernel # go to C code
|
|
#
|
|
# We returned from start_kernel ?!? PANIK
|
|
#
|
|
basr %r13,0
|
|
lpswe .Ldw-.(%r13) # load disabled wait psw
|
|
|
|
.align 8
|
|
.Ldw: .quad 0x0002000180000000,0x0000000000000000
|
|
.Laregs:.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|