2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* Based on arch/arm/mm/fault.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1995 Linus Torvalds
|
|
|
|
* Copyright (C) 1995-2004 Russell King
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
2016-09-20 04:38:55 +07:00
|
|
|
#include <linux/extable.h>
|
2012-03-05 18:49:27 +07:00
|
|
|
#include <linux/signal.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/kprobes.h>
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <linux/page-flags.h>
|
2017-02-09 00:51:30 +07:00
|
|
|
#include <linux/sched/signal.h>
|
2017-02-09 00:51:35 +07:00
|
|
|
#include <linux/sched/debug.h>
|
2012-03-05 18:49:27 +07:00
|
|
|
#include <linux/highmem.h>
|
|
|
|
#include <linux/perf_event.h>
|
2016-10-18 17:27:47 +07:00
|
|
|
#include <linux/preempt.h>
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2016-10-18 17:27:47 +07:00
|
|
|
#include <asm/bug.h>
|
2015-07-23 01:05:54 +07:00
|
|
|
#include <asm/cpufeature.h>
|
2012-03-05 18:49:27 +07:00
|
|
|
#include <asm/exception.h>
|
|
|
|
#include <asm/debug-monitors.h>
|
2014-04-07 05:04:12 +07:00
|
|
|
#include <asm/esr.h>
|
2015-07-23 01:05:54 +07:00
|
|
|
#include <asm/sysreg.h>
|
2012-03-05 18:49:27 +07:00
|
|
|
#include <asm/system_misc.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
2017-04-04 12:51:01 +07:00
|
|
|
struct fault_info {
|
|
|
|
int (*fn)(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs);
|
|
|
|
int sig;
|
|
|
|
int code;
|
|
|
|
const char *name;
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct fault_info fault_info[];
|
|
|
|
|
|
|
|
static inline const struct fault_info *esr_to_fault_info(unsigned int esr)
|
|
|
|
{
|
|
|
|
return fault_info + (esr & 63);
|
|
|
|
}
|
2012-10-24 22:34:02 +07:00
|
|
|
|
arm64: Kprobes with single stepping support
Add support for basic kernel probes(kprobes) and jump probes
(jprobes) for ARM64.
Kprobes utilizes software breakpoint and single step debug
exceptions supported on ARM v8.
A software breakpoint is placed at the probe address to trap the
kernel execution into the kprobe handler.
ARM v8 supports enabling single stepping before the break exception
return (ERET), with next PC in exception return address (ELR_EL1). The
kprobe handler prepares an executable memory slot for out-of-line
execution with a copy of the original instruction being probed, and
enables single stepping. The PC is set to the out-of-line slot address
before the ERET. With this scheme, the instruction is executed with the
exact same register context except for the PC (and DAIF) registers.
Debug mask (PSTATE.D) is enabled only when single stepping a recursive
kprobe, e.g.: during kprobes reenter so that probed instruction can be
single stepped within the kprobe handler -exception- context.
The recursion depth of kprobe is always 2, i.e. upon probe re-entry,
any further re-entry is prevented by not calling handlers and the case
counted as a missed kprobe).
Single stepping from the x-o-l slot has a drawback for PC-relative accesses
like branching and symbolic literals access as the offset from the new PC
(slot address) may not be ensured to fit in the immediate value of
the opcode. Such instructions need simulation, so reject
probing them.
Instructions generating exceptions or cpu mode change are rejected
for probing.
Exclusive load/store instructions are rejected too. Additionally, the
code is checked to see if it is inside an exclusive load/store sequence
(code from Pratyush).
System instructions are mostly enabled for stepping, except MSR/MRS
accesses to "DAIF" flags in PSTATE, which are not safe for
probing.
This also changes arch/arm64/include/asm/ptrace.h to use
include/asm-generic/ptrace.h.
Thanks to Steve Capper and Pratyush Anand for several suggested
Changes.
Signed-off-by: Sandeepa Prabhu <sandeepa.s.prabhu@gmail.com>
Signed-off-by: David A. Long <dave.long@linaro.org>
Signed-off-by: Pratyush Anand <panand@redhat.com>
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-08 23:35:48 +07:00
|
|
|
#ifdef CONFIG_KPROBES
|
|
|
|
static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
|
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
/* kprobe_running() needs smp_processor_id() */
|
|
|
|
if (!user_mode(regs)) {
|
|
|
|
preempt_disable();
|
|
|
|
if (kprobe_running() && kprobe_fault_handler(regs, esr))
|
|
|
|
ret = 1;
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* Dump out the page tables associated with 'addr' in mm 'mm'.
|
|
|
|
*/
|
|
|
|
void show_pte(struct mm_struct *mm, unsigned long addr)
|
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
|
|
|
|
if (!mm)
|
|
|
|
mm = &init_mm;
|
|
|
|
|
|
|
|
pr_alert("pgd = %p\n", mm->pgd);
|
|
|
|
pgd = pgd_offset(mm, addr);
|
|
|
|
pr_alert("[%08lx] *pgd=%016llx", addr, pgd_val(*pgd));
|
|
|
|
|
|
|
|
do {
|
|
|
|
pud_t *pud;
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte;
|
|
|
|
|
2013-04-19 21:49:31 +07:00
|
|
|
if (pgd_none(*pgd) || pgd_bad(*pgd))
|
2012-03-05 18:49:27 +07:00
|
|
|
break;
|
|
|
|
|
|
|
|
pud = pud_offset(pgd, addr);
|
2017-01-03 21:27:26 +07:00
|
|
|
pr_cont(", *pud=%016llx", pud_val(*pud));
|
2013-04-19 21:49:31 +07:00
|
|
|
if (pud_none(*pud) || pud_bad(*pud))
|
2012-03-05 18:49:27 +07:00
|
|
|
break;
|
|
|
|
|
|
|
|
pmd = pmd_offset(pud, addr);
|
2017-01-03 21:27:26 +07:00
|
|
|
pr_cont(", *pmd=%016llx", pmd_val(*pmd));
|
2013-04-19 21:49:31 +07:00
|
|
|
if (pmd_none(*pmd) || pmd_bad(*pmd))
|
2012-03-05 18:49:27 +07:00
|
|
|
break;
|
|
|
|
|
|
|
|
pte = pte_offset_map(pmd, addr);
|
2017-01-03 21:27:26 +07:00
|
|
|
pr_cont(", *pte=%016llx", pte_val(*pte));
|
2012-03-05 18:49:27 +07:00
|
|
|
pte_unmap(pte);
|
|
|
|
} while(0);
|
|
|
|
|
2017-01-03 21:27:26 +07:00
|
|
|
pr_cont("\n");
|
2012-03-05 18:49:27 +07:00
|
|
|
}
|
|
|
|
|
2016-04-13 22:01:22 +07:00
|
|
|
#ifdef CONFIG_ARM64_HW_AFDBM
|
|
|
|
/*
|
|
|
|
* This function sets the access flags (dirty, accessed), as well as write
|
|
|
|
* permission, and only to a more permissive setting.
|
|
|
|
*
|
|
|
|
* It needs to cope with hardware update of the accessed/dirty state by other
|
|
|
|
* agents in the system and can safely skip the __sync_icache_dcache() call as,
|
|
|
|
* like set_pte_at(), the PTE is never changed from no-exec to exec here.
|
|
|
|
*
|
|
|
|
* Returns whether or not the PTE actually changed.
|
|
|
|
*/
|
|
|
|
int ptep_set_access_flags(struct vm_area_struct *vma,
|
|
|
|
unsigned long address, pte_t *ptep,
|
|
|
|
pte_t entry, int dirty)
|
|
|
|
{
|
|
|
|
pteval_t old_pteval;
|
|
|
|
unsigned int tmp;
|
|
|
|
|
|
|
|
if (pte_same(*ptep, entry))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* only preserve the access flags and write permission */
|
|
|
|
pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* PTE_RDONLY is cleared by default in the asm below, so set it in
|
|
|
|
* back if necessary (read-only or clean PTE).
|
|
|
|
*/
|
2016-06-07 23:55:15 +07:00
|
|
|
if (!pte_write(entry) || !pte_sw_dirty(entry))
|
2016-04-13 22:01:22 +07:00
|
|
|
pte_val(entry) |= PTE_RDONLY;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Setting the flags must be done atomically to avoid racing with the
|
|
|
|
* hardware update of the access/dirty state.
|
|
|
|
*/
|
|
|
|
asm volatile("// ptep_set_access_flags\n"
|
|
|
|
" prfm pstl1strm, %2\n"
|
|
|
|
"1: ldxr %0, %2\n"
|
|
|
|
" and %0, %0, %3 // clear PTE_RDONLY\n"
|
|
|
|
" orr %0, %0, %4 // set flags\n"
|
|
|
|
" stxr %w1, %0, %2\n"
|
|
|
|
" cbnz %w1, 1b\n"
|
|
|
|
: "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
|
|
|
|
: "L" (~PTE_RDONLY), "r" (pte_val(entry)));
|
|
|
|
|
|
|
|
flush_tlb_fix_spurious_fault(vma, address);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2016-08-10 08:25:26 +07:00
|
|
|
static bool is_el1_instruction_abort(unsigned int esr)
|
|
|
|
{
|
|
|
|
return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
|
|
|
|
}
|
|
|
|
|
2017-04-06 02:18:31 +07:00
|
|
|
static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs,
|
|
|
|
unsigned long addr)
|
|
|
|
{
|
|
|
|
unsigned int ec = ESR_ELx_EC(esr);
|
|
|
|
unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
|
|
|
|
|
|
|
|
if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (fsc_type == ESR_ELx_FSC_PERM)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (addr < USER_DS && system_uses_ttbr0_pan())
|
|
|
|
return fsc_type == ESR_ELx_FSC_FAULT &&
|
|
|
|
(regs->pstate & PSR_PAN_BIT);
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* The kernel tried to access some page that wasn't present.
|
|
|
|
*/
|
|
|
|
static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
|
|
|
|
unsigned int esr, struct pt_regs *regs)
|
|
|
|
{
|
2017-04-06 02:18:31 +07:00
|
|
|
const char *msg;
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* Are we prepared to handle this kernel fault?
|
2016-08-10 08:25:26 +07:00
|
|
|
* We are almost certainly not prepared to handle instruction faults.
|
2012-03-05 18:49:27 +07:00
|
|
|
*/
|
2016-08-10 08:25:26 +07:00
|
|
|
if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
|
2012-03-05 18:49:27 +07:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No handler, we'll have to terminate things with extreme prejudice.
|
|
|
|
*/
|
|
|
|
bust_spinlocks(1);
|
2017-04-06 02:18:31 +07:00
|
|
|
|
|
|
|
if (is_permission_fault(esr, regs, addr)) {
|
|
|
|
if (esr & ESR_ELx_WNR)
|
|
|
|
msg = "write to read-only memory";
|
|
|
|
else
|
|
|
|
msg = "read from unreadable memory";
|
|
|
|
} else if (addr < PAGE_SIZE) {
|
|
|
|
msg = "NULL pointer dereference";
|
|
|
|
} else {
|
|
|
|
msg = "paging request";
|
|
|
|
}
|
|
|
|
|
|
|
|
pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg,
|
|
|
|
addr);
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
show_pte(mm, addr);
|
|
|
|
die("Oops", regs, esr);
|
|
|
|
bust_spinlocks(0);
|
|
|
|
do_exit(SIGKILL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Something tried to access memory that isn't in our memory map. User mode
|
|
|
|
* accesses just cause a SIGSEGV
|
|
|
|
*/
|
|
|
|
static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
|
|
|
|
unsigned int esr, unsigned int sig, int code,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct siginfo si;
|
2017-04-04 12:51:01 +07:00
|
|
|
const struct fault_info *inf;
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2015-07-03 21:08:08 +07:00
|
|
|
if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) {
|
2017-04-04 12:51:01 +07:00
|
|
|
inf = esr_to_fault_info(esr);
|
2012-10-24 22:34:02 +07:00
|
|
|
pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n",
|
2017-04-04 12:51:01 +07:00
|
|
|
tsk->comm, task_pid_nr(tsk), inf->name, sig,
|
2012-10-24 22:34:02 +07:00
|
|
|
addr, esr);
|
2012-03-05 18:49:27 +07:00
|
|
|
show_pte(tsk->mm, addr);
|
|
|
|
show_regs(regs);
|
|
|
|
}
|
|
|
|
|
|
|
|
tsk->thread.fault_address = addr;
|
2014-04-07 05:04:12 +07:00
|
|
|
tsk->thread.fault_code = esr;
|
2012-03-05 18:49:27 +07:00
|
|
|
si.si_signo = sig;
|
|
|
|
si.si_errno = 0;
|
|
|
|
si.si_code = code;
|
|
|
|
si.si_addr = (void __user *)addr;
|
|
|
|
force_sig_info(sig, &si, tsk);
|
|
|
|
}
|
|
|
|
|
2013-09-16 21:18:28 +07:00
|
|
|
static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
|
2012-03-05 18:49:27 +07:00
|
|
|
{
|
|
|
|
struct task_struct *tsk = current;
|
|
|
|
struct mm_struct *mm = tsk->active_mm;
|
2017-04-04 12:51:01 +07:00
|
|
|
const struct fault_info *inf;
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we are in kernel mode at this point, we have no context to
|
|
|
|
* handle this fault with.
|
|
|
|
*/
|
2017-04-04 12:51:01 +07:00
|
|
|
if (user_mode(regs)) {
|
|
|
|
inf = esr_to_fault_info(esr);
|
|
|
|
__do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs);
|
|
|
|
} else
|
2012-03-05 18:49:27 +07:00
|
|
|
__do_kernel_fault(mm, addr, esr, regs);
|
|
|
|
}
|
|
|
|
|
|
|
|
#define VM_FAULT_BADMAP 0x010000
|
|
|
|
#define VM_FAULT_BADACCESS 0x020000
|
|
|
|
|
|
|
|
static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
|
2013-07-19 21:37:12 +07:00
|
|
|
unsigned int mm_flags, unsigned long vm_flags,
|
2012-03-05 18:49:27 +07:00
|
|
|
struct task_struct *tsk)
|
|
|
|
{
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
int fault;
|
|
|
|
|
|
|
|
vma = find_vma(mm, addr);
|
|
|
|
fault = VM_FAULT_BADMAP;
|
|
|
|
if (unlikely(!vma))
|
|
|
|
goto out;
|
|
|
|
if (unlikely(vma->vm_start > addr))
|
|
|
|
goto check_stack;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ok, we have a good vm_area for this memory access, so we can handle
|
|
|
|
* it.
|
|
|
|
*/
|
|
|
|
good_area:
|
2013-07-19 21:37:12 +07:00
|
|
|
/*
|
|
|
|
* Check that the permissions on the VMA allow for the fault which
|
2016-08-12 00:44:50 +07:00
|
|
|
* occurred.
|
2013-07-19 21:37:12 +07:00
|
|
|
*/
|
|
|
|
if (!(vma->vm_flags & vm_flags)) {
|
2012-03-05 18:49:27 +07:00
|
|
|
fault = VM_FAULT_BADACCESS;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2016-07-27 05:25:18 +07:00
|
|
|
return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags);
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
check_stack:
|
|
|
|
if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
|
|
|
|
goto good_area;
|
|
|
|
out:
|
|
|
|
return fault;
|
|
|
|
}
|
|
|
|
|
arm64: kill ESR_LNX_EXEC
Currently we treat ESR_EL1 bit 24 as software-defined for distinguishing
instruction aborts from data aborts, but this bit is architecturally
RES0 for instruction aborts, and could be allocated for an arbitrary
purpose in future. Additionally, we hard-code the value in entry.S
without the mnemonic, making the code difficult to understand.
Instead, remove ESR_LNX_EXEC, and distinguish aborts based on the esr,
which we already pass to the sole use of ESR_LNX_EXEC. A new helper,
is_el0_instruction_abort() is added to make the logic clear. Any
instruction aborts taken from EL1 will already have been handled by
bad_mode, so we need not handle that case in the helper.
For consistency, the existing permission_fault helper is renamed to
is_permission_fault, and the return type is changed to bool. There
should be no functional changes as the return value was a boolean
expression, and the result is only used in another boolean expression.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Dave P Martin <dave.martin@arm.com>
Cc: Huang Shijie <shijie.huang@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-05-31 18:33:03 +07:00
|
|
|
static bool is_el0_instruction_abort(unsigned int esr)
|
|
|
|
{
|
|
|
|
return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
|
|
|
|
}
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct task_struct *tsk;
|
|
|
|
struct mm_struct *mm;
|
|
|
|
int fault, sig, code;
|
2016-08-12 00:44:50 +07:00
|
|
|
unsigned long vm_flags = VM_READ | VM_WRITE;
|
2013-07-19 21:37:12 +07:00
|
|
|
unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
|
|
|
|
|
arm64: Kprobes with single stepping support
Add support for basic kernel probes(kprobes) and jump probes
(jprobes) for ARM64.
Kprobes utilizes software breakpoint and single step debug
exceptions supported on ARM v8.
A software breakpoint is placed at the probe address to trap the
kernel execution into the kprobe handler.
ARM v8 supports enabling single stepping before the break exception
return (ERET), with next PC in exception return address (ELR_EL1). The
kprobe handler prepares an executable memory slot for out-of-line
execution with a copy of the original instruction being probed, and
enables single stepping. The PC is set to the out-of-line slot address
before the ERET. With this scheme, the instruction is executed with the
exact same register context except for the PC (and DAIF) registers.
Debug mask (PSTATE.D) is enabled only when single stepping a recursive
kprobe, e.g.: during kprobes reenter so that probed instruction can be
single stepped within the kprobe handler -exception- context.
The recursion depth of kprobe is always 2, i.e. upon probe re-entry,
any further re-entry is prevented by not calling handlers and the case
counted as a missed kprobe).
Single stepping from the x-o-l slot has a drawback for PC-relative accesses
like branching and symbolic literals access as the offset from the new PC
(slot address) may not be ensured to fit in the immediate value of
the opcode. Such instructions need simulation, so reject
probing them.
Instructions generating exceptions or cpu mode change are rejected
for probing.
Exclusive load/store instructions are rejected too. Additionally, the
code is checked to see if it is inside an exclusive load/store sequence
(code from Pratyush).
System instructions are mostly enabled for stepping, except MSR/MRS
accesses to "DAIF" flags in PSTATE, which are not safe for
probing.
This also changes arch/arm64/include/asm/ptrace.h to use
include/asm-generic/ptrace.h.
Thanks to Steve Capper and Pratyush Anand for several suggested
Changes.
Signed-off-by: Sandeepa Prabhu <sandeepa.s.prabhu@gmail.com>
Signed-off-by: David A. Long <dave.long@linaro.org>
Signed-off-by: Pratyush Anand <panand@redhat.com>
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-08 23:35:48 +07:00
|
|
|
if (notify_page_fault(regs, esr))
|
|
|
|
return 0;
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
tsk = current;
|
|
|
|
mm = tsk->mm;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we're in an interrupt or have no user context, we must not take
|
|
|
|
* the fault.
|
|
|
|
*/
|
2015-05-11 22:52:11 +07:00
|
|
|
if (faulthandler_disabled() || !mm)
|
2012-03-05 18:49:27 +07:00
|
|
|
goto no_context;
|
|
|
|
|
2013-09-13 05:13:39 +07:00
|
|
|
if (user_mode(regs))
|
|
|
|
mm_flags |= FAULT_FLAG_USER;
|
|
|
|
|
arm64: kill ESR_LNX_EXEC
Currently we treat ESR_EL1 bit 24 as software-defined for distinguishing
instruction aborts from data aborts, but this bit is architecturally
RES0 for instruction aborts, and could be allocated for an arbitrary
purpose in future. Additionally, we hard-code the value in entry.S
without the mnemonic, making the code difficult to understand.
Instead, remove ESR_LNX_EXEC, and distinguish aborts based on the esr,
which we already pass to the sole use of ESR_LNX_EXEC. A new helper,
is_el0_instruction_abort() is added to make the logic clear. Any
instruction aborts taken from EL1 will already have been handled by
bad_mode, so we need not handle that case in the helper.
For consistency, the existing permission_fault helper is renamed to
is_permission_fault, and the return type is changed to bool. There
should be no functional changes as the return value was a boolean
expression, and the result is only used in another boolean expression.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Dave P Martin <dave.martin@arm.com>
Cc: Huang Shijie <shijie.huang@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-05-31 18:33:03 +07:00
|
|
|
if (is_el0_instruction_abort(esr)) {
|
2013-09-13 05:13:39 +07:00
|
|
|
vm_flags = VM_EXEC;
|
2014-11-24 19:31:40 +07:00
|
|
|
} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
|
2013-09-13 05:13:39 +07:00
|
|
|
vm_flags = VM_WRITE;
|
|
|
|
mm_flags |= FAULT_FLAG_WRITE;
|
|
|
|
}
|
|
|
|
|
2017-04-06 02:18:31 +07:00
|
|
|
if (addr < USER_DS && is_permission_fault(esr, regs, addr)) {
|
2016-06-21 00:28:01 +07:00
|
|
|
/* regs->orig_addr_limit may be 0 if we entered from EL0 */
|
|
|
|
if (regs->orig_addr_limit == KERNEL_DS)
|
2016-02-19 21:28:58 +07:00
|
|
|
die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
|
2016-02-05 21:58:50 +07:00
|
|
|
|
2016-08-10 08:25:26 +07:00
|
|
|
if (is_el1_instruction_abort(esr))
|
|
|
|
die("Attempting to execute userspace memory", regs, esr);
|
|
|
|
|
2016-02-05 21:58:48 +07:00
|
|
|
if (!search_exception_tables(regs->pc))
|
2016-02-19 21:28:58 +07:00
|
|
|
die("Accessing user space memory outside uaccess.h routines", regs, esr);
|
2016-02-05 21:58:48 +07:00
|
|
|
}
|
2015-07-23 01:05:54 +07:00
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* As per x86, we may deadlock here. However, since the kernel only
|
|
|
|
* validly references user space from well defined areas of the code,
|
|
|
|
* we can bug out early if this is from code which shouldn't.
|
|
|
|
*/
|
|
|
|
if (!down_read_trylock(&mm->mmap_sem)) {
|
|
|
|
if (!user_mode(regs) && !search_exception_tables(regs->pc))
|
|
|
|
goto no_context;
|
|
|
|
retry:
|
|
|
|
down_read(&mm->mmap_sem);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The above down_read_trylock() might have succeeded in which
|
|
|
|
* case, we'll have missed the might_sleep() from down_read().
|
|
|
|
*/
|
|
|
|
might_sleep();
|
|
|
|
#ifdef CONFIG_DEBUG_VM
|
|
|
|
if (!user_mode(regs) && !search_exception_tables(regs->pc))
|
|
|
|
goto no_context;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2013-07-19 21:37:12 +07:00
|
|
|
fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we need to retry but a fatal signal is pending, handle the
|
|
|
|
* signal first. We do not need to release the mmap_sem because it
|
|
|
|
* would already be released in __lock_page_or_retry in mm/filemap.c.
|
|
|
|
*/
|
|
|
|
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Major/minor page fault accounting is only done on the initial
|
|
|
|
* attempt. If we go through a retry, it is extremely likely that the
|
|
|
|
* page will be found in page cache at that point.
|
|
|
|
*/
|
|
|
|
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
|
2013-07-19 21:37:12 +07:00
|
|
|
if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
|
2012-03-05 18:49:27 +07:00
|
|
|
if (fault & VM_FAULT_MAJOR) {
|
|
|
|
tsk->maj_flt++;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
|
|
|
|
addr);
|
|
|
|
} else {
|
|
|
|
tsk->min_flt++;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
|
|
|
|
addr);
|
|
|
|
}
|
|
|
|
if (fault & VM_FAULT_RETRY) {
|
|
|
|
/*
|
|
|
|
* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
|
|
|
|
* starvation.
|
|
|
|
*/
|
2013-07-19 21:37:12 +07:00
|
|
|
mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
|
arm64: readahead: fault retry breaks mmap file read random detection
This is the arm64 portion of commit 45cac65b0fcd ("readahead: fault
retry breaks mmap file read random detection"), which was absent from
the initial port and has since gone unnoticed. The original commit says:
> .fault now can retry. The retry can break state machine of .fault. In
> filemap_fault, if page is miss, ra->mmap_miss is increased. In the second
> try, since the page is in page cache now, ra->mmap_miss is decreased. And
> these are done in one fault, so we can't detect random mmap file access.
>
> Add a new flag to indicate .fault is tried once. In the second try, skip
> ra->mmap_miss decreasing. The filemap_fault state machine is ok with it.
With this change, Mark reports that:
> Random read improves by 250%, sequential read improves by 40%, and
> random write by 400% to an eMMC device with dm crypto wrapped around it.
Cc: Shaohua Li <shli@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Signed-off-by: Riley Andrews <riandrews@android.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2015-09-22 03:39:50 +07:00
|
|
|
mm_flags |= FAULT_FLAG_TRIED;
|
2012-03-05 18:49:27 +07:00
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
up_read(&mm->mmap_sem);
|
|
|
|
|
|
|
|
/*
|
2016-03-18 04:19:55 +07:00
|
|
|
* Handle the "normal" case first - VM_FAULT_MAJOR
|
2012-03-05 18:49:27 +07:00
|
|
|
*/
|
|
|
|
if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
|
|
|
|
VM_FAULT_BADACCESS))))
|
|
|
|
return 0;
|
|
|
|
|
2013-09-13 05:13:38 +07:00
|
|
|
/*
|
|
|
|
* If we are in kernel mode at this point, we have no context to
|
|
|
|
* handle this fault with.
|
|
|
|
*/
|
|
|
|
if (!user_mode(regs))
|
|
|
|
goto no_context;
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
if (fault & VM_FAULT_OOM) {
|
|
|
|
/*
|
|
|
|
* We ran out of memory, call the OOM killer, and return to
|
|
|
|
* userspace (which will retry the fault, or kill us if we got
|
|
|
|
* oom-killed).
|
|
|
|
*/
|
|
|
|
pagefault_out_of_memory();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fault & VM_FAULT_SIGBUS) {
|
|
|
|
/*
|
|
|
|
* We had some memory, but were unable to successfully fix up
|
|
|
|
* this page fault.
|
|
|
|
*/
|
|
|
|
sig = SIGBUS;
|
|
|
|
code = BUS_ADRERR;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Something tried to access memory that isn't in our memory
|
|
|
|
* map.
|
|
|
|
*/
|
|
|
|
sig = SIGSEGV;
|
|
|
|
code = fault == VM_FAULT_BADACCESS ?
|
|
|
|
SEGV_ACCERR : SEGV_MAPERR;
|
|
|
|
}
|
|
|
|
|
|
|
|
__do_user_fault(tsk, addr, esr, sig, code, regs);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
no_context:
|
|
|
|
__do_kernel_fault(mm, addr, esr, regs);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* First Level Translation Fault Handler
|
|
|
|
*
|
|
|
|
* We enter here because the first level page table doesn't contain a valid
|
|
|
|
* entry for the address.
|
|
|
|
*
|
|
|
|
* If the address is in kernel space (>= TASK_SIZE), then we are probably
|
|
|
|
* faulting in the vmalloc() area.
|
|
|
|
*
|
|
|
|
* If the init_task's first level page tables contains the relevant entry, we
|
|
|
|
* copy the it to this task. If not, we send the process a signal, fixup the
|
|
|
|
* exception, or oops the kernel.
|
|
|
|
*
|
|
|
|
* NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
|
|
|
|
* or a critical region, and should only copy the information from the master
|
|
|
|
* page table, nothing more.
|
|
|
|
*/
|
|
|
|
static int __kprobes do_translation_fault(unsigned long addr,
|
|
|
|
unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
if (addr < TASK_SIZE)
|
|
|
|
return do_page_fault(addr, esr, regs);
|
|
|
|
|
|
|
|
do_bad_area(addr, esr, regs);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-02-16 11:44:35 +07:00
|
|
|
static int do_alignment_fault(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
do_bad_area(addr, esr, regs);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-03-05 18:49:27 +07:00
|
|
|
/*
|
|
|
|
* This abort handler always returns "fault".
|
|
|
|
*/
|
|
|
|
static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2017-06-22 01:17:08 +07:00
|
|
|
/*
|
|
|
|
* This abort handler deals with Synchronous External Abort.
|
|
|
|
* It calls notifiers, and then returns "fault".
|
|
|
|
*/
|
|
|
|
static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct siginfo info;
|
|
|
|
const struct fault_info *inf;
|
|
|
|
|
|
|
|
inf = esr_to_fault_info(esr);
|
|
|
|
pr_err("Synchronous External Abort: %s (0x%08x) at 0x%016lx\n",
|
|
|
|
inf->name, esr, addr);
|
|
|
|
|
|
|
|
info.si_signo = SIGBUS;
|
|
|
|
info.si_errno = 0;
|
|
|
|
info.si_code = 0;
|
|
|
|
if (esr & ESR_ELx_FnV)
|
|
|
|
info.si_addr = NULL;
|
|
|
|
else
|
|
|
|
info.si_addr = (void __user *)addr;
|
|
|
|
arm64_notify_die("", regs, &info, esr);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-04-04 12:51:01 +07:00
|
|
|
static const struct fault_info fault_info[] = {
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "ttbr address size fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "level 1 address size fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "level 2 address size fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "level 3 address size fault" },
|
2014-11-21 21:22:22 +07:00
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
|
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 8" },
|
2013-04-10 19:48:00 +07:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 12" },
|
2013-04-10 19:48:00 +07:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
|
2017-06-22 01:17:08 +07:00
|
|
|
{ do_sea, SIGBUS, 0, "synchronous external abort" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 17" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 18" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 19" },
|
2017-06-22 01:17:08 +07:00
|
|
|
{ do_sea, SIGBUS, 0, "level 0 (translation table walk)" },
|
|
|
|
{ do_sea, SIGBUS, 0, "level 1 (translation table walk)" },
|
|
|
|
{ do_sea, SIGBUS, 0, "level 2 (translation table walk)" },
|
|
|
|
{ do_sea, SIGBUS, 0, "level 3 (translation table walk)" },
|
|
|
|
{ do_sea, SIGBUS, 0, "synchronous parity or ECC error" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 25" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 26" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 27" },
|
2017-06-22 01:17:08 +07:00
|
|
|
{ do_sea, SIGBUS, 0, "level 0 synchronous parity error (translation table walk)" },
|
|
|
|
{ do_sea, SIGBUS, 0, "level 1 synchronous parity error (translation table walk)" },
|
|
|
|
{ do_sea, SIGBUS, 0, "level 2 synchronous parity error (translation table walk)" },
|
|
|
|
{ do_sea, SIGBUS, 0, "level 3 synchronous parity error (translation table walk)" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 32" },
|
2016-02-16 11:44:35 +07:00
|
|
|
{ do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 34" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 35" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 36" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 37" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 38" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 39" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 40" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 41" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 42" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 43" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 44" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 45" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 46" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 47" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "TLB conflict abort" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 49" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 50" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 51" },
|
|
|
|
{ do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 54" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 55" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 56" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 57" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 58" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 59" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 60" },
|
2015-11-23 22:09:36 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "section domain fault" },
|
|
|
|
{ do_bad, SIGBUS, 0, "page domain fault" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 63" },
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Dispatch a data abort to the relevant handler.
|
|
|
|
*/
|
|
|
|
asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
2017-04-04 12:51:01 +07:00
|
|
|
const struct fault_info *inf = esr_to_fault_info(esr);
|
2012-03-05 18:49:27 +07:00
|
|
|
struct siginfo info;
|
|
|
|
|
|
|
|
if (!inf->fn(addr, esr, regs))
|
|
|
|
return;
|
|
|
|
|
|
|
|
pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
|
|
|
|
inf->name, esr, addr);
|
|
|
|
|
|
|
|
info.si_signo = inf->sig;
|
|
|
|
info.si_errno = 0;
|
|
|
|
info.si_code = inf->code;
|
|
|
|
info.si_addr = (void __user *)addr;
|
|
|
|
arm64_notify_die("", regs, &info, esr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Handle stack alignment exceptions.
|
|
|
|
*/
|
|
|
|
asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
|
|
|
|
unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
struct siginfo info;
|
2015-06-19 21:28:16 +07:00
|
|
|
struct task_struct *tsk = current;
|
|
|
|
|
|
|
|
if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS))
|
|
|
|
pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n",
|
|
|
|
tsk->comm, task_pid_nr(tsk),
|
|
|
|
esr_get_class_string(esr), (void *)regs->pc,
|
|
|
|
(void *)regs->sp);
|
2012-03-05 18:49:27 +07:00
|
|
|
|
|
|
|
info.si_signo = SIGBUS;
|
|
|
|
info.si_errno = 0;
|
|
|
|
info.si_code = BUS_ADRALN;
|
|
|
|
info.si_addr = (void __user *)addr;
|
2015-06-19 21:28:16 +07:00
|
|
|
arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr);
|
2012-03-05 18:49:27 +07:00
|
|
|
}
|
|
|
|
|
2015-07-24 22:37:48 +07:00
|
|
|
int __init early_brk64(unsigned long addr, unsigned int esr,
|
|
|
|
struct pt_regs *regs);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* __refdata because early_brk64 is __init, but the reference to it is
|
|
|
|
* clobbered at arch_initcall time.
|
|
|
|
* See traps.c and debug-monitors.c:debug_traps_init().
|
|
|
|
*/
|
|
|
|
static struct fault_info __refdata debug_fault_info[] = {
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
|
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
|
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
|
|
|
|
{ do_bad, SIGBUS, 0, "unknown 3" },
|
|
|
|
{ do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
|
|
|
|
{ do_bad, SIGTRAP, 0, "aarch32 vector catch" },
|
2015-07-24 22:37:48 +07:00
|
|
|
{ early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
|
2012-03-05 18:49:27 +07:00
|
|
|
{ do_bad, SIGBUS, 0, "unknown 7" },
|
|
|
|
};
|
|
|
|
|
|
|
|
void __init hook_debug_fault_code(int nr,
|
|
|
|
int (*fn)(unsigned long, unsigned int, struct pt_regs *),
|
|
|
|
int sig, int code, const char *name)
|
|
|
|
{
|
|
|
|
BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
|
|
|
|
|
|
|
|
debug_fault_info[nr].fn = fn;
|
|
|
|
debug_fault_info[nr].sig = sig;
|
|
|
|
debug_fault_info[nr].code = code;
|
|
|
|
debug_fault_info[nr].name = name;
|
|
|
|
}
|
|
|
|
|
|
|
|
asmlinkage int __exception do_debug_exception(unsigned long addr,
|
|
|
|
unsigned int esr,
|
|
|
|
struct pt_regs *regs)
|
|
|
|
{
|
|
|
|
const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
|
|
|
|
struct siginfo info;
|
2016-04-13 19:40:00 +07:00
|
|
|
int rv;
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2016-04-13 19:40:00 +07:00
|
|
|
/*
|
|
|
|
* Tell lockdep we disabled irqs in entry.S. Do nothing if they were
|
|
|
|
* already disabled to preserve the last enabled/disabled addresses.
|
|
|
|
*/
|
|
|
|
if (interrupts_enabled(regs))
|
|
|
|
trace_hardirqs_off();
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2016-04-13 19:40:00 +07:00
|
|
|
if (!inf->fn(addr, esr, regs)) {
|
|
|
|
rv = 1;
|
|
|
|
} else {
|
|
|
|
pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n",
|
|
|
|
inf->name, esr, addr);
|
|
|
|
|
|
|
|
info.si_signo = inf->sig;
|
|
|
|
info.si_errno = 0;
|
|
|
|
info.si_code = inf->code;
|
|
|
|
info.si_addr = (void __user *)addr;
|
|
|
|
arm64_notify_die("", regs, &info, 0);
|
|
|
|
rv = 0;
|
|
|
|
}
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2016-04-13 19:40:00 +07:00
|
|
|
if (interrupts_enabled(regs))
|
|
|
|
trace_hardirqs_on();
|
2012-03-05 18:49:27 +07:00
|
|
|
|
2016-04-13 19:40:00 +07:00
|
|
|
return rv;
|
2012-03-05 18:49:27 +07:00
|
|
|
}
|
arm64: Kprobes with single stepping support
Add support for basic kernel probes(kprobes) and jump probes
(jprobes) for ARM64.
Kprobes utilizes software breakpoint and single step debug
exceptions supported on ARM v8.
A software breakpoint is placed at the probe address to trap the
kernel execution into the kprobe handler.
ARM v8 supports enabling single stepping before the break exception
return (ERET), with next PC in exception return address (ELR_EL1). The
kprobe handler prepares an executable memory slot for out-of-line
execution with a copy of the original instruction being probed, and
enables single stepping. The PC is set to the out-of-line slot address
before the ERET. With this scheme, the instruction is executed with the
exact same register context except for the PC (and DAIF) registers.
Debug mask (PSTATE.D) is enabled only when single stepping a recursive
kprobe, e.g.: during kprobes reenter so that probed instruction can be
single stepped within the kprobe handler -exception- context.
The recursion depth of kprobe is always 2, i.e. upon probe re-entry,
any further re-entry is prevented by not calling handlers and the case
counted as a missed kprobe).
Single stepping from the x-o-l slot has a drawback for PC-relative accesses
like branching and symbolic literals access as the offset from the new PC
(slot address) may not be ensured to fit in the immediate value of
the opcode. Such instructions need simulation, so reject
probing them.
Instructions generating exceptions or cpu mode change are rejected
for probing.
Exclusive load/store instructions are rejected too. Additionally, the
code is checked to see if it is inside an exclusive load/store sequence
(code from Pratyush).
System instructions are mostly enabled for stepping, except MSR/MRS
accesses to "DAIF" flags in PSTATE, which are not safe for
probing.
This also changes arch/arm64/include/asm/ptrace.h to use
include/asm-generic/ptrace.h.
Thanks to Steve Capper and Pratyush Anand for several suggested
Changes.
Signed-off-by: Sandeepa Prabhu <sandeepa.s.prabhu@gmail.com>
Signed-off-by: David A. Long <dave.long@linaro.org>
Signed-off-by: Pratyush Anand <panand@redhat.com>
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-08 23:35:48 +07:00
|
|
|
NOKPROBE_SYMBOL(do_debug_exception);
|
2015-07-23 01:05:54 +07:00
|
|
|
|
|
|
|
#ifdef CONFIG_ARM64_PAN
|
2016-10-18 17:27:46 +07:00
|
|
|
int cpu_enable_pan(void *__unused)
|
2015-07-23 01:05:54 +07:00
|
|
|
{
|
2016-10-18 17:27:47 +07:00
|
|
|
/*
|
|
|
|
* We modify PSTATE. This won't work from irq context as the PSTATE
|
|
|
|
* is discarded once we return from the exception.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(in_interrupt());
|
|
|
|
|
2015-07-23 01:05:54 +07:00
|
|
|
config_sctlr_el1(SCTLR_EL1_SPAN, 0);
|
2016-10-18 17:27:47 +07:00
|
|
|
asm(SET_PSTATE_PAN(1));
|
2016-10-18 17:27:46 +07:00
|
|
|
return 0;
|
2015-07-23 01:05:54 +07:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_ARM64_PAN */
|