2014-11-18 18:41:24 +07:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2014 ARM Limited
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*/
|
|
|
|
|
2014-11-18 18:41:26 +07:00
|
|
|
#include <linux/cpu.h>
|
2014-11-18 18:41:24 +07:00
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/list.h>
|
2014-11-18 18:41:25 +07:00
|
|
|
#include <linux/perf_event.h>
|
|
|
|
#include <linux/sched.h>
|
2014-11-18 18:41:24 +07:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/sysctl.h>
|
2018-04-13 21:44:35 +07:00
|
|
|
#include <linux/uaccess.h>
|
2014-11-18 18:41:24 +07:00
|
|
|
|
2015-07-23 01:05:54 +07:00
|
|
|
#include <asm/cpufeature.h>
|
2014-11-18 18:41:25 +07:00
|
|
|
#include <asm/insn.h>
|
2015-07-21 19:23:27 +07:00
|
|
|
#include <asm/sysreg.h>
|
2014-11-18 18:41:25 +07:00
|
|
|
#include <asm/system_misc.h>
|
2014-11-18 18:41:24 +07:00
|
|
|
#include <asm/traps.h>
|
2017-02-28 05:26:56 +07:00
|
|
|
#include <asm/kprobes.h>
|
2014-11-18 18:41:24 +07:00
|
|
|
|
2014-11-18 18:41:27 +07:00
|
|
|
#define CREATE_TRACE_POINTS
|
|
|
|
#include "trace-events-emulation.h"
|
|
|
|
|
2014-11-18 18:41:24 +07:00
|
|
|
/*
|
|
|
|
* The runtime support for deprecated instruction support can be in one of
|
|
|
|
* following three states -
|
|
|
|
*
|
|
|
|
* 0 = undef
|
|
|
|
* 1 = emulate (software emulation)
|
|
|
|
* 2 = hw (supported in hardware)
|
|
|
|
*/
|
|
|
|
enum insn_emulation_mode {
|
|
|
|
INSN_UNDEF,
|
|
|
|
INSN_EMULATE,
|
|
|
|
INSN_HW,
|
|
|
|
};
|
|
|
|
|
|
|
|
enum legacy_insn_status {
|
|
|
|
INSN_DEPRECATED,
|
|
|
|
INSN_OBSOLETE,
|
|
|
|
};
|
|
|
|
|
|
|
|
struct insn_emulation_ops {
|
|
|
|
const char *name;
|
|
|
|
enum legacy_insn_status status;
|
|
|
|
struct undef_hook *hooks;
|
|
|
|
int (*set_hw_mode)(bool enable);
|
|
|
|
};
|
|
|
|
|
|
|
|
struct insn_emulation {
|
|
|
|
struct list_head node;
|
|
|
|
struct insn_emulation_ops *ops;
|
|
|
|
int current_mode;
|
|
|
|
int min;
|
|
|
|
int max;
|
|
|
|
};
|
|
|
|
|
|
|
|
static LIST_HEAD(insn_emulation);
|
2015-11-20 16:59:10 +07:00
|
|
|
static int nr_insn_emulated __initdata;
|
2014-11-18 18:41:24 +07:00
|
|
|
static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
|
|
|
|
|
|
|
|
static void register_emulation_hooks(struct insn_emulation_ops *ops)
|
|
|
|
{
|
|
|
|
struct undef_hook *hook;
|
|
|
|
|
|
|
|
BUG_ON(!ops->hooks);
|
|
|
|
|
|
|
|
for (hook = ops->hooks; hook->instr_mask; hook++)
|
|
|
|
register_undef_hook(hook);
|
|
|
|
|
|
|
|
pr_notice("Registered %s emulation handler\n", ops->name);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void remove_emulation_hooks(struct insn_emulation_ops *ops)
|
|
|
|
{
|
|
|
|
struct undef_hook *hook;
|
|
|
|
|
|
|
|
BUG_ON(!ops->hooks);
|
|
|
|
|
|
|
|
for (hook = ops->hooks; hook->instr_mask; hook++)
|
|
|
|
unregister_undef_hook(hook);
|
|
|
|
|
|
|
|
pr_notice("Removed %s emulation handler\n", ops->name);
|
|
|
|
}
|
|
|
|
|
2015-01-21 19:43:10 +07:00
|
|
|
static void enable_insn_hw_mode(void *data)
|
|
|
|
{
|
|
|
|
struct insn_emulation *insn = (struct insn_emulation *)data;
|
|
|
|
if (insn->ops->set_hw_mode)
|
|
|
|
insn->ops->set_hw_mode(true);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void disable_insn_hw_mode(void *data)
|
|
|
|
{
|
|
|
|
struct insn_emulation *insn = (struct insn_emulation *)data;
|
|
|
|
if (insn->ops->set_hw_mode)
|
|
|
|
insn->ops->set_hw_mode(false);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Run set_hw_mode(mode) on all active CPUs */
|
|
|
|
static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
|
|
|
|
{
|
|
|
|
if (!insn->ops->set_hw_mode)
|
|
|
|
return -EINVAL;
|
|
|
|
if (enable)
|
|
|
|
on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
|
|
|
|
else
|
|
|
|
on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Run set_hw_mode for all insns on a starting CPU.
|
|
|
|
* Returns:
|
|
|
|
* 0 - If all the hooks ran successfully.
|
|
|
|
* -EINVAL - At least one hook is not supported by the CPU.
|
|
|
|
*/
|
2016-07-14 00:16:56 +07:00
|
|
|
static int run_all_insn_set_hw_mode(unsigned int cpu)
|
2015-01-21 19:43:10 +07:00
|
|
|
{
|
|
|
|
int rc = 0;
|
|
|
|
unsigned long flags;
|
|
|
|
struct insn_emulation *insn;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
|
|
|
|
list_for_each_entry(insn, &insn_emulation, node) {
|
|
|
|
bool enable = (insn->current_mode == INSN_HW);
|
|
|
|
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
|
2016-07-14 00:16:56 +07:00
|
|
|
pr_warn("CPU[%u] cannot support the emulation of %s",
|
2015-01-21 19:43:10 +07:00
|
|
|
cpu, insn->ops->name);
|
|
|
|
rc = -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2014-11-18 18:41:24 +07:00
|
|
|
static int update_insn_emulation_mode(struct insn_emulation *insn,
|
|
|
|
enum insn_emulation_mode prev)
|
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
switch (prev) {
|
|
|
|
case INSN_UNDEF: /* Nothing to be done */
|
|
|
|
break;
|
|
|
|
case INSN_EMULATE:
|
|
|
|
remove_emulation_hooks(insn->ops);
|
|
|
|
break;
|
|
|
|
case INSN_HW:
|
2015-01-21 19:43:10 +07:00
|
|
|
if (!run_all_cpu_set_hw_mode(insn, false))
|
2014-11-18 18:41:24 +07:00
|
|
|
pr_notice("Disabled %s support\n", insn->ops->name);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
switch (insn->current_mode) {
|
|
|
|
case INSN_UNDEF:
|
|
|
|
break;
|
|
|
|
case INSN_EMULATE:
|
|
|
|
register_emulation_hooks(insn->ops);
|
|
|
|
break;
|
|
|
|
case INSN_HW:
|
2015-01-21 19:43:10 +07:00
|
|
|
ret = run_all_cpu_set_hw_mode(insn, true);
|
|
|
|
if (!ret)
|
2014-11-18 18:41:24 +07:00
|
|
|
pr_notice("Enabled %s support\n", insn->ops->name);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-11-20 16:59:10 +07:00
|
|
|
static void __init register_insn_emulation(struct insn_emulation_ops *ops)
|
2014-11-18 18:41:24 +07:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
struct insn_emulation *insn;
|
|
|
|
|
|
|
|
insn = kzalloc(sizeof(*insn), GFP_KERNEL);
|
|
|
|
insn->ops = ops;
|
|
|
|
insn->min = INSN_UNDEF;
|
|
|
|
|
|
|
|
switch (ops->status) {
|
|
|
|
case INSN_DEPRECATED:
|
|
|
|
insn->current_mode = INSN_EMULATE;
|
2015-01-21 19:43:10 +07:00
|
|
|
/* Disable the HW mode if it was turned on at early boot time */
|
|
|
|
run_all_cpu_set_hw_mode(insn, false);
|
2014-11-18 18:41:24 +07:00
|
|
|
insn->max = INSN_HW;
|
|
|
|
break;
|
|
|
|
case INSN_OBSOLETE:
|
|
|
|
insn->current_mode = INSN_UNDEF;
|
|
|
|
insn->max = INSN_EMULATE;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
|
|
|
|
list_add(&insn->node, &insn_emulation);
|
|
|
|
nr_insn_emulated++;
|
|
|
|
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
|
|
|
|
|
|
|
|
/* Register any handlers if required */
|
|
|
|
update_insn_emulation_mode(insn, INSN_UNDEF);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int emulation_proc_handler(struct ctl_table *table, int write,
|
|
|
|
void __user *buffer, size_t *lenp,
|
|
|
|
loff_t *ppos)
|
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
struct insn_emulation *insn = (struct insn_emulation *) table->data;
|
|
|
|
enum insn_emulation_mode prev_mode = insn->current_mode;
|
|
|
|
|
|
|
|
table->data = &insn->current_mode;
|
|
|
|
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
|
|
|
|
|
|
|
if (ret || !write || prev_mode == insn->current_mode)
|
|
|
|
goto ret;
|
|
|
|
|
|
|
|
ret = update_insn_emulation_mode(insn, prev_mode);
|
2014-11-25 17:05:35 +07:00
|
|
|
if (ret) {
|
2014-11-18 18:41:24 +07:00
|
|
|
/* Mode change failed, revert to previous mode. */
|
|
|
|
insn->current_mode = prev_mode;
|
|
|
|
update_insn_emulation_mode(insn, INSN_UNDEF);
|
|
|
|
}
|
|
|
|
ret:
|
|
|
|
table->data = insn;
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-10-31 22:50:58 +07:00
|
|
|
static void __init register_insn_emulation_sysctl(void)
|
2014-11-18 18:41:24 +07:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
int i = 0;
|
|
|
|
struct insn_emulation *insn;
|
|
|
|
struct ctl_table *insns_sysctl, *sysctl;
|
|
|
|
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-13 04:03:40 +07:00
|
|
|
insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
|
|
|
|
GFP_KERNEL);
|
2014-11-18 18:41:24 +07:00
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
|
|
|
|
list_for_each_entry(insn, &insn_emulation, node) {
|
|
|
|
sysctl = &insns_sysctl[i];
|
|
|
|
|
|
|
|
sysctl->mode = 0644;
|
|
|
|
sysctl->maxlen = sizeof(int);
|
|
|
|
|
|
|
|
sysctl->procname = insn->ops->name;
|
|
|
|
sysctl->data = insn;
|
|
|
|
sysctl->extra1 = &insn->min;
|
|
|
|
sysctl->extra2 = &insn->max;
|
|
|
|
sysctl->proc_handler = emulation_proc_handler;
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
|
|
|
|
|
2017-10-31 22:50:58 +07:00
|
|
|
register_sysctl("abi", insns_sysctl);
|
2014-11-18 18:41:24 +07:00
|
|
|
}
|
|
|
|
|
2014-11-18 18:41:25 +07:00
|
|
|
/*
|
|
|
|
* Implement emulation of the SWP/SWPB instructions using load-exclusive and
|
|
|
|
* store-exclusive.
|
|
|
|
*
|
|
|
|
* Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
|
|
|
|
* Where: Rt = destination
|
|
|
|
* Rt2 = source
|
|
|
|
* Rn = address
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Error-checking SWP macros implemented using ldxr{b}/stxr{b}
|
|
|
|
*/
|
2016-07-04 22:59:43 +07:00
|
|
|
|
|
|
|
/* Arbitrary constant to ensure forward-progress of the LL/SC loop */
|
|
|
|
#define __SWP_LL_SC_LOOPS 4
|
|
|
|
|
|
|
|
#define __user_swpX_asm(data, addr, res, temp, temp2, B) \
|
2016-07-01 20:58:21 +07:00
|
|
|
do { \
|
|
|
|
uaccess_enable(); \
|
2014-11-18 18:41:25 +07:00
|
|
|
__asm__ __volatile__( \
|
2016-07-04 22:59:43 +07:00
|
|
|
" mov %w3, %w7\n" \
|
|
|
|
"0: ldxr"B" %w2, [%4]\n" \
|
|
|
|
"1: stxr"B" %w0, %w1, [%4]\n" \
|
2014-11-18 18:41:25 +07:00
|
|
|
" cbz %w0, 2f\n" \
|
2016-07-04 22:59:43 +07:00
|
|
|
" sub %w3, %w3, #1\n" \
|
|
|
|
" cbnz %w3, 0b\n" \
|
|
|
|
" mov %w0, %w5\n" \
|
2015-10-15 19:55:53 +07:00
|
|
|
" b 3f\n" \
|
2014-11-18 18:41:25 +07:00
|
|
|
"2:\n" \
|
2015-10-15 19:55:53 +07:00
|
|
|
" mov %w1, %w2\n" \
|
|
|
|
"3:\n" \
|
2014-11-18 18:41:25 +07:00
|
|
|
" .pushsection .fixup,\"ax\"\n" \
|
|
|
|
" .align 2\n" \
|
2016-07-04 22:59:43 +07:00
|
|
|
"4: mov %w0, %w6\n" \
|
2015-10-15 19:55:53 +07:00
|
|
|
" b 3b\n" \
|
2014-11-18 18:41:25 +07:00
|
|
|
" .popsection" \
|
2016-01-01 21:02:12 +07:00
|
|
|
_ASM_EXTABLE(0b, 4b) \
|
|
|
|
_ASM_EXTABLE(1b, 4b) \
|
2016-07-04 22:59:43 +07:00
|
|
|
: "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \
|
2017-05-03 22:09:36 +07:00
|
|
|
: "r" ((unsigned long)addr), "i" (-EAGAIN), \
|
|
|
|
"i" (-EFAULT), \
|
2016-07-04 22:59:43 +07:00
|
|
|
"i" (__SWP_LL_SC_LOOPS) \
|
2016-07-01 20:58:21 +07:00
|
|
|
: "memory"); \
|
|
|
|
uaccess_disable(); \
|
|
|
|
} while (0)
|
2014-11-18 18:41:25 +07:00
|
|
|
|
2016-07-04 22:59:43 +07:00
|
|
|
#define __user_swp_asm(data, addr, res, temp, temp2) \
|
|
|
|
__user_swpX_asm(data, addr, res, temp, temp2, "")
|
|
|
|
#define __user_swpb_asm(data, addr, res, temp, temp2) \
|
|
|
|
__user_swpX_asm(data, addr, res, temp, temp2, "b")
|
2014-11-18 18:41:25 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Bit 22 of the instruction encoding distinguishes between
|
|
|
|
* the SWP and SWPB variants (bit set means SWPB).
|
|
|
|
*/
|
|
|
|
#define TYPE_SWPB (1 << 22)
|
|
|
|
|
|
|
|
static int emulate_swpX(unsigned int address, unsigned int *data,
|
|
|
|
unsigned int type)
|
|
|
|
{
|
|
|
|
unsigned int res = 0;
|
|
|
|
|
|
|
|
if ((type != TYPE_SWPB) && (address & 0x3)) {
|
|
|
|
/* SWP to unaligned address not permitted */
|
|
|
|
pr_debug("SWP instruction on unaligned pointer!\n");
|
|
|
|
return -EFAULT;
|
|
|
|
}
|
|
|
|
|
|
|
|
while (1) {
|
2016-07-04 22:59:43 +07:00
|
|
|
unsigned long temp, temp2;
|
2014-11-18 18:41:25 +07:00
|
|
|
|
|
|
|
if (type == TYPE_SWPB)
|
2016-07-04 22:59:43 +07:00
|
|
|
__user_swpb_asm(*data, address, res, temp, temp2);
|
2014-11-18 18:41:25 +07:00
|
|
|
else
|
2016-07-04 22:59:43 +07:00
|
|
|
__user_swp_asm(*data, address, res, temp, temp2);
|
2014-11-18 18:41:25 +07:00
|
|
|
|
|
|
|
if (likely(res != -EAGAIN) || signal_pending(current))
|
|
|
|
break;
|
|
|
|
|
|
|
|
cond_resched();
|
|
|
|
}
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2016-12-01 17:44:33 +07:00
|
|
|
#define ARM_OPCODE_CONDTEST_FAIL 0
|
|
|
|
#define ARM_OPCODE_CONDTEST_PASS 1
|
|
|
|
#define ARM_OPCODE_CONDTEST_UNCOND 2
|
|
|
|
|
2016-07-08 23:35:47 +07:00
|
|
|
#define ARM_OPCODE_CONDITION_UNCOND 0xf
|
|
|
|
|
|
|
|
static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
|
|
|
|
{
|
|
|
|
u32 cc_bits = opcode >> 28;
|
|
|
|
|
|
|
|
if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
|
|
|
|
if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
|
|
|
|
return ARM_OPCODE_CONDTEST_PASS;
|
|
|
|
else
|
|
|
|
return ARM_OPCODE_CONDTEST_FAIL;
|
|
|
|
}
|
|
|
|
return ARM_OPCODE_CONDTEST_UNCOND;
|
|
|
|
}
|
|
|
|
|
2014-11-18 18:41:25 +07:00
|
|
|
/*
|
|
|
|
* swp_handler logs the id of calling process, dissects the instruction, sanity
|
|
|
|
* checks the memory location, calls emulate_swpX for the actual operation and
|
|
|
|
* deals with fixup/error handling before returning
|
|
|
|
*/
|
|
|
|
static int swp_handler(struct pt_regs *regs, u32 instr)
|
|
|
|
{
|
|
|
|
u32 destreg, data, type, address = 0;
|
2018-02-19 20:38:00 +07:00
|
|
|
const void __user *user_ptr;
|
2014-11-18 18:41:25 +07:00
|
|
|
int rn, rt2, res = 0;
|
|
|
|
|
|
|
|
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
|
|
|
|
|
|
|
|
type = instr & TYPE_SWPB;
|
|
|
|
|
2016-07-08 23:35:47 +07:00
|
|
|
switch (aarch32_check_condition(instr, regs->pstate)) {
|
2014-11-18 18:41:25 +07:00
|
|
|
case ARM_OPCODE_CONDTEST_PASS:
|
|
|
|
break;
|
|
|
|
case ARM_OPCODE_CONDTEST_FAIL:
|
|
|
|
/* Condition failed - return to next instruction */
|
|
|
|
goto ret;
|
|
|
|
case ARM_OPCODE_CONDTEST_UNCOND:
|
|
|
|
/* If unconditional encoding - not a SWP, undef */
|
|
|
|
return -EFAULT;
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
|
|
|
|
rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
|
|
|
|
|
|
|
|
address = (u32)regs->user_regs.regs[rn];
|
|
|
|
data = (u32)regs->user_regs.regs[rt2];
|
|
|
|
destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
|
|
|
|
|
|
|
|
pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
|
|
|
|
rn, address, destreg,
|
|
|
|
aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
|
|
|
|
|
|
|
|
/* Check access in reasonable access range for both SWP and SWPB */
|
2018-02-19 20:38:00 +07:00
|
|
|
user_ptr = (const void __user *)(unsigned long)(address & ~3);
|
Remove 'type' argument from access_ok() function
Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument
of the user address range verification function since we got rid of the
old racy i386-only code to walk page tables by hand.
It existed because the original 80386 would not honor the write protect
bit when in kernel mode, so you had to do COW by hand before doing any
user access. But we haven't supported that in a long time, and these
days the 'type' argument is a purely historical artifact.
A discussion about extending 'user_access_begin()' to do the range
checking resulted this patch, because there is no way we're going to
move the old VERIFY_xyz interface to that model. And it's best done at
the end of the merge window when I've done most of my merges, so let's
just get this done once and for all.
This patch was mostly done with a sed-script, with manual fix-ups for
the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form.
There were a couple of notable cases:
- csky still had the old "verify_area()" name as an alias.
- the iter_iov code had magical hardcoded knowledge of the actual
values of VERIFY_{READ,WRITE} (not that they mattered, since nothing
really used it)
- microblaze used the type argument for a debug printout
but other than those oddities this should be a total no-op patch.
I tried to fix up all architectures, did fairly extensive grepping for
access_ok() uses, and the changes are trivial, but I may have missed
something. Any missed conversion should be trivially fixable, though.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2019-01-04 09:57:57 +07:00
|
|
|
if (!access_ok(user_ptr, 4)) {
|
2014-11-18 18:41:25 +07:00
|
|
|
pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
|
|
|
|
address);
|
|
|
|
goto fault;
|
|
|
|
}
|
|
|
|
|
|
|
|
res = emulate_swpX(address, &data, type);
|
|
|
|
if (res == -EFAULT)
|
|
|
|
goto fault;
|
|
|
|
else if (res == 0)
|
|
|
|
regs->user_regs.regs[destreg] = data;
|
|
|
|
|
|
|
|
ret:
|
2014-11-18 18:41:27 +07:00
|
|
|
if (type == TYPE_SWPB)
|
|
|
|
trace_instruction_emulation("swpb", regs->pc);
|
|
|
|
else
|
|
|
|
trace_instruction_emulation("swp", regs->pc);
|
|
|
|
|
2014-11-18 18:41:25 +07:00
|
|
|
pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
|
|
|
|
current->comm, (unsigned long)current->pid, regs->pc);
|
|
|
|
|
2017-10-25 16:04:33 +07:00
|
|
|
arm64_skip_faulting_instruction(regs, 4);
|
2014-11-18 18:41:25 +07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
fault:
|
2016-06-29 00:07:31 +07:00
|
|
|
pr_debug("SWP{B} emulation: access caused memory abort!\n");
|
2018-02-20 21:16:29 +07:00
|
|
|
arm64_notify_segfault(address);
|
2014-11-18 18:41:25 +07:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only emulate SWP/SWPB executed in ARM state/User mode.
|
|
|
|
* The kernel must be SWP free and SWP{B} does not exist in Thumb.
|
|
|
|
*/
|
|
|
|
static struct undef_hook swp_hooks[] = {
|
|
|
|
{
|
|
|
|
.instr_mask = 0x0fb00ff0,
|
|
|
|
.instr_val = 0x01000090,
|
2018-07-05 21:16:52 +07:00
|
|
|
.pstate_mask = PSR_AA32_MODE_MASK,
|
|
|
|
.pstate_val = PSR_AA32_MODE_USR,
|
2014-11-18 18:41:25 +07:00
|
|
|
.fn = swp_handler
|
|
|
|
},
|
|
|
|
{ }
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct insn_emulation_ops swp_ops = {
|
|
|
|
.name = "swp",
|
|
|
|
.status = INSN_OBSOLETE,
|
|
|
|
.hooks = swp_hooks,
|
|
|
|
.set_hw_mode = NULL,
|
|
|
|
};
|
|
|
|
|
2014-11-18 18:41:26 +07:00
|
|
|
static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
|
|
|
|
{
|
|
|
|
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
|
|
|
|
|
2016-07-08 23:35:47 +07:00
|
|
|
switch (aarch32_check_condition(instr, regs->pstate)) {
|
2014-11-18 18:41:26 +07:00
|
|
|
case ARM_OPCODE_CONDTEST_PASS:
|
|
|
|
break;
|
|
|
|
case ARM_OPCODE_CONDTEST_FAIL:
|
|
|
|
/* Condition failed - return to next instruction */
|
|
|
|
goto ret;
|
|
|
|
case ARM_OPCODE_CONDTEST_UNCOND:
|
|
|
|
/* If unconditional encoding - not a barrier instruction */
|
|
|
|
return -EFAULT;
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
switch (aarch32_insn_mcr_extract_crm(instr)) {
|
|
|
|
case 10:
|
|
|
|
/*
|
|
|
|
* dmb - mcr p15, 0, Rt, c7, c10, 5
|
|
|
|
* dsb - mcr p15, 0, Rt, c7, c10, 4
|
|
|
|
*/
|
2014-11-18 18:41:27 +07:00
|
|
|
if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
|
2014-11-18 18:41:26 +07:00
|
|
|
dmb(sy);
|
2014-11-18 18:41:27 +07:00
|
|
|
trace_instruction_emulation(
|
|
|
|
"mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
|
|
|
|
} else {
|
2014-11-18 18:41:26 +07:00
|
|
|
dsb(sy);
|
2014-11-18 18:41:27 +07:00
|
|
|
trace_instruction_emulation(
|
|
|
|
"mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
|
|
|
|
}
|
2014-11-18 18:41:26 +07:00
|
|
|
break;
|
|
|
|
case 5:
|
|
|
|
/*
|
|
|
|
* isb - mcr p15, 0, Rt, c7, c5, 4
|
|
|
|
*
|
|
|
|
* Taking an exception or returning from one acts as an
|
|
|
|
* instruction barrier. So no explicit barrier needed here.
|
|
|
|
*/
|
2014-11-18 18:41:27 +07:00
|
|
|
trace_instruction_emulation(
|
|
|
|
"mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
|
2014-11-18 18:41:26 +07:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret:
|
|
|
|
pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
|
|
|
|
current->comm, (unsigned long)current->pid, regs->pc);
|
|
|
|
|
2017-10-25 16:04:33 +07:00
|
|
|
arm64_skip_faulting_instruction(regs, 4);
|
2014-11-18 18:41:26 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int cp15_barrier_set_hw_mode(bool enable)
|
|
|
|
{
|
2015-01-21 19:43:10 +07:00
|
|
|
if (enable)
|
2018-07-11 20:56:38 +07:00
|
|
|
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
|
2015-01-21 19:43:10 +07:00
|
|
|
else
|
2018-07-11 20:56:38 +07:00
|
|
|
sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
|
2015-01-21 19:43:10 +07:00
|
|
|
return 0;
|
2014-11-18 18:41:26 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct undef_hook cp15_barrier_hooks[] = {
|
|
|
|
{
|
|
|
|
.instr_mask = 0x0fff0fdf,
|
|
|
|
.instr_val = 0x0e070f9a,
|
2018-07-05 21:16:52 +07:00
|
|
|
.pstate_mask = PSR_AA32_MODE_MASK,
|
|
|
|
.pstate_val = PSR_AA32_MODE_USR,
|
2014-11-18 18:41:26 +07:00
|
|
|
.fn = cp15barrier_handler,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.instr_mask = 0x0fff0fff,
|
|
|
|
.instr_val = 0x0e070f95,
|
2018-07-05 21:16:52 +07:00
|
|
|
.pstate_mask = PSR_AA32_MODE_MASK,
|
|
|
|
.pstate_val = PSR_AA32_MODE_USR,
|
2014-11-18 18:41:26 +07:00
|
|
|
.fn = cp15barrier_handler,
|
|
|
|
},
|
|
|
|
{ }
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct insn_emulation_ops cp15_barrier_ops = {
|
|
|
|
.name = "cp15_barrier",
|
|
|
|
.status = INSN_DEPRECATED,
|
|
|
|
.hooks = cp15_barrier_hooks,
|
|
|
|
.set_hw_mode = cp15_barrier_set_hw_mode,
|
|
|
|
};
|
|
|
|
|
2015-01-21 19:43:11 +07:00
|
|
|
static int setend_set_hw_mode(bool enable)
|
|
|
|
{
|
|
|
|
if (!cpu_supports_mixed_endian_el0())
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (enable)
|
2018-07-11 20:56:38 +07:00
|
|
|
sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
|
2015-01-21 19:43:11 +07:00
|
|
|
else
|
2018-07-11 20:56:38 +07:00
|
|
|
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
|
2015-01-21 19:43:11 +07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
|
|
|
|
{
|
|
|
|
char *insn;
|
|
|
|
|
|
|
|
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
|
|
|
|
|
|
|
|
if (big_endian) {
|
|
|
|
insn = "setend be";
|
2018-07-05 21:16:52 +07:00
|
|
|
regs->pstate |= PSR_AA32_E_BIT;
|
2015-01-21 19:43:11 +07:00
|
|
|
} else {
|
|
|
|
insn = "setend le";
|
2018-07-05 21:16:52 +07:00
|
|
|
regs->pstate &= ~PSR_AA32_E_BIT;
|
2015-01-21 19:43:11 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
trace_instruction_emulation(insn, regs->pc);
|
|
|
|
pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
|
|
|
|
current->comm, (unsigned long)current->pid, regs->pc);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int a32_setend_handler(struct pt_regs *regs, u32 instr)
|
|
|
|
{
|
|
|
|
int rc = compat_setend_handler(regs, (instr >> 9) & 1);
|
2017-10-25 16:04:33 +07:00
|
|
|
arm64_skip_faulting_instruction(regs, 4);
|
2015-01-21 19:43:11 +07:00
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int t16_setend_handler(struct pt_regs *regs, u32 instr)
|
|
|
|
{
|
|
|
|
int rc = compat_setend_handler(regs, (instr >> 3) & 1);
|
2017-10-25 16:04:33 +07:00
|
|
|
arm64_skip_faulting_instruction(regs, 2);
|
2015-01-21 19:43:11 +07:00
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct undef_hook setend_hooks[] = {
|
|
|
|
{
|
|
|
|
.instr_mask = 0xfffffdff,
|
|
|
|
.instr_val = 0xf1010000,
|
2018-07-05 21:16:52 +07:00
|
|
|
.pstate_mask = PSR_AA32_MODE_MASK,
|
|
|
|
.pstate_val = PSR_AA32_MODE_USR,
|
2015-01-21 19:43:11 +07:00
|
|
|
.fn = a32_setend_handler,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
/* Thumb mode */
|
|
|
|
.instr_mask = 0x0000fff7,
|
|
|
|
.instr_val = 0x0000b650,
|
2018-07-05 21:16:52 +07:00
|
|
|
.pstate_mask = (PSR_AA32_T_BIT | PSR_AA32_MODE_MASK),
|
|
|
|
.pstate_val = (PSR_AA32_T_BIT | PSR_AA32_MODE_USR),
|
2015-01-21 19:43:11 +07:00
|
|
|
.fn = t16_setend_handler,
|
|
|
|
},
|
|
|
|
{}
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct insn_emulation_ops setend_ops = {
|
|
|
|
.name = "setend",
|
|
|
|
.status = INSN_DEPRECATED,
|
|
|
|
.hooks = setend_hooks,
|
|
|
|
.set_hw_mode = setend_set_hw_mode,
|
|
|
|
};
|
|
|
|
|
2014-11-18 18:41:24 +07:00
|
|
|
/*
|
|
|
|
* Invoked as late_initcall, since not needed before init spawned.
|
|
|
|
*/
|
|
|
|
static int __init armv8_deprecated_init(void)
|
|
|
|
{
|
2014-11-18 18:41:25 +07:00
|
|
|
if (IS_ENABLED(CONFIG_SWP_EMULATION))
|
|
|
|
register_insn_emulation(&swp_ops);
|
|
|
|
|
2014-11-18 18:41:26 +07:00
|
|
|
if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
|
|
|
|
register_insn_emulation(&cp15_barrier_ops);
|
|
|
|
|
2015-01-21 19:43:11 +07:00
|
|
|
if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
|
|
|
|
if(system_supports_mixed_endian_el0())
|
|
|
|
register_insn_emulation(&setend_ops);
|
|
|
|
else
|
2017-01-09 21:13:36 +07:00
|
|
|
pr_info("setend instruction emulation is not supported on this system\n");
|
2015-01-21 19:43:11 +07:00
|
|
|
}
|
|
|
|
|
2016-07-14 00:16:56 +07:00
|
|
|
cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
|
2016-12-22 02:19:54 +07:00
|
|
|
"arm64/isndep:starting",
|
2016-07-14 00:16:56 +07:00
|
|
|
run_all_insn_set_hw_mode, NULL);
|
2017-10-31 22:50:58 +07:00
|
|
|
register_insn_emulation_sysctl();
|
2014-11-18 18:41:24 +07:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-10-06 20:16:52 +07:00
|
|
|
core_initcall(armv8_deprecated_init);
|