linux_dsm_epyc7002/drivers/firmware/arm_sdei.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2017 Arm Ltd.
#define pr_fmt(fmt) "sdei: " fmt

#include <acpi/ghes.h>
#include <linux/acpi.h>
#include <linux/arm_sdei.h>
#include <linux/arm-smccc.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/compiler.h>
#include <linux/cpuhotplug.h>
#include <linux/cpu.h>
#include <linux/cpu_pm.h>
#include <linux/errno.h>
#include <linux/hardirq.h>
#include <linux/kernel.h>
#include <linux/kprobes.h>
#include <linux/kvm_host.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/notifier.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/percpu.h>
#include <linux/platform_device.h>
#include <linux/pm.h>
#include <linux/ptrace.h>
#include <linux/preempt.h>
#include <linux/reboot.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/uaccess.h>

/*
 * The call to use to reach the firmware.
 */
static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
		      unsigned long arg0, unsigned long arg1,
		      unsigned long arg2, unsigned long arg3,
		      unsigned long arg4, struct arm_smccc_res *res);

/* entry point from firmware to arch asm code */
static unsigned long sdei_entry_point;

struct sdei_event {
	/* These three are protected by the sdei_list_lock */
	struct list_head	list;
	bool			reregister;
	bool			reenable;

	u32			event_num;
	u8			type;
	u8			priority;

	/* This pointer is handed to firmware as the event argument. */
	union {
		/* Shared events */
		struct sdei_registered_event *registered;

		/* CPU private events */
		struct sdei_registered_event __percpu *private_registered;
	};
};

/* Take the mutex for any API call or modification. Take the mutex first. */
static DEFINE_MUTEX(sdei_events_lock);

/* and then hold this when modifying the list */
static DEFINE_SPINLOCK(sdei_list_lock);
static LIST_HEAD(sdei_list);

/* Private events are registered/enabled via IPI passing one of these */
struct sdei_crosscall_args {
	struct sdei_event *event;
	atomic_t errors;
	int first_error;
};

#define CROSSCALL_INIT(arg, event)	(arg.event = event, \
					 arg.first_error = 0, \
					 atomic_set(&arg.errors, 0))

static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
{
	struct sdei_crosscall_args arg;

	CROSSCALL_INIT(arg, event);
	on_each_cpu(fn, &arg, true);

	return arg.first_error;
}

static inline void
sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
{
	if (err && (atomic_inc_return(&arg->errors) == 1))
		arg->first_error = err;
}

static int sdei_to_linux_errno(unsigned long sdei_err)
{
	switch (sdei_err) {
	case SDEI_NOT_SUPPORTED:
		return -EOPNOTSUPP;
	case SDEI_INVALID_PARAMETERS:
		return -EINVAL;
	case SDEI_DENIED:
		return -EPERM;
	case SDEI_PENDING:
		return -EINPROGRESS;
	case SDEI_OUT_OF_RESOURCE:
		return -ENOMEM;
	}

	/* Not an error value ... */
	return sdei_err;
}

/*
 * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
 * to translate.
 */
static int sdei_is_err(struct arm_smccc_res *res)
{
	switch (res->a0) {
	case SDEI_NOT_SUPPORTED:
	case SDEI_INVALID_PARAMETERS:
	case SDEI_DENIED:
	case SDEI_PENDING:
	case SDEI_OUT_OF_RESOURCE:
		return true;
	}

	return false;
}

static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
			  unsigned long arg1, unsigned long arg2,
			  unsigned long arg3, unsigned long arg4,
			  u64 *result)
{
	int err = 0;
	struct arm_smccc_res res;

	if (sdei_firmware_call) {
		sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
				   &res);
		if (sdei_is_err(&res))
			err = sdei_to_linux_errno(res.a0);
	} else {
		/*
		 * !sdei_firmware_call means we failed to probe or called
		 * sdei_mark_interface_broken(). -EIO is not an error returned
		 * by sdei_to_linux_errno() and is used to suppress messages
		 * from this driver.
		 */
		err = -EIO;
		res.a0 = SDEI_NOT_SUPPORTED;
	}

	if (result)
		*result = res.a0;

	return err;
}
NOKPROBE_SYMBOL(invoke_sdei_fn);

static struct sdei_event *sdei_event_find(u32 event_num)
{
	struct sdei_event *e, *found = NULL;

	lockdep_assert_held(&sdei_events_lock);

	spin_lock(&sdei_list_lock);
	list_for_each_entry(e, &sdei_list, list) {
		if (e->event_num == event_num) {
			found = e;
			break;
		}
	}
	spin_unlock(&sdei_list_lock);

	return found;
}

int sdei_api_event_context(u32 query, u64 *result)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
			      result);
}
NOKPROBE_SYMBOL(sdei_api_event_context);

static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
			      0, 0, result);
}

static struct sdei_event *sdei_event_create(u32 event_num,
					    sdei_event_callback *cb,
					    void *cb_arg)
{
	int err;
	u64 result;
	struct sdei_event *event;
	struct sdei_registered_event *reg;

	lockdep_assert_held(&sdei_events_lock);

	event = kzalloc(sizeof(*event), GFP_KERNEL);
	if (!event)
		return ERR_PTR(-ENOMEM);

	INIT_LIST_HEAD(&event->list);
	event->event_num = event_num;

	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
				      &result);
	if (err) {
		kfree(event);
		return ERR_PTR(err);
	}
	event->priority = result;

	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
				      &result);
	if (err) {
		kfree(event);
		return ERR_PTR(err);
	}
	event->type = result;

	if (event->type == SDEI_EVENT_TYPE_SHARED) {
		reg = kzalloc(sizeof(*reg), GFP_KERNEL);
		if (!reg) {
			kfree(event);
			return ERR_PTR(-ENOMEM);
		}

		reg->event_num = event_num;
		reg->priority = event->priority;

		reg->callback = cb;
		reg->callback_arg = cb_arg;
		event->registered = reg;
	} else {
		int cpu;
		struct sdei_registered_event __percpu *regs;

		regs = alloc_percpu(struct sdei_registered_event);
		if (!regs) {
			kfree(event);
			return ERR_PTR(-ENOMEM);
		}

		for_each_possible_cpu(cpu) {
			reg = per_cpu_ptr(regs, cpu);

			reg->event_num = event->event_num;
			reg->priority = event->priority;
			reg->callback = cb;
			reg->callback_arg = cb_arg;
		}

		event->private_registered = regs;
	}

	if (sdei_event_find(event_num)) {
		kfree(event->registered);
		kfree(event);
		event = ERR_PTR(-EBUSY);
	} else {
		spin_lock(&sdei_list_lock);
		list_add(&event->list, &sdei_list);
		spin_unlock(&sdei_list_lock);
	}

	return event;
}

static void sdei_event_destroy(struct sdei_event *event)
{
	lockdep_assert_held(&sdei_events_lock);

	spin_lock(&sdei_list_lock);
	list_del(&event->list);
	spin_unlock(&sdei_list_lock);

	if (event->type == SDEI_EVENT_TYPE_SHARED)
		kfree(event->registered);
	else
		free_percpu(event->private_registered);

	kfree(event);
}

static int sdei_api_get_version(u64 *version)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
}

int sdei_mask_local_cpu(void)
{
	int err;

	WARN_ON_ONCE(preemptible());

	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
	if (err && err != -EIO) {
		pr_warn_once("failed to mask CPU[%u]: %d\n",
			      smp_processor_id(), err);
		return err;
	}

	return 0;
}

static void _ipi_mask_cpu(void *ignored)
{
	sdei_mask_local_cpu();
}

int sdei_unmask_local_cpu(void)
{
	int err;

	WARN_ON_ONCE(preemptible());

	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
	if (err && err != -EIO) {
		pr_warn_once("failed to unmask CPU[%u]: %d\n",
			     smp_processor_id(), err);
		return err;
	}

	return 0;
}

static void _ipi_unmask_cpu(void *ignored)
{
	sdei_unmask_local_cpu();
}

static void _ipi_private_reset(void *ignored)
{
	int err;

	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
			     NULL);
	if (err && err != -EIO)
		pr_warn_once("failed to reset CPU[%u]: %d\n",
			     smp_processor_id(), err);
}

static int sdei_api_shared_reset(void)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
			      NULL);
}

static void sdei_mark_interface_broken(void)
{
	pr_err("disabling SDEI firmware interface\n");
	on_each_cpu(&_ipi_mask_cpu, NULL, true);
	sdei_firmware_call = NULL;
}

static int sdei_platform_reset(void)
{
	int err;

	on_each_cpu(&_ipi_private_reset, NULL, true);
	err = sdei_api_shared_reset();
	if (err) {
		pr_err("Failed to reset platform: %d\n", err);
		sdei_mark_interface_broken();
	}

	return err;
}

static int sdei_api_event_enable(u32 event_num)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
			      0, NULL);
}

/* Called directly by the hotplug callbacks */
static void _local_event_enable(void *data)
{
	int err;
	struct sdei_crosscall_args *arg = data;

	WARN_ON_ONCE(preemptible());

	err = sdei_api_event_enable(arg->event->event_num);

	sdei_cross_call_return(arg, err);
}

int sdei_event_enable(u32 event_num)
{
	int err = -EINVAL;
	struct sdei_event *event;

	mutex_lock(&sdei_events_lock);
	event = sdei_event_find(event_num);
	if (!event) {
		mutex_unlock(&sdei_events_lock);
		return -ENOENT;
	}

	spin_lock(&sdei_list_lock);
	event->reenable = true;
	spin_unlock(&sdei_list_lock);

	if (event->type == SDEI_EVENT_TYPE_SHARED)
		err = sdei_api_event_enable(event->event_num);
	else
		err = sdei_do_cross_call(_local_event_enable, event);
	mutex_unlock(&sdei_events_lock);

	return err;
}
EXPORT_SYMBOL(sdei_event_enable);

static int sdei_api_event_disable(u32 event_num)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
			      0, 0, NULL);
}

static void _ipi_event_disable(void *data)
{
	int err;
	struct sdei_crosscall_args *arg = data;

	err = sdei_api_event_disable(arg->event->event_num);

	sdei_cross_call_return(arg, err);
}

int sdei_event_disable(u32 event_num)
{
	int err = -EINVAL;
	struct sdei_event *event;

	mutex_lock(&sdei_events_lock);
	event = sdei_event_find(event_num);
	if (!event) {
		mutex_unlock(&sdei_events_lock);
		return -ENOENT;
	}

	spin_lock(&sdei_list_lock);
	event->reenable = false;
	spin_unlock(&sdei_list_lock);

	if (event->type == SDEI_EVENT_TYPE_SHARED)
		err = sdei_api_event_disable(event->event_num);
	else
		err = sdei_do_cross_call(_ipi_event_disable, event);
	mutex_unlock(&sdei_events_lock);

	return err;
}
EXPORT_SYMBOL(sdei_event_disable);

static int sdei_api_event_unregister(u32 event_num)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
			      0, 0, 0, NULL);
}

/* Called directly by the hotplug callbacks */
static void _local_event_unregister(void *data)
{
	int err;
	struct sdei_crosscall_args *arg = data;

	WARN_ON_ONCE(preemptible());

	err = sdei_api_event_unregister(arg->event->event_num);

	sdei_cross_call_return(arg, err);
}

static int _sdei_event_unregister(struct sdei_event *event)
{
	lockdep_assert_held(&sdei_events_lock);

	spin_lock(&sdei_list_lock);
	event->reregister = false;
	event->reenable = false;
	spin_unlock(&sdei_list_lock);

	if (event->type == SDEI_EVENT_TYPE_SHARED)
		return sdei_api_event_unregister(event->event_num);

	return sdei_do_cross_call(_local_event_unregister, event);
}

int sdei_event_unregister(u32 event_num)
{
	int err;
	struct sdei_event *event;

	WARN_ON(in_nmi());

	mutex_lock(&sdei_events_lock);
	event = sdei_event_find(event_num);
	do {
		if (!event) {
			pr_warn("Event %u not registered\n", event_num);
			err = -ENOENT;
			break;
		}

		err = _sdei_event_unregister(event);
		if (err)
			break;

		sdei_event_destroy(event);
	} while (0);
	mutex_unlock(&sdei_events_lock);

	return err;
}
EXPORT_SYMBOL(sdei_event_unregister);

/*
 * unregister events, but don't destroy them as they are re-registered by
 * sdei_reregister_shared().
 */
static int sdei_unregister_shared(void)
{
	int err = 0;
	struct sdei_event *event;

	mutex_lock(&sdei_events_lock);
	spin_lock(&sdei_list_lock);
	list_for_each_entry(event, &sdei_list, list) {
		if (event->type != SDEI_EVENT_TYPE_SHARED)
			continue;

		err = _sdei_event_unregister(event);
		if (err)
			break;
	}
	spin_unlock(&sdei_list_lock);
	mutex_unlock(&sdei_events_lock);

	return err;
}

static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
				   void *arg, u64 flags, u64 affinity)
{
	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
			      (unsigned long)entry_point, (unsigned long)arg,
			      flags, affinity, NULL);
}

/* Called directly by the hotplug callbacks */
static void _local_event_register(void *data)
{
	int err;
	struct sdei_registered_event *reg;
	struct sdei_crosscall_args *arg = data;

	WARN_ON(preemptible());

	reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
	err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
				      reg, 0, 0);

	sdei_cross_call_return(arg, err);
}

static int _sdei_event_register(struct sdei_event *event)
{
	int err;

	lockdep_assert_held(&sdei_events_lock);

	spin_lock(&sdei_list_lock);
	event->reregister = true;
	spin_unlock(&sdei_list_lock);

	if (event->type == SDEI_EVENT_TYPE_SHARED)
		return sdei_api_event_register(event->event_num,
					       sdei_entry_point,
					       event->registered,
					       SDEI_EVENT_REGISTER_RM_ANY, 0);


	err = sdei_do_cross_call(_local_event_register, event);
	if (err) {
		spin_lock(&sdei_list_lock);
		event->reregister = false;
		event->reenable = false;
		spin_unlock(&sdei_list_lock);

		sdei_do_cross_call(_local_event_unregister, event);
	}

	return err;
}

int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
{
	int err;
	struct sdei_event *event;

	WARN_ON(in_nmi());

	mutex_lock(&sdei_events_lock);
	do {
		if (sdei_event_find(event_num)) {
			pr_warn("Event %u already registered\n", event_num);
			err = -EBUSY;
			break;
		}

		event = sdei_event_create(event_num, cb, arg);
		if (IS_ERR(event)) {
			err = PTR_ERR(event);
			pr_warn("Failed to create event %u: %d\n", event_num,
				err);
			break;
		}

		err = _sdei_event_register(event);
		if (err) {
			sdei_event_destroy(event);
			pr_warn("Failed to register event %u: %d\n", event_num,
				err);
		}
	} while (0);
	mutex_unlock(&sdei_events_lock);

	return err;
}
EXPORT_SYMBOL(sdei_event_register);

static int sdei_reregister_event(struct sdei_event *event)
{
	int err;

	lockdep_assert_held(&sdei_events_lock);

	err = _sdei_event_register(event);
	if (err) {
		pr_err("Failed to re-register event %u\n", event->event_num);
		sdei_event_destroy(event);
		return err;
	}

	if (event->reenable) {
		if (event->type == SDEI_EVENT_TYPE_SHARED)
			err = sdei_api_event_enable(event->event_num);
		else
			err = sdei_do_cross_call(_local_event_enable, event);
	}

	if (err)
		pr_err("Failed to re-enable event %u\n", event->event_num);

	return err;
}

static int sdei_reregister_shared(void)
{
	int err = 0;
	struct sdei_event *event;

	mutex_lock(&sdei_events_lock);
	spin_lock(&sdei_list_lock);
	list_for_each_entry(event, &sdei_list, list) {
		if (event->type != SDEI_EVENT_TYPE_SHARED)
			continue;

		if (event->reregister) {
			err = sdei_reregister_event(event);
			if (err)
				break;
		}
	}
	spin_unlock(&sdei_list_lock);
	mutex_unlock(&sdei_events_lock);

	return err;
}

static int sdei_cpuhp_down(unsigned int cpu)
{
	struct sdei_event *event;
	struct sdei_crosscall_args arg;

	/* un-register private events */
	spin_lock(&sdei_list_lock);
	list_for_each_entry(event, &sdei_list, list) {
		if (event->type == SDEI_EVENT_TYPE_SHARED)
			continue;

		CROSSCALL_INIT(arg, event);
		/* call the cross-call function locally... */
		_local_event_unregister(&arg);
		if (arg.first_error)
			pr_err("Failed to unregister event %u: %d\n",
			       event->event_num, arg.first_error);
	}
	spin_unlock(&sdei_list_lock);

	return sdei_mask_local_cpu();
}

static int sdei_cpuhp_up(unsigned int cpu)
{
	struct sdei_event *event;
	struct sdei_crosscall_args arg;

	/* re-register/enable private events */
	spin_lock(&sdei_list_lock);
	list_for_each_entry(event, &sdei_list, list) {
		if (event->type == SDEI_EVENT_TYPE_SHARED)
			continue;

		if (event->reregister) {
			CROSSCALL_INIT(arg, event);
			/* call the cross-call function locally... */
			_local_event_register(&arg);
			if (arg.first_error)
				pr_err("Failed to re-register event %u: %d\n",
				       event->event_num, arg.first_error);
		}

		if (event->reenable) {
			CROSSCALL_INIT(arg, event);
			_local_event_enable(&arg);
			if (arg.first_error)
				pr_err("Failed to re-enable event %u: %d\n",
				       event->event_num, arg.first_error);
		}
	}
	spin_unlock(&sdei_list_lock);

	return sdei_unmask_local_cpu();
}

/* When entering idle, mask/unmask events for this cpu */
static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
			    void *data)
{
	int rv;

	switch (action) {
	case CPU_PM_ENTER:
		rv = sdei_mask_local_cpu();
		break;
	case CPU_PM_EXIT:
	case CPU_PM_ENTER_FAILED:
		rv = sdei_unmask_local_cpu();
		break;
	default:
		return NOTIFY_DONE;
	}

	if (rv)
		return notifier_from_errno(rv);

	return NOTIFY_OK;
}

static struct notifier_block sdei_pm_nb = {
	.notifier_call = sdei_pm_notifier,
};

static int sdei_device_suspend(struct device *dev)
{
	on_each_cpu(_ipi_mask_cpu, NULL, true);

	return 0;
}

static int sdei_device_resume(struct device *dev)
{
	on_each_cpu(_ipi_unmask_cpu, NULL, true);

	return 0;
}

/*
 * We need all events to be reregistered when we resume from hibernate.
 *
 * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
 * events during freeze, then re-register and re-enable them during thaw
 * and restore.
 */
static int sdei_device_freeze(struct device *dev)
{
	int err;

	/* unregister private events */
	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);

	err = sdei_unregister_shared();
	if (err)
		return err;

	return 0;
}

static int sdei_device_thaw(struct device *dev)
{
	int err;

	/* re-register shared events */
	err = sdei_reregister_shared();
	if (err) {
		pr_warn("Failed to re-register shared events...\n");
		sdei_mark_interface_broken();
		return err;
	}

	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
				&sdei_cpuhp_up, &sdei_cpuhp_down);
	if (err)
		pr_warn("Failed to re-register CPU hotplug notifier...\n");

	return err;
}

static int sdei_device_restore(struct device *dev)
{
	int err;

	err = sdei_platform_reset();
	if (err)
		return err;

	return sdei_device_thaw(dev);
}

static const struct dev_pm_ops sdei_pm_ops = {
	.suspend = sdei_device_suspend,
	.resume = sdei_device_resume,
	.freeze = sdei_device_freeze,
	.thaw = sdei_device_thaw,
	.restore = sdei_device_restore,
};

/*
 * Mask all CPUs and unregister all events on panic, reboot or kexec.
 */
static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
				void *data)
{
	/*
	 * We are going to reset the interface, after this there is no point
	 * doing work when we take CPUs offline.
	 */
	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);

	sdei_platform_reset();

	return NOTIFY_OK;
}

static struct notifier_block sdei_reboot_nb = {
	.notifier_call = sdei_reboot_notifier,
};

static void sdei_smccc_smc(unsigned long function_id,
			   unsigned long arg0, unsigned long arg1,
			   unsigned long arg2, unsigned long arg3,
			   unsigned long arg4, struct arm_smccc_res *res)
{
	arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
}
NOKPROBE_SYMBOL(sdei_smccc_smc);

static void sdei_smccc_hvc(unsigned long function_id,
			   unsigned long arg0, unsigned long arg1,
			   unsigned long arg2, unsigned long arg3,
			   unsigned long arg4, struct arm_smccc_res *res)
{
	arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
}
NOKPROBE_SYMBOL(sdei_smccc_hvc);

int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb,
		       sdei_event_callback *critical_cb)
{
	int err;
	u64 result;
	u32 event_num;
	sdei_event_callback *cb;

	if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
		return -EOPNOTSUPP;

	event_num = ghes->generic->notify.vector;
	if (event_num == 0) {
		/*
		 * Event 0 is reserved by the specification for
		 * SDEI_EVENT_SIGNAL.
		 */
		return -EINVAL;
	}

	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
				      &result);
	if (err)
		return err;

	if (result == SDEI_EVENT_PRIORITY_CRITICAL)
		cb = critical_cb;
	else
		cb = normal_cb;

	err = sdei_event_register(event_num, cb, ghes);
	if (!err)
		err = sdei_event_enable(event_num);

	return err;
}

int sdei_unregister_ghes(struct ghes *ghes)
{
	int i;
	int err;
	u32 event_num = ghes->generic->notify.vector;

	might_sleep();

	if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
		return -EOPNOTSUPP;

	/*
	 * The event may be running on another CPU. Disable it
	 * to stop new events, then try to unregister a few times.
	 */
	err = sdei_event_disable(event_num);
	if (err)
		return err;

	for (i = 0; i < 3; i++) {
		err = sdei_event_unregister(event_num);
		if (err != -EINPROGRESS)
			break;

		schedule();
	}

	return err;
}

static int sdei_get_conduit(struct platform_device *pdev)
{
	const char *method;
	struct device_node *np = pdev->dev.of_node;

	sdei_firmware_call = NULL;
	if (np) {
		if (of_property_read_string(np, "method", &method)) {
			pr_warn("missing \"method\" property\n");
			return CONDUIT_INVALID;
		}

		if (!strcmp("hvc", method)) {
			sdei_firmware_call = &sdei_smccc_hvc;
			return CONDUIT_HVC;
		} else if (!strcmp("smc", method)) {
			sdei_firmware_call = &sdei_smccc_smc;
			return CONDUIT_SMC;
		}

		pr_warn("invalid \"method\" property: %s\n", method);
	} else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
		if (acpi_psci_use_hvc()) {
			sdei_firmware_call = &sdei_smccc_hvc;
			return CONDUIT_HVC;
		} else {
			sdei_firmware_call = &sdei_smccc_smc;
			return CONDUIT_SMC;
		}
	}

	return CONDUIT_INVALID;
}

static int sdei_probe(struct platform_device *pdev)
{
	int err;
	u64 ver = 0;
	int conduit;

	conduit = sdei_get_conduit(pdev);
	if (!sdei_firmware_call)
		return 0;

	err = sdei_api_get_version(&ver);
	if (err == -EOPNOTSUPP)
		pr_err("advertised but not implemented in platform firmware\n");
	if (err) {
		pr_err("Failed to get SDEI version: %d\n", err);
		sdei_mark_interface_broken();
		return err;
	}

	pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
		(int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
		(int)SDEI_VERSION_VENDOR(ver));

	if (SDEI_VERSION_MAJOR(ver) != 1) {
		pr_warn("Conflicting SDEI version detected.\n");
		sdei_mark_interface_broken();
		return -EINVAL;
	}

	err = sdei_platform_reset();
	if (err)
		return err;

	sdei_entry_point = sdei_arch_get_entry_point(conduit);
	if (!sdei_entry_point) {
		/* Not supported due to hardware or boot configuration */
		sdei_mark_interface_broken();
		return 0;
	}

	err = cpu_pm_register_notifier(&sdei_pm_nb);
	if (err) {
		pr_warn("Failed to register CPU PM notifier...\n");
		goto error;
	}

	err = register_reboot_notifier(&sdei_reboot_nb);
	if (err) {
		pr_warn("Failed to register reboot notifier...\n");
		goto remove_cpupm;
	}

	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
				&sdei_cpuhp_up, &sdei_cpuhp_down);
	if (err) {
		pr_warn("Failed to register CPU hotplug notifier...\n");
		goto remove_reboot;
	}

	return 0;

remove_reboot:
	unregister_reboot_notifier(&sdei_reboot_nb);

remove_cpupm:
	cpu_pm_unregister_notifier(&sdei_pm_nb);

error:
	sdei_mark_interface_broken();
	return err;
}

static const struct of_device_id sdei_of_match[] = {
	{ .compatible = "arm,sdei-1.0" },
	{}
};

static struct platform_driver sdei_driver = {
	.driver		= {
		.name			= "sdei",
		.pm			= &sdei_pm_ops,
		.of_match_table		= sdei_of_match,
	},
	.probe		= sdei_probe,
};

static bool __init sdei_present_dt(void)
{
	struct device_node *np, *fw_np;

	fw_np = of_find_node_by_name(NULL, "firmware");
	if (!fw_np)
		return false;

	np = of_find_matching_node(fw_np, sdei_of_match);
	if (!np)
		return false;
	of_node_put(np);

	return true;
}

static bool __init sdei_present_acpi(void)
{
	acpi_status status;
	struct platform_device *pdev;
	struct acpi_table_header *sdei_table_header;

	if (acpi_disabled)
		return false;

	status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
		const char *msg = acpi_format_exception(status);

		pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
	}
	if (ACPI_FAILURE(status))
		return false;

	pdev = platform_device_register_simple(sdei_driver.driver.name, 0, NULL,
					       0);
	if (IS_ERR(pdev))
		return false;

	return true;
}

static int __init sdei_init(void)
{
	if (sdei_present_dt() || sdei_present_acpi())
		platform_driver_register(&sdei_driver);

	return 0;
}

/*
 * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
 * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
 * by device_initcall(). We want to be called in the middle.
 */
subsys_initcall_sync(sdei_init);

int sdei_event_handler(struct pt_regs *regs,
		       struct sdei_registered_event *arg)
{
	int err;
	mm_segment_t orig_addr_limit;
	u32 event_num = arg->event_num;

	orig_addr_limit = get_fs();
	set_fs(USER_DS);

	err = arg->callback(event_num, regs, arg->callback_arg);
	if (err)
		pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
				   event_num, smp_processor_id(), err);

	set_fs(orig_addr_limit);

	return err;
}
NOKPROBE_SYMBOL(sdei_event_handler);