mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-27 09:26:23 +07:00
2874c5fd28
Based on 1 normalized pattern(s): this program is free software you can redistribute it and or modify it under the terms of the gnu general public license as published by the free software foundation either version 2 of the license or at your option any later version extracted by the scancode license scanner the SPDX license identifier GPL-2.0-or-later has been chosen to replace the boilerplate/reference in 3029 file(s). Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Allison Randal <allison@lohutok.net> Cc: linux-spdx@vger.kernel.org Link: https://lkml.kernel.org/r/20190527070032.746973796@linutronix.de Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
419 lines
9.8 KiB
C
419 lines
9.8 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/*
|
|
* SMP support for PowerNV machines.
|
|
*
|
|
* Copyright 2011 IBM Corp.
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/sched/hotplug.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/init.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/cpu.h>
|
|
|
|
#include <asm/irq.h>
|
|
#include <asm/smp.h>
|
|
#include <asm/paca.h>
|
|
#include <asm/machdep.h>
|
|
#include <asm/cputable.h>
|
|
#include <asm/firmware.h>
|
|
#include <asm/vdso_datapage.h>
|
|
#include <asm/cputhreads.h>
|
|
#include <asm/xics.h>
|
|
#include <asm/xive.h>
|
|
#include <asm/opal.h>
|
|
#include <asm/runlatch.h>
|
|
#include <asm/code-patching.h>
|
|
#include <asm/dbell.h>
|
|
#include <asm/kvm_ppc.h>
|
|
#include <asm/ppc-opcode.h>
|
|
#include <asm/cpuidle.h>
|
|
#include <asm/kexec.h>
|
|
#include <asm/reg.h>
|
|
#include <asm/powernv.h>
|
|
|
|
#include "powernv.h"
|
|
|
|
#ifdef DEBUG
|
|
#include <asm/udbg.h>
|
|
#define DBG(fmt...) udbg_printf(fmt)
|
|
#else
|
|
#define DBG(fmt...)
|
|
#endif
|
|
|
|
static void pnv_smp_setup_cpu(int cpu)
|
|
{
|
|
/*
|
|
* P9 workaround for CI vector load (see traps.c),
|
|
* enable the corresponding HMI interrupt
|
|
*/
|
|
if (pvr_version_is(PVR_POWER9))
|
|
mtspr(SPRN_HMEER, mfspr(SPRN_HMEER) | PPC_BIT(17));
|
|
|
|
if (xive_enabled())
|
|
xive_smp_setup_cpu();
|
|
else if (cpu != boot_cpuid)
|
|
xics_setup_cpu();
|
|
}
|
|
|
|
static int pnv_smp_kick_cpu(int nr)
|
|
{
|
|
unsigned int pcpu;
|
|
unsigned long start_here =
|
|
__pa(ppc_function_entry(generic_secondary_smp_init));
|
|
long rc;
|
|
uint8_t status;
|
|
|
|
if (nr < 0 || nr >= nr_cpu_ids)
|
|
return -EINVAL;
|
|
|
|
pcpu = get_hard_smp_processor_id(nr);
|
|
/*
|
|
* If we already started or OPAL is not supported, we just
|
|
* kick the CPU via the PACA
|
|
*/
|
|
if (paca_ptrs[nr]->cpu_start || !firmware_has_feature(FW_FEATURE_OPAL))
|
|
goto kick;
|
|
|
|
/*
|
|
* At this point, the CPU can either be spinning on the way in
|
|
* from kexec or be inside OPAL waiting to be started for the
|
|
* first time. OPAL v3 allows us to query OPAL to know if it
|
|
* has the CPUs, so we do that
|
|
*/
|
|
rc = opal_query_cpu_status(pcpu, &status);
|
|
if (rc != OPAL_SUCCESS) {
|
|
pr_warn("OPAL Error %ld querying CPU %d state\n", rc, nr);
|
|
return -ENODEV;
|
|
}
|
|
|
|
/*
|
|
* Already started, just kick it, probably coming from
|
|
* kexec and spinning
|
|
*/
|
|
if (status == OPAL_THREAD_STARTED)
|
|
goto kick;
|
|
|
|
/*
|
|
* Available/inactive, let's kick it
|
|
*/
|
|
if (status == OPAL_THREAD_INACTIVE) {
|
|
pr_devel("OPAL: Starting CPU %d (HW 0x%x)...\n", nr, pcpu);
|
|
rc = opal_start_cpu(pcpu, start_here);
|
|
if (rc != OPAL_SUCCESS) {
|
|
pr_warn("OPAL Error %ld starting CPU %d\n", rc, nr);
|
|
return -ENODEV;
|
|
}
|
|
} else {
|
|
/*
|
|
* An unavailable CPU (or any other unknown status)
|
|
* shouldn't be started. It should also
|
|
* not be in the possible map but currently it can
|
|
* happen
|
|
*/
|
|
pr_devel("OPAL: CPU %d (HW 0x%x) is unavailable"
|
|
" (status %d)...\n", nr, pcpu, status);
|
|
return -ENODEV;
|
|
}
|
|
|
|
kick:
|
|
return smp_generic_kick_cpu(nr);
|
|
}
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
static int pnv_smp_cpu_disable(void)
|
|
{
|
|
int cpu = smp_processor_id();
|
|
|
|
/* This is identical to pSeries... might consolidate by
|
|
* moving migrate_irqs_away to a ppc_md with default to
|
|
* the generic fixup_irqs. --BenH.
|
|
*/
|
|
set_cpu_online(cpu, false);
|
|
vdso_data->processorCount--;
|
|
if (cpu == boot_cpuid)
|
|
boot_cpuid = cpumask_any(cpu_online_mask);
|
|
if (xive_enabled())
|
|
xive_smp_disable_cpu();
|
|
else
|
|
xics_migrate_irqs_away();
|
|
return 0;
|
|
}
|
|
|
|
static void pnv_smp_cpu_kill_self(void)
|
|
{
|
|
unsigned int cpu;
|
|
unsigned long srr1, wmask;
|
|
u64 lpcr_val;
|
|
|
|
/* Standard hot unplug procedure */
|
|
/*
|
|
* This hard disables local interurpts, ensuring we have no lazy
|
|
* irqs pending.
|
|
*/
|
|
WARN_ON(irqs_disabled());
|
|
hard_irq_disable();
|
|
WARN_ON(lazy_irq_pending());
|
|
|
|
idle_task_exit();
|
|
current->active_mm = NULL; /* for sanity */
|
|
cpu = smp_processor_id();
|
|
DBG("CPU%d offline\n", cpu);
|
|
generic_set_cpu_dead(cpu);
|
|
smp_wmb();
|
|
|
|
wmask = SRR1_WAKEMASK;
|
|
if (cpu_has_feature(CPU_FTR_ARCH_207S))
|
|
wmask = SRR1_WAKEMASK_P8;
|
|
|
|
/*
|
|
* We don't want to take decrementer interrupts while we are
|
|
* offline, so clear LPCR:PECE1. We keep PECE2 (and
|
|
* LPCR_PECE_HVEE on P9) enabled so as to let IPIs in.
|
|
*
|
|
* If the CPU gets woken up by a special wakeup, ensure that
|
|
* the SLW engine sets LPCR with decrementer bit cleared, else
|
|
* the CPU will come back to the kernel due to a spurious
|
|
* wakeup.
|
|
*/
|
|
lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1;
|
|
pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val);
|
|
|
|
while (!generic_check_cpu_restart(cpu)) {
|
|
/*
|
|
* Clear IPI flag, since we don't handle IPIs while
|
|
* offline, except for those when changing micro-threading
|
|
* mode, which are handled explicitly below, and those
|
|
* for coming online, which are handled via
|
|
* generic_check_cpu_restart() calls.
|
|
*/
|
|
kvmppc_set_host_ipi(cpu, 0);
|
|
|
|
srr1 = pnv_cpu_offline(cpu);
|
|
|
|
WARN_ON(lazy_irq_pending());
|
|
|
|
/*
|
|
* If the SRR1 value indicates that we woke up due to
|
|
* an external interrupt, then clear the interrupt.
|
|
* We clear the interrupt before checking for the
|
|
* reason, so as to avoid a race where we wake up for
|
|
* some other reason, find nothing and clear the interrupt
|
|
* just as some other cpu is sending us an interrupt.
|
|
* If we returned from power7_nap as a result of
|
|
* having finished executing in a KVM guest, then srr1
|
|
* contains 0.
|
|
*/
|
|
if (((srr1 & wmask) == SRR1_WAKEEE) ||
|
|
((srr1 & wmask) == SRR1_WAKEHVI)) {
|
|
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
|
|
if (xive_enabled())
|
|
xive_flush_interrupt();
|
|
else
|
|
icp_opal_flush_interrupt();
|
|
} else
|
|
icp_native_flush_interrupt();
|
|
} else if ((srr1 & wmask) == SRR1_WAKEHDBELL) {
|
|
unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
|
|
asm volatile(PPC_MSGCLR(%0) : : "r" (msg));
|
|
} else if ((srr1 & wmask) == SRR1_WAKERESET) {
|
|
irq_set_pending_from_srr1(srr1);
|
|
/* Does not return */
|
|
}
|
|
|
|
smp_mb();
|
|
|
|
/*
|
|
* For kdump kernels, we process the ipi and jump to
|
|
* crash_ipi_callback
|
|
*/
|
|
if (kdump_in_progress()) {
|
|
/*
|
|
* If we got to this point, we've not used
|
|
* NMI's, otherwise we would have gone
|
|
* via the SRR1_WAKERESET path. We are
|
|
* using regular IPI's for waking up offline
|
|
* threads.
|
|
*/
|
|
struct pt_regs regs;
|
|
|
|
ppc_save_regs(®s);
|
|
crash_ipi_callback(®s);
|
|
/* Does not return */
|
|
}
|
|
|
|
if (cpu_core_split_required())
|
|
continue;
|
|
|
|
if (srr1 && !generic_check_cpu_restart(cpu))
|
|
DBG("CPU%d Unexpected exit while offline srr1=%lx!\n",
|
|
cpu, srr1);
|
|
|
|
}
|
|
|
|
/*
|
|
* Re-enable decrementer interrupts in LPCR.
|
|
*
|
|
* Further, we want stop states to be woken up by decrementer
|
|
* for non-hotplug cases. So program the LPCR via stop api as
|
|
* well.
|
|
*/
|
|
lpcr_val = mfspr(SPRN_LPCR) | (u64)LPCR_PECE1;
|
|
pnv_program_cpu_hotplug_lpcr(cpu, lpcr_val);
|
|
|
|
DBG("CPU%d coming online...\n", cpu);
|
|
}
|
|
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
|
|
static int pnv_cpu_bootable(unsigned int nr)
|
|
{
|
|
/*
|
|
* Starting with POWER8, the subcore logic relies on all threads of a
|
|
* core being booted so that they can participate in split mode
|
|
* switches. So on those machines we ignore the smt_enabled_at_boot
|
|
* setting (smt-enabled on the kernel command line).
|
|
*/
|
|
if (cpu_has_feature(CPU_FTR_ARCH_207S))
|
|
return 1;
|
|
|
|
return smp_generic_cpu_bootable(nr);
|
|
}
|
|
|
|
static int pnv_smp_prepare_cpu(int cpu)
|
|
{
|
|
if (xive_enabled())
|
|
return xive_smp_prepare_cpu(cpu);
|
|
return 0;
|
|
}
|
|
|
|
/* Cause IPI as setup by the interrupt controller (xics or xive) */
|
|
static void (*ic_cause_ipi)(int cpu);
|
|
|
|
static void pnv_cause_ipi(int cpu)
|
|
{
|
|
if (doorbell_try_core_ipi(cpu))
|
|
return;
|
|
|
|
ic_cause_ipi(cpu);
|
|
}
|
|
|
|
static void __init pnv_smp_probe(void)
|
|
{
|
|
if (xive_enabled())
|
|
xive_smp_probe();
|
|
else
|
|
xics_smp_probe();
|
|
|
|
if (cpu_has_feature(CPU_FTR_DBELL)) {
|
|
ic_cause_ipi = smp_ops->cause_ipi;
|
|
WARN_ON(!ic_cause_ipi);
|
|
|
|
if (cpu_has_feature(CPU_FTR_ARCH_300))
|
|
smp_ops->cause_ipi = doorbell_global_ipi;
|
|
else
|
|
smp_ops->cause_ipi = pnv_cause_ipi;
|
|
}
|
|
}
|
|
|
|
static int pnv_system_reset_exception(struct pt_regs *regs)
|
|
{
|
|
if (smp_handle_nmi_ipi(regs))
|
|
return 1;
|
|
return 0;
|
|
}
|
|
|
|
static int pnv_cause_nmi_ipi(int cpu)
|
|
{
|
|
int64_t rc;
|
|
|
|
if (cpu >= 0) {
|
|
int h = get_hard_smp_processor_id(cpu);
|
|
|
|
if (opal_check_token(OPAL_QUIESCE))
|
|
opal_quiesce(QUIESCE_HOLD, h);
|
|
|
|
rc = opal_signal_system_reset(h);
|
|
|
|
if (opal_check_token(OPAL_QUIESCE))
|
|
opal_quiesce(QUIESCE_RESUME, h);
|
|
|
|
if (rc != OPAL_SUCCESS)
|
|
return 0;
|
|
return 1;
|
|
|
|
} else if (cpu == NMI_IPI_ALL_OTHERS) {
|
|
bool success = true;
|
|
int c;
|
|
|
|
if (opal_check_token(OPAL_QUIESCE))
|
|
opal_quiesce(QUIESCE_HOLD, -1);
|
|
|
|
/*
|
|
* We do not use broadcasts (yet), because it's not clear
|
|
* exactly what semantics Linux wants or the firmware should
|
|
* provide.
|
|
*/
|
|
for_each_online_cpu(c) {
|
|
if (c == smp_processor_id())
|
|
continue;
|
|
|
|
rc = opal_signal_system_reset(
|
|
get_hard_smp_processor_id(c));
|
|
if (rc != OPAL_SUCCESS)
|
|
success = false;
|
|
}
|
|
|
|
if (opal_check_token(OPAL_QUIESCE))
|
|
opal_quiesce(QUIESCE_RESUME, -1);
|
|
|
|
if (success)
|
|
return 1;
|
|
|
|
/*
|
|
* Caller will fall back to doorbells, which may pick
|
|
* up the remainders.
|
|
*/
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct smp_ops_t pnv_smp_ops = {
|
|
.message_pass = NULL, /* Use smp_muxed_ipi_message_pass */
|
|
.cause_ipi = NULL, /* Filled at runtime by pnv_smp_probe() */
|
|
.cause_nmi_ipi = NULL,
|
|
.probe = pnv_smp_probe,
|
|
.prepare_cpu = pnv_smp_prepare_cpu,
|
|
.kick_cpu = pnv_smp_kick_cpu,
|
|
.setup_cpu = pnv_smp_setup_cpu,
|
|
.cpu_bootable = pnv_cpu_bootable,
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
.cpu_disable = pnv_smp_cpu_disable,
|
|
.cpu_die = generic_cpu_die,
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
};
|
|
|
|
/* This is called very early during platform setup_arch */
|
|
void __init pnv_smp_init(void)
|
|
{
|
|
if (opal_check_token(OPAL_SIGNAL_SYSTEM_RESET)) {
|
|
ppc_md.system_reset_exception = pnv_system_reset_exception;
|
|
pnv_smp_ops.cause_nmi_ipi = pnv_cause_nmi_ipi;
|
|
}
|
|
smp_ops = &pnv_smp_ops;
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
ppc_md.cpu_die = pnv_smp_cpu_kill_self;
|
|
#ifdef CONFIG_KEXEC_CORE
|
|
crash_wake_offline = 1;
|
|
#endif
|
|
#endif
|
|
}
|