mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-28 11:18:45 +07:00
bb48711800
The Kryo CPUs are also affected by the Falkor 1003 errata, so
we need to do the same workaround on Kryo CPUs. The MIDR is
slightly more complicated here, where the PART number is not
always the same when looking at all the bits from 15 to 4. Drop
the lower 8 bits and just look at the top 4 to see if it's '2'
and then consider those as Kryo CPUs. This covers all the
combinations without having to list them all out.
Fixes: 38fd94b027 ("arm64: Work around Falkor erratum 1003")
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Stephen Boyd <sboyd@codeaurora.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
396 lines
10 KiB
C
396 lines
10 KiB
C
/*
|
|
* Contains CPU specific errata definitions
|
|
*
|
|
* Copyright (C) 2014 ARM Ltd.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <asm/cpu.h>
|
|
#include <asm/cputype.h>
|
|
#include <asm/cpufeature.h>
|
|
|
|
static bool __maybe_unused
|
|
is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
|
|
{
|
|
WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
|
|
return MIDR_IS_CPU_MODEL_RANGE(read_cpuid_id(), entry->midr_model,
|
|
entry->midr_range_min,
|
|
entry->midr_range_max);
|
|
}
|
|
|
|
static bool __maybe_unused
|
|
is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
|
|
{
|
|
u32 model;
|
|
|
|
WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
|
|
|
|
model = read_cpuid_id();
|
|
model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
|
|
MIDR_ARCHITECTURE_MASK;
|
|
|
|
return model == entry->midr_model;
|
|
}
|
|
|
|
static bool
|
|
has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
|
|
int scope)
|
|
{
|
|
WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
|
|
return (read_cpuid_cachetype() & arm64_ftr_reg_ctrel0.strict_mask) !=
|
|
(arm64_ftr_reg_ctrel0.sys_val & arm64_ftr_reg_ctrel0.strict_mask);
|
|
}
|
|
|
|
static int cpu_enable_trap_ctr_access(void *__unused)
|
|
{
|
|
/* Clear SCTLR_EL1.UCT */
|
|
config_sctlr_el1(SCTLR_EL1_UCT, 0);
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
|
|
#include <asm/mmu_context.h>
|
|
#include <asm/cacheflush.h>
|
|
|
|
DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
|
|
|
|
#ifdef CONFIG_KVM
|
|
extern char __psci_hyp_bp_inval_start[], __psci_hyp_bp_inval_end[];
|
|
extern char __qcom_hyp_sanitize_link_stack_start[];
|
|
extern char __qcom_hyp_sanitize_link_stack_end[];
|
|
|
|
static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
|
|
const char *hyp_vecs_end)
|
|
{
|
|
void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
|
|
int i;
|
|
|
|
for (i = 0; i < SZ_2K; i += 0x80)
|
|
memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
|
|
|
|
flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
|
|
}
|
|
|
|
static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
|
|
const char *hyp_vecs_start,
|
|
const char *hyp_vecs_end)
|
|
{
|
|
static int last_slot = -1;
|
|
static DEFINE_SPINLOCK(bp_lock);
|
|
int cpu, slot = -1;
|
|
|
|
spin_lock(&bp_lock);
|
|
for_each_possible_cpu(cpu) {
|
|
if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
|
|
slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (slot == -1) {
|
|
last_slot++;
|
|
BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start)
|
|
/ SZ_2K) <= last_slot);
|
|
slot = last_slot;
|
|
__copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
|
|
}
|
|
|
|
__this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
|
|
__this_cpu_write(bp_hardening_data.fn, fn);
|
|
spin_unlock(&bp_lock);
|
|
}
|
|
#else
|
|
#define __psci_hyp_bp_inval_start NULL
|
|
#define __psci_hyp_bp_inval_end NULL
|
|
#define __qcom_hyp_sanitize_link_stack_start NULL
|
|
#define __qcom_hyp_sanitize_link_stack_end NULL
|
|
|
|
static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
|
|
const char *hyp_vecs_start,
|
|
const char *hyp_vecs_end)
|
|
{
|
|
__this_cpu_write(bp_hardening_data.fn, fn);
|
|
}
|
|
#endif /* CONFIG_KVM */
|
|
|
|
static void install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
|
|
bp_hardening_cb_t fn,
|
|
const char *hyp_vecs_start,
|
|
const char *hyp_vecs_end)
|
|
{
|
|
u64 pfr0;
|
|
|
|
if (!entry->matches(entry, SCOPE_LOCAL_CPU))
|
|
return;
|
|
|
|
pfr0 = read_cpuid(ID_AA64PFR0_EL1);
|
|
if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
|
|
return;
|
|
|
|
__install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
|
|
}
|
|
|
|
#include <linux/psci.h>
|
|
|
|
static int enable_psci_bp_hardening(void *data)
|
|
{
|
|
const struct arm64_cpu_capabilities *entry = data;
|
|
|
|
if (psci_ops.get_version)
|
|
install_bp_hardening_cb(entry,
|
|
(bp_hardening_cb_t)psci_ops.get_version,
|
|
__psci_hyp_bp_inval_start,
|
|
__psci_hyp_bp_inval_end);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void qcom_link_stack_sanitization(void)
|
|
{
|
|
u64 tmp;
|
|
|
|
asm volatile("mov %0, x30 \n"
|
|
".rept 16 \n"
|
|
"bl . + 4 \n"
|
|
".endr \n"
|
|
"mov x30, %0 \n"
|
|
: "=&r" (tmp));
|
|
}
|
|
|
|
static int qcom_enable_link_stack_sanitization(void *data)
|
|
{
|
|
const struct arm64_cpu_capabilities *entry = data;
|
|
|
|
install_bp_hardening_cb(entry, qcom_link_stack_sanitization,
|
|
__qcom_hyp_sanitize_link_stack_start,
|
|
__qcom_hyp_sanitize_link_stack_end);
|
|
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */
|
|
|
|
#define MIDR_RANGE(model, min, max) \
|
|
.def_scope = SCOPE_LOCAL_CPU, \
|
|
.matches = is_affected_midr_range, \
|
|
.midr_model = model, \
|
|
.midr_range_min = min, \
|
|
.midr_range_max = max
|
|
|
|
#define MIDR_ALL_VERSIONS(model) \
|
|
.def_scope = SCOPE_LOCAL_CPU, \
|
|
.matches = is_affected_midr_range, \
|
|
.midr_model = model, \
|
|
.midr_range_min = 0, \
|
|
.midr_range_max = (MIDR_VARIANT_MASK | MIDR_REVISION_MASK)
|
|
|
|
const struct arm64_cpu_capabilities arm64_errata[] = {
|
|
#if defined(CONFIG_ARM64_ERRATUM_826319) || \
|
|
defined(CONFIG_ARM64_ERRATUM_827319) || \
|
|
defined(CONFIG_ARM64_ERRATUM_824069)
|
|
{
|
|
/* Cortex-A53 r0p[012] */
|
|
.desc = "ARM errata 826319, 827319, 824069",
|
|
.capability = ARM64_WORKAROUND_CLEAN_CACHE,
|
|
MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02),
|
|
.enable = cpu_enable_cache_maint_trap,
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_ARM64_ERRATUM_819472
|
|
{
|
|
/* Cortex-A53 r0p[01] */
|
|
.desc = "ARM errata 819472",
|
|
.capability = ARM64_WORKAROUND_CLEAN_CACHE,
|
|
MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x01),
|
|
.enable = cpu_enable_cache_maint_trap,
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_ARM64_ERRATUM_832075
|
|
{
|
|
/* Cortex-A57 r0p0 - r1p2 */
|
|
.desc = "ARM erratum 832075",
|
|
.capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
|
|
MIDR_RANGE(MIDR_CORTEX_A57,
|
|
MIDR_CPU_VAR_REV(0, 0),
|
|
MIDR_CPU_VAR_REV(1, 2)),
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_ARM64_ERRATUM_834220
|
|
{
|
|
/* Cortex-A57 r0p0 - r1p2 */
|
|
.desc = "ARM erratum 834220",
|
|
.capability = ARM64_WORKAROUND_834220,
|
|
MIDR_RANGE(MIDR_CORTEX_A57,
|
|
MIDR_CPU_VAR_REV(0, 0),
|
|
MIDR_CPU_VAR_REV(1, 2)),
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_ARM64_ERRATUM_845719
|
|
{
|
|
/* Cortex-A53 r0p[01234] */
|
|
.desc = "ARM erratum 845719",
|
|
.capability = ARM64_WORKAROUND_845719,
|
|
MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04),
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_CAVIUM_ERRATUM_23154
|
|
{
|
|
/* Cavium ThunderX, pass 1.x */
|
|
.desc = "Cavium erratum 23154",
|
|
.capability = ARM64_WORKAROUND_CAVIUM_23154,
|
|
MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01),
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_CAVIUM_ERRATUM_27456
|
|
{
|
|
/* Cavium ThunderX, T88 pass 1.x - 2.1 */
|
|
.desc = "Cavium erratum 27456",
|
|
.capability = ARM64_WORKAROUND_CAVIUM_27456,
|
|
MIDR_RANGE(MIDR_THUNDERX,
|
|
MIDR_CPU_VAR_REV(0, 0),
|
|
MIDR_CPU_VAR_REV(1, 1)),
|
|
},
|
|
{
|
|
/* Cavium ThunderX, T81 pass 1.0 */
|
|
.desc = "Cavium erratum 27456",
|
|
.capability = ARM64_WORKAROUND_CAVIUM_27456,
|
|
MIDR_RANGE(MIDR_THUNDERX_81XX, 0x00, 0x00),
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_CAVIUM_ERRATUM_30115
|
|
{
|
|
/* Cavium ThunderX, T88 pass 1.x - 2.2 */
|
|
.desc = "Cavium erratum 30115",
|
|
.capability = ARM64_WORKAROUND_CAVIUM_30115,
|
|
MIDR_RANGE(MIDR_THUNDERX, 0x00,
|
|
(1 << MIDR_VARIANT_SHIFT) | 2),
|
|
},
|
|
{
|
|
/* Cavium ThunderX, T81 pass 1.0 - 1.2 */
|
|
.desc = "Cavium erratum 30115",
|
|
.capability = ARM64_WORKAROUND_CAVIUM_30115,
|
|
MIDR_RANGE(MIDR_THUNDERX_81XX, 0x00, 0x02),
|
|
},
|
|
{
|
|
/* Cavium ThunderX, T83 pass 1.0 */
|
|
.desc = "Cavium erratum 30115",
|
|
.capability = ARM64_WORKAROUND_CAVIUM_30115,
|
|
MIDR_RANGE(MIDR_THUNDERX_83XX, 0x00, 0x00),
|
|
},
|
|
#endif
|
|
{
|
|
.desc = "Mismatched cache line size",
|
|
.capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
|
|
.matches = has_mismatched_cache_line_size,
|
|
.def_scope = SCOPE_LOCAL_CPU,
|
|
.enable = cpu_enable_trap_ctr_access,
|
|
},
|
|
#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
|
|
{
|
|
.desc = "Qualcomm Technologies Falkor erratum 1003",
|
|
.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
|
|
MIDR_RANGE(MIDR_QCOM_FALKOR_V1,
|
|
MIDR_CPU_VAR_REV(0, 0),
|
|
MIDR_CPU_VAR_REV(0, 0)),
|
|
},
|
|
{
|
|
.desc = "Qualcomm Technologies Kryo erratum 1003",
|
|
.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
|
|
.def_scope = SCOPE_LOCAL_CPU,
|
|
.midr_model = MIDR_QCOM_KRYO,
|
|
.matches = is_kryo_midr,
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
|
|
{
|
|
.desc = "Qualcomm Technologies Falkor erratum 1009",
|
|
.capability = ARM64_WORKAROUND_REPEAT_TLBI,
|
|
MIDR_RANGE(MIDR_QCOM_FALKOR_V1,
|
|
MIDR_CPU_VAR_REV(0, 0),
|
|
MIDR_CPU_VAR_REV(0, 0)),
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_ARM64_ERRATUM_858921
|
|
{
|
|
/* Cortex-A73 all versions */
|
|
.desc = "ARM erratum 858921",
|
|
.capability = ARM64_WORKAROUND_858921,
|
|
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
|
|
},
|
|
#endif
|
|
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
|
|
{
|
|
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
|
|
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
|
|
.enable = enable_psci_bp_hardening,
|
|
},
|
|
{
|
|
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
|
|
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
|
|
.enable = enable_psci_bp_hardening,
|
|
},
|
|
{
|
|
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
|
|
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
|
|
.enable = enable_psci_bp_hardening,
|
|
},
|
|
{
|
|
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
|
|
MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
|
|
.enable = enable_psci_bp_hardening,
|
|
},
|
|
{
|
|
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
|
|
MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
|
|
.enable = qcom_enable_link_stack_sanitization,
|
|
},
|
|
{
|
|
.capability = ARM64_HARDEN_BP_POST_GUEST_EXIT,
|
|
MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
|
|
},
|
|
#endif
|
|
{
|
|
}
|
|
};
|
|
|
|
/*
|
|
* The CPU Errata work arounds are detected and applied at boot time
|
|
* and the related information is freed soon after. If the new CPU requires
|
|
* an errata not detected at boot, fail this CPU.
|
|
*/
|
|
void verify_local_cpu_errata_workarounds(void)
|
|
{
|
|
const struct arm64_cpu_capabilities *caps = arm64_errata;
|
|
|
|
for (; caps->matches; caps++)
|
|
if (!cpus_have_cap(caps->capability) &&
|
|
caps->matches(caps, SCOPE_LOCAL_CPU)) {
|
|
pr_crit("CPU%d: Requires work around for %s, not detected"
|
|
" at boot time\n",
|
|
smp_processor_id(),
|
|
caps->desc ? : "an erratum");
|
|
cpu_die_early();
|
|
}
|
|
}
|
|
|
|
void update_cpu_errata_workarounds(void)
|
|
{
|
|
update_cpu_capabilities(arm64_errata, "enabling workaround for");
|
|
}
|
|
|
|
void __init enable_errata_workarounds(void)
|
|
{
|
|
enable_cpu_capabilities(arm64_errata);
|
|
}
|