mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-03 04:36:51 +07:00
7604537bbb
Current implementation of cpu_{suspend}/cpu_{resume} relies on the MPIDR
to index the array of pointers where the context is saved and restored.
The current approach works as long as the MPIDR can be considered a
linear index, so that the pointers array can simply be dereferenced by
using the MPIDR[7:0] value.
On ARM multi-cluster systems, where the MPIDR may not be a linear index,
to properly dereference the stack pointer array, a mapping function should
be applied to it so that it can be used for arrays look-ups.
This patch adds code in the cpu_{suspend}/cpu_{resume} implementation
that relies on shifting and ORing hashing method to map a MPIDR value to a
set of buckets precomputed at boot to have a collision free mapping from
MPIDR to context pointers.
The hashing algorithm must be simple, fast, and implementable with few
instructions since in the cpu_resume path the mapping is carried out with
the MMU off and the I-cache off, hence code and data are fetched from DRAM
with no-caching available. Simplicity is counterbalanced with a little
increase of memory (allocated dynamically) for stack pointers buckets, that
should be anyway fairly limited on most systems.
Memory for context pointers is allocated in a early_initcall with
size precomputed and stashed previously in kernel data structures.
Memory for context pointers is allocated through kmalloc; this
guarantees contiguous physical addresses for the allocated memory which
is fundamental to the correct functioning of the resume mechanism that
relies on the context pointer array to be a chunk of contiguous physical
memory. Virtual to physical address conversion for the context pointer
array base is carried out at boot to avoid fiddling with virt_to_phys
conversions in the cpu_resume path which is quite fragile and should be
optimized to execute as few instructions as possible.
Virtual and physical context pointer base array addresses are stashed in a
struct that is accessible from assembly using values generated through the
asm-offsets.c mechanism.
Cc: Will Deacon <will.deacon@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Colin Cross <ccross@android.com>
Cc: Santosh Shilimkar <santosh.shilimkar@ti.com>
Cc: Daniel Lezcano <daniel.lezcano@linaro.org>
Cc: Amit Kucheria <amit.kucheria@linaro.org>
Signed-off-by: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Reviewed-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Nicolas Pitre <nico@linaro.org>
Tested-by: Shawn Guo <shawn.guo@linaro.org>
Tested-by: Kevin Hilman <khilman@linaro.org>
Tested-by: Stephen Warren <swarren@wwwdotorg.org>
105 lines
2.7 KiB
C
105 lines
2.7 KiB
C
#include <linux/init.h>
|
|
#include <linux/slab.h>
|
|
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/idmap.h>
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/memory.h>
|
|
#include <asm/smp_plat.h>
|
|
#include <asm/suspend.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
extern int __cpu_suspend(unsigned long, int (*)(unsigned long));
|
|
extern void cpu_resume_mmu(void);
|
|
|
|
#ifdef CONFIG_MMU
|
|
/*
|
|
* Hide the first two arguments to __cpu_suspend - these are an implementation
|
|
* detail which platform code shouldn't have to know about.
|
|
*/
|
|
int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
|
|
{
|
|
struct mm_struct *mm = current->active_mm;
|
|
int ret;
|
|
|
|
if (!idmap_pgd)
|
|
return -EINVAL;
|
|
|
|
/*
|
|
* Provide a temporary page table with an identity mapping for
|
|
* the MMU-enable code, required for resuming. On successful
|
|
* resume (indicated by a zero return code), we need to switch
|
|
* back to the correct page tables.
|
|
*/
|
|
ret = __cpu_suspend(arg, fn);
|
|
if (ret == 0) {
|
|
cpu_switch_mm(mm->pgd, mm);
|
|
local_flush_bp_all();
|
|
local_flush_tlb_all();
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
#else
|
|
int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
|
|
{
|
|
return __cpu_suspend(arg, fn);
|
|
}
|
|
#define idmap_pgd NULL
|
|
#endif
|
|
|
|
/*
|
|
* This is called by __cpu_suspend() to save the state, and do whatever
|
|
* flushing is required to ensure that when the CPU goes to sleep we have
|
|
* the necessary data available when the caches are not searched.
|
|
*/
|
|
void __cpu_suspend_save(u32 *ptr, u32 ptrsz, u32 sp, u32 *save_ptr)
|
|
{
|
|
u32 *ctx = ptr;
|
|
|
|
*save_ptr = virt_to_phys(ptr);
|
|
|
|
/* This must correspond to the LDM in cpu_resume() assembly */
|
|
*ptr++ = virt_to_phys(idmap_pgd);
|
|
*ptr++ = sp;
|
|
*ptr++ = virt_to_phys(cpu_do_resume);
|
|
|
|
cpu_do_suspend(ptr);
|
|
|
|
flush_cache_louis();
|
|
|
|
/*
|
|
* flush_cache_louis does not guarantee that
|
|
* save_ptr and ptr are cleaned to main memory,
|
|
* just up to the Level of Unification Inner Shareable.
|
|
* Since the context pointer and context itself
|
|
* are to be retrieved with the MMU off that
|
|
* data must be cleaned from all cache levels
|
|
* to main memory using "area" cache primitives.
|
|
*/
|
|
__cpuc_flush_dcache_area(ctx, ptrsz);
|
|
__cpuc_flush_dcache_area(save_ptr, sizeof(*save_ptr));
|
|
|
|
outer_clean_range(*save_ptr, *save_ptr + ptrsz);
|
|
outer_clean_range(virt_to_phys(save_ptr),
|
|
virt_to_phys(save_ptr) + sizeof(*save_ptr));
|
|
}
|
|
|
|
extern struct sleep_save_sp sleep_save_sp;
|
|
|
|
static int cpu_suspend_alloc_sp(void)
|
|
{
|
|
void *ctx_ptr;
|
|
/* ctx_ptr is an array of physical addresses */
|
|
ctx_ptr = kcalloc(mpidr_hash_size(), sizeof(u32), GFP_KERNEL);
|
|
|
|
if (WARN_ON(!ctx_ptr))
|
|
return -ENOMEM;
|
|
sleep_save_sp.save_ptr_stash = ctx_ptr;
|
|
sleep_save_sp.save_ptr_stash_phys = virt_to_phys(ctx_ptr);
|
|
sync_cache_w(&sleep_save_sp);
|
|
return 0;
|
|
}
|
|
early_initcall(cpu_suspend_alloc_sp);
|