AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-12-17 07:47:11 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity
c1e8d7c6a7
linux_dsm_epyc7002/arch/nds32/mm/proc.c
Mike Rapoport e05c7b1f2b mm: pgtable: add shortcuts for accessing kernel PMD and PTE 
The powerpc 32-bit implementation of pgtable has nice shortcuts for
accessing kernel PMD and PTE for a given virtual address.  Make these
helpers available for all architectures.

[rppt@linux.ibm.com: microblaze: fix page table traversal in setup_rt_frame()]
  Link: http://lkml.kernel.org/r/20200518191511.GD1118872@kernel.org
[akpm@linux-foundation.org: s/pmd_ptr_k/pmd_off_k/ in various powerpc places]

Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Cain <bcain@codeaurora.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Zankel <chris@zankel.net>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Greentime Hu <green.hu@gmail.com>
Cc: Greg Ungerer <gerg@linux-m68k.org>
Cc: Guan Xuetao <gxt@pku.edu.cn>
Cc: Guo Ren <guoren@kernel.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Ley Foon Tan <ley.foon.tan@intel.com>
Cc: Mark Salter <msalter@redhat.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Nick Hu <nickhu@andestech.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Rich Felker <dalias@libc.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stafford Horne <shorne@gmail.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Vincent Chen <deanbo422@gmail.com>
Cc: Vineet Gupta <vgupta@synopsys.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Link: http://lkml.kernel.org/r/20200514170327.31389-9-rppt@kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2020-06-09 09:39:13 -07:00

537 lines
12 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2005-2017 Andes Technology Corporation
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <asm/nds32.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
#include <asm/l2_cache.h>
#include <nds32_intrinsic.h>
#include <asm/cache_info.h>
extern struct cache_info L1_cache_info[2];
int va_kernel_present(unsigned long addr)
{
pmd_t *pmd;
pte_t *ptep, pte;
pmd = pmd_off_k(addr);
if (!pmd_none(*pmd)) {
ptep = pte_offset_map(pmd, addr);
pte = *ptep;
if (pte_present(pte))
return pte;
}
return 0;
}
pte_t va_present(struct mm_struct * mm, unsigned long addr)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *ptep, pte;
pgd = pgd_offset(mm, addr);
if (!pgd_none(*pgd)) {
p4d = p4d_offset(pgd, addr);
if (!p4d_none(*p4d)) {
pud = pud_offset(p4d, addr);
if (!pud_none(*pud)) {
pmd = pmd_offset(pud, addr);
if (!pmd_none(*pmd)) {
ptep = pte_offset_map(pmd, addr);
pte = *ptep;
if (pte_present(pte))
return pte;
}
}
}
}
return 0;
}
int va_readable(struct pt_regs *regs, unsigned long addr)
{
struct mm_struct *mm = current->mm;
pte_t pte;
int ret = 0;
if (user_mode(regs)) {
/* user mode */
pte = va_present(mm, addr);
if (!pte && pte_read(pte))
ret = 1;
} else {
/* superuser mode is always readable, so we can only
* check it is present or not*/
return (! !va_kernel_present(addr));
}
return ret;
}
int va_writable(struct pt_regs *regs, unsigned long addr)
{
struct mm_struct *mm = current->mm;
pte_t pte;
int ret = 0;
if (user_mode(regs)) {
/* user mode */
pte = va_present(mm, addr);
if (!pte && pte_write(pte))
ret = 1;
} else {
/* superuser mode */
pte = va_kernel_present(addr);
if (!pte && pte_kernel_write(pte))
ret = 1;
}
return ret;
}
/*
* All
*/
void cpu_icache_inval_all(void)
{
unsigned long end, line_size;
line_size = L1_cache_info[ICACHE].line_size;
end =
line_size * L1_cache_info[ICACHE].ways * L1_cache_info[ICACHE].sets;
do {
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
} while (end > 0);
__nds32__isb();
}
void cpu_dcache_inval_all(void)
{
__nds32__cctl_l1d_invalall();
}
#ifdef CONFIG_CACHE_L2
void dcache_wb_all_level(void)
{
unsigned long flags, cmd;
local_irq_save(flags);
__nds32__cctl_l1d_wball_alvl();
/* Section 1: Ensure the section 2 & 3 program code execution after */
__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
/* Section 2: Confirm the writeback all level is done in CPU and L2C */
cmd = CCTL_CMD_L2_SYNC;
L2_CMD_RDY();
L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
L2_CMD_RDY();
/* Section 3: Writeback whole L2 cache */
cmd = CCTL_ALL_CMD | CCTL_CMD_L2_IX_WB;
L2_CMD_RDY();
L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
L2_CMD_RDY();
__nds32__msync_all();
local_irq_restore(flags);
}
EXPORT_SYMBOL(dcache_wb_all_level);
#endif
void cpu_dcache_wb_all(void)
{
__nds32__cctl_l1d_wball_one_lvl();
__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
}
void cpu_dcache_wbinval_all(void)
{
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
unsigned long flags;
local_irq_save(flags);
#endif
cpu_dcache_wb_all();
cpu_dcache_inval_all();
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
local_irq_restore(flags);
#endif
}
/*
* Page
*/
void cpu_icache_inval_page(unsigned long start)
{
unsigned long line_size, end;
line_size = L1_cache_info[ICACHE].line_size;
end = start + PAGE_SIZE;
do {
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
} while (end != start);
__nds32__isb();
}
void cpu_dcache_inval_page(unsigned long start)
{
unsigned long line_size, end;
line_size = L1_cache_info[DCACHE].line_size;
end = start + PAGE_SIZE;
do {
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
} while (end != start);
}
void cpu_dcache_wb_page(unsigned long start)
{
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
unsigned long line_size, end;
line_size = L1_cache_info[DCACHE].line_size;
end = start + PAGE_SIZE;
do {
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
end -= line_size;
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
} while (end != start);
__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
#endif
}
void cpu_dcache_wbinval_page(unsigned long start)
{
unsigned long line_size, end;
line_size = L1_cache_info[DCACHE].line_size;
end = start + PAGE_SIZE;
do {
end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
} while (end != start);
__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
}
void cpu_cache_wbinval_page(unsigned long page, int flushi)
{
cpu_dcache_wbinval_page(page);
if (flushi)
cpu_icache_inval_page(page);
}
/*
* Range
*/
void cpu_icache_inval_range(unsigned long start, unsigned long end)
{
unsigned long line_size;
line_size = L1_cache_info[ICACHE].line_size;
while (end > start) {
__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (start));
start += line_size;
}
__nds32__isb();
}
void cpu_dcache_inval_range(unsigned long start, unsigned long end)
{
unsigned long line_size;
line_size = L1_cache_info[DCACHE].line_size;
while (end > start) {
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start));
start += line_size;
}
}
void cpu_dcache_wb_range(unsigned long start, unsigned long end)
{
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
unsigned long line_size;
line_size = L1_cache_info[DCACHE].line_size;
while (end > start) {
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start));
start += line_size;
}
__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
#endif
}
void cpu_dcache_wbinval_range(unsigned long start, unsigned long end)
{
unsigned long line_size;
line_size = L1_cache_info[DCACHE].line_size;
while (end > start) {
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start));
#endif
__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start));
start += line_size;
}
__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
}
void cpu_cache_wbinval_range(unsigned long start, unsigned long end, int flushi)
{
unsigned long line_size, align_start, align_end;
line_size = L1_cache_info[DCACHE].line_size;
align_start = start & ~(line_size - 1);
align_end = (end + line_size - 1) & ~(line_size - 1);
cpu_dcache_wbinval_range(align_start, align_end);
if (flushi) {
line_size = L1_cache_info[ICACHE].line_size;
align_start = start & ~(line_size - 1);
align_end = (end + line_size - 1) & ~(line_size - 1);
cpu_icache_inval_range(align_start, align_end);
}
}
void cpu_cache_wbinval_range_check(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
bool flushi, bool wbd)
{
unsigned long line_size, t_start, t_end;
if (!flushi && !wbd)
return;
line_size = L1_cache_info[DCACHE].line_size;
start = start & ~(line_size - 1);
end = (end + line_size - 1) & ~(line_size - 1);
if ((end - start) > (8 * PAGE_SIZE)) {
if (wbd)
cpu_dcache_wbinval_all();
if (flushi)
cpu_icache_inval_all();
return;
}
t_start = (start + PAGE_SIZE) & PAGE_MASK;
t_end = ((end - 1) & PAGE_MASK);
if ((start & PAGE_MASK) == t_end) {
if (va_present(vma->vm_mm, start)) {
if (wbd)
cpu_dcache_wbinval_range(start, end);
if (flushi)
cpu_icache_inval_range(start, end);
}
return;
}
if (va_present(vma->vm_mm, start)) {
if (wbd)
cpu_dcache_wbinval_range(start, t_start);
if (flushi)
cpu_icache_inval_range(start, t_start);
}
if (va_present(vma->vm_mm, end - 1)) {
if (wbd)
cpu_dcache_wbinval_range(t_end, end);
if (flushi)
cpu_icache_inval_range(t_end, end);
}
while (t_start < t_end) {
if (va_present(vma->vm_mm, t_start)) {
if (wbd)
cpu_dcache_wbinval_page(t_start);
if (flushi)
cpu_icache_inval_page(t_start);
}
t_start += PAGE_SIZE;
}
}
#ifdef CONFIG_CACHE_L2
static inline void cpu_l2cache_op(unsigned long start, unsigned long end, unsigned long op)
{
if (atl2c_base) {
unsigned long p_start = __pa(start);
unsigned long p_end = __pa(end);
unsigned long cmd;
unsigned long line_size;
/* TODO Can Use PAGE Mode to optimize if range large than PAGE_SIZE */
line_size = L2_CACHE_LINE_SIZE();
p_start = p_start & (~(line_size - 1));
p_end = (p_end + line_size - 1) & (~(line_size - 1));
cmd =
(p_start & ~(line_size - 1)) | op |
CCTL_SINGLE_CMD;
do {
L2_CMD_RDY();
L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
cmd += line_size;
p_start += line_size;
} while (p_end > p_start);
cmd = CCTL_CMD_L2_SYNC;
L2_CMD_RDY();
L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
L2_CMD_RDY();
}
}
#else
#define cpu_l2cache_op(start,end,op) do { } while (0)
#endif
/*
* DMA
*/
void cpu_dma_wb_range(unsigned long start, unsigned long end)
{
unsigned long line_size;
unsigned long flags;
line_size = L1_cache_info[DCACHE].line_size;
start = start & (~(line_size - 1));
end = (end + line_size - 1) & (~(line_size - 1));
if (unlikely(start == end))
return;
local_irq_save(flags);
cpu_dcache_wb_range(start, end);
cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WB);
__nds32__msync_all();
local_irq_restore(flags);
}
void cpu_dma_inval_range(unsigned long start, unsigned long end)
{
unsigned long line_size;
unsigned long old_start = start;
unsigned long old_end = end;
unsigned long flags;
line_size = L1_cache_info[DCACHE].line_size;
start = start & (~(line_size - 1));
end = (end + line_size - 1) & (~(line_size - 1));
if (unlikely(start == end))
return;
local_irq_save(flags);
if (start != old_start) {
cpu_dcache_wbinval_range(start, start + line_size);
cpu_l2cache_op(start, start + line_size, CCTL_CMD_L2_PA_WBINVAL);
}
if (end != old_end) {
cpu_dcache_wbinval_range(end - line_size, end);
cpu_l2cache_op(end - line_size, end, CCTL_CMD_L2_PA_WBINVAL);
}
cpu_dcache_inval_range(start, end);
cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_INVAL);
__nds32__msync_all();
local_irq_restore(flags);
}
void cpu_dma_wbinval_range(unsigned long start, unsigned long end)
{
unsigned long line_size;
unsigned long flags;
line_size = L1_cache_info[DCACHE].line_size;
start = start & (~(line_size - 1));
end = (end + line_size - 1) & (~(line_size - 1));
if (unlikely(start == end))
return;
local_irq_save(flags);
cpu_dcache_wbinval_range(start, end);
cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WBINVAL);
__nds32__msync_all();
local_irq_restore(flags);
}
void cpu_proc_init(void)
{
}
void cpu_proc_fin(void)
{
}
void cpu_do_idle(void)
{
__nds32__standby_no_wake_grant();
}
void cpu_reset(unsigned long reset)
{
u32 tmp;
GIE_DISABLE();
tmp = __nds32__mfsr(NDS32_SR_CACHE_CTL);
tmp &= ~(CACHE_CTL_mskIC_EN | CACHE_CTL_mskDC_EN);
__nds32__mtsr_isb(tmp, NDS32_SR_CACHE_CTL);
cpu_dcache_wbinval_all();
cpu_icache_inval_all();
__asm__ __volatile__("jr.toff %0\n\t"::"r"(reset));
}
void cpu_switch_mm(struct mm_struct *mm)
{
unsigned long cid;
cid = __nds32__mfsr(NDS32_SR_TLB_MISC);
cid = (cid & ~TLB_MISC_mskCID) | mm->context.id;
__nds32__mtsr_dsb(cid, NDS32_SR_TLB_MISC);
__nds32__mtsr_isb(__pa(mm->pgd), NDS32_SR_L1_PPTB);
}
Reference in New Issue View Git Blame Copy Permalink