mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-28 11:18:45 +07:00
f9040773b7
This moves the module area to right before the vmalloc area, and moves the kernel image to the base of the vmalloc area. This is an intermediate step towards implementing KASLR, which allows the kernel image to be located anywhere in the vmalloc area. Since other subsystems such as hibernate may still need to refer to the kernel text or data segments via their linears addresses, both are mapped in the linear region as well. The linear alias of the text region is mapped read-only/non-executable to prevent inadvertent modification or execution. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
202 lines
5.6 KiB
C
202 lines
5.6 KiB
C
/*
|
|
* This file contains kasan initialization code for ARM64.
|
|
*
|
|
* Copyright (c) 2015 Samsung Electronics Co., Ltd.
|
|
* Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
*/
|
|
|
|
#define pr_fmt(fmt) "kasan: " fmt
|
|
#include <linux/kasan.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/start_kernel.h>
|
|
|
|
#include <asm/mmu_context.h>
|
|
#include <asm/kernel-pgtable.h>
|
|
#include <asm/page.h>
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/sections.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
|
|
|
|
static void __init kasan_early_pte_populate(pmd_t *pmd, unsigned long addr,
|
|
unsigned long end)
|
|
{
|
|
pte_t *pte;
|
|
unsigned long next;
|
|
|
|
if (pmd_none(*pmd))
|
|
pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
|
|
|
|
pte = pte_offset_kimg(pmd, addr);
|
|
do {
|
|
next = addr + PAGE_SIZE;
|
|
set_pte(pte, pfn_pte(virt_to_pfn(kasan_zero_page),
|
|
PAGE_KERNEL));
|
|
} while (pte++, addr = next, addr != end && pte_none(*pte));
|
|
}
|
|
|
|
static void __init kasan_early_pmd_populate(pud_t *pud,
|
|
unsigned long addr,
|
|
unsigned long end)
|
|
{
|
|
pmd_t *pmd;
|
|
unsigned long next;
|
|
|
|
if (pud_none(*pud))
|
|
pud_populate(&init_mm, pud, kasan_zero_pmd);
|
|
|
|
pmd = pmd_offset_kimg(pud, addr);
|
|
do {
|
|
next = pmd_addr_end(addr, end);
|
|
kasan_early_pte_populate(pmd, addr, next);
|
|
} while (pmd++, addr = next, addr != end && pmd_none(*pmd));
|
|
}
|
|
|
|
static void __init kasan_early_pud_populate(pgd_t *pgd,
|
|
unsigned long addr,
|
|
unsigned long end)
|
|
{
|
|
pud_t *pud;
|
|
unsigned long next;
|
|
|
|
if (pgd_none(*pgd))
|
|
pgd_populate(&init_mm, pgd, kasan_zero_pud);
|
|
|
|
pud = pud_offset_kimg(pgd, addr);
|
|
do {
|
|
next = pud_addr_end(addr, end);
|
|
kasan_early_pmd_populate(pud, addr, next);
|
|
} while (pud++, addr = next, addr != end && pud_none(*pud));
|
|
}
|
|
|
|
static void __init kasan_map_early_shadow(void)
|
|
{
|
|
unsigned long addr = KASAN_SHADOW_START;
|
|
unsigned long end = KASAN_SHADOW_END;
|
|
unsigned long next;
|
|
pgd_t *pgd;
|
|
|
|
pgd = pgd_offset_k(addr);
|
|
do {
|
|
next = pgd_addr_end(addr, end);
|
|
kasan_early_pud_populate(pgd, addr, next);
|
|
} while (pgd++, addr = next, addr != end);
|
|
}
|
|
|
|
asmlinkage void __init kasan_early_init(void)
|
|
{
|
|
BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61));
|
|
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE));
|
|
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
|
|
kasan_map_early_shadow();
|
|
}
|
|
|
|
/*
|
|
* Copy the current shadow region into a new pgdir.
|
|
*/
|
|
void __init kasan_copy_shadow(pgd_t *pgdir)
|
|
{
|
|
pgd_t *pgd, *pgd_new, *pgd_end;
|
|
|
|
pgd = pgd_offset_k(KASAN_SHADOW_START);
|
|
pgd_end = pgd_offset_k(KASAN_SHADOW_END);
|
|
pgd_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START);
|
|
do {
|
|
set_pgd(pgd_new, *pgd);
|
|
} while (pgd++, pgd_new++, pgd != pgd_end);
|
|
}
|
|
|
|
static void __init clear_pgds(unsigned long start,
|
|
unsigned long end)
|
|
{
|
|
/*
|
|
* Remove references to kasan page tables from
|
|
* swapper_pg_dir. pgd_clear() can't be used
|
|
* here because it's nop on 2,3-level pagetable setups
|
|
*/
|
|
for (; start < end; start += PGDIR_SIZE)
|
|
set_pgd(pgd_offset_k(start), __pgd(0));
|
|
}
|
|
|
|
void __init kasan_init(void)
|
|
{
|
|
u64 kimg_shadow_start, kimg_shadow_end;
|
|
struct memblock_region *reg;
|
|
int i;
|
|
|
|
kimg_shadow_start = (u64)kasan_mem_to_shadow(_text);
|
|
kimg_shadow_end = (u64)kasan_mem_to_shadow(_end);
|
|
|
|
/*
|
|
* We are going to perform proper setup of shadow memory.
|
|
* At first we should unmap early shadow (clear_pgds() call bellow).
|
|
* However, instrumented code couldn't execute without shadow memory.
|
|
* tmp_pg_dir used to keep early shadow mapped until full shadow
|
|
* setup will be finished.
|
|
*/
|
|
memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
|
|
dsb(ishst);
|
|
cpu_replace_ttbr1(tmp_pg_dir);
|
|
|
|
clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
|
|
|
|
vmemmap_populate(kimg_shadow_start, kimg_shadow_end, NUMA_NO_NODE);
|
|
|
|
/*
|
|
* vmemmap_populate() has populated the shadow region that covers the
|
|
* kernel image with SWAPPER_BLOCK_SIZE mappings, so we have to round
|
|
* the start and end addresses to SWAPPER_BLOCK_SIZE as well, to prevent
|
|
* kasan_populate_zero_shadow() from replacing the PMD block mappings
|
|
* with PMD table mappings at the edges of the shadow region for the
|
|
* kernel image.
|
|
*/
|
|
if (ARM64_SWAPPER_USES_SECTION_MAPS)
|
|
kimg_shadow_end = round_up(kimg_shadow_end, SWAPPER_BLOCK_SIZE);
|
|
|
|
kasan_populate_zero_shadow((void *)KASAN_SHADOW_START,
|
|
kasan_mem_to_shadow((void *)MODULES_VADDR));
|
|
kasan_populate_zero_shadow((void *)kimg_shadow_end,
|
|
kasan_mem_to_shadow((void *)PAGE_OFFSET));
|
|
|
|
for_each_memblock(memory, reg) {
|
|
void *start = (void *)__phys_to_virt(reg->base);
|
|
void *end = (void *)__phys_to_virt(reg->base + reg->size);
|
|
|
|
if (start >= end)
|
|
break;
|
|
|
|
/*
|
|
* end + 1 here is intentional. We check several shadow bytes in
|
|
* advance to slightly speed up fastpath. In some rare cases
|
|
* we could cross boundary of mapped shadow, so we just map
|
|
* some more here.
|
|
*/
|
|
vmemmap_populate((unsigned long)kasan_mem_to_shadow(start),
|
|
(unsigned long)kasan_mem_to_shadow(end) + 1,
|
|
pfn_to_nid(virt_to_pfn(start)));
|
|
}
|
|
|
|
/*
|
|
* KAsan may reuse the contents of kasan_zero_pte directly, so we
|
|
* should make sure that it maps the zero page read-only.
|
|
*/
|
|
for (i = 0; i < PTRS_PER_PTE; i++)
|
|
set_pte(&kasan_zero_pte[i],
|
|
pfn_pte(virt_to_pfn(kasan_zero_page), PAGE_KERNEL_RO));
|
|
|
|
memset(kasan_zero_page, 0, PAGE_SIZE);
|
|
cpu_replace_ttbr1(swapper_pg_dir);
|
|
|
|
/* At this point kasan is fully initialized. Enable error messages */
|
|
init_task.kasan_depth = 0;
|
|
pr_info("KernelAddressSanitizer initialized\n");
|
|
}
|