mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-06 11:36:39 +07:00
1e01979c8f
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b
(x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
842 lines
21 KiB
C
842 lines
21 KiB
C
/* Common code for 32 and 64-bit NUMA */
|
|
#include <linux/kernel.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/string.h>
|
|
#include <linux/init.h>
|
|
#include <linux/bootmem.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/mmzone.h>
|
|
#include <linux/ctype.h>
|
|
#include <linux/module.h>
|
|
#include <linux/nodemask.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/topology.h>
|
|
|
|
#include <asm/e820.h>
|
|
#include <asm/proto.h>
|
|
#include <asm/dma.h>
|
|
#include <asm/acpi.h>
|
|
#include <asm/amd_nb.h>
|
|
|
|
#include "numa_internal.h"
|
|
|
|
int __initdata numa_off;
|
|
nodemask_t numa_nodes_parsed __initdata;
|
|
|
|
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
|
|
EXPORT_SYMBOL(node_data);
|
|
|
|
static struct numa_meminfo numa_meminfo
|
|
#ifndef CONFIG_MEMORY_HOTPLUG
|
|
__initdata
|
|
#endif
|
|
;
|
|
|
|
static int numa_distance_cnt;
|
|
static u8 *numa_distance;
|
|
|
|
static __init int numa_setup(char *opt)
|
|
{
|
|
if (!opt)
|
|
return -EINVAL;
|
|
if (!strncmp(opt, "off", 3))
|
|
numa_off = 1;
|
|
#ifdef CONFIG_NUMA_EMU
|
|
if (!strncmp(opt, "fake=", 5))
|
|
numa_emu_cmdline(opt + 5);
|
|
#endif
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
if (!strncmp(opt, "noacpi", 6))
|
|
acpi_numa = -1;
|
|
#endif
|
|
return 0;
|
|
}
|
|
early_param("numa", numa_setup);
|
|
|
|
/*
|
|
* apicid, cpu, node mappings
|
|
*/
|
|
s16 __apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
|
|
[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
|
|
};
|
|
|
|
int __cpuinit numa_cpu_node(int cpu)
|
|
{
|
|
int apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
|
|
|
|
if (apicid != BAD_APICID)
|
|
return __apicid_to_node[apicid];
|
|
return NUMA_NO_NODE;
|
|
}
|
|
|
|
cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
|
EXPORT_SYMBOL(node_to_cpumask_map);
|
|
|
|
/*
|
|
* Map cpu index to node index
|
|
*/
|
|
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
|
|
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
|
|
|
|
void __cpuinit numa_set_node(int cpu, int node)
|
|
{
|
|
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
|
|
|
|
/* early setting, no percpu area yet */
|
|
if (cpu_to_node_map) {
|
|
cpu_to_node_map[cpu] = node;
|
|
return;
|
|
}
|
|
|
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
|
if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
|
|
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
|
|
dump_stack();
|
|
return;
|
|
}
|
|
#endif
|
|
per_cpu(x86_cpu_to_node_map, cpu) = node;
|
|
|
|
if (node != NUMA_NO_NODE)
|
|
set_cpu_numa_node(cpu, node);
|
|
}
|
|
|
|
void __cpuinit numa_clear_node(int cpu)
|
|
{
|
|
numa_set_node(cpu, NUMA_NO_NODE);
|
|
}
|
|
|
|
/*
|
|
* Allocate node_to_cpumask_map based on number of available nodes
|
|
* Requires node_possible_map to be valid.
|
|
*
|
|
* Note: node_to_cpumask() is not valid until after this is done.
|
|
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
|
|
*/
|
|
void __init setup_node_to_cpumask_map(void)
|
|
{
|
|
unsigned int node, num = 0;
|
|
|
|
/* setup nr_node_ids if not done yet */
|
|
if (nr_node_ids == MAX_NUMNODES) {
|
|
for_each_node_mask(node, node_possible_map)
|
|
num = node;
|
|
nr_node_ids = num + 1;
|
|
}
|
|
|
|
/* allocate the map */
|
|
for (node = 0; node < nr_node_ids; node++)
|
|
alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
|
|
|
|
/* cpumask_of_node() will now work */
|
|
pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
|
|
}
|
|
|
|
static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
|
|
struct numa_meminfo *mi)
|
|
{
|
|
/* ignore zero length blks */
|
|
if (start == end)
|
|
return 0;
|
|
|
|
/* whine about and ignore invalid blks */
|
|
if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
|
|
pr_warning("NUMA: Warning: invalid memblk node %d (%Lx-%Lx)\n",
|
|
nid, start, end);
|
|
return 0;
|
|
}
|
|
|
|
if (mi->nr_blks >= NR_NODE_MEMBLKS) {
|
|
pr_err("NUMA: too many memblk ranges\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
mi->blk[mi->nr_blks].start = start;
|
|
mi->blk[mi->nr_blks].end = end;
|
|
mi->blk[mi->nr_blks].nid = nid;
|
|
mi->nr_blks++;
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo
|
|
* @idx: Index of memblk to remove
|
|
* @mi: numa_meminfo to remove memblk from
|
|
*
|
|
* Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and
|
|
* decrementing @mi->nr_blks.
|
|
*/
|
|
void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi)
|
|
{
|
|
mi->nr_blks--;
|
|
memmove(&mi->blk[idx], &mi->blk[idx + 1],
|
|
(mi->nr_blks - idx) * sizeof(mi->blk[0]));
|
|
}
|
|
|
|
/**
|
|
* numa_add_memblk - Add one numa_memblk to numa_meminfo
|
|
* @nid: NUMA node ID of the new memblk
|
|
* @start: Start address of the new memblk
|
|
* @end: End address of the new memblk
|
|
*
|
|
* Add a new memblk to the default numa_meminfo.
|
|
*
|
|
* RETURNS:
|
|
* 0 on success, -errno on failure.
|
|
*/
|
|
int __init numa_add_memblk(int nid, u64 start, u64 end)
|
|
{
|
|
return numa_add_memblk_to(nid, start, end, &numa_meminfo);
|
|
}
|
|
|
|
/* Initialize NODE_DATA for a node on the local memory */
|
|
static void __init setup_node_data(int nid, u64 start, u64 end)
|
|
{
|
|
const u64 nd_low = PFN_PHYS(MAX_DMA_PFN);
|
|
const u64 nd_high = PFN_PHYS(max_pfn_mapped);
|
|
const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
|
|
bool remapped = false;
|
|
u64 nd_pa;
|
|
void *nd;
|
|
int tnid;
|
|
|
|
/*
|
|
* Don't confuse VM with a node that doesn't have the
|
|
* minimum amount of memory:
|
|
*/
|
|
if (end && (end - start) < NODE_MIN_SIZE)
|
|
return;
|
|
|
|
/* initialize remap allocator before aligning to ZONE_ALIGN */
|
|
init_alloc_remap(nid, start, end);
|
|
|
|
start = roundup(start, ZONE_ALIGN);
|
|
|
|
printk(KERN_INFO "Initmem setup node %d %016Lx-%016Lx\n",
|
|
nid, start, end);
|
|
|
|
/*
|
|
* Allocate node data. Try remap allocator first, node-local
|
|
* memory and then any node. Never allocate in DMA zone.
|
|
*/
|
|
nd = alloc_remap(nid, nd_size);
|
|
if (nd) {
|
|
nd_pa = __pa(nd);
|
|
remapped = true;
|
|
} else {
|
|
nd_pa = memblock_x86_find_in_range_node(nid, nd_low, nd_high,
|
|
nd_size, SMP_CACHE_BYTES);
|
|
if (nd_pa == MEMBLOCK_ERROR)
|
|
nd_pa = memblock_find_in_range(nd_low, nd_high,
|
|
nd_size, SMP_CACHE_BYTES);
|
|
if (nd_pa == MEMBLOCK_ERROR) {
|
|
pr_err("Cannot find %zu bytes in node %d\n",
|
|
nd_size, nid);
|
|
return;
|
|
}
|
|
memblock_x86_reserve_range(nd_pa, nd_pa + nd_size, "NODE_DATA");
|
|
nd = __va(nd_pa);
|
|
}
|
|
|
|
/* report and initialize */
|
|
printk(KERN_INFO " NODE_DATA [%016Lx - %016Lx]%s\n",
|
|
nd_pa, nd_pa + nd_size - 1, remapped ? " (remapped)" : "");
|
|
tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
|
|
if (!remapped && tnid != nid)
|
|
printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nid, tnid);
|
|
|
|
node_data[nid] = nd;
|
|
memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
|
|
NODE_DATA(nid)->node_id = nid;
|
|
NODE_DATA(nid)->node_start_pfn = start >> PAGE_SHIFT;
|
|
NODE_DATA(nid)->node_spanned_pages = (end - start) >> PAGE_SHIFT;
|
|
|
|
node_set_online(nid);
|
|
}
|
|
|
|
/**
|
|
* numa_cleanup_meminfo - Cleanup a numa_meminfo
|
|
* @mi: numa_meminfo to clean up
|
|
*
|
|
* Sanitize @mi by merging and removing unncessary memblks. Also check for
|
|
* conflicts and clear unused memblks.
|
|
*
|
|
* RETURNS:
|
|
* 0 on success, -errno on failure.
|
|
*/
|
|
int __init numa_cleanup_meminfo(struct numa_meminfo *mi)
|
|
{
|
|
const u64 low = 0;
|
|
const u64 high = PFN_PHYS(max_pfn);
|
|
int i, j, k;
|
|
|
|
/* first, trim all entries */
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
struct numa_memblk *bi = &mi->blk[i];
|
|
|
|
/* make sure all blocks are inside the limits */
|
|
bi->start = max(bi->start, low);
|
|
bi->end = min(bi->end, high);
|
|
|
|
/* and there's no empty block */
|
|
if (bi->start >= bi->end)
|
|
numa_remove_memblk_from(i--, mi);
|
|
}
|
|
|
|
/* merge neighboring / overlapping entries */
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
struct numa_memblk *bi = &mi->blk[i];
|
|
|
|
for (j = i + 1; j < mi->nr_blks; j++) {
|
|
struct numa_memblk *bj = &mi->blk[j];
|
|
u64 start, end;
|
|
|
|
/*
|
|
* See whether there are overlapping blocks. Whine
|
|
* about but allow overlaps of the same nid. They
|
|
* will be merged below.
|
|
*/
|
|
if (bi->end > bj->start && bi->start < bj->end) {
|
|
if (bi->nid != bj->nid) {
|
|
pr_err("NUMA: node %d (%Lx-%Lx) overlaps with node %d (%Lx-%Lx)\n",
|
|
bi->nid, bi->start, bi->end,
|
|
bj->nid, bj->start, bj->end);
|
|
return -EINVAL;
|
|
}
|
|
pr_warning("NUMA: Warning: node %d (%Lx-%Lx) overlaps with itself (%Lx-%Lx)\n",
|
|
bi->nid, bi->start, bi->end,
|
|
bj->start, bj->end);
|
|
}
|
|
|
|
/*
|
|
* Join together blocks on the same node, holes
|
|
* between which don't overlap with memory on other
|
|
* nodes.
|
|
*/
|
|
if (bi->nid != bj->nid)
|
|
continue;
|
|
start = min(bi->start, bj->start);
|
|
end = max(bi->end, bj->end);
|
|
for (k = 0; k < mi->nr_blks; k++) {
|
|
struct numa_memblk *bk = &mi->blk[k];
|
|
|
|
if (bi->nid == bk->nid)
|
|
continue;
|
|
if (start < bk->end && end > bk->start)
|
|
break;
|
|
}
|
|
if (k < mi->nr_blks)
|
|
continue;
|
|
printk(KERN_INFO "NUMA: Node %d [%Lx,%Lx) + [%Lx,%Lx) -> [%Lx,%Lx)\n",
|
|
bi->nid, bi->start, bi->end, bj->start, bj->end,
|
|
start, end);
|
|
bi->start = start;
|
|
bi->end = end;
|
|
numa_remove_memblk_from(j--, mi);
|
|
}
|
|
}
|
|
|
|
/* clear unused ones */
|
|
for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) {
|
|
mi->blk[i].start = mi->blk[i].end = 0;
|
|
mi->blk[i].nid = NUMA_NO_NODE;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Set nodes, which have memory in @mi, in *@nodemask.
|
|
*/
|
|
static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
|
|
const struct numa_meminfo *mi)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
|
|
if (mi->blk[i].start != mi->blk[i].end &&
|
|
mi->blk[i].nid != NUMA_NO_NODE)
|
|
node_set(mi->blk[i].nid, *nodemask);
|
|
}
|
|
|
|
/**
|
|
* numa_reset_distance - Reset NUMA distance table
|
|
*
|
|
* The current table is freed. The next numa_set_distance() call will
|
|
* create a new one.
|
|
*/
|
|
void __init numa_reset_distance(void)
|
|
{
|
|
size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]);
|
|
|
|
/* numa_distance could be 1LU marking allocation failure, test cnt */
|
|
if (numa_distance_cnt)
|
|
memblock_x86_free_range(__pa(numa_distance),
|
|
__pa(numa_distance) + size);
|
|
numa_distance_cnt = 0;
|
|
numa_distance = NULL; /* enable table creation */
|
|
}
|
|
|
|
static int __init numa_alloc_distance(void)
|
|
{
|
|
nodemask_t nodes_parsed;
|
|
size_t size;
|
|
int i, j, cnt = 0;
|
|
u64 phys;
|
|
|
|
/* size the new table and allocate it */
|
|
nodes_parsed = numa_nodes_parsed;
|
|
numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
|
|
|
|
for_each_node_mask(i, nodes_parsed)
|
|
cnt = i;
|
|
cnt++;
|
|
size = cnt * cnt * sizeof(numa_distance[0]);
|
|
|
|
phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped),
|
|
size, PAGE_SIZE);
|
|
if (phys == MEMBLOCK_ERROR) {
|
|
pr_warning("NUMA: Warning: can't allocate distance table!\n");
|
|
/* don't retry until explicitly reset */
|
|
numa_distance = (void *)1LU;
|
|
return -ENOMEM;
|
|
}
|
|
memblock_x86_reserve_range(phys, phys + size, "NUMA DIST");
|
|
|
|
numa_distance = __va(phys);
|
|
numa_distance_cnt = cnt;
|
|
|
|
/* fill with the default distances */
|
|
for (i = 0; i < cnt; i++)
|
|
for (j = 0; j < cnt; j++)
|
|
numa_distance[i * cnt + j] = i == j ?
|
|
LOCAL_DISTANCE : REMOTE_DISTANCE;
|
|
printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* numa_set_distance - Set NUMA distance from one NUMA to another
|
|
* @from: the 'from' node to set distance
|
|
* @to: the 'to' node to set distance
|
|
* @distance: NUMA distance
|
|
*
|
|
* Set the distance from node @from to @to to @distance. If distance table
|
|
* doesn't exist, one which is large enough to accommodate all the currently
|
|
* known nodes will be created.
|
|
*
|
|
* If such table cannot be allocated, a warning is printed and further
|
|
* calls are ignored until the distance table is reset with
|
|
* numa_reset_distance().
|
|
*
|
|
* If @from or @to is higher than the highest known node at the time of
|
|
* table creation or @distance doesn't make sense, the call is ignored.
|
|
* This is to allow simplification of specific NUMA config implementations.
|
|
*/
|
|
void __init numa_set_distance(int from, int to, int distance)
|
|
{
|
|
if (!numa_distance && numa_alloc_distance() < 0)
|
|
return;
|
|
|
|
if (from >= numa_distance_cnt || to >= numa_distance_cnt) {
|
|
printk_once(KERN_DEBUG "NUMA: Debug: distance out of bound, from=%d to=%d distance=%d\n",
|
|
from, to, distance);
|
|
return;
|
|
}
|
|
|
|
if ((u8)distance != distance ||
|
|
(from == to && distance != LOCAL_DISTANCE)) {
|
|
pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
|
|
from, to, distance);
|
|
return;
|
|
}
|
|
|
|
numa_distance[from * numa_distance_cnt + to] = distance;
|
|
}
|
|
|
|
int __node_distance(int from, int to)
|
|
{
|
|
if (from >= numa_distance_cnt || to >= numa_distance_cnt)
|
|
return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
|
|
return numa_distance[from * numa_distance_cnt + to];
|
|
}
|
|
EXPORT_SYMBOL(__node_distance);
|
|
|
|
/*
|
|
* Sanity check to catch more bad NUMA configurations (they are amazingly
|
|
* common). Make sure the nodes cover all memory.
|
|
*/
|
|
static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
|
|
{
|
|
u64 numaram, e820ram;
|
|
int i;
|
|
|
|
numaram = 0;
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
u64 s = mi->blk[i].start >> PAGE_SHIFT;
|
|
u64 e = mi->blk[i].end >> PAGE_SHIFT;
|
|
numaram += e - s;
|
|
numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
|
|
if ((s64)numaram < 0)
|
|
numaram = 0;
|
|
}
|
|
|
|
e820ram = max_pfn - (memblock_x86_hole_size(0,
|
|
PFN_PHYS(max_pfn)) >> PAGE_SHIFT);
|
|
/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
|
|
if ((s64)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) {
|
|
printk(KERN_ERR "NUMA: nodes only cover %LuMB of your %LuMB e820 RAM. Not used.\n",
|
|
(numaram << PAGE_SHIFT) >> 20,
|
|
(e820ram << PAGE_SHIFT) >> 20);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static int __init numa_register_memblks(struct numa_meminfo *mi)
|
|
{
|
|
unsigned long uninitialized_var(pfn_align);
|
|
int i, nid;
|
|
|
|
/* Account for nodes with cpus and no memory */
|
|
node_possible_map = numa_nodes_parsed;
|
|
numa_nodemask_from_meminfo(&node_possible_map, mi);
|
|
if (WARN_ON(nodes_empty(node_possible_map)))
|
|
return -EINVAL;
|
|
|
|
for (i = 0; i < mi->nr_blks; i++)
|
|
memblock_x86_register_active_regions(mi->blk[i].nid,
|
|
mi->blk[i].start >> PAGE_SHIFT,
|
|
mi->blk[i].end >> PAGE_SHIFT);
|
|
|
|
/* for out of order entries */
|
|
sort_node_map();
|
|
|
|
/*
|
|
* If sections array is gonna be used for pfn -> nid mapping, check
|
|
* whether its granularity is fine enough.
|
|
*/
|
|
#ifdef NODE_NOT_IN_PAGE_FLAGS
|
|
pfn_align = node_map_pfn_alignment();
|
|
if (pfn_align && pfn_align < PAGES_PER_SECTION) {
|
|
printk(KERN_WARNING "Node alignment %LuMB < min %LuMB, rejecting NUMA config\n",
|
|
PFN_PHYS(pfn_align) >> 20,
|
|
PFN_PHYS(PAGES_PER_SECTION) >> 20);
|
|
return -EINVAL;
|
|
}
|
|
#endif
|
|
if (!numa_meminfo_cover_memory(mi))
|
|
return -EINVAL;
|
|
|
|
/* Finally register nodes. */
|
|
for_each_node_mask(nid, node_possible_map) {
|
|
u64 start = PFN_PHYS(max_pfn);
|
|
u64 end = 0;
|
|
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
if (nid != mi->blk[i].nid)
|
|
continue;
|
|
start = min(mi->blk[i].start, start);
|
|
end = max(mi->blk[i].end, end);
|
|
}
|
|
|
|
if (start < end)
|
|
setup_node_data(nid, start, end);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* There are unfortunately some poorly designed mainboards around that
|
|
* only connect memory to a single CPU. This breaks the 1:1 cpu->node
|
|
* mapping. To avoid this fill in the mapping for all possible CPUs,
|
|
* as the number of CPUs is not known yet. We round robin the existing
|
|
* nodes.
|
|
*/
|
|
static void __init numa_init_array(void)
|
|
{
|
|
int rr, i;
|
|
|
|
rr = first_node(node_online_map);
|
|
for (i = 0; i < nr_cpu_ids; i++) {
|
|
if (early_cpu_to_node(i) != NUMA_NO_NODE)
|
|
continue;
|
|
numa_set_node(i, rr);
|
|
rr = next_node(rr, node_online_map);
|
|
if (rr == MAX_NUMNODES)
|
|
rr = first_node(node_online_map);
|
|
}
|
|
}
|
|
|
|
static int __init numa_init(int (*init_func)(void))
|
|
{
|
|
int i;
|
|
int ret;
|
|
|
|
for (i = 0; i < MAX_LOCAL_APIC; i++)
|
|
set_apicid_to_node(i, NUMA_NO_NODE);
|
|
|
|
nodes_clear(numa_nodes_parsed);
|
|
nodes_clear(node_possible_map);
|
|
nodes_clear(node_online_map);
|
|
memset(&numa_meminfo, 0, sizeof(numa_meminfo));
|
|
remove_all_active_ranges();
|
|
numa_reset_distance();
|
|
|
|
ret = init_func();
|
|
if (ret < 0)
|
|
return ret;
|
|
ret = numa_cleanup_meminfo(&numa_meminfo);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
numa_emulation(&numa_meminfo, numa_distance_cnt);
|
|
|
|
ret = numa_register_memblks(&numa_meminfo);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
for (i = 0; i < nr_cpu_ids; i++) {
|
|
int nid = early_cpu_to_node(i);
|
|
|
|
if (nid == NUMA_NO_NODE)
|
|
continue;
|
|
if (!node_online(nid))
|
|
numa_clear_node(i);
|
|
}
|
|
numa_init_array();
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* dummy_numa_init - Fallback dummy NUMA init
|
|
*
|
|
* Used if there's no underlying NUMA architecture, NUMA initialization
|
|
* fails, or NUMA is disabled on the command line.
|
|
*
|
|
* Must online at least one node and add memory blocks that cover all
|
|
* allowed memory. This function must not fail.
|
|
*/
|
|
static int __init dummy_numa_init(void)
|
|
{
|
|
printk(KERN_INFO "%s\n",
|
|
numa_off ? "NUMA turned off" : "No NUMA configuration found");
|
|
printk(KERN_INFO "Faking a node at %016Lx-%016Lx\n",
|
|
0LLU, PFN_PHYS(max_pfn));
|
|
|
|
node_set(0, numa_nodes_parsed);
|
|
numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* x86_numa_init - Initialize NUMA
|
|
*
|
|
* Try each configured NUMA initialization method until one succeeds. The
|
|
* last fallback is dummy single node config encomapssing whole memory and
|
|
* never fails.
|
|
*/
|
|
void __init x86_numa_init(void)
|
|
{
|
|
if (!numa_off) {
|
|
#ifdef CONFIG_X86_NUMAQ
|
|
if (!numa_init(numaq_numa_init))
|
|
return;
|
|
#endif
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
if (!numa_init(x86_acpi_numa_init))
|
|
return;
|
|
#endif
|
|
#ifdef CONFIG_AMD_NUMA
|
|
if (!numa_init(amd_numa_init))
|
|
return;
|
|
#endif
|
|
}
|
|
|
|
numa_init(dummy_numa_init);
|
|
}
|
|
|
|
static __init int find_near_online_node(int node)
|
|
{
|
|
int n, val;
|
|
int min_val = INT_MAX;
|
|
int best_node = -1;
|
|
|
|
for_each_online_node(n) {
|
|
val = node_distance(node, n);
|
|
|
|
if (val < min_val) {
|
|
min_val = val;
|
|
best_node = n;
|
|
}
|
|
}
|
|
|
|
return best_node;
|
|
}
|
|
|
|
/*
|
|
* Setup early cpu_to_node.
|
|
*
|
|
* Populate cpu_to_node[] only if x86_cpu_to_apicid[],
|
|
* and apicid_to_node[] tables have valid entries for a CPU.
|
|
* This means we skip cpu_to_node[] initialisation for NUMA
|
|
* emulation and faking node case (when running a kernel compiled
|
|
* for NUMA on a non NUMA box), which is OK as cpu_to_node[]
|
|
* is already initialized in a round robin manner at numa_init_array,
|
|
* prior to this call, and this initialization is good enough
|
|
* for the fake NUMA cases.
|
|
*
|
|
* Called before the per_cpu areas are setup.
|
|
*/
|
|
void __init init_cpu_to_node(void)
|
|
{
|
|
int cpu;
|
|
u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
|
|
|
|
BUG_ON(cpu_to_apicid == NULL);
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
int node = numa_cpu_node(cpu);
|
|
|
|
if (node == NUMA_NO_NODE)
|
|
continue;
|
|
if (!node_online(node))
|
|
node = find_near_online_node(node);
|
|
numa_set_node(cpu, node);
|
|
}
|
|
}
|
|
|
|
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
|
|
|
|
# ifndef CONFIG_NUMA_EMU
|
|
void __cpuinit numa_add_cpu(int cpu)
|
|
{
|
|
cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
|
|
}
|
|
|
|
void __cpuinit numa_remove_cpu(int cpu)
|
|
{
|
|
cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
|
|
}
|
|
# endif /* !CONFIG_NUMA_EMU */
|
|
|
|
#else /* !CONFIG_DEBUG_PER_CPU_MAPS */
|
|
|
|
int __cpu_to_node(int cpu)
|
|
{
|
|
if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
|
|
printk(KERN_WARNING
|
|
"cpu_to_node(%d): usage too early!\n", cpu);
|
|
dump_stack();
|
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
}
|
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
}
|
|
EXPORT_SYMBOL(__cpu_to_node);
|
|
|
|
/*
|
|
* Same function as cpu_to_node() but used if called before the
|
|
* per_cpu areas are setup.
|
|
*/
|
|
int early_cpu_to_node(int cpu)
|
|
{
|
|
if (early_per_cpu_ptr(x86_cpu_to_node_map))
|
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
|
|
if (!cpu_possible(cpu)) {
|
|
printk(KERN_WARNING
|
|
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
|
|
dump_stack();
|
|
return NUMA_NO_NODE;
|
|
}
|
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
}
|
|
|
|
void debug_cpumask_set_cpu(int cpu, int node, bool enable)
|
|
{
|
|
struct cpumask *mask;
|
|
char buf[64];
|
|
|
|
if (node == NUMA_NO_NODE) {
|
|
/* early_cpu_to_node() already emits a warning and trace */
|
|
return;
|
|
}
|
|
mask = node_to_cpumask_map[node];
|
|
if (!mask) {
|
|
pr_err("node_to_cpumask_map[%i] NULL\n", node);
|
|
dump_stack();
|
|
return;
|
|
}
|
|
|
|
if (enable)
|
|
cpumask_set_cpu(cpu, mask);
|
|
else
|
|
cpumask_clear_cpu(cpu, mask);
|
|
|
|
cpulist_scnprintf(buf, sizeof(buf), mask);
|
|
printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
|
|
enable ? "numa_add_cpu" : "numa_remove_cpu",
|
|
cpu, node, buf);
|
|
return;
|
|
}
|
|
|
|
# ifndef CONFIG_NUMA_EMU
|
|
static void __cpuinit numa_set_cpumask(int cpu, bool enable)
|
|
{
|
|
debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable);
|
|
}
|
|
|
|
void __cpuinit numa_add_cpu(int cpu)
|
|
{
|
|
numa_set_cpumask(cpu, true);
|
|
}
|
|
|
|
void __cpuinit numa_remove_cpu(int cpu)
|
|
{
|
|
numa_set_cpumask(cpu, false);
|
|
}
|
|
# endif /* !CONFIG_NUMA_EMU */
|
|
|
|
/*
|
|
* Returns a pointer to the bitmask of CPUs on Node 'node'.
|
|
*/
|
|
const struct cpumask *cpumask_of_node(int node)
|
|
{
|
|
if (node >= nr_node_ids) {
|
|
printk(KERN_WARNING
|
|
"cpumask_of_node(%d): node > nr_node_ids(%d)\n",
|
|
node, nr_node_ids);
|
|
dump_stack();
|
|
return cpu_none_mask;
|
|
}
|
|
if (node_to_cpumask_map[node] == NULL) {
|
|
printk(KERN_WARNING
|
|
"cpumask_of_node(%d): no node_to_cpumask_map!\n",
|
|
node);
|
|
dump_stack();
|
|
return cpu_online_mask;
|
|
}
|
|
return node_to_cpumask_map[node];
|
|
}
|
|
EXPORT_SYMBOL(cpumask_of_node);
|
|
|
|
#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
|
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
|
int memory_add_physaddr_to_nid(u64 start)
|
|
{
|
|
struct numa_meminfo *mi = &numa_meminfo;
|
|
int nid = mi->blk[0].nid;
|
|
int i;
|
|
|
|
for (i = 0; i < mi->nr_blks; i++)
|
|
if (mi->blk[i].start <= start && mi->blk[i].end > start)
|
|
nid = mi->blk[i].nid;
|
|
return nid;
|
|
}
|
|
EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
|
|
#endif
|