Commit 1a2db300 authored by Ganapatrao Kulkarni's avatar Ganapatrao Kulkarni Committed by Will Deacon

arm64, numa: Add NUMA support for arm64 platforms.

Attempt to get the memory and CPU NUMA node via of_numa.  If that
fails, default the dummy NUMA node and map all memory and CPUs to node
0.
Tested-by: default avatarShannon Zhao <shannon.zhao@linaro.org>
Reviewed-by: default avatarRobert Richter <rrichter@cavium.com>
Signed-off-by: default avatarGanapatrao Kulkarni <gkulkarni@caviumnetworks.com>
Signed-off-by: default avatarDavid Daney <david.daney@cavium.com>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent 3194ac6e
......@@ -76,6 +76,7 @@ config ARM64
select HAVE_HW_BREAKPOINT if PERF_EVENTS
select HAVE_IRQ_TIME_ACCOUNTING
select HAVE_MEMBLOCK
select HAVE_MEMBLOCK_NODE_MAP if NUMA
select HAVE_PATA_PLATFORM
select HAVE_PERF_EVENTS
select HAVE_PERF_REGS
......@@ -98,6 +99,7 @@ config ARM64
select SYSCTL_EXCEPTION_TRACE
select HAVE_CONTEXT_TRACKING
select HAVE_ARM_SMCCC
select OF_NUMA if NUMA && OF
help
ARM 64-bit (AArch64) Linux support.
......@@ -546,6 +548,30 @@ config HOTPLUG_CPU
Say Y here to experiment with turning CPUs off and on. CPUs
can be controlled through /sys/devices/system/cpu.
# Common NUMA Features
config NUMA
bool "Numa Memory Allocation and Scheduler Support"
depends on SMP
help
Enable NUMA (Non Uniform Memory Access) support.
The kernel will try to allocate memory used by a CPU on the
local memory of the CPU and add some more
NUMA awareness to the kernel.
config NODES_SHIFT
int "Maximum NUMA Nodes (as a power of 2)"
range 1 10
default "2"
depends on NEED_MULTIPLE_NODES
help
Specify the maximum number of NUMA Nodes available on the target
system. Increases memory reserved to accommodate various tables.
config USE_PERCPU_NUMA_NODE_ID
def_bool y
depends on NUMA
source kernel/Kconfig.preempt
source kernel/Kconfig.hz
......
#ifndef __ASM_MMZONE_H
#define __ASM_MMZONE_H
#ifdef CONFIG_NUMA
#include <asm/numa.h>
extern struct pglist_data *node_data[];
#define NODE_DATA(nid) (node_data[(nid)])
#endif /* CONFIG_NUMA */
#endif /* __ASM_MMZONE_H */
#ifndef __ASM_NUMA_H
#define __ASM_NUMA_H
#include <asm/topology.h>
#ifdef CONFIG_NUMA
/* currently, arm64 implements flat NUMA topology */
#define parent_node(node) (node)
int __node_distance(int from, int to);
#define node_distance(a, b) __node_distance(a, b)
extern nodemask_t numa_nodes_parsed __initdata;
/* Mappings between node number and cpus on that node. */
extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
void numa_clear_node(unsigned int cpu);
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
const struct cpumask *cpumask_of_node(int node);
#else
/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
static inline const struct cpumask *cpumask_of_node(int node)
{
return node_to_cpumask_map[node];
}
#endif
void __init arm64_numa_init(void);
int __init numa_add_memblk(int nodeid, u64 start, u64 end);
void __init numa_set_distance(int from, int to, int distance);
void __init numa_free_distance(void);
void __init early_map_cpu_to_node(unsigned int cpu, int nid);
void numa_store_cpu_info(unsigned int cpu);
#else /* CONFIG_NUMA */
static inline void numa_store_cpu_info(unsigned int cpu) { }
static inline void arm64_numa_init(void) { }
static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
#endif /* CONFIG_NUMA */
#endif /* __ASM_NUMA_H */
......@@ -22,6 +22,16 @@ void init_cpu_topology(void);
void store_cpu_topology(unsigned int cpuid);
const struct cpumask *cpu_coregroup_mask(int cpu);
#ifdef CONFIG_NUMA
struct pci_bus;
int pcibus_to_node(struct pci_bus *bus);
#define cpumask_of_pcibus(bus) (pcibus_to_node(bus) == -1 ? \
cpu_all_mask : \
cpumask_of_node(pcibus_to_node(bus)))
#endif /* CONFIG_NUMA */
#include <asm-generic/topology.h>
#endif /* _ASM_ARM_TOPOLOGY_H */
......@@ -74,6 +74,16 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
return -ENXIO;
}
#ifdef CONFIG_NUMA
int pcibus_to_node(struct pci_bus *bus)
{
return dev_to_node(&bus->dev);
}
EXPORT_SYMBOL(pcibus_to_node);
#endif
#ifdef CONFIG_ACPI
/* Root bridge scanning */
struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)
......
......@@ -53,6 +53,7 @@
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
#include <asm/kasan.h>
#include <asm/numa.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/smp_plat.h>
......@@ -319,6 +320,9 @@ static int __init topology_init(void)
{
int i;
for_each_online_node(i)
register_one_node(i);
for_each_possible_cpu(i) {
struct cpu *cpu = &per_cpu(cpu_data.cpu, i);
cpu->hotpluggable = 1;
......
......@@ -45,6 +45,7 @@
#include <asm/cputype.h>
#include <asm/cpu_ops.h>
#include <asm/mmu_context.h>
#include <asm/numa.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/processor.h>
......@@ -203,6 +204,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
static void smp_store_cpu_info(unsigned int cpuid)
{
store_cpu_topology(cpuid);
numa_store_cpu_info(cpuid);
}
/*
......@@ -633,6 +635,8 @@ static void __init of_parse_and_init_cpus(void)
pr_debug("cpu logical map 0x%llx\n", hwid);
cpu_logical_map(cpu_count) = hwid;
early_map_cpu_to_node(cpu_count, of_node_to_nid(dn));
next:
cpu_count++;
}
......
......@@ -4,6 +4,7 @@ obj-y := dma-mapping.o extable.o fault.o init.o \
context.o proc.o pageattr.o
obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
obj-$(CONFIG_ARM64_PTDUMP) += dump.o
obj-$(CONFIG_NUMA) += numa.o
obj-$(CONFIG_KASAN) += kasan_init.o
KASAN_SANITIZE_kasan_init.o := n
......@@ -40,6 +40,7 @@
#include <asm/kasan.h>
#include <asm/kernel-pgtable.h>
#include <asm/memory.h>
#include <asm/numa.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/sizes.h>
......@@ -86,6 +87,21 @@ static phys_addr_t __init max_zone_dma_phys(void)
return min(offset + (1ULL << 32), memblock_end_of_DRAM());
}
#ifdef CONFIG_NUMA
static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
unsigned long max_zone_pfns[MAX_NR_ZONES] = {0};
if (IS_ENABLED(CONFIG_ZONE_DMA))
max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
max_zone_pfns[ZONE_NORMAL] = max;
free_area_init_nodes(max_zone_pfns);
}
#else
static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
struct memblock_region *reg;
......@@ -126,6 +142,8 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
free_area_init_node(0, zone_size, min, zhole_size);
}
#endif /* CONFIG_NUMA */
#ifdef CONFIG_HAVE_ARCH_PFN_VALID
int pfn_valid(unsigned long pfn)
{
......@@ -142,10 +160,15 @@ static void __init arm64_memory_present(void)
static void __init arm64_memory_present(void)
{
struct memblock_region *reg;
int nid = 0;
for_each_memblock(memory, reg)
memory_present(0, memblock_region_memory_base_pfn(reg),
memblock_region_memory_end_pfn(reg));
for_each_memblock(memory, reg) {
#ifdef CONFIG_NUMA
nid = reg->nid;
#endif
memory_present(nid, memblock_region_memory_base_pfn(reg),
memblock_region_memory_end_pfn(reg));
}
}
#endif
......@@ -278,7 +301,6 @@ void __init arm64_memblock_init(void)
dma_contiguous_reserve(arm64_dma_phys_limit);
memblock_allow_resize();
memblock_dump_all();
}
void __init bootmem_init(void)
......@@ -290,6 +312,9 @@ void __init bootmem_init(void)
early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT);
max_pfn = max_low_pfn = max;
arm64_numa_init();
/*
* Sparsemem tries to allocate bootmem in memory_present(), so must be
* done after the fixed reservations.
......@@ -300,7 +325,7 @@ void __init bootmem_init(void)
zone_sizes_init(min, max);
high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
max_pfn = max_low_pfn = max;
memblock_dump_all();
}
#ifndef CONFIG_SPARSEMEM_VMEMMAP
......
/*
* NUMA support, based on the x86 implementation.
*
* Copyright (C) 2015 Cavium Inc.
* Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/bootmem.h>
#include <linux/memblock.h>
#include <linux/module.h>
#include <linux/of.h>
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
EXPORT_SYMBOL(node_data);
nodemask_t numa_nodes_parsed __initdata;
static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
static int numa_distance_cnt;
static u8 *numa_distance;
static int numa_off;
static __init int numa_parse_early_param(char *opt)
{
if (!opt)
return -EINVAL;
if (!strncmp(opt, "off", 3)) {
pr_info("%s\n", "NUMA turned off");
numa_off = 1;
}
return 0;
}
early_param("numa", numa_parse_early_param);
cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
EXPORT_SYMBOL(node_to_cpumask_map);
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
/*
* Returns a pointer to the bitmask of CPUs on Node 'node'.
*/
const struct cpumask *cpumask_of_node(int node)
{
if (WARN_ON(node >= nr_node_ids))
return cpu_none_mask;
if (WARN_ON(node_to_cpumask_map[node] == NULL))
return cpu_online_mask;
return node_to_cpumask_map[node];
}
EXPORT_SYMBOL(cpumask_of_node);
#endif
static void map_cpu_to_node(unsigned int cpu, int nid)
{
set_cpu_numa_node(cpu, nid);
if (nid >= 0)
cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
}
void numa_clear_node(unsigned int cpu)
{
int nid = cpu_to_node(cpu);
if (nid >= 0)
cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
set_cpu_numa_node(cpu, NUMA_NO_NODE);
}
/*
* Allocate node_to_cpumask_map based on number of available nodes
* Requires node_possible_map to be valid.
*
* Note: cpumask_of_node() is not valid until after this is done.
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
*/
static void __init setup_node_to_cpumask_map(void)
{
unsigned int cpu;
int node;
/* setup nr_node_ids if not done yet */
if (nr_node_ids == MAX_NUMNODES)
setup_nr_node_ids();
/* allocate and clear the mapping */
for (node = 0; node < nr_node_ids; node++) {
alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
cpumask_clear(node_to_cpumask_map[node]);
}
for_each_possible_cpu(cpu)
set_cpu_numa_node(cpu, NUMA_NO_NODE);
/* cpumask_of_node() will now work */
pr_debug("NUMA: Node to cpumask map for %d nodes\n", nr_node_ids);
}
/*
* Set the cpu to node and mem mapping
*/
void numa_store_cpu_info(unsigned int cpu)
{
map_cpu_to_node(cpu, numa_off ? 0 : cpu_to_node_map[cpu]);
}
void __init early_map_cpu_to_node(unsigned int cpu, int nid)
{
/* fallback to node 0 */
if (nid < 0 || nid >= MAX_NUMNODES)
nid = 0;
cpu_to_node_map[cpu] = nid;
}
/**
* numa_add_memblk - Set node id to memblk
* @nid: NUMA node ID of the new memblk
* @start: Start address of the new memblk
* @size: Size of the new memblk
*
* RETURNS:
* 0 on success, -errno on failure.
*/
int __init numa_add_memblk(int nid, u64 start, u64 size)
{
int ret;
ret = memblock_set_node(start, size, &memblock.memory, nid);
if (ret < 0) {
pr_err("NUMA: memblock [0x%llx - 0x%llx] failed to add on node %d\n",
start, (start + size - 1), nid);
return ret;
}
node_set(nid, numa_nodes_parsed);
pr_info("NUMA: Adding memblock [0x%llx - 0x%llx] on node %d\n",
start, (start + size - 1), nid);
return ret;
}
/**
* Initialize NODE_DATA for a node on the local memory
*/
static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
{
const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES);
u64 nd_pa;
void *nd;
int tnid;
pr_info("NUMA: Initmem setup node %d [mem %#010Lx-%#010Lx]\n",
nid, start_pfn << PAGE_SHIFT,
(end_pfn << PAGE_SHIFT) - 1);
nd_pa = memblock_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid);
nd = __va(nd_pa);
/* report and initialize */
pr_info("NUMA: NODE_DATA [mem %#010Lx-%#010Lx]\n",
nd_pa, nd_pa + nd_size - 1);
tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
if (tnid != nid)
pr_info("NUMA: NODE_DATA(%d) on node %d\n", nid, tnid);
node_data[nid] = nd;
memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
NODE_DATA(nid)->node_id = nid;
NODE_DATA(nid)->node_start_pfn = start_pfn;
NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
}
/**
* numa_free_distance
*
* The current table is freed.
*/
void __init numa_free_distance(void)
{
size_t size;
if (!numa_distance)
return;
size = numa_distance_cnt * numa_distance_cnt *
sizeof(numa_distance[0]);
memblock_free(__pa(numa_distance), size);
numa_distance_cnt = 0;
numa_distance = NULL;
}
/**
*
* Create a new NUMA distance table.
*
*/
static int __init numa_alloc_distance(void)
{
size_t size;
u64 phys;
int i, j;
size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]);
phys = memblock_find_in_range(0, PFN_PHYS(max_pfn),
size, PAGE_SIZE);
if (WARN_ON(!phys))
return -ENOMEM;
memblock_reserve(phys, size);
numa_distance = __va(phys);
numa_distance_cnt = nr_node_ids;
/* fill with the default distances */
for (i = 0; i < numa_distance_cnt; i++)
for (j = 0; j < numa_distance_cnt; j++)
numa_distance[i * numa_distance_cnt + j] = i == j ?
LOCAL_DISTANCE : REMOTE_DISTANCE;
pr_debug("NUMA: Initialized distance table, cnt=%d\n",
numa_distance_cnt);
return 0;
}
/**
* numa_set_distance - Set inter node NUMA distance from node to node.
* @from: the 'from' node to set distance
* @to: the 'to' node to set distance
* @distance: NUMA distance
*
* Set the distance from node @from to @to to @distance.
* If distance table doesn't exist, a warning is printed.
*
* If @from or @to is higher than the highest known node or lower than zero
* or @distance doesn't make sense, the call is ignored.
*
*/
void __init numa_set_distance(int from, int to, int distance)
{
if (!numa_distance) {
pr_warn_once("NUMA: Warning: distance table not allocated yet\n");
return;
}
if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
from < 0 || to < 0) {
pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
from, to, distance);
return;
}
if ((u8)distance != distance ||
(from == to && distance != LOCAL_DISTANCE)) {
pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
from, to, distance);
return;
}
numa_distance[from * numa_distance_cnt + to] = distance;
}
/**
* Return NUMA distance @from to @to
*/
int __node_distance(int from, int to)
{
if (from >= numa_distance_cnt || to >= numa_distance_cnt)
return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
return numa_distance[from * numa_distance_cnt + to];
}
EXPORT_SYMBOL(__node_distance);
static int __init numa_register_nodes(void)
{
int nid;
struct memblock_region *mblk;
/* Check that valid nid is set to memblks */
for_each_memblock(memory, mblk)
if (mblk->nid == NUMA_NO_NODE || mblk->nid >= MAX_NUMNODES) {
pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
mblk->nid, mblk->base,
mblk->base + mblk->size - 1);
return -EINVAL;
}
/* Finally register nodes. */
for_each_node_mask(nid, numa_nodes_parsed) {
unsigned long start_pfn, end_pfn;
get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
setup_node_data(nid, start_pfn, end_pfn);
node_set_online(nid);
}
/* Setup online nodes to actual nodes*/
node_possible_map = numa_nodes_parsed;
return 0;
}
static int __init numa_init(int (*init_func)(void))
{
int ret;
nodes_clear(numa_nodes_parsed);
nodes_clear(node_possible_map);
nodes_clear(node_online_map);
numa_free_distance();
ret = numa_alloc_distance();
if (ret < 0)
return ret;
ret = init_func();
if (ret < 0)
return ret;
if (nodes_empty(numa_nodes_parsed))
return -EINVAL;
ret = numa_register_nodes();
if (ret < 0)
return ret;
setup_node_to_cpumask_map();
/* init boot processor */
cpu_to_node_map[0] = 0;
map_cpu_to_node(0, 0);
return 0;
}
/**
* dummy_numa_init - Fallback dummy NUMA init
*
* Used if there's no underlying NUMA architecture, NUMA initialization
* fails, or NUMA is disabled on the command line.
*
* Must online at least one node (node 0) and add memory blocks that cover all
* allowed memory. It is unlikely that this function fails.
*/
static int __init dummy_numa_init(void)