Commit a7f8de16 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: allow kernel Image to be loaded anywhere in physical memory

This relaxes the kernel Image placement requirements, so that it
may be placed at any 2 MB aligned offset in physical memory.

This is accomplished by ignoring PHYS_OFFSET when installing
memblocks, and accounting for the apparent virtual offset of
the kernel Image. As a result, virtual address references
below PAGE_OFFSET are correctly mapped onto physical references
into the kernel Image regardless of where it sits in memory.

Special care needs to be taken for dealing with memory limits passed
via mem=, since the generic implementation clips memory top down, which
may clip the kernel image itself if it is loaded high up in memory. To
deal with this case, we simply add back the memory covering the kernel
image, which may result in more memory to be retained than was passed
as a mem= parameter.

Since mem= should not be considered a production feature, a panic notifier
handler is installed that dumps the memory limit at panic time if one was
Signed-off-by: default avatarArd Biesheuvel <>
Signed-off-by: default avatarCatalin Marinas <>
parent a89dea58
......@@ -109,7 +109,13 @@ Header notes:
1 - 4K
2 - 16K
3 - 64K
Bits 3-63: Reserved.
Bit 3: Kernel physical placement
0 - 2MB aligned base should be as close as possible
to the base of DRAM, since memory below it is not
accessible via the linear mapping
1 - 2MB aligned base may be anywhere in physical
Bits 4-63: Reserved.
- When image_size is zero, a bootloader should attempt to keep as much
memory as possible free for use by the kernel immediately after the
......@@ -117,14 +123,14 @@ Header notes:
depending on selected features, and is effectively unbound.
The Image must be placed text_offset bytes from a 2MB aligned base
address near the start of usable system RAM and called there. Memory
below that base address is currently unusable by Linux, and therefore it
is strongly recommended that this location is the start of system RAM.
The region between the 2 MB aligned base address and the start of the
image has no special significance to the kernel, and may be used for
other purposes.
address anywhere in usable system RAM and called there. The region
between the 2 MB aligned base address and the start of the image has no
special significance to the kernel, and may be used for other purposes.
At least image_size bytes from the start of the image must be free for
use by the kernel.
NOTE: versions prior to v4.6 cannot make use of memory below the
physical offset of the Image so it is recommended that the Image be
placed as close as possible to the start of system RAM.
Any memory described to the kernel (even that below the start of the
image) which is not marked as reserved from the kernel (e.g., with a
......@@ -11,4 +11,10 @@
#define MIN_FDT_ALIGN 8
#define MAX_FDT_SIZE SZ_2M
* arm64 requires the kernel image to placed
* TEXT_OFFSET bytes beyond a 2 MB aligned base
......@@ -79,5 +79,17 @@
* To make optimal use of block mappings when laying out the linear
* mapping, round down the base of physical memory to a size that can
* be mapped efficiently, i.e., either PUD_SIZE (4k granule) or PMD_SIZE
* (64k granule), or a multiple that can be mapped using contiguous bits
* in the page tables: 32 * PMD_SIZE (16k granule)
#endif /* __ASM_KERNEL_PGTABLE_H */
......@@ -26,24 +26,9 @@
#define kvm_ksym_ref(sym) ((void *)&sym + kvm_ksym_shift)
#define kvm_ksym_ref(sym) phys_to_virt((u64)&sym - kimage_voffset)
#ifndef __ASSEMBLY__
#if __GNUC__ > 4
#define kvm_ksym_shift (PAGE_OFFSET - KIMAGE_VADDR)
* GCC versions 4.9 and older will fold the constant below into the addend of
* the reference to 'sym' above if kvm_ksym_shift is declared static or if the
* constant is used directly. However, since we use the small code model for
* the core kernel, the reference to 'sym' will be emitted as a adrp/add pair,
* with a +/- 4 GB range, resulting in linker relocation errors if the shift
* is sufficiently large. So prevent the compiler from folding the shift into
* the addend, by making the shift a variable with external linkage.
__weak u64 kvm_ksym_shift = PAGE_OFFSET - KIMAGE_VADDR;
struct kvm;
struct kvm_vcpu;
......@@ -24,6 +24,7 @@
#include <linux/compiler.h>
#include <linux/const.h>
#include <linux/types.h>
#include <asm/bug.h>
#include <asm/sizes.h>
......@@ -88,10 +89,10 @@
#define __virt_to_phys(x) ({ \
phys_addr_t __x = (phys_addr_t)(x); \
(__x - kimage_voffset); })
#define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET + PAGE_OFFSET))
#define __phys_to_kimg(x) ((unsigned long)((x) - PHYS_OFFSET + KIMAGE_VADDR))
#define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset))
* Convert a page to/from a physical address
......@@ -133,15 +134,16 @@
extern phys_addr_t memstart_addr;
/* PHYS_OFFSET - the physical address of the start of memory. */
#define PHYS_OFFSET ({ memstart_addr; })
#define PHYS_OFFSET ({ BUG_ON(memstart_addr & 1); memstart_addr; })
/* the offset between the kernel virtual and physical mappings */
extern u64 kimage_voffset;
* The maximum physical address that the linear direct mapping
* of system RAM can cover. (PAGE_OFFSET can be interpreted as
* a 2's complement signed quantity and negated to derive the
* maximum size of the linear mapping.)
* Allow all memory at the discovery stage. We will clip it later.
#define MAX_MEMBLOCK_ADDR ({ memstart_addr - PAGE_OFFSET - 1; })
* PFNs are used to describe any physical page; this means
......@@ -428,7 +428,11 @@ __mmap_switched:
and x4, x4, #~(THREAD_SIZE - 1)
msr sp_el0, x4 // Save thread_info
str_l x21, __fdt_pointer, x5 // Save FDT pointer
str_l x24, memstart_addr, x6 // Save PHYS_OFFSET
ldr x4, =KIMAGE_VADDR // Save the offset between
sub x4, x4, x24 // the kernel virtual and
str_l x4, kimage_voffset, x5 // physical mappings
mov x29, #0
bl kasan_early_init
......@@ -42,15 +42,18 @@
#define __HEAD_FLAG_BE 1
#define __HEAD_FLAG_BE 1
#define __HEAD_FLAG_BE 0
#define __HEAD_FLAG_BE 0
#define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2)
#define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2)
#define __HEAD_FLAGS ((__HEAD_FLAG_BE << 0) | \
#define __HEAD_FLAGS ((__HEAD_FLAG_BE << 0) | \
(__HEAD_FLAG_PAGE_SIZE << 1) | \
* These will output as part of the Image header, which should be little-endian
......@@ -35,8 +35,10 @@
#include <linux/efi.h>
#include <linux/swiotlb.h>
#include <asm/boot.h>
#include <asm/fixmap.h>
#include <asm/kasan.h>
#include <asm/kernel-pgtable.h>
#include <asm/memory.h>
#include <asm/sections.h>
#include <asm/setup.h>
......@@ -46,7 +48,13 @@
#include "mm.h"
phys_addr_t memstart_addr __read_mostly = 0;
* We need to be able to catch inadvertent references to memstart_addr
* that occur (potentially in generic code) before arm64_memblock_init()
* executes, which assigns it its actual value. So use a default value
* that cannot be mistaken for a real physical address.
phys_addr_t memstart_addr __read_mostly = ~0ULL;
phys_addr_t arm64_dma_phys_limit __read_mostly;
......@@ -160,7 +168,33 @@ early_param("mem", early_mem);
void __init arm64_memblock_init(void)
const s64 linear_region_size = -(s64)PAGE_OFFSET;
* Select a suitable value for the base of physical memory.
memstart_addr = round_down(memblock_start_of_DRAM(),
* Remove the memory that we will not be able to cover with the
* linear mapping. Take care not to clip the kernel which may be
* high in memory.
memblock_remove(max(memstart_addr + linear_region_size, __pa(_end)),
if (memblock_end_of_DRAM() > linear_region_size)
memblock_remove(0, memblock_end_of_DRAM() - linear_region_size);
* Apply the memory limit if it was set. Since the kernel may be loaded
* high up in memory, add back the kernel region that must be accessible
* via the linear mapping.
if (memory_limit != (phys_addr_t)ULLONG_MAX) {
memblock_add(__pa(_text), (u64)(_end - _text));
* Register the kernel text, kernel data, initrd, and initial
......@@ -386,3 +420,28 @@ static int __init keepinitrd_setup(char *__unused)
__setup("keepinitrd", keepinitrd_setup);
* Dump out memory limit information on panic.
static int dump_mem_limit(struct notifier_block *self, unsigned long v, void *p)
if (memory_limit != (phys_addr_t)ULLONG_MAX) {
pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20);
} else {
pr_emerg("Memory Limit: none\n");
return 0;
static struct notifier_block mem_limit_notifier = {
.notifier_call = dump_mem_limit,
static int __init register_mem_limit_dumper(void)
return 0;
......@@ -46,6 +46,9 @@
u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
u64 kimage_voffset __read_mostly;
* Empty_zero_page is a special page that is used for zero-initialized data
* and COW.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment