Commit 2979d763 authored by Charlie Jacobsen's avatar Charlie Jacobsen Committed by Vikram Narayanan

liblcd-v2: Fix generalized page alloc metadata alloc/free.

It seemed dumb to have the generalized page allocator code
repeatedly call out to the user to get memory for metadata.
For one thing, it was buggy. For another, in many use cases
we can alloc the metadata in one shot.

RAM map allocator partially started.
parent b9ce0034
......@@ -78,8 +78,12 @@ int __liblcd_heap_init(void);
*
* The maximum address space block you can allocate from the RAM map
* area is 1 GB (2^18 = 262,144 pages).
*
* In hindsight, maybe a simple bitmap would have been just as good
* here since there are only 8GB/64MB = 128 allocation blocks. Oh well.
*/
#define LCD_RAM_MAP_NR_PAGES_ORDER 26
#define LCD_RAM_MAP_NR_PAGES_ORDER \
(ilog2(LCD_RAM_MAP_REGION_SIZE >> PAGE_SHIFT))
#define LCD_RAM_MAP_SIZE LCD_RAM_MAP_REGION_SIZE
#define LCD_RAM_MAP_MIN_ORDER 14
#define LCD_RAM_MAP_MAX_ORDER 18
......
......@@ -102,14 +102,14 @@ struct lcd_page_allocator_cbs {
* trying to understand the internals.
*/
struct lcd_page_allocator {
/* Size of metadata in bytes (includes this struct) */
unsigned long metadata_sz;
/* The page allocator covers 2^nr_pages_order pages of memory. */
unsigned int nr_pages_order;
/* The minimum number of pages you can allocate is 2^min_order. */
unsigned int min_order;
/* The maximum number of pages during metadata allocation that
* will be attempted is 2^metadata_malloc_order. */
unsigned int metadata_malloc_order;
/* The maximum number of pages you can allocate is 2^max_order. */
unsigned int max_order;
......@@ -181,24 +181,34 @@ struct lcd_page_allocator {
* map and allocation interface that the page allocator will call out
* to at specific times:
*
* -- alloc_map_metadata_memory_chunk
* -- alloc_map_metadata_memory
* This is called when the code is allocating memory for
* the page allocator metadata (the struct lcd_page_allocator,
* the giant array of page blocks, etc.). The code will
* only try to allocate in chunks of 2^metadata_malloc_order
* pages.
*
* This function is expected to allocate the real memory and
* map it at the correct offset. The function will be provided
* with the size (2^order) and the offset from the beginning.
* The function is expected to map offsets contiguously.
*
* IMPORTANT #1: If you wanted metadata embedded, you should map
* the giant array of page blocks, etc.).
*
* This function is expected to allocate the real memory needed
* for the metadata.
*
* This function will be invoked only once, with the
* required memory specified as an argument (sz).
*
* IMPORTANT #1: If you are embedding the metadata in the
* page allocator's memory region, you must allocate the
* memory in chunks of 2^max_order pages. First, the metadata
* memory you allocate must be a multiple of 2^max_order pages
* because the remainder that is unused by the metadata will
* be shared with real data. Second, during initialization,
* the code will attach these chunks of memory to the
* appropriate page block ranges, and it expects the memory
* to be chunked in 2^max_order page chunks. (Just take my
* word for it for now haha).
*
* IMPORTANT #2: If you wanted metadata embedded, you should map
* the memory from the beginning of the memory area. If you
* get this wrong, it's not a serious problem, so long as you
* are consistent.
*
* IMPORTANT #2: You should ensure that the starting address
* IMPORTANT #3: You should ensure that the starting address
* of the metadata is aligned at a 2^max_order-page boundary
* (i.e., 2^max_order * 4096 byte boundary). If you don't,
* things will still sort of work, but when you allocate pages,
......@@ -206,13 +216,13 @@ struct lcd_page_allocator {
* 4 KBs of memory, but it won't be aligned at multiples
* of 0x1000).
*
* Return lcd_resource_node that refers to allocated memory
* as an out parameter.
* Return pointer to start of metadata memory as an out
* parameter (void **).
*
* -- free_unmap_metadata_memory_chunk
* Free memory allocated for metadata. This is only called
* in error situations during initialization to free up
* allocated memory.
* -- free_unmap_metadata_memory
* Free memory allocated for metadata. This is called when
* the page allocator is destroyed (error situations in
* initialization or during tear down).
*
* -- alloc_map_regular_mem_chunk
* This will be called when:
......@@ -274,7 +284,6 @@ struct lcd_page_allocator {
*/
int lcd_page_allocator_create(unsigned long nr_pages_order,
unsigned int min_order,
unsigned int metadata_malloc_order,
unsigned int max_order,
const struct lcd_page_allocator_cbs *cbs,
int embed_metadata,
......@@ -288,11 +297,9 @@ int lcd_page_allocator_create(unsigned long nr_pages_order,
* memory area - except those that contain metadata (if the page allocator
* was configured for embedding metadata).
*
* IMPORTANT: The caller is responsible for freeing the resources that
* contain the page allocator metadata. This may involve traversing
* a resource tree and deleting the nodes that refer to the pages
* that contain the metadata (and returning the memory to the host).
* Or it may mean returning the pages to *another* page allocator.
* This will invoke the free_unmap_metadata_memory callback so that
* the caller can free up metadata memory allocated for the page
* allocator at the right point in allocator tear down.
*/
void lcd_page_allocator_destroy(struct lcd_page_allocator *pa);
......
......@@ -287,18 +287,18 @@ static void do_free(struct lcd_page_allocator *pa,
static struct lcd_page_allocator*
init_struct_pa(void *metadata_addr,
unsigned long metadata_sz,
unsigned long nr_pages_order,
unsigned int min_order,
unsigned int metadata_malloc_order,
unsigned int max_order,
const struct lcd_page_allocator_cbs *cbs,
int embed_metadata)
{
struct lcd_page_allocator *pa = metadata_addr;
pa->metadata_sz = metadata_sz;
pa->nr_pages_order = nr_pages_order;
pa->min_order = min_order;
pa->metadata_malloc_order = metadata_malloc_order;
pa->max_order = max_order;
pa->cbs = cbs;
pa->embed_metadata = embed_metadata;
......@@ -383,11 +383,11 @@ static int alloc_metadata_page_blocks(struct lcd_page_allocator *pa,
/*
* Allocate the big chunks first
*/
while (metadata_sz > (1UL << (pa->metadata_malloc_order + PAGE_SIZE))) {
while (metadata_sz > (1UL << (pa->max_order + PAGE_SIZE))) {
/*
* Allocate 2^max_order bytes
*/
pb = alloc_page_blocks(pa, pa->metadata_malloc_order);
pb = alloc_page_blocks(pa, pa->max_order);
if (!pb) {
LIBLCD_ERR("error allocating blocks for metadata");
return -EIO;
......@@ -410,7 +410,7 @@ static int alloc_metadata_page_blocks(struct lcd_page_allocator *pa,
/*
* Shift to next 2^max_order chunk
*/
metadata_sz -= (1UL << (pa->metadata_malloc_order + PAGE_SIZE));
metadata_sz -= (1UL << (pa->max_order + PAGE_SIZE));
}
/*
* Allocate the remainder, rounded up to the closest power of 2
......@@ -455,7 +455,6 @@ static int allocator_init(void *metadata_addr,
unsigned long metadata_sz,
unsigned long nr_pages_order,
unsigned int min_order,
unsigned int metadata_malloc_order,
unsigned int max_order,
const struct lcd_page_allocator_cbs *cbs,
int embed_metadata,
......@@ -466,6 +465,7 @@ static int allocator_init(void *metadata_addr,
* Init top level struct
*/
pa = init_struct_pa(metadata_addr,
metadata_sz,
nr_pages_order,
min_order,
max_order,
......@@ -501,92 +501,32 @@ fail1:
/* METADATA FREE -------------------------------------------------- */
static void free_metadata(void *metadata_addr,
unsigned int alloc_order,
unsigned long metadata_sz,
unsigned int alloc_order,
const struct lcd_page_allocator_cbs *cbs)
{
struct lcd_resource_node *n;
unsigned long nr_blocks;
unsigned long i;
int ret;
gpa_t base;
gpa_t addr;
/*
* Calculate number of memory chunks we need to free
*/
nr_blocks = ALIGN(metadata_sz, (1UL << (alloc_order + PAGE_SIZE)));
nr_blocks >>= (alloc_order + PAGE_SIZE);
/*
* Calculate starting guest physical address where metadata is
*
* XXX: Assumes guest virtual == guest physical
*/
base = __gpa((unsigned long)metadata_addr);
/*
* Look up chunks, and pass to free callback
*/
for (i = 0; i < nr_blocks; i++) {
addr = gpa_add(base, i * (1UL << (alloc_order + PAGE_SIZE)));
ret = lcd_phys_to_resource_node(
addr,
&n);
if (ret) {
LIBLCD_ERR("error looking up metadata resource node for gpa = %lx", gpa_val(addr));
continue;
}
cbs->free_unmap_metadata_memory_chunk(cbs, n);
}
cbs->free_unmap_metadata_memory(cbs, metadata_addr, metadat_sz,
alloc_order);
}
/* METADATA MALLOC -------------------------------------------------- */
static void* get_start_addr(struct lcd_resource_node *n)
{
/* Assumes guest virtual == guest physical */
return (void *)gpa_val(lcd_resource_node_start);
}
static void* malloc_metadata(unsigned int alloc_order,
unsigned long metadata_sz,
const struct lcd_page_allocator_cbs *cbs)
{
struct lcd_resource_node *n;
unsigned long nr_blocks;
unsigned long i, j;
int ret;
void *out;
/*
* Calculate number of memory chunks we need, rounded up.
* Get memory
*/
nr_blocks = ALIGN(metadata_sz, (1UL << alloc_order)) >> alloc_order;
for (i = 0; i < nr_blocks; i++) {
/*
* Allocate and map a memory chunk
*/
ret = cbs->alloc_map_metadata_memory_chunk(
cbs,
i * (1UL << (alloc_order + PAGE_SIZE)),
alloc_order,
&n);
if (ret) {
LIBLCD_ERR("error allocating metadata chunk %ul of %ul", i, nr_blocks);
goto fail1;
}
/*
* For first chunk, remember starting address (beginning
* of metadata).
*/
if (i == 0)
out = get_start_addr(n)
ret = cbs->alloc_map_metadata_memory(cbs, alloc_order, metadata_sz,
&out);
if (ret) {
LIBLCD_ERR("failed to get metadata memory");
return NULL;
}
return out;
fail1:
for (j = 0; j < i; j++)
cbs->free_unmap_metadata_memory_chunk(cbs, n);
return NULL;
}
/* METADATA SIZE CALCULATIONS ------------------------------ */
......@@ -599,7 +539,6 @@ static unsigned long calc_nr_lcd_page_blocks(unsigned int nr_pages_order,
static unsigned long calc_metadata_size(unsigned int nr_pages_order,
unsigned int min_order,
unsigned int metadata_malloc_order,
unsigned int max_order,
int embed_metadata,
unsigned long *metadata_sz_out)
......@@ -612,38 +551,14 @@ static unsigned long calc_metadata_size(unsigned int nr_pages_order,
*
* min_order <= max_order <= nr_pages_order
*
* Furthermore, if we are embedding the metadata into the memory
* area, the metadata malloc order must satisfy:
*
* min_order <= metadata_malloc_order <= max_order
*
* This is because we will mark those page blocks that contain
* metadata as occupied. It needs to make sense for the page
* allocator; the page allocator uses an internal algorithm to
* track a correspondence between page blocks and resource nodes.
* A resource node cannot span less than 2^min_order memory or
* more than 2^max_order memory.
*
* We don't use BUG_ON's so that we don't crash the caller.
*/
if (unlikely(min_order > max_order)) {
LIBLCD_ERR("min order > max order, impossible");
return -EINVAL;
}
if (unlikely(min_order > metadata_malloc_order)) {
LIBLCD_ERR("min_order > metadata malloc order, impossible");
return -EINVAL;
}
if (unlikely(metadata_malloc_order > max_order)) {
LIBLCD_ERR("metadata malloc order > max order, impossible");
return -EINVAL;
}
if (unlikely(embed_metadata && metadata_malloc_order == max_order)) {
LIBLCD_ERR("trying to embed metadata, but metadata malloc order != max order, not allowed");
return -EINVAL;
}
if (unlikely(max_order > nr_pages_order)) {
LIBLCD_ERR("trying to embed metadata, and max order > total memory area order, impossible");
LIBLCD_ERR("max order > total memory area order, impossible");
return -EINVAL;
}
/*
......@@ -693,7 +608,6 @@ static unsigned long calc_metadata_size(unsigned int nr_pages_order,
int lcd_page_allocator_create(unsigned long nr_pages_order,
unsigned int min_order,
unsigned int metadata_malloc_order,
unsigned int max_order,
const struct lcd_page_allocator_cbs *cbs,
int embed_metadata,
......@@ -709,7 +623,6 @@ int lcd_page_allocator_create(unsigned long nr_pages_order,
*/
ret = calc_metadata_size(nr_pages_order,
min_order,
metadata_malloc_order,
max_order,
embed_metadata,
&metadata_sz);
......@@ -718,7 +631,7 @@ int lcd_page_allocator_create(unsigned long nr_pages_order,
/*
* Get memory for metadata
*/
metadata_addr = malloc_metadata(metadata_malloc_order,
metadata_addr = malloc_metadata(max_order,
metadata_sz,
cbs);
if (!metadata_addr) {
......@@ -732,7 +645,6 @@ int lcd_page_allocator_create(unsigned long nr_pages_order,
metadata_sz,
nr_pages_order,
min_order,
metadata_malloc_order,
max_order,
cbs,
embed_metadata);
......@@ -742,10 +654,7 @@ int lcd_page_allocator_create(unsigned long nr_pages_order,
return 0;
fail3:
free_metadata(metadata_addr,
metadata_malloc_order,
metadata_sz,
cbs);
free_metadata(metadata_addr, metadata_sz, max_order, cbs);
fail2:
fail1:
return ret;
......@@ -773,8 +682,11 @@ void lcd_page_allocator_destroy(struct lcd_page_allocator *pa)
lcd_page_allocator_free(pa, &cursor[i], 0);
}
/*
* The caller is responsible for freeing the metadata
* Free metadata (pa becomes invalid as this call is executed)
*/
free_metadata(pa, pa->metadata_sz, pa->max_order, pa->cbs);
pa = NULL;
return;
}
......
......@@ -115,15 +115,43 @@ fail1:
/* PAGE ALLOCATOR INTERNALS ---------------------------------------- */
static int
heap_alloc_map_metadata_memory_chunk(struct lcd_page_allocator_cbs *cbs,
unsigned long mapping_offset,
unsigned int alloc_order,
struct lcd_resource_node **n_out)
static void __do_one_heap_free(struct lcd_resource_node *n)
{
cptr_t pages = n->cptr;
gpa_t base = __gpa(lcd_resource_node_start(n));
/*
* Unmap from guest physical
*/
_lcd_munmap(pages, base);
/*
* Remove from resource tree
*/
__liblcd_mem_itree_delete(n);
/*
* Free pages from host
*/
lcd_cap_delete(pages);
}
static void do_one_heap_free(gpa_t dest)
{
int ret;
struct lcd_resource_node *n;
/*
* Use resource tree to look up node for memory object
*/
ret = lcd_phys_to_resource_node(dest, &n);
if (ret) {
LIBLCD_ERR("error looking up resource node");
return;
}
__do_one_heap_free(n);
}
static int do_one_heap_alloc(gpa_t dest, unsigned int alloc_order)
{
int ret;
cptr_t pages;
gpa_t dest = gpa_add(LCD_HEAP_GP_ADDR, mapping_offset);
/*
* Do low-level page alloc out into microkernel
*/
......@@ -140,43 +168,72 @@ heap_alloc_map_metadata_memory_chunk(struct lcd_page_allocator_cbs *cbs,
LIBLCD_ERR("low level mmap failed");
goto fail2;
}
return 0;
fail2:
lcd_cap_delete(pages);
fail1:
return ret;
}
static int
heap_alloc_map_metadata_memory_chunk(struct lcd_page_allocator_cbs *cbs,
unsigned int alloc_order,
unsigned long metadata_sz,
void **metadata_addr)
{
int ret;
unsigned long i, j;
unsigned long nr_allocs;
unsigned total;
gpa_t dest;
/*
* Look up resource node for new pages
* Since we are embedding, we need to allocate in 2^alloc_order
* chunks of pages.
*/
ret = lcd_phys_to_resource_node(dest, n_out);
if (ret) {
LIBLCD_ERR("failed to get new resource node");
goto fail3;
total = ALIGN(metadata_sz, (1UL << (alloc_order + PAGE_SHIFT)));
nr_allocs = total >> (alloc_order + PAGE_SHIFT); /* > 0 */
for (i = 0; i < nr_allocs; i++) {
dest = gpa_add(LCD_HEAP_GP_ADDR,
i * (1UL << (alloc_order + PAGE_SHIFT)));
ret = do_one_heap_meta_alloc(dest, alloc_order);
if (ret) {
LIBLCD_ERR("metadata alloca failed at i = %lx", i);
goto fail1;
}
}
*metadata_addr = (void *)gva_val(LCD_HEAP_GV_ADDR);
return 0;
fail3:
_lcd_munmap(pages, dest);
fail2:
lcd_cap_delete(pages);
fail1:
for (j = 0; j < i; j++) {
dest = gpa_add(LCD_HEAP_GP_ADDR,
j * (1UL << (alloc_order + PAGE_SHIFT)));
do_one_heap_free(dest);
}
return ret;
}
static void
heap_free_unmap_metadata_memory_chunk(struct lcd_page_allocator_cbs *cbs,
struct lcd_resource_node *n_to_delete)
void *metadata_addr,
unsigned long metadata_sz,
unsigned int alloc_order)
{
cptr_t pages = n_to_delete->cptr;
gpa_t base = __gpa(lcd_resource_node_start(n_to_delete));
/*
* Unmap from guest physical
*/
_lcd_munmap(pages, base);
/*
* Remove from resource tree
*/
__liblcd_mem_itree_delete(n_to_delete);
/*
* Free pages from host
*/
lcd_cap_delete(pages);
do_one_heap_free(lcd_gva2gpa(__gva((unsigned long)metadata_addr)));
}
static int
......@@ -186,12 +243,9 @@ heap_alloc_map_regular_mem_chunk(struct lcd_page_allocator *pa,
unsigned int alloc_order,
struct lcd_resource_node **n_out)
{
/*
* For now, we just re-use the metadata func, since there
* is no difference.
*/
return heap_alloc_map_metadata_memory_chunk(&pa->cbs, mapping_offset,
alloc_order, n_out);
gpa_t dest = gpa_add(LCD_HEAP_GP_ADDR, mapping_offset);
return do_one_heap_alloc(dest, alloc_order);
}
static void
......@@ -201,11 +255,7 @@ heap_free_unmap_regular_mem_chunk(struct lcd_page_allocator *pa,
unsigned long mapping_offset,
unsigned int order)
{
/*
* Again, we re-use the metadata funcs since they're the same
* right now.
*/
heap_free_unmap_metadata_memory_chunk(&pa->cbs, n_to_delete);
__do_one_heap_free(n_to_delete);
}
static inline gva_t heap_page_block_to_addr(struct lcd_page_block *pb)
......@@ -397,8 +447,8 @@ static void __init_refok kmalloc_init(void)
}
struct lcd_page_allocator_cbs heap_page_allocator_cbs = {
.alloc_map_metadata_memory_chunk = heap_alloc_map_metadata_memory_chunk,
.free_unmap_metadata_memory_chunk = heap_free_unmap_metadata_memory_chunk,
.alloc_map_metadata_memory = heap_alloc_map_metadata_memory,
.free_unmap_metadata_memory = heap_free_unmap_metadata_memory,
.alloc_map_regular_mem_chunk = heap_alloc_map_regular_mem_chunk,
.free_unmap_regular_mem_chunk = heap_free_unmap_regular_mem_chunk,
};
......@@ -409,10 +459,9 @@ int __liblcd_heap_init(void)
/*
* Create new page allocator in heap region
*/
ret = lcd_page_allocator_create(LCD_HEAP_SIZE,
ret = lcd_page_allocator_create(LCD_HEAP_NR_PAGES_ORDER,
LCD_HEAP_MIN_ORDER,
LCD_HEAP_MAX_ORDER,
LCD_HEAP_MAX_ORDER,
&heap_page_allocator_cbs,
1, /* embed metadata */
&heap_allocator);
......@@ -441,7 +490,7 @@ int __liblcd_heap_init(void)
return 0;
fail2:
lcd_page_allocator_destroy(heap_allocator);
lcd_page_allocator_destroy(heap_allocator); /* frees metadata */
heap_allocator = NULL;
fail1:
return ret;
......
......@@ -5,6 +5,15 @@
* spaces.
*/
#include <lcd_config/pre_hook.h>
#include <liblcd/allocator.h>
#include <liblcd/mem.h>
#include <lcd_config/post_hook.h>
struct lcd_page_allocator *ram_map_allocator;
/* LOW-LEVEL SYSTEM CALLS ---------------------------------------- */
int _lcd_mmap(cptr_t mo, unsigned int order, gpa_t base)
......@@ -59,10 +68,176 @@ void _lcd_munmap(cptr_t mo, gpa_t base)
lcd_syscall_munmap(mo);
}
/* RAM MAP INTERNALS ---------------------------------------- */
static int
ram_alloc_map_metadata_memory_chunk(struct lcd_page_allocator_cbs *cbs,
unsigned long mapping_offset,
unsigned int alloc_order,
struct lcd_resource_node **n_out)
{
int ret;
cptr_t pages;
static int nr_calls = 0; /* should only be called once */
gpa_t dest = gpa_add(LCD_RAM_MAP_GP_ADDR, mapping_offset);
/*
* Just do a kmalloc
*/