heap.c 11.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
/*
 * mem.c
 *
 * Isolated liblcd page allocator and related
 * code.
 */

#include <lcd_config/pre_hook.h>

10
#include <linux/mm.h>
11
#include <linux/slab.h>
12
#include <liblcd/mem.h>
13
#include <liblcd/allocator.h>
14
15
#include <asm/lcd_domains/liblcd.h>
#include <lcd_domains/liblcd.h>
16
17
18

#include <lcd_config/post_hook.h>

19
20
21
struct lcd_page_allocator *heap_allocator;
struct page *heap_page_array;

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
/* LOW-LEVEL SYSCALLS -------------------------------------------------- */

int _lcd_alloc_pages_exact_node(int nid, unsigned int flags, unsigned int order,
				cptr_t *slot_out)
{
	cptr_t slot;
	int ret;
	/*
	 * Get a free cptr
	 */
	ret = lcd_cptr_alloc(&slot);
	if (ret) {
		LIBLCD_ERR("out of cptr's");
		goto fail1;
	}
	/*
	 * Alloc pages
	 */
40
	ret = lcd_syscall_alloc_pages_exact_node(slot, nid, flags, order);
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
	if (ret) {
		LIBLCD_ERR("alloc pages syscall failed");
		goto fail2;
	}

	*slot_out = slot;
	
	return 0;

fail2:
	lcd_cptr_free(slot);
fail1:
	return ret;
}

int _lcd_alloc_pages(unsigned int flags, unsigned int order,
		cptr_t *slot_out)
{
	cptr_t slot;
	int ret;
	/*
	 * Get a free cptr
	 */
	ret = lcd_cptr_alloc(&slot);
	if (ret) {
		LIBLCD_ERR("out of cptr's");
		goto fail1;
	}
	/*
	 * Alloc pages
	 */
72
	ret = lcd_syscall_alloc_pages(slot, flags, order);
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
	if (ret) {
		LIBLCD_ERR("alloc pages syscall failed");
		goto fail2;
	}

	*slot_out = slot;
	
	return 0;

fail2:
	lcd_cptr_free(slot);
fail1:
	return ret;
}

88
int _lcd_vmalloc(unsigned long nr_pages, cptr_t *slot_out)
89
90
91
92
93
94
95
96
97
98
99
100
101
102
{
	cptr_t slot;
	int ret;
	/*
	 * Get a free cptr
	 */
	ret = lcd_cptr_alloc(&slot);
	if (ret) {
		LIBLCD_ERR("out of cptr's");
		goto fail1;
	}
	/*
	 * Alloc pages
	 */
103
	ret = lcd_syscall_vmalloc(slot, nr_pages);
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
	if (ret) {
		LIBLCD_ERR("vmalloc syscall failed");
		goto fail2;
	}

	*slot_out = slot;
	
	return 0;

fail2:
	lcd_cptr_free(slot);
fail1:
	return ret;
}

119
120
/* PAGE ALLOCATOR INTERNALS ---------------------------------------- */

121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
static void __do_one_heap_free(struct lcd_resource_node *n)
{
	cptr_t pages = n->cptr;
	gpa_t base = __gpa(lcd_resource_node_start(n));
	/*
	 * Unmap from guest physical
	 */
	_lcd_munmap(pages, base);
	/*
	 * Free pages from host
	 */
	lcd_cap_delete(pages);
}

static void do_one_heap_free(gpa_t dest)
{
	int ret;
	struct lcd_resource_node *n;
	/*
	 * Use resource tree to look up node for memory object
	 */
	ret = lcd_phys_to_resource_node(dest, &n);
	if (ret) {
		LIBLCD_ERR("error looking up resource node");
		return;
	}
	__do_one_heap_free(n);
}

150
151
static int do_one_heap_alloc(gpa_t dest, unsigned int alloc_order,
			struct lcd_resource_node **n_out)
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
{
	int ret;
	cptr_t pages;
	/*
	 * Do low-level page alloc out into microkernel
	 */
	ret = _lcd_alloc_pages(0, alloc_order, &pages);
	if (ret) {
		LIBLCD_ERR("low level alloc failed");
		goto fail1;
	}
	/*
	 * Map in guest physical at the right offset into the heap region
	 */
	ret = _lcd_mmap(pages, alloc_order, dest);
	if (ret) {
		LIBLCD_ERR("low level mmap failed");
		goto fail2;
	}
171
172
173
174
175
176
177
178
	/*
	 * Get the resource node for the mapped memory
	 */
	ret = lcd_phys_to_resource_node(dest, n_out);
	if (ret) {
		LIBLCD_ERR("error getting resource node");
		goto fail3;
	}
179
180
181

	return 0;

182
183
fail3:
	_lcd_munmap(pages, dest);
184
185
186
187
188
189
190
fail2:
	lcd_cap_delete(pages);
fail1:
	return ret;
}

static int 
191
heap_alloc_map_metadata_memory(const struct lcd_page_allocator_cbs *cbs,
192
193
194
195
196
197
198
199
200
				unsigned int alloc_order,
				unsigned long metadata_sz,
				void **metadata_addr)
{
	int ret;
	unsigned long i, j;
	unsigned long nr_allocs;
	unsigned total;
	gpa_t dest;
201
	struct lcd_resource_node *unused;
202

203
	/*
204
205
	 * Since we are embedding, we need to allocate in 2^alloc_order
	 * chunks of pages.
206
	 */
207
208
209
210
211
212
213
214
	total = ALIGN(metadata_sz, (1UL << (alloc_order + PAGE_SHIFT)));
	nr_allocs = total >> (alloc_order + PAGE_SHIFT); /* > 0 */

	for (i = 0; i < nr_allocs; i++) {

		dest = gpa_add(LCD_HEAP_GP_ADDR, 
			i * (1UL << (alloc_order + PAGE_SHIFT)));

215
		ret = do_one_heap_alloc(dest, alloc_order, &unused);
216
217
218
219
220
		if (ret) {
			LIBLCD_ERR("metadata alloca failed at i = %lx", i);
			goto fail1;
		}

221
	}
222
223
	
	*metadata_addr = (void *)gva_val(LCD_HEAP_GV_ADDR);
224
225
226
227

	return 0;

fail1:
228
229
230
231
232
233
234
235
236
	for (j = 0; j < i; j++) {

		dest = gpa_add(LCD_HEAP_GP_ADDR, 
			j * (1UL << (alloc_order + PAGE_SHIFT)));

		do_one_heap_free(dest);

	}

237
238
239
240
	return ret;
}

static void
241
heap_free_unmap_metadata_memory(const struct lcd_page_allocator_cbs *cbs,
242
243
244
				void *metadata_addr,
				unsigned long metadata_sz,
				unsigned int alloc_order)
245
{
246
247
248
249
250
251
252
253
254
255
256
257
	unsigned long i;
	unsigned long nr_frees;
	unsigned total;
	gpa_t dest;

	/*
	 * Ignore the metadata address; should be at predefined spot.
	 *
	 * We alloc'd the metadata in chunks; free them.
	 */

	total = ALIGN(metadata_sz, (1UL << (alloc_order + PAGE_SHIFT)));
258
	nr_frees = total >> (alloc_order + PAGE_SHIFT); /* > 0 */
259

260
	for (i = 0; i < nr_frees; i++) {
261
262
263
264
265
266
		dest = gpa_add(LCD_HEAP_GP_ADDR, 
			i * (1UL << (alloc_order + PAGE_SHIFT)));
		do_one_heap_free(dest);
	}

	return;
267
268
269
270
271
272
273
274
275
}

static int 
heap_alloc_map_regular_mem_chunk(struct lcd_page_allocator *pa,
				struct lcd_page_block *dest_blocks,
				unsigned long mapping_offset,
				unsigned int alloc_order,
				struct lcd_resource_node **n_out)
{
276
277
	gpa_t dest = gpa_add(LCD_HEAP_GP_ADDR, mapping_offset);

278
	return do_one_heap_alloc(dest, alloc_order, n_out);
279
280
281
282
283
284
285
286
287
}

static void
heap_free_unmap_regular_mem_chunk(struct lcd_page_allocator *pa,
				struct lcd_page_block *page_blocks,
				struct lcd_resource_node *n_to_delete,
				unsigned long mapping_offset,
				unsigned int order)
{
288
	__do_one_heap_free(n_to_delete);
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
}

static inline gva_t heap_page_block_to_addr(struct lcd_page_block *pb)
{
	return gva_add(LCD_HEAP_GV_ADDR, 
		lcd_page_block_to_offset(heap_allocator, pb));
}

static inline struct lcd_page_block *heap_addr_to_page_block(gva_t addr)
{
	return lcd_offset_to_page_block(
		heap_allocator,
		gva_val(addr) - gva_val(LCD_HEAP_GV_ADDR));
}

static inline struct page *heap_addr_to_struct_page(gva_t addr)
{
	unsigned long idx;
	idx = (gva_val(addr) - gva_val(LCD_HEAP_GV_ADDR)) >> PAGE_SHIFT;
	return &heap_page_array[idx];
}

311
static inline gva_t heap_struct_page_to_addr(const struct page *p)
312
313
314
315
316
317
{
	unsigned long idx;
	idx = p - heap_page_array;
	return gva_add(LCD_HEAP_GV_ADDR, idx * PAGE_SIZE);
}

318
319
320
321
322
323
324
325
static inline struct page *heap_page_block_to_struct_page(
	struct lcd_page_block *pb)
{
	return heap_addr_to_struct_page(
		heap_page_block_to_addr(pb));
}

static inline struct lcd_page_block *heap_struct_page_to_page_block(
326
	const struct page *p)
327
328
329
330
331
332
333
{
	return heap_addr_to_page_block(
		heap_struct_page_to_addr(p));
}

/* PAGE ALLOC INTERFACE ---------------------------------------- */

334
335
336
337
338
339
340
341
342
struct page *lcd_alloc_pages_node(int nid, unsigned int flags,
					unsigned int order)
{
	/*
	 * For now, we ignore the node id (not numa aware).
	 */
	return lcd_alloc_pages(flags, order);
}

343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
struct page *lcd_alloc_pages_exact_node(int nid, unsigned int flags, 
					unsigned int order)
{
	/*
	 * For now, we ignore the node id (not numa aware).
	 */
	return lcd_alloc_pages(flags, order);
}

struct page *lcd_alloc_pages(unsigned int flags, unsigned int order)
{
	struct lcd_page_block *pb;
	/*
	 * Do heap alloc. Flags are ignored for now.
	 */
	pb = lcd_page_allocator_alloc(heap_allocator, order);
	if (!pb) {
		LIBLCD_ERR("alloc failed");
		goto fail1;
	}
	/*
	 * Convert to struct page
	 */
	return heap_page_block_to_struct_page(pb);

fail1:
	return NULL;
}

void lcd_free_pages(struct page *base, unsigned int order)
{
	lcd_page_allocator_free(heap_allocator,
375
				heap_struct_page_to_page_block(base),
376
377
378
379
380
381
382
				order);
}

void* lcd_vmalloc(unsigned long sz)
{
	/* Not implemented for now */
	BUG();
383
	return NULL;
384
385
386
387
388
389
390
391
}

void lcd_vfree(void *ptr)
{
	/* Not implemented for now */
	BUG();
}

392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
struct page *lcd_virt_to_head_page(const void *addr)
{
	unsigned long i_addr = (unsigned long)addr;

	/* If outside heap, error out and return NULL */
	if (i_addr < gva_val(LCD_HEAP_GV_ADDR) ||
		i_addr >= gva_val(LCD_HEAP_GV_ADDR) + LCD_HEAP_SIZE) {
		LIBLCD_ERR("address %p falls outside heap; only addresses in the heap can be converted to a struct page pointer (we don't use the costly struct page metadata for all bits of the physical address space)", addr);
		return NULL;
	}

	return heap_addr_to_struct_page(__gva((unsigned long)addr));
}

void *lcd_page_address(const struct page *page)
{
	return (void *)gva_val(heap_struct_page_to_addr(page));
}

void lcd_free_memcg_kmem_pages(unsigned long addr, unsigned int order)
{
	lcd_free_pages(heap_addr_to_struct_page(__gva(addr)), order);
}

416
417
418
419
420
/* VOLUNTEERING -------------------------------------------------- */

int lcd_volunteer_pages(struct page *base, unsigned int order,
			cptr_t *slot_out)
{
421
	unsigned long size, unused;
422
423
	return lcd_phys_to_cptr(
		lcd_gva2gpa(__gva((unsigned long)lcd_page_address(base))), 
424
		slot_out, &size, &unused);
425
426
427
428
429
430
431
432
433
434
}

void lcd_unvolunteer_pages(cptr_t pages)
{
	return;
}

int lcd_volunteer_dev_mem(gpa_t base, unsigned int order,
			cptr_t *slot_out)
{
435
436
	unsigned long size, unused;
	return lcd_phys_to_cptr(base, slot_out, &size, &unused);
437
438
439
440
441
442
443
}

void lcd_unvolunteer_dev_mem(cptr_t devmem)
{
	return;
}

444
int lcd_volunteer_vmalloc_mem(gva_t base, unsigned long nr_pages,
445
446
			cptr_t *slot_out)
{
447
448
	unsigned long size, unused;
	return lcd_virt_to_cptr(base, slot_out, &size, &unused);
449
450
451
452
453
454
455
456
457
458
459
}

void lcd_unvolunteer_vmalloc_mem(cptr_t vmalloc_mem)
{
	return;
}

/* ADDRESS TRANSLATION -------------------------------------------------- */

gpa_t lcd_gva2gpa(gva_t gva)
{
460
	return isolated_lcd_gva2gpa(gva);
461
462
463
464
}

gva_t lcd_gpa2gva(gpa_t gpa)
{
465
	return isolated_lcd_gpa2gva(gpa);
466
467
468
469
}

/* INIT/EXIT -------------------------------------------------- */

470
471
472
473
474
static int setup_struct_page_array(void)
{
	struct lcd_page_block *pb;
	unsigned int order;
	unsigned long bytes;
475
	int ret;
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
	/*
	 * Compute number of struct pages we need
	 */
	bytes = roundup_pow_of_two((1UL << LCD_HEAP_NR_PAGES_ORDER) *
				sizeof(struct page));
	order = ilog2(bytes >> PAGE_SHIFT);
	/*
	 * Do the alloc
	 */
	pb = lcd_page_allocator_alloc(heap_allocator, order);
	if (!pb) {
		LIBLCD_ERR("error setting up struct page array for heap");
		ret = -ENOMEM;
		goto fail1;
	}
	/*
	 * Zero out the array (unnecessary right now, but just in case)
	 */
494
	heap_page_array = (void *)gva_val(heap_page_block_to_addr(pb));
495
496
	memset(heap_page_array,
		0,
497
		(1UL << (order + PAGE_SHIFT)));
498
499
500
501
502
503
504
505
506

	return 0;

fail1:
	return ret;
}

void cpucache_init(void);

507
static void __ref kmalloc_init(void)
508
509
510
511
512
513
514
{
	kmem_cache_init();
	kmem_cache_init_late();
	cpucache_init();
}

struct lcd_page_allocator_cbs heap_page_allocator_cbs = {
515
516
	.alloc_map_metadata_memory = heap_alloc_map_metadata_memory,
	.free_unmap_metadata_memory = heap_free_unmap_metadata_memory,
517
518
519
520
521
	.alloc_map_regular_mem_chunk = heap_alloc_map_regular_mem_chunk,
	.free_unmap_regular_mem_chunk = heap_free_unmap_regular_mem_chunk,
};

int __liblcd_heap_init(void)
522
{
523
	int ret;
524
525
526
	/*
	 * Create new page allocator in heap region
	 */
527
	ret = lcd_page_allocator_create(LCD_HEAP_NR_PAGES_ORDER,
528
529
530
531
532
					LCD_HEAP_MIN_ORDER,
					LCD_HEAP_MAX_ORDER,
					&heap_page_allocator_cbs,
					1, /* embed metadata */
					&heap_allocator);
533
	if (ret) {
534
		LIBLCD_ERR("error initializing heap allocator");
535
536
		goto fail1;
	}
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
	/*
	 * Set up struct page array
	 */
	ret = setup_struct_page_array();
	if (ret) {
		LIBLCD_ERR("error setting up struct page array for heap");
		goto fail2;
	}
	/*
	 * Initialize kmalloc
	 */
	kmalloc_init();
	/*
	 * Inform mem itree the page and slab allocators are up (and so
	 * it can start using kmalloc for allocating nodes)
	 */
	__liblcd_mem_itree_booted();
554

555
	return 0;
556

557
fail2:
558
	lcd_page_allocator_destroy(heap_allocator); /* frees metadata */
559
	heap_allocator = NULL;
560
561
fail1:
	return ret;
562
}
563
564
565
566
567
568
569
570
571
572
573
574

void *__lcd_get_free_pages(gfp_t mask, unsigned int order)
{
	struct page *p = lcd_alloc_pages(mask, order);
	return p ? lcd_page_address(p) : NULL;
}

void __lcd_free_pages(unsigned long addr, unsigned int order)
{
	if (addr)
		lcd_free_pages(heap_addr_to_struct_page(__gva(addr)), order);
}