mem.c 8.67 KB
Newer Older
1
/*
Jeff Dike's avatar
Jeff Dike committed
2
 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
Linus Torvalds's avatar
Linus Torvalds committed
3
4
5
 * Licensed under the GPL
 */

Jeff Dike's avatar
Jeff Dike committed
6
7
8
9
10
11
12
13
#include <linux/stddef.h>
#include <linux/bootmem.h>
#include <linux/gfp.h>
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <asm/fixmap.h>
#include <asm/page.h>
Jeff Dike's avatar
Jeff Dike committed
14
#include "as-layout.h"
Jeff Dike's avatar
Jeff Dike committed
15
#include "init.h"
Linus Torvalds's avatar
Linus Torvalds committed
16
#include "kern.h"
Jeff Dike's avatar
Jeff Dike committed
17
#include "kern_util.h"
Linus Torvalds's avatar
Linus Torvalds committed
18
19
20
#include "mem_user.h"
#include "os.h"

21
/* allocated in paging_init, zeroed in mem_init, and unchanged thereafter */
Linus Torvalds's avatar
Linus Torvalds committed
22
unsigned long *empty_zero_page = NULL;
23
/* allocated in paging_init and unchanged thereafter */
Linus Torvalds's avatar
Linus Torvalds committed
24
unsigned long *empty_bad_page = NULL;
Jeff Dike's avatar
Jeff Dike committed
25
26
27
28
29

/*
 * Initialized during boot, and readonly for initializing page tables
 * afterwards
 */
Linus Torvalds's avatar
Linus Torvalds committed
30
pgd_t swapper_pg_dir[PTRS_PER_PGD];
Jeff Dike's avatar
Jeff Dike committed
31
32

/* Initialized at boot time, and readonly after that */
Jeff Dike's avatar
Jeff Dike committed
33
unsigned long long highmem;
Linus Torvalds's avatar
Linus Torvalds committed
34
35
int kmalloc_ok = 0;

Jeff Dike's avatar
Jeff Dike committed
36
/* Used during early boot */
Linus Torvalds's avatar
Linus Torvalds committed
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
static unsigned long brk_end;

static void map_cb(void *unused)
{
	map_memory(brk_end, __pa(brk_end), uml_reserved - brk_end, 1, 1, 0);
}

#ifdef CONFIG_HIGHMEM
static void setup_highmem(unsigned long highmem_start,
			  unsigned long highmem_len)
{
	struct page *page;
	unsigned long highmem_pfn;
	int i;

	highmem_pfn = __pa(highmem_start) >> PAGE_SHIFT;
Jeff Dike's avatar
Jeff Dike committed
53
	for (i = 0; i < highmem_len >> PAGE_SHIFT; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
54
55
		page = &mem_map[highmem_pfn + i];
		ClearPageReserved(page);
56
		init_page_count(page);
Linus Torvalds's avatar
Linus Torvalds committed
57
58
59
60
61
		__free_page(page);
	}
}
#endif

Jeff Dike's avatar
Jeff Dike committed
62
void __init mem_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
63
{
64
	/* clear the zero-page */
WANG Cong's avatar
WANG Cong committed
65
	memset(empty_zero_page, 0, PAGE_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
66
67
68
69
70
71
72
73
74
75
76
77

	/* Map in the area just after the brk now that kmalloc is about
	 * to be turned on.
	 */
	brk_end = (unsigned long) UML_ROUND_UP(sbrk(0));
	map_cb(NULL);
	initial_thread_cb(map_cb, NULL);
	free_bootmem(__pa(brk_end), uml_reserved - brk_end);
	uml_reserved = brk_end;

	/* this will put all low memory onto the freelists */
	totalram_pages = free_all_bootmem();
Jason Lunz's avatar
Jason Lunz committed
78
	max_low_pfn = totalram_pages;
79
#ifdef CONFIG_HIGHMEM
Linus Torvalds's avatar
Linus Torvalds committed
80
81
	totalhigh_pages = highmem >> PAGE_SHIFT;
	totalram_pages += totalhigh_pages;
82
#endif
Linus Torvalds's avatar
Linus Torvalds committed
83
84
	num_physpages = totalram_pages;
	max_pfn = totalram_pages;
Jeff Dike's avatar
Jeff Dike committed
85
	printk(KERN_INFO "Memory: %luk available\n",
Linus Torvalds's avatar
Linus Torvalds committed
86
87
88
89
90
91
92
93
	       (unsigned long) nr_free_pages() << (PAGE_SHIFT-10));
	kmalloc_ok = 1;

#ifdef CONFIG_HIGHMEM
	setup_highmem(end_iomem, highmem);
#endif
}

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
/*
 * Create a page table and place a pointer to it in a middle page
 * directory entry.
 */
static void __init one_page_table_init(pmd_t *pmd)
{
	if (pmd_none(*pmd)) {
		pte_t *pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
		set_pmd(pmd, __pmd(_KERNPG_TABLE +
					   (unsigned long) __pa(pte)));
		if (pte != pte_offset_kernel(pmd, 0))
			BUG();
	}
}

static void __init one_md_table_init(pud_t *pud)
{
#ifdef CONFIG_3_LEVEL_PGTABLES
	pmd_t *pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE);
	set_pud(pud, __pud(_KERNPG_TABLE + (unsigned long) __pa(pmd_table)));
	if (pmd_table != pmd_offset(pud, 0))
		BUG();
#endif
}

Jeff Dike's avatar
Jeff Dike committed
119
static void __init fixrange_init(unsigned long start, unsigned long end,
Linus Torvalds's avatar
Linus Torvalds committed
120
121
122
				 pgd_t *pgd_base)
{
	pgd_t *pgd;
123
	pud_t *pud;
Linus Torvalds's avatar
Linus Torvalds committed
124
125
126
127
128
129
130
131
132
133
	pmd_t *pmd;
	int i, j;
	unsigned long vaddr;

	vaddr = start;
	i = pgd_index(vaddr);
	j = pmd_index(vaddr);
	pgd = pgd_base + i;

	for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) {
134
135
136
137
		pud = pud_offset(pgd, vaddr);
		if (pud_none(*pud))
			one_md_table_init(pud);
		pmd = pmd_offset(pud, vaddr);
Jeff Dike's avatar
Jeff Dike committed
138
		for (; (j < PTRS_PER_PMD) && (vaddr < end); pmd++, j++) {
139
			one_page_table_init(pmd);
Linus Torvalds's avatar
Linus Torvalds committed
140
141
142
143
144
145
146
147
148
149
150
151
			vaddr += PMD_SIZE;
		}
		j = 0;
	}
}

#ifdef CONFIG_HIGHMEM
pte_t *kmap_pte;
pgprot_t kmap_prot;

#define kmap_get_fixmap_pte(vaddr)					\
	pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)),\
Jeff Dike's avatar
Jeff Dike committed
152
				     (vaddr)), (vaddr))
Linus Torvalds's avatar
Linus Torvalds committed
153
154
155
156
157
158
159
160
161
162
163
164

static void __init kmap_init(void)
{
	unsigned long kmap_vstart;

	/* cache the first kmap pte */
	kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
	kmap_pte = kmap_get_fixmap_pte(kmap_vstart);

	kmap_prot = PAGE_KERNEL;
}

165
static void __init init_highmem(void)
Linus Torvalds's avatar
Linus Torvalds committed
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
	unsigned long vaddr;

	/*
	 * Permanent kmaps:
	 */
	vaddr = PKMAP_BASE;
	fixrange_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, swapper_pg_dir);

	pgd = swapper_pg_dir + pgd_index(vaddr);
	pud = pud_offset(pgd, vaddr);
	pmd = pmd_offset(pud, vaddr);
	pte = pte_offset_kernel(pmd, vaddr);
	pkmap_page_table = pte;

	kmap_init();
}
#endif /* CONFIG_HIGHMEM */

static void __init fixaddr_user_init( void)
{
191
#ifdef CONFIG_ARCH_REUSE_HOST_VSYSCALL_AREA
Linus Torvalds's avatar
Linus Torvalds committed
192
193
194
195
196
	long size = FIXADDR_USER_END - FIXADDR_USER_START;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
Jeff Dike's avatar
Jeff Dike committed
197
198
	phys_t p;
	unsigned long v, vaddr = FIXADDR_USER_START;
Linus Torvalds's avatar
Linus Torvalds committed
199

Jeff Dike's avatar
Jeff Dike committed
200
	if (!size)
Linus Torvalds's avatar
Linus Torvalds committed
201
202
203
		return;

	fixrange_init( FIXADDR_USER_START, FIXADDR_USER_END, swapper_pg_dir);
Jeff Dike's avatar
Jeff Dike committed
204
205
206
	v = (unsigned long) alloc_bootmem_low_pages(size);
	memcpy((void *) v , (void *) FIXADDR_USER_START, size);
	p = __pa(v);
Jeff Dike's avatar
Jeff Dike committed
207
	for ( ; size > 0; size -= PAGE_SIZE, vaddr += PAGE_SIZE,
Jeff Dike's avatar
Jeff Dike committed
208
		      p += PAGE_SIZE) {
Linus Torvalds's avatar
Linus Torvalds committed
209
210
211
212
		pgd = swapper_pg_dir + pgd_index(vaddr);
		pud = pud_offset(pgd, vaddr);
		pmd = pmd_offset(pud, vaddr);
		pte = pte_offset_kernel(pmd, vaddr);
Jeff Dike's avatar
Jeff Dike committed
213
		pte_set_val(*pte, p, PAGE_READONLY);
Linus Torvalds's avatar
Linus Torvalds committed
214
215
216
217
	}
#endif
}

218
void __init paging_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
219
220
221
222
223
224
{
	unsigned long zones_size[MAX_NR_ZONES], vaddr;
	int i;

	empty_zero_page = (unsigned long *) alloc_bootmem_low_pages(PAGE_SIZE);
	empty_bad_page = (unsigned long *) alloc_bootmem_low_pages(PAGE_SIZE);
Jeff Dike's avatar
Jeff Dike committed
225
	for (i = 0; i < ARRAY_SIZE(zones_size); i++)
Linus Torvalds's avatar
Linus Torvalds committed
226
		zones_size[i] = 0;
227

228
229
	zones_size[ZONE_NORMAL] = (end_iomem >> PAGE_SHIFT) -
		(uml_physmem >> PAGE_SHIFT);
230
#ifdef CONFIG_HIGHMEM
231
	zones_size[ZONE_HIGHMEM] = highmem >> PAGE_SHIFT;
232
#endif
Linus Torvalds's avatar
Linus Torvalds committed
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
	free_area_init(zones_size);

	/*
	 * Fixed mappings, only the page table structure has to be
	 * created - mappings will be set by set_fixmap():
	 */
	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
	fixrange_init(vaddr, FIXADDR_TOP, swapper_pg_dir);

	fixaddr_user_init();

#ifdef CONFIG_HIGHMEM
	init_highmem();
#endif
}

249
struct page *arch_validate(struct page *page, gfp_t mask, int order)
Linus Torvalds's avatar
Linus Torvalds committed
250
251
252
253
254
{
	unsigned long addr, zero = 0;
	int i;

 again:
Jeff Dike's avatar
Jeff Dike committed
255
	if (page == NULL)
256
		return page;
Jeff Dike's avatar
Jeff Dike committed
257
	if (PageHighMem(page))
258
		return page;
Linus Torvalds's avatar
Linus Torvalds committed
259
260

	addr = (unsigned long) page_address(page);
Jeff Dike's avatar
Jeff Dike committed
261
	for (i = 0; i < (1 << order); i++) {
Linus Torvalds's avatar
Linus Torvalds committed
262
		current->thread.fault_addr = (void *) addr;
Jeff Dike's avatar
Jeff Dike committed
263
		if (__do_copy_to_user((void __user *) addr, &zero,
Linus Torvalds's avatar
Linus Torvalds committed
264
265
				     sizeof(zero),
				     &current->thread.fault_addr,
Jeff Dike's avatar
Jeff Dike committed
266
267
				     &current->thread.fault_catcher)) {
			if (!(mask & __GFP_WAIT))
268
				return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
269
270
271
272
273
			else break;
		}
		addr += PAGE_SIZE;
	}

Jeff Dike's avatar
Jeff Dike committed
274
	if (i == (1 << order))
275
		return page;
Linus Torvalds's avatar
Linus Torvalds committed
276
277
278
279
	page = alloc_pages(mask, order);
	goto again;
}

Jeff Dike's avatar
Jeff Dike committed
280
281
/*
 * This can't do anything because nothing in the kernel image can be freed
Linus Torvalds's avatar
Linus Torvalds committed
282
283
284
285
286
287
288
289
290
291
292
 * since it's not in kernel physical memory.
 */

void free_initmem(void)
{
}

#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
	if (start < end)
Jeff Dike's avatar
Jeff Dike committed
293
294
		printk(KERN_INFO "Freeing initrd memory: %ldk freed\n",
		       (end - start) >> 10);
Linus Torvalds's avatar
Linus Torvalds committed
295
296
	for (; start < end; start += PAGE_SIZE) {
		ClearPageReserved(virt_to_page(start));
297
		init_page_count(virt_to_page(start));
Linus Torvalds's avatar
Linus Torvalds committed
298
299
300
301
302
303
304
305
		free_page(start);
		totalram_pages++;
	}
}
#endif

void show_mem(void)
{
306
307
308
	int pfn, total = 0, reserved = 0;
	int shared = 0, cached = 0;
	int highmem = 0;
Linus Torvalds's avatar
Linus Torvalds committed
309
310
	struct page *page;

Jeff Dike's avatar
Jeff Dike committed
311
	printk(KERN_INFO "Mem-info:\n");
312
	show_free_areas();
Jeff Dike's avatar
Jeff Dike committed
313
314
	printk(KERN_INFO "Free swap:       %6ldkB\n",
	       nr_swap_pages<<(PAGE_SHIFT-10));
315
	pfn = max_mapnr;
Jeff Dike's avatar
Jeff Dike committed
316
	while (pfn-- > 0) {
Linus Torvalds's avatar
Linus Torvalds committed
317
		page = pfn_to_page(pfn);
318
		total++;
Jeff Dike's avatar
Jeff Dike committed
319
		if (PageHighMem(page))
320
			highmem++;
Jeff Dike's avatar
Jeff Dike committed
321
		if (PageReserved(page))
322
			reserved++;
Jeff Dike's avatar
Jeff Dike committed
323
		else if (PageSwapCache(page))
324
			cached++;
Jeff Dike's avatar
Jeff Dike committed
325
		else if (page_count(page))
326
327
			shared += page_count(page) - 1;
	}
Jeff Dike's avatar
Jeff Dike committed
328
329
330
331
332
	printk(KERN_INFO "%d pages of RAM\n", total);
	printk(KERN_INFO "%d pages of HIGHMEM\n", highmem);
	printk(KERN_INFO "%d reserved pages\n", reserved);
	printk(KERN_INFO "%d pages shared\n", shared);
	printk(KERN_INFO "%d pages swap cached\n", cached);
Linus Torvalds's avatar
Linus Torvalds committed
333
334
}

Jeff Dike's avatar
Jeff Dike committed
335
/* Allocate and free page tables. */
Linus Torvalds's avatar
Linus Torvalds committed
336
337
338
339
340
341
342

pgd_t *pgd_alloc(struct mm_struct *mm)
{
	pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL);

	if (pgd) {
		memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
Jeff Dike's avatar
Jeff Dike committed
343
344
		memcpy(pgd + USER_PTRS_PER_PGD,
		       swapper_pg_dir + USER_PTRS_PER_PGD,
Linus Torvalds's avatar
Linus Torvalds committed
345
346
347
348
349
		       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
	}
	return pgd;
}

350
void pgd_free(struct mm_struct *mm, pgd_t *pgd)
Linus Torvalds's avatar
Linus Torvalds committed
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
{
	free_page((unsigned long) pgd);
}

pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
	pte_t *pte;

	pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
	return pte;
}

struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
	struct page *pte;
366

Linus Torvalds's avatar
Linus Torvalds committed
367
368
369
	pte = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
	return pte;
}
Jeff Dike's avatar
Jeff Dike committed
370
371
372
373
374
375
376
377
378
379
380
381

#ifdef CONFIG_3_LEVEL_PGTABLES
pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
{
	pmd_t *pmd = (pmd_t *) __get_free_page(GFP_KERNEL);

	if (pmd)
		memset(pmd, 0, PAGE_SIZE);

	return pmd;
}
#endif