page_alloc.c 183 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 *  linux/mm/page_alloc.c
 *
 *  Manages the free list, the system allocates free pages here.
 *  Note that kmalloc() lives in slab.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
22
#include <linux/jiffies.h>
Linus Torvalds's avatar
Linus Torvalds committed
23
#include <linux/bootmem.h>
24
#include <linux/memblock.h>
Linus Torvalds's avatar
Linus Torvalds committed
25
#include <linux/compiler.h>
26
#include <linux/kernel.h>
27
#include <linux/kmemcheck.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29 30 31 32
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
33
#include <linux/ratelimit.h>
34
#include <linux/oom.h>
Linus Torvalds's avatar
Linus Torvalds committed
35 36 37 38 39
#include <linux/notifier.h>
#include <linux/topology.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
40
#include <linux/memory_hotplug.h>
Linus Torvalds's avatar
Linus Torvalds committed
41 42
#include <linux/nodemask.h>
#include <linux/vmalloc.h>
43
#include <linux/vmstat.h>
44
#include <linux/mempolicy.h>
45
#include <linux/stop_machine.h>
46 47
#include <linux/sort.h>
#include <linux/pfn.h>
48
#include <linux/backing-dev.h>
49
#include <linux/fault-inject.h>
50
#include <linux/page-isolation.h>
51
#include <linux/page_ext.h>
52
#include <linux/debugobjects.h>
53
#include <linux/kmemleak.h>
54
#include <linux/compaction.h>
55
#include <trace/events/kmem.h>
56
#include <linux/prefetch.h>
57
#include <linux/mm_inline.h>
58
#include <linux/migrate.h>
59
#include <linux/page_ext.h>
60
#include <linux/hugetlb.h>
61
#include <linux/sched/rt.h>
Linus Torvalds's avatar
Linus Torvalds committed
62

63
#include <asm/sections.h>
Linus Torvalds's avatar
Linus Torvalds committed
64
#include <asm/tlbflush.h>
65
#include <asm/div64.h>
Linus Torvalds's avatar
Linus Torvalds committed
66 67
#include "internal.h"

68 69
/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
static DEFINE_MUTEX(pcp_batch_high_lock);
70
#define MIN_PERCPU_PAGELIST_FRACTION	(8)
71

72 73 74 75 76
#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DEFINE_PER_CPU(int, numa_node);
EXPORT_PER_CPU_SYMBOL(numa_node);
#endif

77 78 79 80 81 82 83 84 85
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 * defined in <linux/topology.h>.
 */
DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */
EXPORT_PER_CPU_SYMBOL(_numa_mem_);
86
int _node_numa_mem_[MAX_NUMNODES];
87 88
#endif

Linus Torvalds's avatar
Linus Torvalds committed
89
/*
90
 * Array of node states.
Linus Torvalds's avatar
Linus Torvalds committed
91
 */
92 93 94 95 96 97 98
nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
	[N_POSSIBLE] = NODE_MASK_ALL,
	[N_ONLINE] = { { [0] = 1UL } },
#ifndef CONFIG_NUMA
	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
#ifdef CONFIG_HIGHMEM
	[N_HIGH_MEMORY] = { { [0] = 1UL } },
99 100 101
#endif
#ifdef CONFIG_MOVABLE_NODE
	[N_MEMORY] = { { [0] = 1UL } },
102 103 104 105 106 107
#endif
	[N_CPU] = { { [0] = 1UL } },
#endif	/* NUMA */
};
EXPORT_SYMBOL(node_states);

108 109 110
/* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock);

111
unsigned long totalram_pages __read_mostly;
112
unsigned long totalreserve_pages __read_mostly;
113 114 115 116 117 118 119 120
/*
 * When calculating the number of globally allowed dirty pages, there
 * is a certain number of per-zone reserves that should not be
 * considered dirtyable memory.  This is the sum of those reserves
 * over all existing zones that contribute dirtyable memory.
 */
unsigned long dirty_balance_reserve __read_mostly;

121
int percpu_pagelist_fraction;
122
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
Linus Torvalds's avatar
Linus Torvalds committed
123

124 125 126 127 128 129 130 131 132
#ifdef CONFIG_PM_SLEEP
/*
 * The following functions are used by the suspend/hibernate code to temporarily
 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 * while devices are suspended.  To avoid races with the suspend/hibernate code,
 * they should always be called with pm_mutex held (gfp_allowed_mask also should
 * only be modified with pm_mutex held, unless the suspend/hibernate code is
 * guaranteed not to run in parallel with that modification).
 */
133 134 135 136

static gfp_t saved_gfp_mask;

void pm_restore_gfp_mask(void)
137 138
{
	WARN_ON(!mutex_is_locked(&pm_mutex));
139 140 141 142
	if (saved_gfp_mask) {
		gfp_allowed_mask = saved_gfp_mask;
		saved_gfp_mask = 0;
	}
143 144
}

145
void pm_restrict_gfp_mask(void)
146 147
{
	WARN_ON(!mutex_is_locked(&pm_mutex));
148 149 150
	WARN_ON(saved_gfp_mask);
	saved_gfp_mask = gfp_allowed_mask;
	gfp_allowed_mask &= ~GFP_IOFS;
151
}
152 153 154 155 156 157 158

bool pm_suspended_storage(void)
{
	if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
		return false;
	return true;
}
159 160
#endif /* CONFIG_PM_SLEEP */

161 162 163 164
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
int pageblock_order __read_mostly;
#endif

165
static void __free_pages_ok(struct page *page, unsigned int order);
166

Linus Torvalds's avatar
Linus Torvalds committed
167 168 169 170 171 172 173
/*
 * results with 256, 32 in the lowmem_reserve sysctl:
 *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 *	1G machine -> (16M dma, 784M normal, 224M high)
 *	NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 *	HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
 *	HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
174 175 176
 *
 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 * don't need any ZONE_NORMAL reservation
Linus Torvalds's avatar
Linus Torvalds committed
177
 */
178
int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
179
#ifdef CONFIG_ZONE_DMA
180
	 256,
181
#endif
182
#ifdef CONFIG_ZONE_DMA32
183
	 256,
184
#endif
185
#ifdef CONFIG_HIGHMEM
Mel Gorman's avatar
Mel Gorman committed
186
	 32,
187
#endif
Mel Gorman's avatar
Mel Gorman committed
188
	 32,
189
};
Linus Torvalds's avatar
Linus Torvalds committed
190 191 192

EXPORT_SYMBOL(totalram_pages);

193
static char * const zone_names[MAX_NR_ZONES] = {
194
#ifdef CONFIG_ZONE_DMA
195
	 "DMA",
196
#endif
197
#ifdef CONFIG_ZONE_DMA32
198
	 "DMA32",
199
#endif
200
	 "Normal",
201
#ifdef CONFIG_HIGHMEM
Mel Gorman's avatar
Mel Gorman committed
202
	 "HighMem",
203
#endif
Mel Gorman's avatar
Mel Gorman committed
204
	 "Movable",
205 206
};

Linus Torvalds's avatar
Linus Torvalds committed
207
int min_free_kbytes = 1024;
208
int user_min_free_kbytes = -1;
Linus Torvalds's avatar
Linus Torvalds committed
209

210 211
static unsigned long __meminitdata nr_kernel_pages;
static unsigned long __meminitdata nr_all_pages;
212
static unsigned long __meminitdata dma_reserve;
Linus Torvalds's avatar
Linus Torvalds committed
213

Tejun Heo's avatar
Tejun Heo committed
214 215 216 217 218 219 220 221 222 223 224
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
static unsigned long __initdata required_kernelcore;
static unsigned long __initdata required_movablecore;
static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];

/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
int movable_zone;
EXPORT_SYMBOL(movable_zone);
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
225

Miklos Szeredi's avatar
Miklos Szeredi committed
226 227
#if MAX_NUMNODES > 1
int nr_node_ids __read_mostly = MAX_NUMNODES;
228
int nr_online_nodes __read_mostly = 1;
Miklos Szeredi's avatar
Miklos Szeredi committed
229
EXPORT_SYMBOL(nr_node_ids);
230
EXPORT_SYMBOL(nr_online_nodes);
Miklos Szeredi's avatar
Miklos Szeredi committed
231 232
#endif

233 234
int page_group_by_mobility_disabled __read_mostly;

235
void set_pageblock_migratetype(struct page *page, int migratetype)
236
{
237 238
	if (unlikely(page_group_by_mobility_disabled &&
		     migratetype < MIGRATE_PCPTYPES))
239 240
		migratetype = MIGRATE_UNMOVABLE;

241 242 243 244
	set_pageblock_flags_group(page, (unsigned long)migratetype,
					PB_migrate, PB_migrate_end);
}

245 246
bool oom_killer_disabled __read_mostly;

Nick Piggin's avatar
Nick Piggin committed
247
#ifdef CONFIG_DEBUG_VM
248
static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
249
{
250 251 252
	int ret = 0;
	unsigned seq;
	unsigned long pfn = page_to_pfn(page);
253
	unsigned long sp, start_pfn;
254

255 256
	do {
		seq = zone_span_seqbegin(zone);
257 258
		start_pfn = zone->zone_start_pfn;
		sp = zone->spanned_pages;
259
		if (!zone_spans_pfn(zone, pfn))
260 261 262
			ret = 1;
	} while (zone_span_seqretry(zone, seq));

263
	if (ret)
264 265 266
		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
			pfn, zone_to_nid(zone), zone->name,
			start_pfn, start_pfn + sp);
267

268
	return ret;
269 270 271 272
}

static int page_is_consistent(struct zone *zone, struct page *page)
{
273
	if (!pfn_valid_within(page_to_pfn(page)))
274
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
275
	if (zone != page_zone(page))
276 277 278 279 280 281 282 283 284 285
		return 0;

	return 1;
}
/*
 * Temporary debugging check for pages not lying within a given zone.
 */
static int bad_range(struct zone *zone, struct page *page)
{
	if (page_outside_zone_boundaries(zone, page))
Linus Torvalds's avatar
Linus Torvalds committed
286
		return 1;
287 288 289
	if (!page_is_consistent(zone, page))
		return 1;

Linus Torvalds's avatar
Linus Torvalds committed
290 291
	return 0;
}
Nick Piggin's avatar
Nick Piggin committed
292 293 294 295 296 297 298
#else
static inline int bad_range(struct zone *zone, struct page *page)
{
	return 0;
}
#endif

299 300
static void bad_page(struct page *page, const char *reason,
		unsigned long bad_flags)
Linus Torvalds's avatar
Linus Torvalds committed
301
{
302 303 304 305
	static unsigned long resume;
	static unsigned long nr_shown;
	static unsigned long nr_unshown;

306 307
	/* Don't complain about poisoned pages */
	if (PageHWPoison(page)) {
308
		page_mapcount_reset(page); /* remove PageBuddy */
309 310 311
		return;
	}

312 313 314 315 316 317 318 319 320 321
	/*
	 * Allow a burst of 60 reports, then keep quiet for that minute;
	 * or allow a steady drip of one report per second.
	 */
	if (nr_shown == 60) {
		if (time_before(jiffies, resume)) {
			nr_unshown++;
			goto out;
		}
		if (nr_unshown) {
322 323
			printk(KERN_ALERT
			      "BUG: Bad page state: %lu messages suppressed\n",
324 325 326 327 328 329 330 331
				nr_unshown);
			nr_unshown = 0;
		}
		nr_shown = 0;
	}
	if (nr_shown++ == 0)
		resume = jiffies + 60 * HZ;

332
	printk(KERN_ALERT "BUG: Bad page state in process %s  pfn:%05lx\n",
333
		current->comm, page_to_pfn(page));
334
	dump_page_badflags(page, reason, bad_flags);
335

336
	print_modules();
Linus Torvalds's avatar
Linus Torvalds committed
337
	dump_stack();
338
out:
339
	/* Leave bad fields for debug, except PageBuddy could make trouble */
340
	page_mapcount_reset(page); /* remove PageBuddy */
341
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds's avatar
Linus Torvalds committed
342 343 344 345 346 347 348 349 350
}

/*
 * Higher-order pages are called "compound pages".  They are structured thusly:
 *
 * The first PAGE_SIZE page is called the "head page".
 *
 * The remaining PAGE_SIZE pages are called "tail pages".
 *
351 352
 * All pages have PG_compound set.  All tail pages have their ->first_page
 * pointing at the head page.
Linus Torvalds's avatar
Linus Torvalds committed
353
 *
354 355 356
 * The first tail page's ->lru.next holds the address of the compound page's
 * put_page() function.  Its ->lru.prev holds the order of allocation.
 * This usage means that zero-order pages may not be compound.
Linus Torvalds's avatar
Linus Torvalds committed
357
 */
358 359 360

static void free_compound_page(struct page *page)
{
361
	__free_pages_ok(page, compound_order(page));
362 363
}

364
void prep_compound_page(struct page *page, unsigned long order)
365 366 367 368 369 370 371 372 373
{
	int i;
	int nr_pages = 1 << order;

	set_compound_page_dtor(page, free_compound_page);
	set_compound_order(page, order);
	__SetPageHead(page);
	for (i = 1; i < nr_pages; i++) {
		struct page *p = page + i;
374
		set_page_count(p, 0);
375
		p->first_page = page;
David Rientjes's avatar
David Rientjes committed
376 377 378
		/* Make sure p->first_page is always valid for PageTail() */
		smp_wmb();
		__SetPageTail(p);
379 380 381
	}
}

382
/* update __split_huge_page_refcount if you change this function */
383
static int destroy_compound_page(struct page *page, unsigned long order)
Linus Torvalds's avatar
Linus Torvalds committed
384 385 386
{
	int i;
	int nr_pages = 1 << order;
387
	int bad = 0;
Linus Torvalds's avatar
Linus Torvalds committed
388

389
	if (unlikely(compound_order(page) != order)) {
390
		bad_page(page, "wrong compound order", 0);
391 392
		bad++;
	}
Linus Torvalds's avatar
Linus Torvalds committed
393

394
	__ClearPageHead(page);
395

396 397
	for (i = 1; i < nr_pages; i++) {
		struct page *p = page + i;
Linus Torvalds's avatar
Linus Torvalds committed
398

399 400 401 402 403
		if (unlikely(!PageTail(p))) {
			bad_page(page, "PageTail not set", 0);
			bad++;
		} else if (unlikely(p->first_page != page)) {
			bad_page(page, "first_page not consistent", 0);
404 405
			bad++;
		}
406
		__ClearPageTail(p);
Linus Torvalds's avatar
Linus Torvalds committed
407
	}
408 409

	return bad;
Linus Torvalds's avatar
Linus Torvalds committed
410 411
}

412 413
static inline void prep_zero_page(struct page *page, unsigned int order,
							gfp_t gfp_flags)
Nick Piggin's avatar
Nick Piggin committed
414 415 416
{
	int i;

417 418 419 420
	/*
	 * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
	 * and __GFP_HIGHMEM from hard or soft interrupt context.
	 */
Nick Piggin's avatar
Nick Piggin committed
421
	VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
Nick Piggin's avatar
Nick Piggin committed
422 423 424 425
	for (i = 0; i < (1 << order); i++)
		clear_highpage(page + i);
}

426 427
#ifdef CONFIG_DEBUG_PAGEALLOC
unsigned int _debug_guardpage_minorder;
428
bool _debug_pagealloc_enabled __read_mostly;
429 430
bool _debug_guardpage_enabled __read_mostly;

431 432 433 434 435 436 437 438 439 440 441 442
static int __init early_debug_pagealloc(char *buf)
{
	if (!buf)
		return -EINVAL;

	if (strcmp(buf, "on") == 0)
		_debug_pagealloc_enabled = true;

	return 0;
}
early_param("debug_pagealloc", early_debug_pagealloc);

443 444
static bool need_debug_guardpage(void)
{
445 446 447 448
	/* If we don't use debug_pagealloc, we don't need guard page */
	if (!debug_pagealloc_enabled())
		return false;

449 450 451 452 453
	return true;
}

static void init_debug_guardpage(void)
{
454 455 456
	if (!debug_pagealloc_enabled())
		return;

457 458 459 460 461 462 463
	_debug_guardpage_enabled = true;
}

struct page_ext_operations debug_guardpage_ops = {
	.need = need_debug_guardpage,
	.init = init_debug_guardpage,
};
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478

static int __init debug_guardpage_minorder_setup(char *buf)
{
	unsigned long res;

	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
		printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
		return 0;
	}
	_debug_guardpage_minorder = res;
	printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
	return 0;
}
__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);

479 480
static inline void set_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
481
{
482 483 484 485 486 487 488 489
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
	__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

490 491 492 493
	INIT_LIST_HEAD(&page->lru);
	set_page_private(page, order);
	/* Guard pages are not available for any usage */
	__mod_zone_freepage_state(zone, -(1 << order), migratetype);
494 495
}

496 497
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
498
{
499 500 501 502 503 504 505 506
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
	__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

507 508 509
	set_page_private(page, 0);
	if (!is_migrate_isolate(migratetype))
		__mod_zone_freepage_state(zone, (1 << order), migratetype);
510 511
}
#else
512
struct page_ext_operations debug_guardpage_ops = { NULL, };
513 514 515 516
static inline void set_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
517 518
#endif

519
static inline void set_page_order(struct page *page, unsigned int order)
520
{
521
	set_page_private(page, order);
522
	__SetPageBuddy(page);
Linus Torvalds's avatar
Linus Torvalds committed
523 524 525 526
}

static inline void rmv_page_order(struct page *page)
{
527
	__ClearPageBuddy(page);
528
	set_page_private(page, 0);
Linus Torvalds's avatar
Linus Torvalds committed
529 530 531 532 533
}

/*
 * This function checks whether a page is free && is the buddy
 * we can do coalesce a page and its buddy if
Nick Piggin's avatar
Nick Piggin committed
534
 * (a) the buddy is not in a hole &&
535
 * (b) the buddy is in the buddy system &&
536 537
 * (c) a page and its buddy have the same order &&
 * (d) a page and its buddy are in the same zone.
538
 *
539 540 541 542
 * For recording whether a page is in the buddy system, we set ->_mapcount
 * PAGE_BUDDY_MAPCOUNT_VALUE.
 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
 * serialized by zone->lock.
Linus Torvalds's avatar
Linus Torvalds committed
543
 *
544
 * For recording page's order, we use page_private(page).
Linus Torvalds's avatar
Linus Torvalds committed
545
 */
546
static inline int page_is_buddy(struct page *page, struct page *buddy,
547
							unsigned int order)
Linus Torvalds's avatar
Linus Torvalds committed
548
{
549
	if (!pfn_valid_within(page_to_pfn(buddy)))
Nick Piggin's avatar
Nick Piggin committed
550 551
		return 0;

552
	if (page_is_guard(buddy) && page_order(buddy) == order) {
553
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
554 555 556 557

		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

558 559 560
		return 1;
	}

561
	if (PageBuddy(buddy) && page_order(buddy) == order) {
562
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
563 564 565 566 567 568 569 570 571

		/*
		 * zone check is done late to avoid uselessly
		 * calculating zone/node ids for pages that could
		 * never merge.
		 */
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

572
		return 1;
573
	}
574
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
}

/*
 * Freeing function for a buddy system allocator.
 *
 * The concept of a buddy system is to maintain direct-mapped table
 * (containing bit values) for memory blocks of various "orders".
 * The bottom level table contains the map for the smallest allocatable
 * units of memory (here, pages), and each level above it describes
 * pairs of units from the levels below, hence, "buddies".
 * At a high level, all that happens here is marking the table entry
 * at the bottom level available, and propagating the changes upward
 * as necessary, plus some accounting needed to play nicely with other
 * parts of the VM system.
 * At each level, we keep a list of pages, which are heads of continuous
590 591 592
 * free pages of length of (1 << order) and marked with _mapcount
 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
 * field.
Linus Torvalds's avatar
Linus Torvalds committed
593
 * So when we are allocating or freeing one, we can derive the state of the
594 595
 * other.  That is, if we allocate a small block, and both were
 * free, the remainder of the region must be split into blocks.
Linus Torvalds's avatar
Linus Torvalds committed
596
 * If a block is freed, and its buddy is also free, then this
597
 * triggers coalescing into a block of larger size.
Linus Torvalds's avatar
Linus Torvalds committed
598
 *
599
 * -- nyc
Linus Torvalds's avatar
Linus Torvalds committed
600 601
 */

Nick Piggin's avatar
Nick Piggin committed
602
static inline void __free_one_page(struct page *page,
603
		unsigned long pfn,
604 605
		struct zone *zone, unsigned int order,
		int migratetype)
Linus Torvalds's avatar
Linus Torvalds committed
606 607
{
	unsigned long page_idx;
608
	unsigned long combined_idx;
609
	unsigned long uninitialized_var(buddy_idx);
610
	struct page *buddy;
611
	int max_order = MAX_ORDER;
Linus Torvalds's avatar
Linus Torvalds committed
612

613 614
	VM_BUG_ON(!zone_is_initialized(zone));

615
	if (unlikely(PageCompound(page)))
616 617
		if (unlikely(destroy_compound_page(page, order)))
			return;
Linus Torvalds's avatar
Linus Torvalds committed
618

619
	VM_BUG_ON(migratetype == -1);
620 621 622 623 624 625 626 627 628
	if (is_migrate_isolate(migratetype)) {
		/*
		 * We restrict max order of merging to prevent merge
		 * between freepages on isolate pageblock and normal
		 * pageblock. Without this, pageblock isolation
		 * could cause incorrect freepage accounting.
		 */
		max_order = min(MAX_ORDER, pageblock_order + 1);
	} else {
629
		__mod_zone_freepage_state(zone, 1 << order, migratetype);
630
	}
631

632
	page_idx = pfn & ((1 << max_order) - 1);
Linus Torvalds's avatar
Linus Torvalds committed
633

634 635
	VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
	VM_BUG_ON_PAGE(bad_range(zone, page), page);
Linus Torvalds's avatar
Linus Torvalds committed
636

637
	while (order < max_order - 1) {
638 639
		buddy_idx = __find_buddy_index(page_idx, order);
		buddy = page + (buddy_idx - page_idx);
640
		if (!page_is_buddy(page, buddy, order))
641
			break;
642 643 644 645 646
		/*
		 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
		 * merge with it and move up one order.
		 */
		if (page_is_guard(buddy)) {
647
			clear_page_guard(zone, buddy, order, migratetype);
648 649 650 651 652
		} else {
			list_del(&buddy->lru);
			zone->free_area[order].nr_free--;
			rmv_page_order(buddy);
		}
653
		combined_idx = buddy_idx & page_idx;
Linus Torvalds's avatar
Linus Torvalds committed
654 655 656 657 658
		page = page + (combined_idx - page_idx);
		page_idx = combined_idx;
		order++;
	}
	set_page_order(page, order);
659 660 661 662 663 664 665 666 667

	/*
	 * If this is not the largest possible page, check if the buddy
	 * of the next-highest order is free. If it is, it's possible
	 * that pages are being freed that will coalesce soon. In case,
	 * that is happening, add the free page to the tail of the list
	 * so it's less likely to be used soon and more likely to be merged
	 * as a higher order page
	 */
668
	if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
669
		struct page *higher_page, *higher_buddy;
670 671 672
		combined_idx = buddy_idx & page_idx;
		higher_page = page + (combined_idx - page_idx);
		buddy_idx = __find_buddy_index(combined_idx, order + 1);
673
		higher_buddy = higher_page + (buddy_idx - combined_idx);
674 675 676 677 678 679 680 681 682
		if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
			list_add_tail(&page->lru,
				&zone->free_area[order].free_list[migratetype]);
			goto out;
		}
	}

	list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
out:
Linus Torvalds's avatar
Linus Torvalds committed
683 684 685
	zone->free_area[order].nr_free++;
}

686
static inline int free_pages_check(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
687
{
688
	const char *bad_reason = NULL;
689 690 691 692 693 694 695 696 697 698 699 700
	unsigned long bad_flags = 0;

	if (unlikely(page_mapcount(page)))
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
	if (unlikely(atomic_read(&page->_count) != 0))
		bad_reason = "nonzero _count";
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
		bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
	}
701 702 703 704
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
705 706
	if (unlikely(bad_reason)) {
		bad_page(page, bad_reason, bad_flags);
707
		return 1;
708
	}
709
	page_cpupid_reset_last(page);
710 711 712
	if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
		page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
713 714 715
}

/*
716
 * Frees a number of pages from the PCP lists
Linus Torvalds's avatar
Linus Torvalds committed
717
 * Assumes all pages on list are in same zone, and of same order.
718
 * count is the number of pages to free.
Linus Torvalds's avatar
Linus Torvalds committed
719 720 721 722 723 724 725
 *
 * If the zone was previously in an "all pages pinned" state then look to
 * see if this freeing clears that state.
 *
 * And clear the zone's pages_scanned counter, to hold off the "all pages are
 * pinned" detection logic.
 */
726 727
static void free_pcppages_bulk(struct zone *zone, int count,
					struct per_cpu_pages *pcp)
Linus Torvalds's avatar
Linus Torvalds committed
728
{
729
	int migratetype = 0;
730
	int batch_free = 0;
731
	int to_free = count;
732
	unsigned long nr_scanned;
733

Nick Piggin's avatar
Nick Piggin committed
734
	spin_lock(&zone->lock);
735 736 737
	nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
	if (nr_scanned)
		__mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
738

739
	while (to_free) {
Nick Piggin's avatar
Nick Piggin committed
740
		struct page *page;
741 742 743
		struct list_head *list;

		/*
744 745 746 747 748
		 * Remove pages from lists in a round-robin fashion. A
		 * batch_free count is maintained that is incremented when an
		 * empty list is encountered.  This is so more pages are freed
		 * off fuller lists instead of spinning excessively around empty
		 * lists
749 750
		 */
		do {
751
			batch_free++;
752 753 754 755
			if (++migratetype == MIGRATE_PCPTYPES)
				migratetype = 0;
			list = &pcp->lists[migratetype];
		} while (list_empty(list));
Nick Piggin's avatar
Nick Piggin committed
756

757 758 759 760
		/* This is the only non-empty list. Free them all. */
		if (batch_free == MIGRATE_PCPTYPES)
			batch_free = to_free;

761
		do {
762 763
			int mt;	/* migratetype of the to-be-freed page */

764 765 766
			page = list_entry(list->prev, struct page, lru);
			/* must delete as __free_one_page list manipulates */
			list_del(&page->lru);
767
			mt = get_freepage_migratetype(page);
768
			if (unlikely(has_isolate_pageblock(zone)))
769 770
				mt = get_pageblock_migratetype(page);

771
			/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
772
			__free_one_page(page, page_to_pfn(page), zone, 0, mt);
773
			trace_mm_page_pcpu_drain(page, 0, mt);
774
		} while (--to_free && --batch_free && !list_empty(list));
Linus Torvalds's avatar
Linus Torvalds committed
775
	}
Nick Piggin's avatar
Nick Piggin committed
776
	spin_unlock(&zone->lock);
Linus Torvalds's avatar
Linus Torvalds committed
777 778
}

779 780
static void free_one_page(struct zone *zone,
				struct page *page, unsigned long pfn,
781
				unsigned int order,
782
				int migratetype)
Linus Torvalds's avatar
Linus Torvalds committed
783
{
784
	unsigned long nr_scanned;
785
	spin_lock(&zone->lock);
786 787 788
	nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
	if (nr_scanned)
		__mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
789

790 791 792 793
	if (unlikely(has_isolate_pageblock(zone) ||
		is_migrate_isolate(migratetype))) {
		migratetype = get_pfnblock_migratetype(page, pfn);
	}
794
	__free_one_page(page, pfn, zone, order, migratetype);
795
	spin_unlock(&zone->lock);
Nick Piggin's avatar
Nick Piggin committed
796 797
}

798
static bool free_pages_prepare(struct page *page, unsigned int order)
Nick Piggin's avatar
Nick Piggin committed
799
{
Linus Torvalds's avatar
Linus Torvalds committed
800
	int i;
801
	int bad = 0;
Linus Torvalds's avatar
Linus Torvalds committed
802

803 804 805
	VM_BUG_ON_PAGE(PageTail(page), page);
	VM_BUG_ON_PAGE(PageHead(page) && compound_order(page) != order, page);

806
	trace_mm_page_free(page, order);
807 808
	kmemcheck_free_shadow(page, order);

Andrea Arcangeli's avatar
Andrea Arcangeli committed
809 810 811 812
	if (PageAnon(page))
		page->mapping = NULL;
	for (i = 0; i < (1 << order); i++)
		bad += free_pages_check(page + i);
813
	if (bad)
814
		return false;
815

816
	if (!PageHighMem(page)) {
817 818
		debug_check_no_locks_freed(page_address(page),
					   PAGE_SIZE << order);
819 820 821
		debug_check_no_obj_freed(page_address(page),
					   PAGE_SIZE << order);
	}
Nick Piggin's avatar
Nick Piggin committed
822
	arch_free_page(page, order);
Nick Piggin's avatar
Nick Piggin committed
823
	kernel_map_pages(page, 1 << order, 0);
Nick Piggin's avatar
Nick Piggin committed
824

825 826 827 828 829 830
	return true;
}

static void __free_pages_ok(struct page *page, unsigned int order)
{
	unsigned long flags;
831
	int migratetype;
832
	unsigned long pfn = page_to_pfn(page);
833 834 835 836

	if (!free_pages_prepare(page, order))
		return;

837
	migratetype = get_pfnblock_migratetype(page, pfn);
Nick Piggin's avatar
Nick Piggin committed
838
	local_irq_save(flags);
839
	__count_vm_events(PGFREE, 1 << order);
840
	set_freepage_migratetype(page, migratetype);
841
	free_one_page(page_zone(page), page, pfn, order, migratetype);
Nick Piggin's avatar
Nick Piggin committed
842
	local_irq_restore(flags);
Linus Torvalds's avatar
Linus Torvalds committed
843 844
}

845
void __init __free_pages_bootmem(struct page *page, unsigned int order)
846
{
847
	unsigned int nr_pages = 1 << order;
848
	struct page *p = page;
849
	unsigned int loop;
850

851 852 853
	prefetchw(p);
	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
		prefetchw(p + 1);
854 855
		__ClearPageReserved(p);
		set_page_count(p, 0);
856
	}
857 858
	__ClearPageReserved(p);
	set_page_count(p, 0);
859

860
	page_zone(page)->managed_pages += nr_pages;
861 862
	set_page_refcounted(page);
	__free_pages(page, order);
863 864
}

865
#ifdef CONFIG_CMA
866
/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
867 868 869 870 871 872 873 874 875 876 877
void __init init_cma_reserved_pageblock(struct page *page)
{
	unsigned i = pageblock_nr_pages;
	struct page *p = page;

	do {
		__ClearPageReserved(p);
		set_page_count(p, 0);
	} while (++p, --i);

	set_pageblock_migratetype(page, MIGRATE_CMA);
878 879 880 881 882 883 884 885 886 887 888 889 890 891

	if (pageblock_order >= MAX_ORDER) {
		i = pageblock_nr_pages;
		p = page;
		do {
			set_page_refcounted(p);