page_alloc.c 182 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 *  linux/mm/page_alloc.c
 *
 *  Manages the free list, the system allocates free pages here.
 *  Note that kmalloc() lives in slab.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
22
#include <linux/jiffies.h>
Linus Torvalds's avatar
Linus Torvalds committed
23
#include <linux/bootmem.h>
24
#include <linux/memblock.h>
Linus Torvalds's avatar
Linus Torvalds committed
25
#include <linux/compiler.h>
26
#include <linux/kernel.h>
27
#include <linux/kmemcheck.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29 30 31 32
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
33
#include <linux/ratelimit.h>
34
#include <linux/oom.h>
Linus Torvalds's avatar
Linus Torvalds committed
35 36 37 38 39
#include <linux/notifier.h>
#include <linux/topology.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
40
#include <linux/memory_hotplug.h>
Linus Torvalds's avatar
Linus Torvalds committed
41 42
#include <linux/nodemask.h>
#include <linux/vmalloc.h>
43
#include <linux/vmstat.h>
44
#include <linux/mempolicy.h>
45
#include <linux/stop_machine.h>
46 47
#include <linux/sort.h>
#include <linux/pfn.h>
48
#include <linux/backing-dev.h>
49
#include <linux/fault-inject.h>
50
#include <linux/page-isolation.h>
51
#include <linux/page_ext.h>
52
#include <linux/debugobjects.h>
53
#include <linux/kmemleak.h>
54
#include <linux/compaction.h>
55
#include <trace/events/kmem.h>
56
#include <linux/prefetch.h>
57
#include <linux/mm_inline.h>
58
#include <linux/migrate.h>
59
#include <linux/page_ext.h>
60
#include <linux/hugetlb.h>
61
#include <linux/sched/rt.h>
62
#include <linux/page_owner.h>
Linus Torvalds's avatar
Linus Torvalds committed
63

64
#include <asm/sections.h>
Linus Torvalds's avatar
Linus Torvalds committed
65
#include <asm/tlbflush.h>
66
#include <asm/div64.h>
Linus Torvalds's avatar
Linus Torvalds committed
67 68
#include "internal.h"

69 70
/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
static DEFINE_MUTEX(pcp_batch_high_lock);
71
#define MIN_PERCPU_PAGELIST_FRACTION	(8)
72

73 74 75 76 77
#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DEFINE_PER_CPU(int, numa_node);
EXPORT_PER_CPU_SYMBOL(numa_node);
#endif

78 79 80 81 82 83 84 85 86
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 * defined in <linux/topology.h>.
 */
DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */
EXPORT_PER_CPU_SYMBOL(_numa_mem_);
87
int _node_numa_mem_[MAX_NUMNODES];
88 89
#endif

Linus Torvalds's avatar
Linus Torvalds committed
90
/*
91
 * Array of node states.
Linus Torvalds's avatar
Linus Torvalds committed
92
 */
93 94 95 96 97 98 99
nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
	[N_POSSIBLE] = NODE_MASK_ALL,
	[N_ONLINE] = { { [0] = 1UL } },
#ifndef CONFIG_NUMA
	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
#ifdef CONFIG_HIGHMEM
	[N_HIGH_MEMORY] = { { [0] = 1UL } },
100 101 102
#endif
#ifdef CONFIG_MOVABLE_NODE
	[N_MEMORY] = { { [0] = 1UL } },
103 104 105 106 107 108
#endif
	[N_CPU] = { { [0] = 1UL } },
#endif	/* NUMA */
};
EXPORT_SYMBOL(node_states);

109 110 111
/* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock);

112
unsigned long totalram_pages __read_mostly;
113
unsigned long totalreserve_pages __read_mostly;
114
unsigned long totalcma_pages __read_mostly;
115 116 117 118 119 120 121 122
/*
 * When calculating the number of globally allowed dirty pages, there
 * is a certain number of per-zone reserves that should not be
 * considered dirtyable memory.  This is the sum of those reserves
 * over all existing zones that contribute dirtyable memory.
 */
unsigned long dirty_balance_reserve __read_mostly;

123
int percpu_pagelist_fraction;
124
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
Linus Torvalds's avatar
Linus Torvalds committed
125

126 127 128 129 130 131 132 133 134
#ifdef CONFIG_PM_SLEEP
/*
 * The following functions are used by the suspend/hibernate code to temporarily
 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 * while devices are suspended.  To avoid races with the suspend/hibernate code,
 * they should always be called with pm_mutex held (gfp_allowed_mask also should
 * only be modified with pm_mutex held, unless the suspend/hibernate code is
 * guaranteed not to run in parallel with that modification).
 */
135 136 137 138

static gfp_t saved_gfp_mask;

void pm_restore_gfp_mask(void)
139 140
{
	WARN_ON(!mutex_is_locked(&pm_mutex));
141 142 143 144
	if (saved_gfp_mask) {
		gfp_allowed_mask = saved_gfp_mask;
		saved_gfp_mask = 0;
	}
145 146
}

147
void pm_restrict_gfp_mask(void)
148 149
{
	WARN_ON(!mutex_is_locked(&pm_mutex));
150 151 152
	WARN_ON(saved_gfp_mask);
	saved_gfp_mask = gfp_allowed_mask;
	gfp_allowed_mask &= ~GFP_IOFS;
153
}
154 155 156 157 158 159 160

bool pm_suspended_storage(void)
{
	if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
		return false;
	return true;
}
161 162
#endif /* CONFIG_PM_SLEEP */

163 164 165 166
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
int pageblock_order __read_mostly;
#endif

167
static void __free_pages_ok(struct page *page, unsigned int order);
168

Linus Torvalds's avatar
Linus Torvalds committed
169 170 171 172 173 174 175
/*
 * results with 256, 32 in the lowmem_reserve sysctl:
 *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 *	1G machine -> (16M dma, 784M normal, 224M high)
 *	NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 *	HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
 *	HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
176 177 178
 *
 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 * don't need any ZONE_NORMAL reservation
Linus Torvalds's avatar
Linus Torvalds committed
179
 */
180
int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
181
#ifdef CONFIG_ZONE_DMA
182
	 256,
183
#endif
184
#ifdef CONFIG_ZONE_DMA32
185
	 256,
186
#endif
187
#ifdef CONFIG_HIGHMEM
Mel Gorman's avatar
Mel Gorman committed
188
	 32,
189
#endif
Mel Gorman's avatar
Mel Gorman committed
190
	 32,
191
};
Linus Torvalds's avatar
Linus Torvalds committed
192 193 194

EXPORT_SYMBOL(totalram_pages);

195
static char * const zone_names[MAX_NR_ZONES] = {
196
#ifdef CONFIG_ZONE_DMA
197
	 "DMA",
198
#endif
199
#ifdef CONFIG_ZONE_DMA32
200
	 "DMA32",
201
#endif
202
	 "Normal",
203
#ifdef CONFIG_HIGHMEM
Mel Gorman's avatar
Mel Gorman committed
204
	 "HighMem",
205
#endif
Mel Gorman's avatar
Mel Gorman committed
206
	 "Movable",
207 208
};

Linus Torvalds's avatar
Linus Torvalds committed
209
int min_free_kbytes = 1024;
210
int user_min_free_kbytes = -1;
Linus Torvalds's avatar
Linus Torvalds committed
211

212 213
static unsigned long __meminitdata nr_kernel_pages;
static unsigned long __meminitdata nr_all_pages;
214
static unsigned long __meminitdata dma_reserve;
Linus Torvalds's avatar
Linus Torvalds committed
215

Tejun Heo's avatar
Tejun Heo committed
216 217 218 219 220 221 222 223 224 225 226
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
static unsigned long __initdata required_kernelcore;
static unsigned long __initdata required_movablecore;
static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];

/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
int movable_zone;
EXPORT_SYMBOL(movable_zone);
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
227

Miklos Szeredi's avatar
Miklos Szeredi committed
228 229
#if MAX_NUMNODES > 1
int nr_node_ids __read_mostly = MAX_NUMNODES;
230
int nr_online_nodes __read_mostly = 1;
Miklos Szeredi's avatar
Miklos Szeredi committed
231
EXPORT_SYMBOL(nr_node_ids);
232
EXPORT_SYMBOL(nr_online_nodes);
Miklos Szeredi's avatar
Miklos Szeredi committed
233 234
#endif

235 236
int page_group_by_mobility_disabled __read_mostly;

237
void set_pageblock_migratetype(struct page *page, int migratetype)
238
{
239 240
	if (unlikely(page_group_by_mobility_disabled &&
		     migratetype < MIGRATE_PCPTYPES))
241 242
		migratetype = MIGRATE_UNMOVABLE;

243 244 245 246
	set_pageblock_flags_group(page, (unsigned long)migratetype,
					PB_migrate, PB_migrate_end);
}

Nick Piggin's avatar
Nick Piggin committed
247
#ifdef CONFIG_DEBUG_VM
248
static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
249
{
250 251 252
	int ret = 0;
	unsigned seq;
	unsigned long pfn = page_to_pfn(page);
253
	unsigned long sp, start_pfn;
254

255 256
	do {
		seq = zone_span_seqbegin(zone);
257 258
		start_pfn = zone->zone_start_pfn;
		sp = zone->spanned_pages;
259
		if (!zone_spans_pfn(zone, pfn))
260 261 262
			ret = 1;
	} while (zone_span_seqretry(zone, seq));

263
	if (ret)
264 265 266
		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
			pfn, zone_to_nid(zone), zone->name,
			start_pfn, start_pfn + sp);
267

268
	return ret;
269 270 271 272
}

static int page_is_consistent(struct zone *zone, struct page *page)
{
273
	if (!pfn_valid_within(page_to_pfn(page)))
274
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
275
	if (zone != page_zone(page))
276 277 278 279 280 281 282 283 284 285
		return 0;

	return 1;
}
/*
 * Temporary debugging check for pages not lying within a given zone.
 */
static int bad_range(struct zone *zone, struct page *page)
{
	if (page_outside_zone_boundaries(zone, page))
Linus Torvalds's avatar
Linus Torvalds committed
286
		return 1;
287 288 289
	if (!page_is_consistent(zone, page))
		return 1;

Linus Torvalds's avatar
Linus Torvalds committed
290 291
	return 0;
}
Nick Piggin's avatar
Nick Piggin committed
292 293 294 295 296 297 298
#else
static inline int bad_range(struct zone *zone, struct page *page)
{
	return 0;
}
#endif

299 300
static void bad_page(struct page *page, const char *reason,
		unsigned long bad_flags)
Linus Torvalds's avatar
Linus Torvalds committed
301
{
302 303 304 305
	static unsigned long resume;
	static unsigned long nr_shown;
	static unsigned long nr_unshown;

306 307
	/* Don't complain about poisoned pages */
	if (PageHWPoison(page)) {
308
		page_mapcount_reset(page); /* remove PageBuddy */
309 310 311
		return;
	}

312 313 314 315 316 317 318 319 320 321
	/*
	 * Allow a burst of 60 reports, then keep quiet for that minute;
	 * or allow a steady drip of one report per second.
	 */
	if (nr_shown == 60) {
		if (time_before(jiffies, resume)) {
			nr_unshown++;
			goto out;
		}
		if (nr_unshown) {
322 323
			printk(KERN_ALERT
			      "BUG: Bad page state: %lu messages suppressed\n",
324 325 326 327 328 329 330 331
				nr_unshown);
			nr_unshown = 0;
		}
		nr_shown = 0;
	}
	if (nr_shown++ == 0)
		resume = jiffies + 60 * HZ;

332
	printk(KERN_ALERT "BUG: Bad page state in process %s  pfn:%05lx\n",
333
		current->comm, page_to_pfn(page));
334
	dump_page_badflags(page, reason, bad_flags);
335

336
	print_modules();
Linus Torvalds's avatar
Linus Torvalds committed
337
	dump_stack();
338
out:
339
	/* Leave bad fields for debug, except PageBuddy could make trouble */
340
	page_mapcount_reset(page); /* remove PageBuddy */
341
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds's avatar
Linus Torvalds committed
342 343 344 345 346 347 348 349 350
}

/*
 * Higher-order pages are called "compound pages".  They are structured thusly:
 *
 * The first PAGE_SIZE page is called the "head page".
 *
 * The remaining PAGE_SIZE pages are called "tail pages".
 *
351 352
 * All pages have PG_compound set.  All tail pages have their ->first_page
 * pointing at the head page.
Linus Torvalds's avatar
Linus Torvalds committed
353
 *
354 355 356
 * The first tail page's ->lru.next holds the address of the compound page's
 * put_page() function.  Its ->lru.prev holds the order of allocation.
 * This usage means that zero-order pages may not be compound.
Linus Torvalds's avatar
Linus Torvalds committed
357
 */
358 359 360

static void free_compound_page(struct page *page)
{
361
	__free_pages_ok(page, compound_order(page));
362 363
}

364
void prep_compound_page(struct page *page, unsigned long order)
365 366 367 368 369 370 371 372 373
{
	int i;
	int nr_pages = 1 << order;

	set_compound_page_dtor(page, free_compound_page);
	set_compound_order(page, order);
	__SetPageHead(page);
	for (i = 1; i < nr_pages; i++) {
		struct page *p = page + i;
374
		set_page_count(p, 0);
375
		p->first_page = page;
David Rientjes's avatar
David Rientjes committed
376 377 378
		/* Make sure p->first_page is always valid for PageTail() */
		smp_wmb();
		__SetPageTail(p);
379 380 381
	}
}

382 383
static inline void prep_zero_page(struct page *page, unsigned int order,
							gfp_t gfp_flags)
Nick Piggin's avatar
Nick Piggin committed
384 385 386
{
	int i;

387 388 389 390
	/*
	 * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
	 * and __GFP_HIGHMEM from hard or soft interrupt context.
	 */
Nick Piggin's avatar
Nick Piggin committed
391
	VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
Nick Piggin's avatar
Nick Piggin committed
392 393 394 395
	for (i = 0; i < (1 << order); i++)
		clear_highpage(page + i);
}

396 397
#ifdef CONFIG_DEBUG_PAGEALLOC
unsigned int _debug_guardpage_minorder;
398
bool _debug_pagealloc_enabled __read_mostly;
399 400
bool _debug_guardpage_enabled __read_mostly;

401 402 403 404 405 406 407 408 409 410 411 412
static int __init early_debug_pagealloc(char *buf)
{
	if (!buf)
		return -EINVAL;

	if (strcmp(buf, "on") == 0)
		_debug_pagealloc_enabled = true;

	return 0;
}
early_param("debug_pagealloc", early_debug_pagealloc);

413 414
static bool need_debug_guardpage(void)
{
415 416 417 418
	/* If we don't use debug_pagealloc, we don't need guard page */
	if (!debug_pagealloc_enabled())
		return false;

419 420 421 422 423
	return true;
}

static void init_debug_guardpage(void)
{
424 425 426
	if (!debug_pagealloc_enabled())
		return;

427 428 429 430 431 432 433
	_debug_guardpage_enabled = true;
}

struct page_ext_operations debug_guardpage_ops = {
	.need = need_debug_guardpage,
	.init = init_debug_guardpage,
};
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448

static int __init debug_guardpage_minorder_setup(char *buf)
{
	unsigned long res;

	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
		printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
		return 0;
	}
	_debug_guardpage_minorder = res;
	printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
	return 0;
}
__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);

449 450
static inline void set_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
451
{
452 453 454 455 456 457 458 459
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
	__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

460 461 462 463
	INIT_LIST_HEAD(&page->lru);
	set_page_private(page, order);
	/* Guard pages are not available for any usage */
	__mod_zone_freepage_state(zone, -(1 << order), migratetype);
464 465
}

466 467
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
468
{
469 470 471 472 473 474 475 476
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
	__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

477 478 479
	set_page_private(page, 0);
	if (!is_migrate_isolate(migratetype))
		__mod_zone_freepage_state(zone, (1 << order), migratetype);
480 481
}
#else
482
struct page_ext_operations debug_guardpage_ops = { NULL, };
483 484 485 486
static inline void set_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
487 488
#endif

489
static inline void set_page_order(struct page *page, unsigned int order)
490
{
491
	set_page_private(page, order);
492
	__SetPageBuddy(page);
Linus Torvalds's avatar
Linus Torvalds committed
493 494 495 496
}

static inline void rmv_page_order(struct page *page)
{
497
	__ClearPageBuddy(page);
498
	set_page_private(page, 0);
Linus Torvalds's avatar
Linus Torvalds committed
499 500 501 502 503
}

/*
 * This function checks whether a page is free && is the buddy
 * we can do coalesce a page and its buddy if
Nick Piggin's avatar
Nick Piggin committed
504
 * (a) the buddy is not in a hole &&
505
 * (b) the buddy is in the buddy system &&
506 507
 * (c) a page and its buddy have the same order &&
 * (d) a page and its buddy are in the same zone.
508
 *
509 510 511 512
 * For recording whether a page is in the buddy system, we set ->_mapcount
 * PAGE_BUDDY_MAPCOUNT_VALUE.
 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
 * serialized by zone->lock.
Linus Torvalds's avatar
Linus Torvalds committed
513
 *
514
 * For recording page's order, we use page_private(page).
Linus Torvalds's avatar
Linus Torvalds committed
515
 */
516
static inline int page_is_buddy(struct page *page, struct page *buddy,
517
							unsigned int order)
Linus Torvalds's avatar
Linus Torvalds committed
518
{
519
	if (!pfn_valid_within(page_to_pfn(buddy)))
Nick Piggin's avatar
Nick Piggin committed
520 521
		return 0;

522
	if (page_is_guard(buddy) && page_order(buddy) == order) {
523 524 525
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

526 527
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

528 529 530
		return 1;
	}

531
	if (PageBuddy(buddy) && page_order(buddy) == order) {
532 533 534 535 536 537 538 539
		/*
		 * zone check is done late to avoid uselessly
		 * calculating zone/node ids for pages that could
		 * never merge.
		 */
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

540 541
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

542
		return 1;
543
	}
544
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
}

/*
 * Freeing function for a buddy system allocator.
 *
 * The concept of a buddy system is to maintain direct-mapped table
 * (containing bit values) for memory blocks of various "orders".
 * The bottom level table contains the map for the smallest allocatable
 * units of memory (here, pages), and each level above it describes
 * pairs of units from the levels below, hence, "buddies".
 * At a high level, all that happens here is marking the table entry
 * at the bottom level available, and propagating the changes upward
 * as necessary, plus some accounting needed to play nicely with other
 * parts of the VM system.
 * At each level, we keep a list of pages, which are heads of continuous
560 561 562
 * free pages of length of (1 << order) and marked with _mapcount
 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
 * field.
Linus Torvalds's avatar
Linus Torvalds committed
563
 * So when we are allocating or freeing one, we can derive the state of the
564 565
 * other.  That is, if we allocate a small block, and both were
 * free, the remainder of the region must be split into blocks.
Linus Torvalds's avatar
Linus Torvalds committed
566
 * If a block is freed, and its buddy is also free, then this
567
 * triggers coalescing into a block of larger size.
Linus Torvalds's avatar
Linus Torvalds committed
568
 *
569
 * -- nyc
Linus Torvalds's avatar
Linus Torvalds committed
570 571
 */

Nick Piggin's avatar
Nick Piggin committed
572
static inline void __free_one_page(struct page *page,
573
		unsigned long pfn,
574 575
		struct zone *zone, unsigned int order,
		int migratetype)
Linus Torvalds's avatar
Linus Torvalds committed
576 577
{
	unsigned long page_idx;
578
	unsigned long combined_idx;
579
	unsigned long uninitialized_var(buddy_idx);
580
	struct page *buddy;
581
	int max_order = MAX_ORDER;
Linus Torvalds's avatar
Linus Torvalds committed
582

583
	VM_BUG_ON(!zone_is_initialized(zone));
584
	VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
Linus Torvalds's avatar
Linus Torvalds committed
585

586
	VM_BUG_ON(migratetype == -1);
587 588 589 590 591 592 593 594 595
	if (is_migrate_isolate(migratetype)) {
		/*
		 * We restrict max order of merging to prevent merge
		 * between freepages on isolate pageblock and normal
		 * pageblock. Without this, pageblock isolation
		 * could cause incorrect freepage accounting.
		 */
		max_order = min(MAX_ORDER, pageblock_order + 1);
	} else {
596
		__mod_zone_freepage_state(zone, 1 << order, migratetype);
597
	}
598

599
	page_idx = pfn & ((1 << max_order) - 1);
Linus Torvalds's avatar
Linus Torvalds committed
600

601 602
	VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
	VM_BUG_ON_PAGE(bad_range(zone, page), page);
Linus Torvalds's avatar
Linus Torvalds committed
603

604
	while (order < max_order - 1) {
605 606
		buddy_idx = __find_buddy_index(page_idx, order);
		buddy = page + (buddy_idx - page_idx);
607
		if (!page_is_buddy(page, buddy, order))
608
			break;
609 610 611 612 613
		/*
		 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
		 * merge with it and move up one order.
		 */
		if (page_is_guard(buddy)) {
614
			clear_page_guard(zone, buddy, order, migratetype);
615 616 617 618 619
		} else {
			list_del(&buddy->lru);
			zone->free_area[order].nr_free--;
			rmv_page_order(buddy);
		}
620
		combined_idx = buddy_idx & page_idx;
Linus Torvalds's avatar
Linus Torvalds committed
621 622 623 624 625
		page = page + (combined_idx - page_idx);
		page_idx = combined_idx;
		order++;
	}
	set_page_order(page, order);
626 627 628 629 630 631 632 633 634

	/*
	 * If this is not the largest possible page, check if the buddy
	 * of the next-highest order is free. If it is, it's possible
	 * that pages are being freed that will coalesce soon. In case,
	 * that is happening, add the free page to the tail of the list
	 * so it's less likely to be used soon and more likely to be merged
	 * as a higher order page
	 */
635
	if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
636
		struct page *higher_page, *higher_buddy;
637 638 639
		combined_idx = buddy_idx & page_idx;
		higher_page = page + (combined_idx - page_idx);
		buddy_idx = __find_buddy_index(combined_idx, order + 1);
640
		higher_buddy = higher_page + (buddy_idx - combined_idx);
641 642 643 644 645 646 647 648 649
		if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
			list_add_tail(&page->lru,
				&zone->free_area[order].free_list[migratetype]);
			goto out;
		}
	}

	list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
out:
Linus Torvalds's avatar
Linus Torvalds committed
650 651 652
	zone->free_area[order].nr_free++;
}

653
static inline int free_pages_check(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
654
{
655
	const char *bad_reason = NULL;
656 657 658 659 660 661 662 663 664 665 666 667
	unsigned long bad_flags = 0;

	if (unlikely(page_mapcount(page)))
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
	if (unlikely(atomic_read(&page->_count) != 0))
		bad_reason = "nonzero _count";
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
		bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
	}
668 669 670 671
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
672 673
	if (unlikely(bad_reason)) {
		bad_page(page, bad_reason, bad_flags);
674
		return 1;
675
	}
676
	page_cpupid_reset_last(page);
677 678 679
	if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
		page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
680 681 682
}

/*
683
 * Frees a number of pages from the PCP lists
Linus Torvalds's avatar
Linus Torvalds committed
684
 * Assumes all pages on list are in same zone, and of same order.
685
 * count is the number of pages to free.
Linus Torvalds's avatar
Linus Torvalds committed
686 687 688 689 690 691 692
 *
 * If the zone was previously in an "all pages pinned" state then look to
 * see if this freeing clears that state.
 *
 * And clear the zone's pages_scanned counter, to hold off the "all pages are
 * pinned" detection logic.
 */
693 694
static void free_pcppages_bulk(struct zone *zone, int count,
					struct per_cpu_pages *pcp)
Linus Torvalds's avatar
Linus Torvalds committed
695
{
696
	int migratetype = 0;
697
	int batch_free = 0;
698
	int to_free = count;
699
	unsigned long nr_scanned;
700

Nick Piggin's avatar
Nick Piggin committed
701
	spin_lock(&zone->lock);
702 703 704
	nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
	if (nr_scanned)
		__mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
705

706
	while (to_free) {
Nick Piggin's avatar
Nick Piggin committed
707
		struct page *page;
708 709 710
		struct list_head *list;

		/*
711 712 713 714 715
		 * Remove pages from lists in a round-robin fashion. A
		 * batch_free count is maintained that is incremented when an
		 * empty list is encountered.  This is so more pages are freed
		 * off fuller lists instead of spinning excessively around empty
		 * lists
716 717
		 */
		do {
718
			batch_free++;
719 720 721 722
			if (++migratetype == MIGRATE_PCPTYPES)
				migratetype = 0;
			list = &pcp->lists[migratetype];
		} while (list_empty(list));
Nick Piggin's avatar
Nick Piggin committed
723

724 725 726 727
		/* This is the only non-empty list. Free them all. */
		if (batch_free == MIGRATE_PCPTYPES)
			batch_free = to_free;

728
		do {
729 730
			int mt;	/* migratetype of the to-be-freed page */

731 732 733
			page = list_entry(list->prev, struct page, lru);
			/* must delete as __free_one_page list manipulates */
			list_del(&page->lru);
734
			mt = get_freepage_migratetype(page);
735
			if (unlikely(has_isolate_pageblock(zone)))
736 737
				mt = get_pageblock_migratetype(page);

738
			/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
739
			__free_one_page(page, page_to_pfn(page), zone, 0, mt);
740
			trace_mm_page_pcpu_drain(page, 0, mt);
741
		} while (--to_free && --batch_free && !list_empty(list));
Linus Torvalds's avatar
Linus Torvalds committed
742
	}
Nick Piggin's avatar
Nick Piggin committed
743
	spin_unlock(&zone->lock);
Linus Torvalds's avatar
Linus Torvalds committed
744 745
}

746 747
static void free_one_page(struct zone *zone,
				struct page *page, unsigned long pfn,
748
				unsigned int order,
749
				int migratetype)
Linus Torvalds's avatar
Linus Torvalds committed
750
{
751
	unsigned long nr_scanned;
752
	spin_lock(&zone->lock);
753 754 755
	nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
	if (nr_scanned)
		__mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
756

757 758 759 760
	if (unlikely(has_isolate_pageblock(zone) ||
		is_migrate_isolate(migratetype))) {
		migratetype = get_pfnblock_migratetype(page, pfn);
	}
761
	__free_one_page(page, pfn, zone, order, migratetype);
762
	spin_unlock(&zone->lock);
Nick Piggin's avatar
Nick Piggin committed
763 764
}

765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
static int free_tail_pages_check(struct page *head_page, struct page *page)
{
	if (!IS_ENABLED(CONFIG_DEBUG_VM))
		return 0;
	if (unlikely(!PageTail(page))) {
		bad_page(page, "PageTail not set", 0);
		return 1;
	}
	if (unlikely(page->first_page != head_page)) {
		bad_page(page, "first_page not consistent", 0);
		return 1;
	}
	return 0;
}

780
static bool free_pages_prepare(struct page *page, unsigned int order)
Nick Piggin's avatar
Nick Piggin committed
781
{
782 783
	bool compound = PageCompound(page);
	int i, bad = 0;
Linus Torvalds's avatar
Linus Torvalds committed
784

785
	VM_BUG_ON_PAGE(PageTail(page), page);
786
	VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
787

788
	trace_mm_page_free(page, order);
789 790
	kmemcheck_free_shadow(page, order);

Andrea Arcangeli's avatar
Andrea Arcangeli committed
791 792
	if (PageAnon(page))
		page->mapping = NULL;
793 794 795 796
	bad += free_pages_check(page);
	for (i = 1; i < (1 << order); i++) {
		if (compound)
			bad += free_tail_pages_check(page, page + i);
Andrea Arcangeli's avatar
Andrea Arcangeli committed
797
		bad += free_pages_check(page + i);
798
	}
799
	if (bad)
800
		return false;
801

802 803
	reset_page_owner(page, order);

804
	if (!PageHighMem(page)) {
805 806
		debug_check_no_locks_freed(page_address(page),
					   PAGE_SIZE << order);
807 808 809
		debug_check_no_obj_freed(page_address(page),
					   PAGE_SIZE << order);
	}
Nick Piggin's avatar
Nick Piggin committed
810
	arch_free_page(page, order);
Nick Piggin's avatar
Nick Piggin committed
811
	kernel_map_pages(page, 1 << order, 0);
Nick Piggin's avatar
Nick Piggin committed
812

813 814 815 816 817 818
	return true;
}

static void __free_pages_ok(struct page *page, unsigned int order)
{
	unsigned long flags;
819
	int migratetype;
820
	unsigned long pfn = page_to_pfn(page);
821 822 823 824

	if (!free_pages_prepare(page, order))
		return;

825
	migratetype = get_pfnblock_migratetype(page, pfn);
Nick Piggin's avatar
Nick Piggin committed
826
	local_irq_save(flags);
827
	__count_vm_events(PGFREE, 1 << order);
828
	set_freepage_migratetype(page, migratetype);
829
	free_one_page(page_zone(page), page, pfn, order, migratetype);
Nick Piggin's avatar
Nick Piggin committed
830
	local_irq_restore(flags);
Linus Torvalds's avatar
Linus Torvalds committed
831 832
}

833
void __init __free_pages_bootmem(struct page *page, unsigned int order)
834
{
835
	unsigned int nr_pages = 1 << order;
836
	struct page *p = page;
837
	unsigned int loop;
838

839 840 841
	prefetchw(p);
	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
		prefetchw(p + 1);
842 843
		__ClearPageReserved(p);
		set_page_count(p, 0);
844
	}
845 846
	__ClearPageReserved(p);
	set_page_count(p, 0);
847

848
	page_zone(page)->managed_pages += nr_pages;
849 850
	set_page_refcounted(page);
	__free_pages(page, order);
851 852
}

853
#ifdef CONFIG_CMA
854
/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
855 856 857 858 859 860 861 862 863 864 865
void __init init_cma_reserved_pageblock(struct page *page)
{
	unsigned i = pageblock_nr_pages;
	struct page *p = page;

	do {
		__ClearPageReserved(p);
		set_page_count(p, 0);
	} while (++p, --i);

	set_pageblock_migratetype(page, MIGRATE_CMA);
866 867 868 869 870 871 872 873 874 875 876 877 878 879

	if (pageblock_order >= MAX_ORDER) {
		i = pageblock_nr_pages;
		p = page;
		do {
			set_page_refcounted(p);
			__free_pages(p, MAX_ORDER - 1);
			p += MAX_ORDER_NR_PAGES;
		} while (i -= MAX_ORDER_NR_PAGES);
	} else {
		set_page_refcounted(page);
		__free_pages(page, pageblock_order);
	}

880
	adjust_managed_page_count(page, pageblock_nr_pages);
881 882
}
#endif
Linus Torvalds's avatar
Linus Torvalds committed
883 884 885 886 887 888 889 890 891 892 893 894 895

/*
 * The order of subdivision here is critical for the IO subsystem.
 * Please do not alter this order without good reasons and regression
 * testing. Specifically, as large blocks of memory are subdivided,
 * the order in which smaller blocks are delivered depends on the order
 * they're subdivided in this function. This is the primary factor
 * influencing the order in which pages are delivered to the IO
 * subsystem according to empirical testing, and this is also justified
 * by considering the behavior of a buddy system containing a single
 * large block of memory acted on by a series of small allocations.
 * This behavior is a critical factor in sglist merging's success.
 *
896
 * -- nyc
Linus Torvalds's avatar
Linus Torvalds committed
897
 */
Nick Piggin's avatar
Nick Piggin committed
898
static inline void expand(struct zone *zone, struct page *page,
899 900
	int low, int high, struct free_area *area,
	int migratetype)