filemap.c 75.2 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11
/*
 *	linux/mm/filemap.c
 *
 * Copyright (C) 1994-1999  Linus Torvalds
 */

/*
 * This file handles the generic file mmap semantics used by
 * most "normal" filesystems (but you don't /have/ to use this:
 * the NFS filesystem used to do this differently, for example)
 */
12
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <linux/compiler.h>
14
#include <linux/dax.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/fs.h>
16
#include <linux/uaccess.h>
17
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
#include <linux/kernel_stat.h>
19
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
20 21 22 23 24 25 26 27
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/hash.h>
#include <linux/writeback.h>
28
#include <linux/backing-dev.h>
Linus Torvalds's avatar
Linus Torvalds committed
29 30 31
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/security.h>
32
#include <linux/cpuset.h>
Nick Piggin's avatar
Nick Piggin committed
33
#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
34
#include <linux/hugetlb.h>
35
#include <linux/memcontrol.h>
36
#include <linux/cleancache.h>
37
#include <linux/rmap.h>
38 39
#include "internal.h"

40 41 42
#define CREATE_TRACE_POINTS
#include <trace/events/filemap.h>

Linus Torvalds's avatar
Linus Torvalds committed
43 44 45
/*
 * FIXME: remove all knowledge of the buffer layer from the core VM
 */
46
#include <linux/buffer_head.h> /* for try_to_free_buffers */
Linus Torvalds's avatar
Linus Torvalds committed
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64

#include <asm/mman.h>

/*
 * Shared mappings implemented 30.11.1994. It's not fully working yet,
 * though.
 *
 * Shared mappings now work. 15.8.1995  Bruno.
 *
 * finished 'unifying' the page and buffer cache and SMP-threaded the
 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
 *
 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
 */

/*
 * Lock ordering:
 *
65
 *  ->i_mmap_rwsem		(truncate_pagecache)
Linus Torvalds's avatar
Linus Torvalds committed
66
 *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
67 68
 *      ->swap_lock		(exclusive_swap_page, others)
 *        ->mapping->tree_lock
Linus Torvalds's avatar
Linus Torvalds committed
69
 *
70
 *  ->i_mutex
71
 *    ->i_mmap_rwsem		(truncate->unmap_mapping_range)
Linus Torvalds's avatar
Linus Torvalds committed
72 73
 *
 *  ->mmap_sem
74
 *    ->i_mmap_rwsem
75
 *      ->page_table_lock or pte_lock	(various, mainly in memory.c)
Linus Torvalds's avatar
Linus Torvalds committed
76 77 78 79 80
 *        ->mapping->tree_lock	(arch-dependent flush_dcache_mmap_lock)
 *
 *  ->mmap_sem
 *    ->lock_page		(access_process_vm)
 *
Al Viro's avatar
Al Viro committed
81
 *  ->i_mutex			(generic_perform_write)
82
 *    ->mmap_sem		(fault_in_pages_readable->do_page_fault)
Linus Torvalds's avatar
Linus Torvalds committed
83
 *
84
 *  bdi->wb.list_lock
85
 *    sb_lock			(fs/fs-writeback.c)
Linus Torvalds's avatar
Linus Torvalds committed
86 87
 *    ->mapping->tree_lock	(__sync_single_inode)
 *
88
 *  ->i_mmap_rwsem
Linus Torvalds's avatar
Linus Torvalds committed
89 90 91
 *    ->anon_vma.lock		(vma_adjust)
 *
 *  ->anon_vma.lock
92
 *    ->page_table_lock or pte_lock	(anon_vma_prepare and various)
Linus Torvalds's avatar
Linus Torvalds committed
93
 *
94
 *  ->page_table_lock or pte_lock
95
 *    ->swap_lock		(try_to_unmap_one)
Linus Torvalds's avatar
Linus Torvalds committed
96 97 98
 *    ->private_lock		(try_to_unmap_one)
 *    ->tree_lock		(try_to_unmap_one)
 *    ->zone.lru_lock		(follow_page->mark_page_accessed)
99
 *    ->zone.lru_lock		(check_pte_range->isolate_lru_page)
Linus Torvalds's avatar
Linus Torvalds committed
100 101
 *    ->private_lock		(page_remove_rmap->set_page_dirty)
 *    ->tree_lock		(page_remove_rmap->set_page_dirty)
102
 *    bdi.wb->list_lock		(page_remove_rmap->set_page_dirty)
103
 *    ->inode->i_lock		(page_remove_rmap->set_page_dirty)
104
 *    ->memcg->move_lock	(page_remove_rmap->lock_page_memcg)
105
 *    bdi.wb->list_lock		(zap_pte_range->set_page_dirty)
106
 *    ->inode->i_lock		(zap_pte_range->set_page_dirty)
Linus Torvalds's avatar
Linus Torvalds committed
107 108
 *    ->private_lock		(zap_pte_range->__set_page_dirty_buffers)
 *
109
 * ->i_mmap_rwsem
110
 *   ->tasklist_lock            (memory_failure, collect_procs_ao)
Linus Torvalds's avatar
Linus Torvalds committed
111 112
 */

113 114 115
static void page_cache_tree_delete(struct address_space *mapping,
				   struct page *page, void *shadow)
{
116 117 118 119 120
	struct radix_tree_node *node;
	unsigned long index;
	unsigned int offset;
	unsigned int tag;
	void **slot;
121

122 123 124 125 126
	VM_BUG_ON(!PageLocked(page));

	__radix_tree_lookup(&mapping->page_tree, page->index, &node, &slot);

	if (shadow) {
127
		mapping->nrexceptional++;
128
		/*
129
		 * Make sure the nrexceptional update is committed before
130 131 132 133 134
		 * the nrpages update so that final truncate racing
		 * with reclaim does not see both counters 0 at the
		 * same time and miss a shadow entry.
		 */
		smp_wmb();
135
	}
136
	mapping->nrpages--;
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173

	if (!node) {
		/* Clear direct pointer tags in root node */
		mapping->page_tree.gfp_mask &= __GFP_BITS_MASK;
		radix_tree_replace_slot(slot, shadow);
		return;
	}

	/* Clear tree tags for the removed page */
	index = page->index;
	offset = index & RADIX_TREE_MAP_MASK;
	for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {
		if (test_bit(offset, node->tags[tag]))
			radix_tree_tag_clear(&mapping->page_tree, index, tag);
	}

	/* Delete page, swap shadow entry */
	radix_tree_replace_slot(slot, shadow);
	workingset_node_pages_dec(node);
	if (shadow)
		workingset_node_shadows_inc(node);
	else
		if (__radix_tree_delete_node(&mapping->page_tree, node))
			return;

	/*
	 * Track node that only contains shadow entries.
	 *
	 * Avoid acquiring the list_lru lock if already tracked.  The
	 * list_empty() test is safe as node->private_list is
	 * protected by mapping->tree_lock.
	 */
	if (!workingset_node_pages(node) &&
	    list_empty(&node->private_list)) {
		node->private_data = mapping;
		list_lru_add(&workingset_shadow_nodes, &node->private_list);
	}
174 175
}

Linus Torvalds's avatar
Linus Torvalds committed
176
/*
177
 * Delete a page from the page cache and free it. Caller has to make
Linus Torvalds's avatar
Linus Torvalds committed
178
 * sure the page is locked and that nobody else uses it - or that usage
179
 * is safe.  The caller must hold the mapping's tree_lock.
Linus Torvalds's avatar
Linus Torvalds committed
180
 */
181
void __delete_from_page_cache(struct page *page, void *shadow)
Linus Torvalds's avatar
Linus Torvalds committed
182 183 184
{
	struct address_space *mapping = page->mapping;

185
	trace_mm_filemap_delete_from_page_cache(page);
186 187 188 189 190 191 192 193
	/*
	 * if we're uptodate, flush out into the cleancache, otherwise
	 * invalidate any existing cleancache entries.  We can't leave
	 * stale data around in the cleancache once our page is gone
	 */
	if (PageUptodate(page) && PageMappedToDisk(page))
		cleancache_put_page(page);
	else
194
		cleancache_invalidate_page(mapping, page);
195

196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
	VM_BUG_ON_PAGE(page_mapped(page), page);
	if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
		int mapcount;

		pr_alert("BUG: Bad page cache in process %s  pfn:%05lx\n",
			 current->comm, page_to_pfn(page));
		dump_page(page, "still mapped when deleted");
		dump_stack();
		add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);

		mapcount = page_mapcount(page);
		if (mapping_exiting(mapping) &&
		    page_count(page) >= mapcount + 2) {
			/*
			 * All vmas have already been torn down, so it's
			 * a good bet that actually the page is unmapped,
			 * and we'd prefer not to leak it: if we're wrong,
			 * some other bad page check should catch it later.
			 */
			page_mapcount_reset(page);
			atomic_sub(mapcount, &page->_count);
		}
	}

220 221
	page_cache_tree_delete(mapping, page, shadow);

Linus Torvalds's avatar
Linus Torvalds committed
222
	page->mapping = NULL;
223
	/* Leave page->index set: truncation lookup relies upon it */
224

225 226 227
	/* hugetlb pages do not participate in page cache accounting. */
	if (!PageHuge(page))
		__dec_zone_page_state(page, NR_FILE_PAGES);
228 229
	if (PageSwapBacked(page))
		__dec_zone_page_state(page, NR_SHMEM);
230 231

	/*
232 233
	 * At this point page must be either written or cleaned by truncate.
	 * Dirty page here signals a bug and loss of unwritten data.
234
	 *
235 236 237
	 * This fixes dirty accounting after removing the page entirely but
	 * leaves PageDirty set: it has no effect for truncated page and
	 * anyway will be cleared before returning page into buddy allocator.
238
	 */
239
	if (WARN_ON_ONCE(PageDirty(page)))
240
		account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
Linus Torvalds's avatar
Linus Torvalds committed
241 242
}

243 244 245 246 247 248 249 250 251
/**
 * delete_from_page_cache - delete page from page cache
 * @page: the page which the kernel is trying to remove from page cache
 *
 * This must be called only on pages that have been verified to be in the page
 * cache and locked.  It will never put the page into the free list, the caller
 * has a reference on the page.
 */
void delete_from_page_cache(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
252 253
{
	struct address_space *mapping = page->mapping;
254 255
	unsigned long flags;

256
	void (*freepage)(struct page *);
Linus Torvalds's avatar
Linus Torvalds committed
257

Matt Mackall's avatar
Matt Mackall committed
258
	BUG_ON(!PageLocked(page));
Linus Torvalds's avatar
Linus Torvalds committed
259

260
	freepage = mapping->a_ops->freepage;
261 262

	spin_lock_irqsave(&mapping->tree_lock, flags);
263
	__delete_from_page_cache(page, NULL);
264
	spin_unlock_irqrestore(&mapping->tree_lock, flags);
265 266 267

	if (freepage)
		freepage(page);
268
	put_page(page);
269 270 271
}
EXPORT_SYMBOL(delete_from_page_cache);

272 273 274 275
static int filemap_check_errors(struct address_space *mapping)
{
	int ret = 0;
	/* Check for outstanding write errors */
276 277
	if (test_bit(AS_ENOSPC, &mapping->flags) &&
	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))
278
		ret = -ENOSPC;
279 280
	if (test_bit(AS_EIO, &mapping->flags) &&
	    test_and_clear_bit(AS_EIO, &mapping->flags))
281 282 283 284
		ret = -EIO;
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
285
/**
286
 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
287 288
 * @mapping:	address space structure to write
 * @start:	offset in bytes where the range starts
289
 * @end:	offset in bytes where the range ends (inclusive)
290
 * @sync_mode:	enable synchronous operation
Linus Torvalds's avatar
Linus Torvalds committed
291
 *
292 293 294
 * Start writeback against all of a mapping's dirty pages that lie
 * within the byte offsets <start, end> inclusive.
 *
Linus Torvalds's avatar
Linus Torvalds committed
295
 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
296
 * opposed to a regular memory cleansing writeback.  The difference between
Linus Torvalds's avatar
Linus Torvalds committed
297 298 299
 * these two operations is that if a dirty page/buffer is encountered, it must
 * be waited upon, and not just skipped over.
 */
300 301
int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
				loff_t end, int sync_mode)
Linus Torvalds's avatar
Linus Torvalds committed
302 303 304 305
{
	int ret;
	struct writeback_control wbc = {
		.sync_mode = sync_mode,
306
		.nr_to_write = LONG_MAX,
307 308
		.range_start = start,
		.range_end = end,
Linus Torvalds's avatar
Linus Torvalds committed
309 310 311 312 313
	};

	if (!mapping_cap_writeback_dirty(mapping))
		return 0;

314
	wbc_attach_fdatawrite_inode(&wbc, mapping->host);
Linus Torvalds's avatar
Linus Torvalds committed
315
	ret = do_writepages(mapping, &wbc);
316
	wbc_detach_inode(&wbc);
Linus Torvalds's avatar
Linus Torvalds committed
317 318 319 320 321 322
	return ret;
}

static inline int __filemap_fdatawrite(struct address_space *mapping,
	int sync_mode)
{
323
	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
Linus Torvalds's avatar
Linus Torvalds committed
324 325 326 327 328 329 330 331
}

int filemap_fdatawrite(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite);

332
int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
333
				loff_t end)
Linus Torvalds's avatar
Linus Torvalds committed
334 335 336
{
	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
}
337
EXPORT_SYMBOL(filemap_fdatawrite_range);
Linus Torvalds's avatar
Linus Torvalds committed
338

339 340 341 342
/**
 * filemap_flush - mostly a non-blocking flush
 * @mapping:	target address_space
 *
Linus Torvalds's avatar
Linus Torvalds committed
343 344 345 346 347 348 349 350 351
 * This is a mostly non-blocking flush.  Not suitable for data-integrity
 * purposes - I/O may not be started against all dirty pages.
 */
int filemap_flush(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
}
EXPORT_SYMBOL(filemap_flush);

352 353
static int __filemap_fdatawait_range(struct address_space *mapping,
				     loff_t start_byte, loff_t end_byte)
Linus Torvalds's avatar
Linus Torvalds committed
354
{
355 356
	pgoff_t index = start_byte >> PAGE_SHIFT;
	pgoff_t end = end_byte >> PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
357 358
	struct pagevec pvec;
	int nr_pages;
359
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
360

361
	if (end_byte < start_byte)
362
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378

	pagevec_init(&pvec, 0);
	while ((index <= end) &&
			(nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
			PAGECACHE_TAG_WRITEBACK,
			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
		unsigned i;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/* until radix tree lookup accepts end_index */
			if (page->index > end)
				continue;

			wait_on_page_writeback(page);
379
			if (TestClearPageError(page))
Linus Torvalds's avatar
Linus Torvalds committed
380 381 382 383 384
				ret = -EIO;
		}
		pagevec_release(&pvec);
		cond_resched();
	}
385
out:
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
	return ret;
}

/**
 * filemap_fdatawait_range - wait for writeback to complete
 * @mapping:		address space structure to wait for
 * @start_byte:		offset in bytes where the range starts
 * @end_byte:		offset in bytes where the range ends (inclusive)
 *
 * Walk the list of under-writeback pages of the given address space
 * in the given range and wait for all of them.  Check error status of
 * the address space and return it.
 *
 * Since the error status of the address space is cleared by this function,
 * callers are responsible for checking the return value and handling and/or
 * reporting the error.
 */
int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
			    loff_t end_byte)
{
	int ret, ret2;

	ret = __filemap_fdatawait_range(mapping, start_byte, end_byte);
409 410 411
	ret2 = filemap_check_errors(mapping);
	if (!ret)
		ret = ret2;
Linus Torvalds's avatar
Linus Torvalds committed
412 413 414

	return ret;
}
415 416
EXPORT_SYMBOL(filemap_fdatawait_range);

417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
/**
 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
 * @mapping: address space structure to wait for
 *
 * Walk the list of under-writeback pages of the given address space
 * and wait for all of them.  Unlike filemap_fdatawait(), this function
 * does not clear error status of the address space.
 *
 * Use this function if callers don't handle errors themselves.  Expected
 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
 * fsfreeze(8)
 */
void filemap_fdatawait_keep_errors(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return;

	__filemap_fdatawait_range(mapping, 0, i_size - 1);
}

Linus Torvalds's avatar
Linus Torvalds committed
439
/**
440
 * filemap_fdatawait - wait for all under-writeback pages to complete
Linus Torvalds's avatar
Linus Torvalds committed
441
 * @mapping: address space structure to wait for
442 443
 *
 * Walk the list of under-writeback pages of the given address space
444 445 446 447 448 449
 * and wait for all of them.  Check error status of the address space
 * and return it.
 *
 * Since the error status of the address space is cleared by this function,
 * callers are responsible for checking the return value and handling and/or
 * reporting the error.
Linus Torvalds's avatar
Linus Torvalds committed
450 451 452 453 454 455 456 457
 */
int filemap_fdatawait(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return 0;

458
	return filemap_fdatawait_range(mapping, 0, i_size - 1);
Linus Torvalds's avatar
Linus Torvalds committed
459 460 461 462 463
}
EXPORT_SYMBOL(filemap_fdatawait);

int filemap_write_and_wait(struct address_space *mapping)
{
464
	int err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
465

466 467
	if ((!dax_mapping(mapping) && mapping->nrpages) ||
	    (dax_mapping(mapping) && mapping->nrexceptional)) {
468 469 470 471 472 473 474 475 476 477 478 479
		err = filemap_fdatawrite(mapping);
		/*
		 * Even if the above returned error, the pages may be
		 * written partially (e.g. -ENOSPC), so we wait for it.
		 * But the -EIO is special case, it may indicate the worst
		 * thing (e.g. bug) happened, so we avoid waiting for it.
		 */
		if (err != -EIO) {
			int err2 = filemap_fdatawait(mapping);
			if (!err)
				err = err2;
		}
480 481
	} else {
		err = filemap_check_errors(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
482
	}
483
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
484
}
485
EXPORT_SYMBOL(filemap_write_and_wait);
Linus Torvalds's avatar
Linus Torvalds committed
486

487 488 489 490 491 492
/**
 * filemap_write_and_wait_range - write out & wait on a file range
 * @mapping:	the address_space for the pages
 * @lstart:	offset in bytes where the range starts
 * @lend:	offset in bytes where the range ends (inclusive)
 *
493 494 495 496 497
 * Write out and wait upon file offsets lstart->lend, inclusive.
 *
 * Note that `lend' is inclusive (describes the last byte to be written) so
 * that this function can be used to write to the very end-of-file (end = -1).
 */
Linus Torvalds's avatar
Linus Torvalds committed
498 499 500
int filemap_write_and_wait_range(struct address_space *mapping,
				 loff_t lstart, loff_t lend)
{
501
	int err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
502

503 504
	if ((!dax_mapping(mapping) && mapping->nrpages) ||
	    (dax_mapping(mapping) && mapping->nrexceptional)) {
505 506 507 508
		err = __filemap_fdatawrite_range(mapping, lstart, lend,
						 WB_SYNC_ALL);
		/* See comment of filemap_write_and_wait() */
		if (err != -EIO) {
509 510
			int err2 = filemap_fdatawait_range(mapping,
						lstart, lend);
511 512 513
			if (!err)
				err = err2;
		}
514 515
	} else {
		err = filemap_check_errors(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
516
	}
517
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
518
}
519
EXPORT_SYMBOL(filemap_write_and_wait_range);
Linus Torvalds's avatar
Linus Torvalds committed
520

521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
/**
 * replace_page_cache_page - replace a pagecache page with a new one
 * @old:	page to be replaced
 * @new:	page to replace with
 * @gfp_mask:	allocation mode
 *
 * This function replaces a page in the pagecache with a new one.  On
 * success it acquires the pagecache reference for the new page and
 * drops it for the old page.  Both the old and new pages must be
 * locked.  This function does not add the new page to the LRU, the
 * caller must do that.
 *
 * The remove + add is atomic.  The only way this function can fail is
 * memory allocation failure.
 */
int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
{
	int error;

540 541 542
	VM_BUG_ON_PAGE(!PageLocked(old), old);
	VM_BUG_ON_PAGE(!PageLocked(new), new);
	VM_BUG_ON_PAGE(new->mapping, new);
543 544 545 546 547

	error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
	if (!error) {
		struct address_space *mapping = old->mapping;
		void (*freepage)(struct page *);
548
		unsigned long flags;
549 550 551 552

		pgoff_t offset = old->index;
		freepage = mapping->a_ops->freepage;

553
		get_page(new);
554 555 556
		new->mapping = mapping;
		new->index = offset;

557
		spin_lock_irqsave(&mapping->tree_lock, flags);
558
		__delete_from_page_cache(old, NULL);
559 560 561
		error = radix_tree_insert(&mapping->page_tree, offset, new);
		BUG_ON(error);
		mapping->nrpages++;
562 563 564 565 566 567

		/*
		 * hugetlb pages do not participate in page cache accounting.
		 */
		if (!PageHuge(new))
			__inc_zone_page_state(new, NR_FILE_PAGES);
568 569
		if (PageSwapBacked(new))
			__inc_zone_page_state(new, NR_SHMEM);
570
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
571
		mem_cgroup_migrate(old, new);
572 573 574
		radix_tree_preload_end();
		if (freepage)
			freepage(old);
575
		put_page(old);
576 577 578 579 580 581
	}

	return error;
}
EXPORT_SYMBOL_GPL(replace_page_cache_page);

582
static int page_cache_tree_insert(struct address_space *mapping,
583
				  struct page *page, void **shadowp)
584
{
585
	struct radix_tree_node *node;
586 587 588
	void **slot;
	int error;

589
	error = __radix_tree_create(&mapping->page_tree, page->index, 0,
590 591 592 593
				    &node, &slot);
	if (error)
		return error;
	if (*slot) {
594 595 596 597 598
		void *p;

		p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
		if (!radix_tree_exceptional_entry(p))
			return -EEXIST;
599 600 601 602

		if (WARN_ON(dax_mapping(mapping)))
			return -EINVAL;

603 604
		if (shadowp)
			*shadowp = p;
605
		mapping->nrexceptional--;
606 607
		if (node)
			workingset_node_shadows_dec(node);
608
	}
609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
	radix_tree_replace_slot(slot, page);
	mapping->nrpages++;
	if (node) {
		workingset_node_pages_inc(node);
		/*
		 * Don't track node that contains actual pages.
		 *
		 * Avoid acquiring the list_lru lock if already
		 * untracked.  The list_empty() test is safe as
		 * node->private_list is protected by
		 * mapping->tree_lock.
		 */
		if (!list_empty(&node->private_list))
			list_lru_del(&workingset_shadow_nodes,
				     &node->private_list);
	}
	return 0;
626 627
}

628 629 630 631
static int __add_to_page_cache_locked(struct page *page,
				      struct address_space *mapping,
				      pgoff_t offset, gfp_t gfp_mask,
				      void **shadowp)
Linus Torvalds's avatar
Linus Torvalds committed
632
{
633 634
	int huge = PageHuge(page);
	struct mem_cgroup *memcg;
635 636
	int error;

637 638
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(PageSwapBacked(page), page);
639

640 641
	if (!huge) {
		error = mem_cgroup_try_charge(page, current->mm,
642
					      gfp_mask, &memcg, false);
643 644 645
		if (error)
			return error;
	}
Linus Torvalds's avatar
Linus Torvalds committed
646

647
	error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
648
	if (error) {
649
		if (!huge)
650
			mem_cgroup_cancel_charge(page, memcg, false);
651 652 653
		return error;
	}

654
	get_page(page);
655 656 657 658
	page->mapping = mapping;
	page->index = offset;

	spin_lock_irq(&mapping->tree_lock);
659
	error = page_cache_tree_insert(mapping, page, shadowp);
660 661 662
	radix_tree_preload_end();
	if (unlikely(error))
		goto err_insert;
663 664 665 666

	/* hugetlb pages do not participate in page cache accounting. */
	if (!huge)
		__inc_zone_page_state(page, NR_FILE_PAGES);
667
	spin_unlock_irq(&mapping->tree_lock);
668
	if (!huge)
669
		mem_cgroup_commit_charge(page, memcg, false, false);
670 671 672 673 674 675
	trace_mm_filemap_add_to_page_cache(page);
	return 0;
err_insert:
	page->mapping = NULL;
	/* Leave page->index set: truncation relies upon it */
	spin_unlock_irq(&mapping->tree_lock);
676
	if (!huge)
677
		mem_cgroup_cancel_charge(page, memcg, false);
678
	put_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
679 680
	return error;
}
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697

/**
 * add_to_page_cache_locked - add a locked page to the pagecache
 * @page:	page to add
 * @mapping:	the page's address_space
 * @offset:	page index
 * @gfp_mask:	page allocation mode
 *
 * This function is used to add a page to the pagecache. It must be locked.
 * This function does not add the page to the LRU.  The caller must do that.
 */
int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
		pgoff_t offset, gfp_t gfp_mask)
{
	return __add_to_page_cache_locked(page, mapping, offset,
					  gfp_mask, NULL);
}
698
EXPORT_SYMBOL(add_to_page_cache_locked);
Linus Torvalds's avatar
Linus Torvalds committed
699 700

int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
Al Viro's avatar
Al Viro committed
701
				pgoff_t offset, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
702
{
703
	void *shadow = NULL;
704 705
	int ret;

706
	__SetPageLocked(page);
707 708 709
	ret = __add_to_page_cache_locked(page, mapping, offset,
					 gfp_mask, &shadow);
	if (unlikely(ret))
710
		__ClearPageLocked(page);
711 712 713 714 715 716 717 718 719 720 721 722 723
	else {
		/*
		 * The page might have been evicted from cache only
		 * recently, in which case it should be activated like
		 * any other repeatedly accessed page.
		 */
		if (shadow && workingset_refault(shadow)) {
			SetPageActive(page);
			workingset_activation(page);
		} else
			ClearPageActive(page);
		lru_cache_add(page);
	}
Linus Torvalds's avatar
Linus Torvalds committed
724 725
	return ret;
}
726
EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
Linus Torvalds's avatar
Linus Torvalds committed
727

728
#ifdef CONFIG_NUMA
729
struct page *__page_cache_alloc(gfp_t gfp)
730
{
731 732 733
	int n;
	struct page *page;

734
	if (cpuset_do_page_mem_spread()) {
735 736
		unsigned int cpuset_mems_cookie;
		do {
737
			cpuset_mems_cookie = read_mems_allowed_begin();
738
			n = cpuset_mem_spread_node();
739
			page = __alloc_pages_node(n, gfp, 0);
740
		} while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
741

742
		return page;
743
	}
744
	return alloc_pages(gfp, 0);
745
}
746
EXPORT_SYMBOL(__page_cache_alloc);
747 748
#endif

Linus Torvalds's avatar
Linus Torvalds committed
749 750 751 752 753 754 755 756 757 758
/*
 * In order to wait for pages to become available there must be
 * waitqueues associated with pages. By using a hash table of
 * waitqueues where the bucket discipline is to maintain all
 * waiters on the same queue and wake all when any of the pages
 * become available, and for the woken contexts to check to be
 * sure the appropriate page became available, this saves space
 * at a cost of "thundering herd" phenomena during rare hash
 * collisions.
 */
NeilBrown's avatar
NeilBrown committed
759
wait_queue_head_t *page_waitqueue(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
760 761 762 763 764
{
	const struct zone *zone = page_zone(page);

	return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
}
NeilBrown's avatar
NeilBrown committed
765
EXPORT_SYMBOL(page_waitqueue);
Linus Torvalds's avatar
Linus Torvalds committed
766

767
void wait_on_page_bit(struct page *page, int bit_nr)
Linus Torvalds's avatar
Linus Torvalds committed
768 769 770 771
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (test_bit(bit_nr, &page->flags))
772
		__wait_on_bit(page_waitqueue(page), &wait, bit_wait_io,
Linus Torvalds's avatar
Linus Torvalds committed
773 774 775 776
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(wait_on_page_bit);

777 778 779 780 781 782 783 784
int wait_on_page_bit_killable(struct page *page, int bit_nr)
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (!test_bit(bit_nr, &page->flags))
		return 0;

	return __wait_on_bit(page_waitqueue(page), &wait,
785
			     bit_wait_io, TASK_KILLABLE);
786 787
}

788 789 790 791 792 793 794 795 796 797 798 799 800
int wait_on_page_bit_killable_timeout(struct page *page,
				       int bit_nr, unsigned long timeout)
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	wait.key.timeout = jiffies + timeout;
	if (!test_bit(bit_nr, &page->flags))
		return 0;
	return __wait_on_bit(page_waitqueue(page), &wait,
			     bit_wait_io_timeout, TASK_KILLABLE);
}
EXPORT_SYMBOL_GPL(wait_on_page_bit_killable_timeout);

801 802
/**
 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
803 804
 * @page: Page defining the wait queue of interest
 * @waiter: Waiter to add to the queue
805 806 807 808 809 810 811 812 813 814 815 816 817 818
 *
 * Add an arbitrary @waiter to the wait queue for the nominated @page.
 */
void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
{
	wait_queue_head_t *q = page_waitqueue(page);
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, waiter);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(add_page_wait_queue);

Linus Torvalds's avatar
Linus Torvalds committed
819
/**
820
 * unlock_page - unlock a locked page
Linus Torvalds's avatar
Linus Torvalds committed
821 822 823 824
 * @page: the page
 *
 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
825
 * mechanism between PageLocked pages and PageWriteback pages is shared.
Linus Torvalds's avatar
Linus Torvalds committed
826 827
 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
 *
Nick Piggin's avatar
Nick Piggin committed
828 829
 * The mb is necessary to enforce ordering between the clear_bit and the read
 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
Linus Torvalds's avatar
Linus Torvalds committed
830
 */
831
void unlock_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
832
{
833
	page = compound_head(page);
834
	VM_BUG_ON_PAGE(!PageLocked(page), page);
Nick Piggin's avatar
Nick Piggin committed
835
	clear_bit_unlock(PG_locked, &page->flags);
836
	smp_mb__after_atomic();
Linus Torvalds's avatar
Linus Torvalds committed
837 838 839 840
	wake_up_page(page, PG_locked);
}
EXPORT_SYMBOL(unlock_page);

841 842 843
/**
 * end_page_writeback - end writeback against a page
 * @page: the page
Linus Torvalds's avatar
Linus Torvalds committed
844 845 846
 */
void end_page_writeback(struct page *page)
{
847 848 849 850 851 852 853 854 855
	/*
	 * TestClearPageReclaim could be used here but it is an atomic
	 * operation and overkill in this particular case. Failing to
	 * shuffle a page marked for immediate reclaim is too mild to
	 * justify taking an atomic operation penalty at the end of
	 * ever page writeback.
	 */
	if (PageReclaim(page)) {
		ClearPageReclaim(page);
856
		rotate_reclaimable_page(page);
857
	}
858 859 860 861

	if (!test_clear_page_writeback(page))
		BUG();

862
	smp_mb__after_atomic();
Linus Torvalds's avatar
Linus Torvalds committed
863 864 865 866
	wake_up_page(page, PG_writeback);
}
EXPORT_SYMBOL(end_page_writeback);

867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891
/*
 * After completing I/O on a page, call this routine to update the page
 * flags appropriately
 */
void page_endio(struct page *page, int rw, int err)
{
	if (rw == READ) {
		if (!err) {
			SetPageUptodate(page);
		} else {
			ClearPageUptodate(page);
			SetPageError(page);
		}
		unlock_page(page);
	} else { /* rw == WRITE */
		if (err) {
			SetPageError(page);
			if (page->mapping)
				mapping_set_error(page->mapping, err);
		}
		end_page_writeback(page);
	}
}
EXPORT_SYMBOL_GPL(page_endio);

892 893 894
/**
 * __lock_page - get a lock on the page, assuming we need to sleep to get it
 * @page: the page to lock
Linus Torvalds's avatar
Linus Torvalds committed
895
 */
896
void __lock_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
897
{
898 899
	struct page *page_head = compound_head(page);
	DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
Linus Torvalds's avatar
Linus Torvalds committed
900

901
	__wait_on_bit_lock(page_waitqueue(page_head), &wait, bit_wait_io,
Linus Torvalds's avatar
Linus Torvalds committed
902 903 904 905
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__lock_page);

906
int __lock_page_killable(struct page *page)
Matthew Wilcox's avatar
Matthew Wilcox committed
907
{
908 909
	struct page *page_head = compound_head(page);
	DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
Matthew Wilcox's avatar
Matthew Wilcox committed
910

911
	return __wait_on_bit_lock(page_waitqueue(page_head), &wait,
912
					bit_wait_io, TASK_KILLABLE);
Matthew Wilcox's avatar
Matthew Wilcox committed
913
}
914
EXPORT_SYMBOL_GPL(__lock_page_killable);
Matthew Wilcox's avatar
Matthew Wilcox committed
915

916 917 918 919 920 921 922 923 924 925 926
/*
 * Return values:
 * 1 - page is locked; mmap_sem is still held.
 * 0 - page is not locked.
 *     mmap_sem has been released (up_read()), unless flags had both
 *     FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
 *     which case mmap_sem is still held.
 *
 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
 * with the page locked and the mmap_sem unperturbed.
 */
927 928 929
int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
			 unsigned int flags)
{
930 931 932 933 934 935 936 937 938 939 940 941
	if (flags & FAULT_FLAG_ALLOW_RETRY) {
		/*
		 * CAUTION! In this case, mmap_sem is not released
		 * even though return 0.
		 */
		if (flags & FAULT_FLAG_RETRY_NOWAIT)
			return 0;

		up_read(&mm->mmap_sem);
		if (flags & FAULT_FLAG_KILLABLE)
			wait_on_page_locked_killable(page);
		else
942
			wait_on_page_locked(page);
943
		return 0;
944 945 946 947 948 949 950 951 952 953 954 955
	} else {
		if (flags & FAULT_FLAG_KILLABLE) {
			int ret;

			ret = __lock_page_killable(page);
			if (ret) {
				up_read(&mm->mmap_sem);
				return 0;
			}
		} else
			__lock_page(page);
		return 1;
956 957 958
	}
}

959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
/**
 * page_cache_next_hole - find the next hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
 * lowest indexed hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'return - index >=
 * max_scan' will be true). In rare cases of index wrap-around, 0 will
 * be returned.
 *
 * page_cache_next_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 5, then subsequently a hole is created at
 * index 10, page_cache_next_hole covering both indexes may return 10
 * if called under rcu_read_lock.
 */
pgoff_t page_cache_next_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
986 987 988 989
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
			break;
		index++;
		if (index == 0)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_next_hole);

/**
 * page_cache_prev_hole - find the prev hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search backwards in the range [max(index-max_scan+1, 0), index] for
 * the first hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'index - return >=
 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
 * will be returned.
 *
 * page_cache_prev_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 10, then subsequently a hole is created at
 * index 5, page_cache_prev_hole covering both indexes may return 5 if
 * called under rcu_read_lock.
 */
pgoff_t page_cache_prev_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
1027 1028 1029 1030
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
			break;
		index--;
		if (index == ULONG_MAX)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_prev_hole);

1041
/**
1042
 * find_get_entry - find and get a page cache entry
1043
 * @mapping: the address_space to search
1044 1045 1046 1047
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned with an increased refcount.
1048
 *
1049 1050
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
1051 1052
 *
 * Otherwise, %NULL is returned.
Linus Torvalds's avatar
Linus Torvalds committed
1053
 */
1054
struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds's avatar
Linus Torvalds committed
1055
{
Nick Piggin's avatar
Nick Piggin committed
1056
	void **pagep;
Linus Torvalds's avatar
Linus Torvalds committed
1057 1058
	struct page *page;

Nick Piggin's avatar
Nick Piggin committed
1059 1060 1061 1062 1063 1064
	rcu_read_lock();
repeat:
	page = NULL;
	pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
	if (pagep) {
		page = radix_tree_deref_slot(pagep);
Nick Piggin's avatar
Nick Piggin committed
1065 1066
		if (unlikely(!page))
			goto out;
1067
		if (radix_tree_exception(page)) {
1068 1069 1070
			if (radix_tree_deref_retry(page))
				goto repeat;
			/*
1071 1072 1073
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Return
			 * it without attempting to raise page count.
1074 1075
			 */
			goto out;
1076
		}
Nick Piggin's avatar
Nick Piggin committed
1077 1078 1079 1080 1081 1082 1083 1084 1085
		if (!page_cache_get_speculative(page))
			goto repeat;

		/*
		 * Has the page moved?
		 * This is part of the lockless pagecache protocol. See
		 * include/linux/pagemap.h for details.
		 */
		if (unlikely(page != *pagep)) {
1086
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1087 1088 1089
			goto repeat;
		}
	}
Nick Piggin's avatar
Nick Piggin committed
1090
out:
Nick Piggin's avatar
Nick Piggin committed
1091 1092
	rcu_read_unlock();

Linus Torvalds's avatar
Linus Torvalds committed
1093 1094
	return page;
}
1095
EXPORT_SYMBOL(find_get_entry);
Linus Torvalds's avatar
Linus Torvalds committed
1096

1097 1098 1099 1100 1101 1102 1103 1104 1105
/**
 * find_lock_entry - locate, pin and lock a page cache entry
 * @mapping: the address_space to search
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned locked and with an increased
 * refcount.
 *
1106 1107
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
1108 1109 1110 1111 1112 1113
 *
 * Otherwise, %NULL is returned.
 *
 * find_lock_entry() may sleep.
 */
struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds's avatar
Linus Torvalds committed
1114 1115 1116 1117
{
	struct page *page;

repeat:
1118
	page = find_get_entry(mapping, offset);
1119
	if (page && !radix_tree_exception(page)) {
Nick Piggin's avatar
Nick Piggin committed
1120 1121 1122 1123
		lock_page(page);
		/* Has the page been truncated? */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
1124
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1125
			goto repeat;
Linus Torvalds's avatar
Linus Torvalds committed
1126
		}
1127
		VM_BUG_ON_PAGE(page->index != offset, page);
Linus Torvalds's avatar
Linus Torvalds committed
1128 1129 1130
	}
	return page;
}
1131 1132 1133
EXPORT_SYMBOL(find_lock_entry);

/**
1134
 * pagecache_get_page - find and get a page reference
1135 1136
 * @mapping: the address_space to search
 * @offset: the page index
1137
 * @fgp_flags: PCG flags
1138
 * @gfp_mask: gfp mask to use for the page cache data page allocation
1139
 *
1140
 * Looks up the page cache slot at @mapping & @offset.
Linus Torvalds's avatar
Linus Torvalds committed
1141
 *
1142
 * PCG flags modify how the page is returned.
1143
 *
1144 1145 1146
 * FGP_ACCESSED: the page will be marked accessed
 * FGP_LOCK: Page is return locked
 * FGP_CREAT: If page is not present then a new page is allocated using
1147 1148 1149
 *		@gfp_mask and added to the page cache and the VM's LRU
 *		list. The page is returned locked and with an increased
 *		refcount. Otherwise, %NULL is returned.
Linus Torvalds's avatar
Linus Torvalds committed
1150
 *
1151 1152
 * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
 * if the GFP flags specified for FGP_CREAT are atomic.
Linus Torvalds's avatar
Linus Torvalds committed
1153
 *
1154
 * If there is a page cache page, it is returned with an increased refcount.
Linus Torvalds's avatar
Linus Torvalds committed
1155
 */
1156
struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
1157
	int fgp_flags, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1158
{
Nick Piggin's avatar
Nick Piggin committed
1159
	struct page *page;
1160

Linus Torvalds's avatar
Linus Torvalds committed
1161
repeat:
1162 1163 1164 1165 1166 1167 1168 1169 1170
	page = find_get_entry(mapping, offset);
	if (radix_tree_exceptional_entry(page))
		page = NULL;
	if (!page)
		goto no_page;

	if (fgp_flags & FGP_LOCK) {
		if (fgp_flags & FGP_NOWAIT) {
			if (!trylock_page(page)) {
1171
				put_page(page);
1172 1173 1174 1175 1176 1177 1178 1179 1180
				return NULL;
			}
		} else {
			lock_page(page);
		}

		/* Has the page been truncated? */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
1181
			put_page(page);
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193
			goto repeat;
		}
		VM_BUG_ON_PAGE(page->index != offset, page);
	}

	if (page && (fgp_flags & FGP_ACCESSED))
		mark_page_accessed(page);

no_page:
	if (!page && (fgp_flags & FGP_CREAT)) {
		int err;
		if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
1194 1195 1196
			gfp_mask |= __GFP_WRITE;
		if (fgp_flags & FGP_NOFS)
			gfp_mask &= ~__GFP_FS;
1197

1198
		page = __page_cache_alloc(gfp_mask);
Nick Piggin's avatar
Nick Piggin committed
1199 1200
		if (!page)
			return NULL;
1201 1202 1203 1204

		if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
			fgp_flags |= FGP_LOCK;

1205
		/* Init accessed so avoid atomic mark_page_accessed later */
1206
		if (fgp_flags & FGP_ACCESSED)
1207
			__SetPageReferenced(page);
1208

1209 1210
		err = add_to_page_cache_lru(page, mapping, offset,
				gfp_mask & GFP_RECLAIM_MASK);
Nick Piggin's avatar
Nick Piggin committed
1211
		if (unlikely(err)) {
1212
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1213 1214 1215
			page = NULL;
			if (err == -EEXIST)
				goto repeat;
Linus Torvalds's avatar
Linus Torvalds committed
1216 1217
		}
	}
1218

Linus Torvalds's avatar
Linus Torvalds committed
1219 1220
	return page;
}
1221
EXPORT_SYMBOL(pagecache_get_page);
Linus Torvalds's avatar
Linus Torvalds committed
1222

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
/**
 * find_get_entries - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page cache index
 * @nr_entries:	The maximum number of entries
 * @entries:	Where the resulting entries are placed
 * @indices:	The cache indices corresponding to the entries in @entries
 *
 * find_get_entries() will search for and return a group of up to
 * @nr_entries entries in the mapping.  The entries are placed at
 * @entries.  find_get_entries() takes a reference against any actual
 * pages it returns.
 *
 * The search returns a group of mapping-contiguous page cache entries
 * with ascending indexes.  There may be holes in the indices due to
 * not-present pages.
 *
1240 1241
 * Any shadow entries of evicted pages, or swap entries from
 * shmem/tmpfs, are included in the returned array.
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264
 *
 * find_get_entries() returns the number of pages and shadow entries
 * which were found.
 */
unsigned find_get_entries(struct address_space *mapping,
			  pgoff_t start, unsigned int nr_entries,
			  struct page **entries, pgoff_t *indices)
{
	void **slot;
	unsigned int ret = 0;
	struct radix_tree_iter iter;

	if (!nr_entries)
		return 0;

	rcu_read_lock();
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
		struct page *page;
repeat:
		page = radix_tree_deref_slot(slot);
		if (unlikely(!page))
			continue;
		if (radix_tree_exception(page)) {
1265 1266 1267 1268
			if (radix_tree_deref_retry(page)) {
				slot = radix_tree_iter_retry(&iter);
				continue;
			}
1269
			/*
1270 1271 1272
			 * A shadow entry of a recently evicted page, a swap
			 * entry from shmem/tmpfs or a DAX entry.  Return it
			 * without attempting to raise page count.
1273 1274 1275 1276 1277 1278 1279 1280
			 */
			goto export;
		}
		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
		if (unlikely(page != *slot)) {
1281
			put_page(page);
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
			goto repeat;
		}
export:
		indices[ret] = iter.index;
		entries[ret] = page;
		if (++ret == nr_entries)
			break;
	}
	rcu_read_unlock();
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312
/**
 * find_get_pages - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages() will search for and return a group of up to
 * @nr_pages pages in the mapping.  The pages are placed at @pages.
 * find_get_pages() takes a reference against the returned pages.
 *
 * The search returns a group of mapping-contiguous pages with ascending
 * indexes.  There may be holes in the indices due to not-present pages.
 *
 * find_get_pages() returns the number of pages which were found.
 */
unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
			    unsigned int nr_pages, struct page **pages)
{
1313 1314 1315 1316 1317 1318
	struct radix_tree_iter iter;
	void **slot;
	unsigned ret = 0;

	if (unlikely(!nr_pages))
		return 0;
Nick Piggin's avatar
Nick Piggin committed
1319 1320

	rcu_read_lock();
1321
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
Nick Piggin's avatar
Nick Piggin committed
1322 1323
		struct page *page;
repeat:
1324
		page = radix_tree_deref_slot(slot);
Nick Piggin's avatar
Nick Piggin committed
1325 1326
		if (unlikely(!page))
			continue;
1327

1328
		if (radix_tree_exception(page)) {
1329
			if (radix_tree_deref_retry(page)) {