filemap.c 74.1 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11
/*
 *	linux/mm/filemap.c
 *
 * Copyright (C) 1994-1999  Linus Torvalds
 */

/*
 * This file handles the generic file mmap semantics used by
 * most "normal" filesystems (but you don't /have/ to use this:
 * the NFS filesystem used to do this differently, for example)
 */
12
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <linux/compiler.h>
14
#include <linux/dax.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/fs.h>
16
#include <linux/uaccess.h>
17
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
#include <linux/kernel_stat.h>
19
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
20 21 22 23 24 25 26 27
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/hash.h>
#include <linux/writeback.h>
28
#include <linux/backing-dev.h>
Linus Torvalds's avatar
Linus Torvalds committed
29 30 31
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/security.h>
32
#include <linux/cpuset.h>
Nick Piggin's avatar
Nick Piggin committed
33
#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
34
#include <linux/hugetlb.h>
35
#include <linux/memcontrol.h>
36
#include <linux/cleancache.h>
37
#include <linux/rmap.h>
38 39
#include "internal.h"

40 41 42
#define CREATE_TRACE_POINTS
#include <trace/events/filemap.h>

Linus Torvalds's avatar
Linus Torvalds committed
43 44 45
/*
 * FIXME: remove all knowledge of the buffer layer from the core VM
 */
46
#include <linux/buffer_head.h> /* for try_to_free_buffers */
Linus Torvalds's avatar
Linus Torvalds committed
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64

#include <asm/mman.h>

/*
 * Shared mappings implemented 30.11.1994. It's not fully working yet,
 * though.
 *
 * Shared mappings now work. 15.8.1995  Bruno.
 *
 * finished 'unifying' the page and buffer cache and SMP-threaded the
 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
 *
 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
 */

/*
 * Lock ordering:
 *
65
 *  ->i_mmap_rwsem		(truncate_pagecache)
Linus Torvalds's avatar
Linus Torvalds committed
66
 *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
67 68
 *      ->swap_lock		(exclusive_swap_page, others)
 *        ->mapping->tree_lock
Linus Torvalds's avatar
Linus Torvalds committed
69
 *
70
 *  ->i_mutex
71
 *    ->i_mmap_rwsem		(truncate->unmap_mapping_range)
Linus Torvalds's avatar
Linus Torvalds committed
72 73
 *
 *  ->mmap_sem
74
 *    ->i_mmap_rwsem
75
 *      ->page_table_lock or pte_lock	(various, mainly in memory.c)
Linus Torvalds's avatar
Linus Torvalds committed
76 77 78 79 80
 *        ->mapping->tree_lock	(arch-dependent flush_dcache_mmap_lock)
 *
 *  ->mmap_sem
 *    ->lock_page		(access_process_vm)
 *
Al Viro's avatar
Al Viro committed
81
 *  ->i_mutex			(generic_perform_write)
82
 *    ->mmap_sem		(fault_in_pages_readable->do_page_fault)
Linus Torvalds's avatar
Linus Torvalds committed
83
 *
84
 *  bdi->wb.list_lock
85
 *    sb_lock			(fs/fs-writeback.c)
Linus Torvalds's avatar
Linus Torvalds committed
86 87
 *    ->mapping->tree_lock	(__sync_single_inode)
 *
88
 *  ->i_mmap_rwsem
Linus Torvalds's avatar
Linus Torvalds committed
89 90 91
 *    ->anon_vma.lock		(vma_adjust)
 *
 *  ->anon_vma.lock
92
 *    ->page_table_lock or pte_lock	(anon_vma_prepare and various)
Linus Torvalds's avatar
Linus Torvalds committed
93
 *
94
 *  ->page_table_lock or pte_lock
95
 *    ->swap_lock		(try_to_unmap_one)
Linus Torvalds's avatar
Linus Torvalds committed
96 97 98
 *    ->private_lock		(try_to_unmap_one)
 *    ->tree_lock		(try_to_unmap_one)
 *    ->zone.lru_lock		(follow_page->mark_page_accessed)
99
 *    ->zone.lru_lock		(check_pte_range->isolate_lru_page)
Linus Torvalds's avatar
Linus Torvalds committed
100 101
 *    ->private_lock		(page_remove_rmap->set_page_dirty)
 *    ->tree_lock		(page_remove_rmap->set_page_dirty)
102
 *    bdi.wb->list_lock		(page_remove_rmap->set_page_dirty)
103
 *    ->inode->i_lock		(page_remove_rmap->set_page_dirty)
104
 *    ->memcg->move_lock	(page_remove_rmap->mem_cgroup_begin_page_stat)
105
 *    bdi.wb->list_lock		(zap_pte_range->set_page_dirty)
106
 *    ->inode->i_lock		(zap_pte_range->set_page_dirty)
Linus Torvalds's avatar
Linus Torvalds committed
107 108
 *    ->private_lock		(zap_pte_range->__set_page_dirty_buffers)
 *
109
 * ->i_mmap_rwsem
110
 *   ->tasklist_lock            (memory_failure, collect_procs_ao)
Linus Torvalds's avatar
Linus Torvalds committed
111 112
 */

113 114 115
static void page_cache_tree_delete(struct address_space *mapping,
				   struct page *page, void *shadow)
{
116 117 118 119 120
	struct radix_tree_node *node;
	unsigned long index;
	unsigned int offset;
	unsigned int tag;
	void **slot;
121

122 123 124 125 126
	VM_BUG_ON(!PageLocked(page));

	__radix_tree_lookup(&mapping->page_tree, page->index, &node, &slot);

	if (shadow) {
127
		mapping->nrexceptional++;
128
		/*
129
		 * Make sure the nrexceptional update is committed before
130 131 132 133 134
		 * the nrpages update so that final truncate racing
		 * with reclaim does not see both counters 0 at the
		 * same time and miss a shadow entry.
		 */
		smp_wmb();
135
	}
136
	mapping->nrpages--;
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173

	if (!node) {
		/* Clear direct pointer tags in root node */
		mapping->page_tree.gfp_mask &= __GFP_BITS_MASK;
		radix_tree_replace_slot(slot, shadow);
		return;
	}

	/* Clear tree tags for the removed page */
	index = page->index;
	offset = index & RADIX_TREE_MAP_MASK;
	for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {
		if (test_bit(offset, node->tags[tag]))
			radix_tree_tag_clear(&mapping->page_tree, index, tag);
	}

	/* Delete page, swap shadow entry */
	radix_tree_replace_slot(slot, shadow);
	workingset_node_pages_dec(node);
	if (shadow)
		workingset_node_shadows_inc(node);
	else
		if (__radix_tree_delete_node(&mapping->page_tree, node))
			return;

	/*
	 * Track node that only contains shadow entries.
	 *
	 * Avoid acquiring the list_lru lock if already tracked.  The
	 * list_empty() test is safe as node->private_list is
	 * protected by mapping->tree_lock.
	 */
	if (!workingset_node_pages(node) &&
	    list_empty(&node->private_list)) {
		node->private_data = mapping;
		list_lru_add(&workingset_shadow_nodes, &node->private_list);
	}
174 175
}

Linus Torvalds's avatar
Linus Torvalds committed
176
/*
177
 * Delete a page from the page cache and free it. Caller has to make
Linus Torvalds's avatar
Linus Torvalds committed
178
 * sure the page is locked and that nobody else uses it - or that usage
179 180
 * is safe.  The caller must hold the mapping's tree_lock and
 * mem_cgroup_begin_page_stat().
Linus Torvalds's avatar
Linus Torvalds committed
181
 */
182 183
void __delete_from_page_cache(struct page *page, void *shadow,
			      struct mem_cgroup *memcg)
Linus Torvalds's avatar
Linus Torvalds committed
184 185 186
{
	struct address_space *mapping = page->mapping;

187
	trace_mm_filemap_delete_from_page_cache(page);
188 189 190 191 192 193 194 195
	/*
	 * if we're uptodate, flush out into the cleancache, otherwise
	 * invalidate any existing cleancache entries.  We can't leave
	 * stale data around in the cleancache once our page is gone
	 */
	if (PageUptodate(page) && PageMappedToDisk(page))
		cleancache_put_page(page);
	else
196
		cleancache_invalidate_page(mapping, page);
197

198 199
	page_cache_tree_delete(mapping, page, shadow);

Linus Torvalds's avatar
Linus Torvalds committed
200
	page->mapping = NULL;
201
	/* Leave page->index set: truncation lookup relies upon it */
202

203 204 205
	/* hugetlb pages do not participate in page cache accounting. */
	if (!PageHuge(page))
		__dec_zone_page_state(page, NR_FILE_PAGES);
206 207
	if (PageSwapBacked(page))
		__dec_zone_page_state(page, NR_SHMEM);
208
	VM_BUG_ON_PAGE(page_mapped(page), page);
209 210

	/*
211 212
	 * At this point page must be either written or cleaned by truncate.
	 * Dirty page here signals a bug and loss of unwritten data.
213
	 *
214 215 216
	 * This fixes dirty accounting after removing the page entirely but
	 * leaves PageDirty set: it has no effect for truncated page and
	 * anyway will be cleared before returning page into buddy allocator.
217
	 */
218
	if (WARN_ON_ONCE(PageDirty(page)))
219 220
		account_page_cleaned(page, mapping, memcg,
				     inode_to_wb(mapping->host));
Linus Torvalds's avatar
Linus Torvalds committed
221 222
}

223 224 225 226 227 228 229 230 231
/**
 * delete_from_page_cache - delete page from page cache
 * @page: the page which the kernel is trying to remove from page cache
 *
 * This must be called only on pages that have been verified to be in the page
 * cache and locked.  It will never put the page into the free list, the caller
 * has a reference on the page.
 */
void delete_from_page_cache(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
232 233
{
	struct address_space *mapping = page->mapping;
234 235 236
	struct mem_cgroup *memcg;
	unsigned long flags;

237
	void (*freepage)(struct page *);
Linus Torvalds's avatar
Linus Torvalds committed
238

Matt Mackall's avatar
Matt Mackall committed
239
	BUG_ON(!PageLocked(page));
Linus Torvalds's avatar
Linus Torvalds committed
240

241
	freepage = mapping->a_ops->freepage;
242 243 244 245 246 247

	memcg = mem_cgroup_begin_page_stat(page);
	spin_lock_irqsave(&mapping->tree_lock, flags);
	__delete_from_page_cache(page, NULL, memcg);
	spin_unlock_irqrestore(&mapping->tree_lock, flags);
	mem_cgroup_end_page_stat(memcg);
248 249 250

	if (freepage)
		freepage(page);
251 252 253 254
	page_cache_release(page);
}
EXPORT_SYMBOL(delete_from_page_cache);

255 256 257 258
static int filemap_check_errors(struct address_space *mapping)
{
	int ret = 0;
	/* Check for outstanding write errors */
259 260
	if (test_bit(AS_ENOSPC, &mapping->flags) &&
	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))
261
		ret = -ENOSPC;
262 263
	if (test_bit(AS_EIO, &mapping->flags) &&
	    test_and_clear_bit(AS_EIO, &mapping->flags))
264 265 266 267
		ret = -EIO;
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
268
/**
269
 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
270 271
 * @mapping:	address space structure to write
 * @start:	offset in bytes where the range starts
272
 * @end:	offset in bytes where the range ends (inclusive)
273
 * @sync_mode:	enable synchronous operation
Linus Torvalds's avatar
Linus Torvalds committed
274
 *
275 276 277
 * Start writeback against all of a mapping's dirty pages that lie
 * within the byte offsets <start, end> inclusive.
 *
Linus Torvalds's avatar
Linus Torvalds committed
278
 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
279
 * opposed to a regular memory cleansing writeback.  The difference between
Linus Torvalds's avatar
Linus Torvalds committed
280 281 282
 * these two operations is that if a dirty page/buffer is encountered, it must
 * be waited upon, and not just skipped over.
 */
283 284
int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
				loff_t end, int sync_mode)
Linus Torvalds's avatar
Linus Torvalds committed
285 286 287 288
{
	int ret;
	struct writeback_control wbc = {
		.sync_mode = sync_mode,
289
		.nr_to_write = LONG_MAX,
290 291
		.range_start = start,
		.range_end = end,
Linus Torvalds's avatar
Linus Torvalds committed
292 293 294 295 296
	};

	if (!mapping_cap_writeback_dirty(mapping))
		return 0;

297
	wbc_attach_fdatawrite_inode(&wbc, mapping->host);
Linus Torvalds's avatar
Linus Torvalds committed
298
	ret = do_writepages(mapping, &wbc);
299
	wbc_detach_inode(&wbc);
Linus Torvalds's avatar
Linus Torvalds committed
300 301 302 303 304 305
	return ret;
}

static inline int __filemap_fdatawrite(struct address_space *mapping,
	int sync_mode)
{
306
	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
Linus Torvalds's avatar
Linus Torvalds committed
307 308 309 310 311 312 313 314
}

int filemap_fdatawrite(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite);

315
int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
316
				loff_t end)
Linus Torvalds's avatar
Linus Torvalds committed
317 318 319
{
	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
}
320
EXPORT_SYMBOL(filemap_fdatawrite_range);
Linus Torvalds's avatar
Linus Torvalds committed
321

322 323 324 325
/**
 * filemap_flush - mostly a non-blocking flush
 * @mapping:	target address_space
 *
Linus Torvalds's avatar
Linus Torvalds committed
326 327 328 329 330 331 332 333 334
 * This is a mostly non-blocking flush.  Not suitable for data-integrity
 * purposes - I/O may not be started against all dirty pages.
 */
int filemap_flush(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
}
EXPORT_SYMBOL(filemap_flush);

335 336
static int __filemap_fdatawait_range(struct address_space *mapping,
				     loff_t start_byte, loff_t end_byte)
Linus Torvalds's avatar
Linus Torvalds committed
337
{
338 339
	pgoff_t index = start_byte >> PAGE_CACHE_SHIFT;
	pgoff_t end = end_byte >> PAGE_CACHE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
340 341
	struct pagevec pvec;
	int nr_pages;
342
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
343

344
	if (end_byte < start_byte)
345
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361

	pagevec_init(&pvec, 0);
	while ((index <= end) &&
			(nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
			PAGECACHE_TAG_WRITEBACK,
			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
		unsigned i;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/* until radix tree lookup accepts end_index */
			if (page->index > end)
				continue;

			wait_on_page_writeback(page);
362
			if (TestClearPageError(page))
Linus Torvalds's avatar
Linus Torvalds committed
363 364 365 366 367
				ret = -EIO;
		}
		pagevec_release(&pvec);
		cond_resched();
	}
368
out:
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
	return ret;
}

/**
 * filemap_fdatawait_range - wait for writeback to complete
 * @mapping:		address space structure to wait for
 * @start_byte:		offset in bytes where the range starts
 * @end_byte:		offset in bytes where the range ends (inclusive)
 *
 * Walk the list of under-writeback pages of the given address space
 * in the given range and wait for all of them.  Check error status of
 * the address space and return it.
 *
 * Since the error status of the address space is cleared by this function,
 * callers are responsible for checking the return value and handling and/or
 * reporting the error.
 */
int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
			    loff_t end_byte)
{
	int ret, ret2;

	ret = __filemap_fdatawait_range(mapping, start_byte, end_byte);
392 393 394
	ret2 = filemap_check_errors(mapping);
	if (!ret)
		ret = ret2;
Linus Torvalds's avatar
Linus Torvalds committed
395 396 397

	return ret;
}
398 399
EXPORT_SYMBOL(filemap_fdatawait_range);

400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
/**
 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
 * @mapping: address space structure to wait for
 *
 * Walk the list of under-writeback pages of the given address space
 * and wait for all of them.  Unlike filemap_fdatawait(), this function
 * does not clear error status of the address space.
 *
 * Use this function if callers don't handle errors themselves.  Expected
 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
 * fsfreeze(8)
 */
void filemap_fdatawait_keep_errors(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return;

	__filemap_fdatawait_range(mapping, 0, i_size - 1);
}

Linus Torvalds's avatar
Linus Torvalds committed
422
/**
423
 * filemap_fdatawait - wait for all under-writeback pages to complete
Linus Torvalds's avatar
Linus Torvalds committed
424
 * @mapping: address space structure to wait for
425 426
 *
 * Walk the list of under-writeback pages of the given address space
427 428 429 430 431 432
 * and wait for all of them.  Check error status of the address space
 * and return it.
 *
 * Since the error status of the address space is cleared by this function,
 * callers are responsible for checking the return value and handling and/or
 * reporting the error.
Linus Torvalds's avatar
Linus Torvalds committed
433 434 435 436 437 438 439 440
 */
int filemap_fdatawait(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return 0;

441
	return filemap_fdatawait_range(mapping, 0, i_size - 1);
Linus Torvalds's avatar
Linus Torvalds committed
442 443 444 445 446
}
EXPORT_SYMBOL(filemap_fdatawait);

int filemap_write_and_wait(struct address_space *mapping)
{
447
	int err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
448 449

	if (mapping->nrpages) {
450 451 452 453 454 455 456 457 458 459 460 461
		err = filemap_fdatawrite(mapping);
		/*
		 * Even if the above returned error, the pages may be
		 * written partially (e.g. -ENOSPC), so we wait for it.
		 * But the -EIO is special case, it may indicate the worst
		 * thing (e.g. bug) happened, so we avoid waiting for it.
		 */
		if (err != -EIO) {
			int err2 = filemap_fdatawait(mapping);
			if (!err)
				err = err2;
		}
462 463
	} else {
		err = filemap_check_errors(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
464
	}
465
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
466
}
467
EXPORT_SYMBOL(filemap_write_and_wait);
Linus Torvalds's avatar
Linus Torvalds committed
468

469 470 471 472 473 474
/**
 * filemap_write_and_wait_range - write out & wait on a file range
 * @mapping:	the address_space for the pages
 * @lstart:	offset in bytes where the range starts
 * @lend:	offset in bytes where the range ends (inclusive)
 *
475 476 477 478 479
 * Write out and wait upon file offsets lstart->lend, inclusive.
 *
 * Note that `lend' is inclusive (describes the last byte to be written) so
 * that this function can be used to write to the very end-of-file (end = -1).
 */
Linus Torvalds's avatar
Linus Torvalds committed
480 481 482
int filemap_write_and_wait_range(struct address_space *mapping,
				 loff_t lstart, loff_t lend)
{
483
	int err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
484

485 486 487 488 489 490
	if (dax_mapping(mapping) && mapping->nrexceptional) {
		err = dax_writeback_mapping_range(mapping, lstart, lend);
		if (err)
			return err;
	}

Linus Torvalds's avatar
Linus Torvalds committed
491
	if (mapping->nrpages) {
492 493 494 495
		err = __filemap_fdatawrite_range(mapping, lstart, lend,
						 WB_SYNC_ALL);
		/* See comment of filemap_write_and_wait() */
		if (err != -EIO) {
496 497
			int err2 = filemap_fdatawait_range(mapping,
						lstart, lend);
498 499 500
			if (!err)
				err = err2;
		}
501 502
	} else {
		err = filemap_check_errors(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
503
	}
504
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
505
}
506
EXPORT_SYMBOL(filemap_write_and_wait_range);
Linus Torvalds's avatar
Linus Torvalds committed
507

508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
/**
 * replace_page_cache_page - replace a pagecache page with a new one
 * @old:	page to be replaced
 * @new:	page to replace with
 * @gfp_mask:	allocation mode
 *
 * This function replaces a page in the pagecache with a new one.  On
 * success it acquires the pagecache reference for the new page and
 * drops it for the old page.  Both the old and new pages must be
 * locked.  This function does not add the new page to the LRU, the
 * caller must do that.
 *
 * The remove + add is atomic.  The only way this function can fail is
 * memory allocation failure.
 */
int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
{
	int error;

527 528 529
	VM_BUG_ON_PAGE(!PageLocked(old), old);
	VM_BUG_ON_PAGE(!PageLocked(new), new);
	VM_BUG_ON_PAGE(new->mapping, new);
530 531 532 533 534

	error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
	if (!error) {
		struct address_space *mapping = old->mapping;
		void (*freepage)(struct page *);
535 536
		struct mem_cgroup *memcg;
		unsigned long flags;
537 538 539 540 541 542 543 544

		pgoff_t offset = old->index;
		freepage = mapping->a_ops->freepage;

		page_cache_get(new);
		new->mapping = mapping;
		new->index = offset;

545 546 547
		memcg = mem_cgroup_begin_page_stat(old);
		spin_lock_irqsave(&mapping->tree_lock, flags);
		__delete_from_page_cache(old, NULL, memcg);
548 549 550
		error = radix_tree_insert(&mapping->page_tree, offset, new);
		BUG_ON(error);
		mapping->nrpages++;
551 552 553 554 555 556

		/*
		 * hugetlb pages do not participate in page cache accounting.
		 */
		if (!PageHuge(new))
			__inc_zone_page_state(new, NR_FILE_PAGES);
557 558
		if (PageSwapBacked(new))
			__inc_zone_page_state(new, NR_SHMEM);
559 560
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
		mem_cgroup_end_page_stat(memcg);
561
		mem_cgroup_replace_page(old, new);
562 563 564 565 566 567 568 569 570 571
		radix_tree_preload_end();
		if (freepage)
			freepage(old);
		page_cache_release(old);
	}

	return error;
}
EXPORT_SYMBOL_GPL(replace_page_cache_page);

572
static int page_cache_tree_insert(struct address_space *mapping,
573
				  struct page *page, void **shadowp)
574
{
575
	struct radix_tree_node *node;
576 577 578
	void **slot;
	int error;

579 580 581 582 583
	error = __radix_tree_create(&mapping->page_tree, page->index,
				    &node, &slot);
	if (error)
		return error;
	if (*slot) {
584 585 586 587 588
		void *p;

		p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
		if (!radix_tree_exceptional_entry(p))
			return -EEXIST;
589 590 591 592

		if (WARN_ON(dax_mapping(mapping)))
			return -EINVAL;

593 594
		if (shadowp)
			*shadowp = p;
595
		mapping->nrexceptional--;
596 597
		if (node)
			workingset_node_shadows_dec(node);
598
	}
599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
	radix_tree_replace_slot(slot, page);
	mapping->nrpages++;
	if (node) {
		workingset_node_pages_inc(node);
		/*
		 * Don't track node that contains actual pages.
		 *
		 * Avoid acquiring the list_lru lock if already
		 * untracked.  The list_empty() test is safe as
		 * node->private_list is protected by
		 * mapping->tree_lock.
		 */
		if (!list_empty(&node->private_list))
			list_lru_del(&workingset_shadow_nodes,
				     &node->private_list);
	}
	return 0;
616 617
}

618 619 620 621
static int __add_to_page_cache_locked(struct page *page,
				      struct address_space *mapping,
				      pgoff_t offset, gfp_t gfp_mask,
				      void **shadowp)
Linus Torvalds's avatar
Linus Torvalds committed
622
{
623 624
	int huge = PageHuge(page);
	struct mem_cgroup *memcg;
625 626
	int error;

627 628
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(PageSwapBacked(page), page);
629

630 631
	if (!huge) {
		error = mem_cgroup_try_charge(page, current->mm,
632
					      gfp_mask, &memcg, false);
633 634 635
		if (error)
			return error;
	}
Linus Torvalds's avatar
Linus Torvalds committed
636

637
	error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
638
	if (error) {
639
		if (!huge)
640
			mem_cgroup_cancel_charge(page, memcg, false);
641 642 643 644 645 646 647 648
		return error;
	}

	page_cache_get(page);
	page->mapping = mapping;
	page->index = offset;

	spin_lock_irq(&mapping->tree_lock);
649
	error = page_cache_tree_insert(mapping, page, shadowp);
650 651 652
	radix_tree_preload_end();
	if (unlikely(error))
		goto err_insert;
653 654 655 656

	/* hugetlb pages do not participate in page cache accounting. */
	if (!huge)
		__inc_zone_page_state(page, NR_FILE_PAGES);
657
	spin_unlock_irq(&mapping->tree_lock);
658
	if (!huge)
659
		mem_cgroup_commit_charge(page, memcg, false, false);
660 661 662 663 664 665
	trace_mm_filemap_add_to_page_cache(page);
	return 0;
err_insert:
	page->mapping = NULL;
	/* Leave page->index set: truncation relies upon it */
	spin_unlock_irq(&mapping->tree_lock);
666
	if (!huge)
667
		mem_cgroup_cancel_charge(page, memcg, false);
668
	page_cache_release(page);
Linus Torvalds's avatar
Linus Torvalds committed
669 670
	return error;
}
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687

/**
 * add_to_page_cache_locked - add a locked page to the pagecache
 * @page:	page to add
 * @mapping:	the page's address_space
 * @offset:	page index
 * @gfp_mask:	page allocation mode
 *
 * This function is used to add a page to the pagecache. It must be locked.
 * This function does not add the page to the LRU.  The caller must do that.
 */
int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
		pgoff_t offset, gfp_t gfp_mask)
{
	return __add_to_page_cache_locked(page, mapping, offset,
					  gfp_mask, NULL);
}
688
EXPORT_SYMBOL(add_to_page_cache_locked);
Linus Torvalds's avatar
Linus Torvalds committed
689 690

int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
Al Viro's avatar
Al Viro committed
691
				pgoff_t offset, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
692
{
693
	void *shadow = NULL;
694 695
	int ret;

696
	__SetPageLocked(page);
697 698 699
	ret = __add_to_page_cache_locked(page, mapping, offset,
					 gfp_mask, &shadow);
	if (unlikely(ret))
700
		__ClearPageLocked(page);
701 702 703 704 705 706 707 708 709 710 711 712 713
	else {
		/*
		 * The page might have been evicted from cache only
		 * recently, in which case it should be activated like
		 * any other repeatedly accessed page.
		 */
		if (shadow && workingset_refault(shadow)) {
			SetPageActive(page);
			workingset_activation(page);
		} else
			ClearPageActive(page);
		lru_cache_add(page);
	}
Linus Torvalds's avatar
Linus Torvalds committed
714 715
	return ret;
}
716
EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
Linus Torvalds's avatar
Linus Torvalds committed
717

718
#ifdef CONFIG_NUMA
719
struct page *__page_cache_alloc(gfp_t gfp)
720
{
721 722 723
	int n;
	struct page *page;

724
	if (cpuset_do_page_mem_spread()) {
725 726
		unsigned int cpuset_mems_cookie;
		do {
727
			cpuset_mems_cookie = read_mems_allowed_begin();
728
			n = cpuset_mem_spread_node();
729
			page = __alloc_pages_node(n, gfp, 0);
730
		} while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
731

732
		return page;
733
	}
734
	return alloc_pages(gfp, 0);
735
}
736
EXPORT_SYMBOL(__page_cache_alloc);
737 738
#endif

Linus Torvalds's avatar
Linus Torvalds committed
739 740 741 742 743 744 745 746 747 748
/*
 * In order to wait for pages to become available there must be
 * waitqueues associated with pages. By using a hash table of
 * waitqueues where the bucket discipline is to maintain all
 * waiters on the same queue and wake all when any of the pages
 * become available, and for the woken contexts to check to be
 * sure the appropriate page became available, this saves space
 * at a cost of "thundering herd" phenomena during rare hash
 * collisions.
 */
NeilBrown's avatar
NeilBrown committed
749
wait_queue_head_t *page_waitqueue(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
750 751 752 753 754
{
	const struct zone *zone = page_zone(page);

	return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
}
NeilBrown's avatar
NeilBrown committed
755
EXPORT_SYMBOL(page_waitqueue);
Linus Torvalds's avatar
Linus Torvalds committed
756

757
void wait_on_page_bit(struct page *page, int bit_nr)
Linus Torvalds's avatar
Linus Torvalds committed
758 759 760 761
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (test_bit(bit_nr, &page->flags))
762
		__wait_on_bit(page_waitqueue(page), &wait, bit_wait_io,
Linus Torvalds's avatar
Linus Torvalds committed
763 764 765 766
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(wait_on_page_bit);

767 768 769 770 771 772 773 774
int wait_on_page_bit_killable(struct page *page, int bit_nr)
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (!test_bit(bit_nr, &page->flags))
		return 0;

	return __wait_on_bit(page_waitqueue(page), &wait,
775
			     bit_wait_io, TASK_KILLABLE);
776 777
}

778 779 780 781 782 783 784 785 786 787 788 789 790
int wait_on_page_bit_killable_timeout(struct page *page,
				       int bit_nr, unsigned long timeout)
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	wait.key.timeout = jiffies + timeout;
	if (!test_bit(bit_nr, &page->flags))
		return 0;
	return __wait_on_bit(page_waitqueue(page), &wait,
			     bit_wait_io_timeout, TASK_KILLABLE);
}
EXPORT_SYMBOL_GPL(wait_on_page_bit_killable_timeout);

791 792
/**
 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
793 794
 * @page: Page defining the wait queue of interest
 * @waiter: Waiter to add to the queue
795 796 797 798 799 800 801 802 803 804 805 806 807 808
 *
 * Add an arbitrary @waiter to the wait queue for the nominated @page.
 */
void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
{
	wait_queue_head_t *q = page_waitqueue(page);
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, waiter);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(add_page_wait_queue);

Linus Torvalds's avatar
Linus Torvalds committed
809
/**
810
 * unlock_page - unlock a locked page
Linus Torvalds's avatar
Linus Torvalds committed
811 812 813 814
 * @page: the page
 *
 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
815
 * mechanism between PageLocked pages and PageWriteback pages is shared.
Linus Torvalds's avatar
Linus Torvalds committed
816 817
 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
 *
Nick Piggin's avatar
Nick Piggin committed
818 819
 * The mb is necessary to enforce ordering between the clear_bit and the read
 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
Linus Torvalds's avatar
Linus Torvalds committed
820
 */
821
void unlock_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
822
{
823
	page = compound_head(page);
824
	VM_BUG_ON_PAGE(!PageLocked(page), page);
Nick Piggin's avatar
Nick Piggin committed
825
	clear_bit_unlock(PG_locked, &page->flags);
826
	smp_mb__after_atomic();
Linus Torvalds's avatar
Linus Torvalds committed
827 828 829 830
	wake_up_page(page, PG_locked);
}
EXPORT_SYMBOL(unlock_page);

831 832 833
/**
 * end_page_writeback - end writeback against a page
 * @page: the page
Linus Torvalds's avatar
Linus Torvalds committed
834 835 836
 */
void end_page_writeback(struct page *page)
{
837 838 839 840 841 842 843 844 845
	/*
	 * TestClearPageReclaim could be used here but it is an atomic
	 * operation and overkill in this particular case. Failing to
	 * shuffle a page marked for immediate reclaim is too mild to
	 * justify taking an atomic operation penalty at the end of
	 * ever page writeback.
	 */
	if (PageReclaim(page)) {
		ClearPageReclaim(page);
846
		rotate_reclaimable_page(page);
847
	}
848 849 850 851

	if (!test_clear_page_writeback(page))
		BUG();

852
	smp_mb__after_atomic();
Linus Torvalds's avatar
Linus Torvalds committed
853 854 855 856
	wake_up_page(page, PG_writeback);
}
EXPORT_SYMBOL(end_page_writeback);

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
/*
 * After completing I/O on a page, call this routine to update the page
 * flags appropriately
 */
void page_endio(struct page *page, int rw, int err)
{
	if (rw == READ) {
		if (!err) {
			SetPageUptodate(page);
		} else {
			ClearPageUptodate(page);
			SetPageError(page);
		}
		unlock_page(page);
	} else { /* rw == WRITE */
		if (err) {
			SetPageError(page);
			if (page->mapping)
				mapping_set_error(page->mapping, err);
		}
		end_page_writeback(page);
	}
}
EXPORT_SYMBOL_GPL(page_endio);

882 883 884
/**
 * __lock_page - get a lock on the page, assuming we need to sleep to get it
 * @page: the page to lock
Linus Torvalds's avatar
Linus Torvalds committed
885
 */
886
void __lock_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
887
{
888 889
	struct page *page_head = compound_head(page);
	DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
Linus Torvalds's avatar
Linus Torvalds committed
890

891
	__wait_on_bit_lock(page_waitqueue(page_head), &wait, bit_wait_io,
Linus Torvalds's avatar
Linus Torvalds committed
892 893 894 895
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__lock_page);

896
int __lock_page_killable(struct page *page)
Matthew Wilcox's avatar
Matthew Wilcox committed
897
{
898 899
	struct page *page_head = compound_head(page);
	DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
Matthew Wilcox's avatar
Matthew Wilcox committed
900

901
	return __wait_on_bit_lock(page_waitqueue(page_head), &wait,
902
					bit_wait_io, TASK_KILLABLE);
Matthew Wilcox's avatar
Matthew Wilcox committed
903
}
904
EXPORT_SYMBOL_GPL(__lock_page_killable);
Matthew Wilcox's avatar
Matthew Wilcox committed
905

906 907 908 909 910 911 912 913 914 915 916
/*
 * Return values:
 * 1 - page is locked; mmap_sem is still held.
 * 0 - page is not locked.
 *     mmap_sem has been released (up_read()), unless flags had both
 *     FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
 *     which case mmap_sem is still held.
 *
 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
 * with the page locked and the mmap_sem unperturbed.
 */
917 918 919
int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
			 unsigned int flags)
{
920 921 922 923 924 925 926 927 928 929 930 931
	if (flags & FAULT_FLAG_ALLOW_RETRY) {
		/*
		 * CAUTION! In this case, mmap_sem is not released
		 * even though return 0.
		 */
		if (flags & FAULT_FLAG_RETRY_NOWAIT)
			return 0;

		up_read(&mm->mmap_sem);
		if (flags & FAULT_FLAG_KILLABLE)
			wait_on_page_locked_killable(page);
		else
932
			wait_on_page_locked(page);
933
		return 0;
934 935 936 937 938 939 940 941 942 943 944 945
	} else {
		if (flags & FAULT_FLAG_KILLABLE) {
			int ret;

			ret = __lock_page_killable(page);
			if (ret) {
				up_read(&mm->mmap_sem);
				return 0;
			}
		} else
			__lock_page(page);
		return 1;
946 947 948
	}
}

949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975
/**
 * page_cache_next_hole - find the next hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
 * lowest indexed hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'return - index >=
 * max_scan' will be true). In rare cases of index wrap-around, 0 will
 * be returned.
 *
 * page_cache_next_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 5, then subsequently a hole is created at
 * index 10, page_cache_next_hole covering both indexes may return 10
 * if called under rcu_read_lock.
 */
pgoff_t page_cache_next_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
976 977 978 979
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
			break;
		index++;
		if (index == 0)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_next_hole);

/**
 * page_cache_prev_hole - find the prev hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search backwards in the range [max(index-max_scan+1, 0), index] for
 * the first hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'index - return >=
 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
 * will be returned.
 *
 * page_cache_prev_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 10, then subsequently a hole is created at
 * index 5, page_cache_prev_hole covering both indexes may return 5 if
 * called under rcu_read_lock.
 */
pgoff_t page_cache_prev_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
1017 1018 1019 1020
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
			break;
		index--;
		if (index == ULONG_MAX)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_prev_hole);

1031
/**
1032
 * find_get_entry - find and get a page cache entry
1033
 * @mapping: the address_space to search
1034 1035 1036 1037
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned with an increased refcount.
1038
 *
1039 1040
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
1041 1042
 *
 * Otherwise, %NULL is returned.
Linus Torvalds's avatar
Linus Torvalds committed
1043
 */
1044
struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds's avatar
Linus Torvalds committed
1045
{
Nick Piggin's avatar
Nick Piggin committed
1046
	void **pagep;
Linus Torvalds's avatar
Linus Torvalds committed
1047 1048
	struct page *page;

Nick Piggin's avatar
Nick Piggin committed
1049 1050 1051 1052 1053 1054
	rcu_read_lock();
repeat:
	page = NULL;
	pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
	if (pagep) {
		page = radix_tree_deref_slot(pagep);
Nick Piggin's avatar
Nick Piggin committed
1055 1056
		if (unlikely(!page))
			goto out;
1057
		if (radix_tree_exception(page)) {
1058 1059 1060
			if (radix_tree_deref_retry(page))
				goto repeat;
			/*
1061 1062 1063
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Return
			 * it without attempting to raise page count.
1064 1065
			 */
			goto out;
1066
		}
Nick Piggin's avatar
Nick Piggin committed
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
		if (!page_cache_get_speculative(page))
			goto repeat;

		/*
		 * Has the page moved?
		 * This is part of the lockless pagecache protocol. See
		 * include/linux/pagemap.h for details.
		 */
		if (unlikely(page != *pagep)) {
			page_cache_release(page);
			goto repeat;
		}
	}
Nick Piggin's avatar
Nick Piggin committed
1080
out:
Nick Piggin's avatar
Nick Piggin committed
1081 1082
	rcu_read_unlock();

Linus Torvalds's avatar
Linus Torvalds committed
1083 1084
	return page;
}
1085
EXPORT_SYMBOL(find_get_entry);
Linus Torvalds's avatar
Linus Torvalds committed
1086

1087 1088 1089 1090 1091 1092 1093 1094 1095
/**
 * find_lock_entry - locate, pin and lock a page cache entry
 * @mapping: the address_space to search
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned locked and with an increased
 * refcount.
 *
1096 1097
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
1098 1099 1100 1101 1102 1103
 *
 * Otherwise, %NULL is returned.
 *
 * find_lock_entry() may sleep.
 */
struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds's avatar
Linus Torvalds committed
1104 1105 1106 1107
{
	struct page *page;

repeat:
1108
	page = find_get_entry(mapping, offset);
1109
	if (page && !radix_tree_exception(page)) {
Nick Piggin's avatar
Nick Piggin committed
1110 1111 1112 1113 1114 1115
		lock_page(page);
		/* Has the page been truncated? */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
			page_cache_release(page);
			goto repeat;
Linus Torvalds's avatar
Linus Torvalds committed
1116
		}
1117
		VM_BUG_ON_PAGE(page->index != offset, page);
Linus Torvalds's avatar
Linus Torvalds committed
1118 1119 1120
	}
	return page;
}
1121 1122 1123
EXPORT_SYMBOL(find_lock_entry);

/**
1124
 * pagecache_get_page - find and get a page reference
1125 1126
 * @mapping: the address_space to search
 * @offset: the page index
1127
 * @fgp_flags: PCG flags
1128
 * @gfp_mask: gfp mask to use for the page cache data page allocation
1129
 *
1130
 * Looks up the page cache slot at @mapping & @offset.
Linus Torvalds's avatar
Linus Torvalds committed
1131
 *
1132
 * PCG flags modify how the page is returned.
1133
 *
1134 1135 1136
 * FGP_ACCESSED: the page will be marked accessed
 * FGP_LOCK: Page is return locked
 * FGP_CREAT: If page is not present then a new page is allocated using
1137 1138 1139
 *		@gfp_mask and added to the page cache and the VM's LRU
 *		list. The page is returned locked and with an increased
 *		refcount. Otherwise, %NULL is returned.
Linus Torvalds's avatar
Linus Torvalds committed
1140
 *
1141 1142
 * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
 * if the GFP flags specified for FGP_CREAT are atomic.
Linus Torvalds's avatar
Linus Torvalds committed
1143
 *
1144
 * If there is a page cache page, it is returned with an increased refcount.
Linus Torvalds's avatar
Linus Torvalds committed
1145
 */
1146
struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
1147
	int fgp_flags, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1148
{
Nick Piggin's avatar
Nick Piggin committed
1149
	struct page *page;
1150

Linus Torvalds's avatar
Linus Torvalds committed
1151
repeat:
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
	page = find_get_entry(mapping, offset);
	if (radix_tree_exceptional_entry(page))
		page = NULL;
	if (!page)
		goto no_page;

	if (fgp_flags & FGP_LOCK) {
		if (fgp_flags & FGP_NOWAIT) {
			if (!trylock_page(page)) {
				page_cache_release(page);
				return NULL;
			}
		} else {
			lock_page(page);
		}

		/* Has the page been truncated? */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
			page_cache_release(page);
			goto repeat;
		}
		VM_BUG_ON_PAGE(page->index != offset, page);
	}

	if (page && (fgp_flags & FGP_ACCESSED))
		mark_page_accessed(page);

no_page:
	if (!page && (fgp_flags & FGP_CREAT)) {
		int err;
		if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
1184 1185 1186
			gfp_mask |= __GFP_WRITE;
		if (fgp_flags & FGP_NOFS)
			gfp_mask &= ~__GFP_FS;
1187

1188
		page = __page_cache_alloc(gfp_mask);
Nick Piggin's avatar
Nick Piggin committed
1189 1190
		if (!page)
			return NULL;
1191 1192 1193 1194

		if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
			fgp_flags |= FGP_LOCK;

1195
		/* Init accessed so avoid atomic mark_page_accessed later */
1196
		if (fgp_flags & FGP_ACCESSED)
1197
			__SetPageReferenced(page);
1198

1199 1200
		err = add_to_page_cache_lru(page, mapping, offset,
				gfp_mask & GFP_RECLAIM_MASK);
Nick Piggin's avatar
Nick Piggin committed
1201 1202 1203 1204 1205
		if (unlikely(err)) {
			page_cache_release(page);
			page = NULL;
			if (err == -EEXIST)
				goto repeat;
Linus Torvalds's avatar
Linus Torvalds committed
1206 1207
		}
	}
1208

Linus Torvalds's avatar
Linus Torvalds committed
1209 1210
	return page;
}
1211
EXPORT_SYMBOL(pagecache_get_page);
Linus Torvalds's avatar
Linus Torvalds committed
1212

1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229
/**
 * find_get_entries - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page cache index
 * @nr_entries:	The maximum number of entries
 * @entries:	Where the resulting entries are placed
 * @indices:	The cache indices corresponding to the entries in @entries
 *
 * find_get_entries() will search for and return a group of up to
 * @nr_entries entries in the mapping.  The entries are placed at
 * @entries.  find_get_entries() takes a reference against any actual
 * pages it returns.
 *
 * The search returns a group of mapping-contiguous page cache entries
 * with ascending indexes.  There may be holes in the indices due to
 * not-present pages.
 *
1230 1231
 * Any shadow entries of evicted pages, or swap entries from
 * shmem/tmpfs, are included in the returned array.
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
 *
 * find_get_entries() returns the number of pages and shadow entries
 * which were found.
 */
unsigned find_get_entries(struct address_space *mapping,
			  pgoff_t start, unsigned int nr_entries,
			  struct page **entries, pgoff_t *indices)
{
	void **slot;
	unsigned int ret = 0;
	struct radix_tree_iter iter;

	if (!nr_entries)
		return 0;

	rcu_read_lock();
restart:
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
		struct page *page;
repeat:
		page = radix_tree_deref_slot(slot);
		if (unlikely(!page))
			continue;
		if (radix_tree_exception(page)) {
			if (radix_tree_deref_retry(page))
				goto restart;
			/*
1259 1260 1261
			 * A shadow entry of a recently evicted page, a swap
			 * entry from shmem/tmpfs or a DAX entry.  Return it
			 * without attempting to raise page count.
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
			 */
			goto export;
		}
		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
		if (unlikely(page != *slot)) {
			page_cache_release(page);
			goto repeat;
		}
export:
		indices[ret] = iter.index;
		entries[ret] = page;
		if (++ret == nr_entries)
			break;
	}
	rcu_read_unlock();
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
/**
 * find_get_pages - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages() will search for and return a group of up to
 * @nr_pages pages in the mapping.  The pages are placed at @pages.
 * find_get_pages() takes a reference against the returned pages.
 *
 * The search returns a group of mapping-contiguous pages with ascending
 * indexes.  There may be holes in the indices due to not-present pages.
 *
 * find_get_pages() returns the number of pages which were found.
 */
unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
			    unsigned int nr_pages, struct page **pages)
{
1302 1303 1304 1305 1306 1307
	struct radix_tree_iter iter;
	void **slot;
	unsigned ret = 0;

	if (unlikely(!nr_pages))
		return 0;
Nick Piggin's avatar
Nick Piggin committed
1308 1309 1310

	rcu_read_lock();
restart:
1311
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
Nick Piggin's avatar
Nick Piggin committed
1312 1313
		struct page *page;
repeat:
1314
		page = radix_tree_deref_slot(slot);
Nick Piggin's avatar
Nick Piggin committed
1315 1316
		if (unlikely(!page))
			continue;
1317

1318
		if (radix_tree_exception(page)) {
1319 1320 1321 1322 1323 1324
			if (radix_tree_deref_retry(page)) {
				/*
				 * Transient condition which can only trigger
				 * when entry at index 0 moves out of or back
				 * to root: none yet gotten, safe to restart.
				 */
1325
				WARN_ON(iter.index);
1326 1327
				goto restart;
			}
1328
			/*
1329 1330 1331
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Skip
			 * over it.
1332
			 */
1333
			continue;
Nick Piggin's avatar
Nick Piggin committed
1334
		}
Nick Piggin's avatar
Nick Piggin committed
1335 1336 1337 1338 1339

		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
1340
		if (unlikely(page != *slot)) {
Nick Piggin's avatar
Nick Piggin committed
1341 1342 1343
			page_cache_release(page);
			goto repeat;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1344

Nick Piggin's avatar
Nick Piggin committed
1345
		pages[ret] = page;
1346 1347
		if (++ret == nr_pages)
			break;
Nick Piggin's avatar
Nick Piggin committed
1348
	}
1349

Nick Piggin's avatar
Nick Piggin committed
1350
	rcu_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
1351 1352 1353
	return ret;
}

1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
/**
 * find_get_pages_contig - gang contiguous pagecache lookup
 * @mapping:	The address_space to search
 * @index:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages_contig() works exactly like find_get_pages(), except
 * that the returned number of pages are guaranteed to be contiguous.
 *
 * find_get_pages_contig() returns the number of pages which were found.
 */
unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
			       unsigned int nr_pages, struct page **pages)
{
1369 1370 1371 1372 1373 1374
	struct radix_tree_iter iter;
	void **slot;
	unsigned int ret = 0;

	if (unlikely(!nr_pages))
		return 0;