filemap.c 75.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11
/*
 *	linux/mm/filemap.c
 *
 * Copyright (C) 1994-1999  Linus Torvalds
 */

/*
 * This file handles the generic file mmap semantics used by
 * most "normal" filesystems (but you don't /have/ to use this:
 * the NFS filesystem used to do this differently, for example)
 */
12
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <linux/compiler.h>
14
#include <linux/dax.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/fs.h>
16
#include <linux/uaccess.h>
17
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
#include <linux/kernel_stat.h>
19
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
20 21 22 23 24 25 26 27
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/hash.h>
#include <linux/writeback.h>
28
#include <linux/backing-dev.h>
Linus Torvalds's avatar
Linus Torvalds committed
29 30 31
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/security.h>
32
#include <linux/cpuset.h>
33
#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
34
#include <linux/hugetlb.h>
35
#include <linux/memcontrol.h>
36
#include <linux/cleancache.h>
37
#include <linux/rmap.h>
38 39
#include "internal.h"

40 41 42
#define CREATE_TRACE_POINTS
#include <trace/events/filemap.h>

Linus Torvalds's avatar
Linus Torvalds committed
43 44 45
/*
 * FIXME: remove all knowledge of the buffer layer from the core VM
 */
46
#include <linux/buffer_head.h> /* for try_to_free_buffers */
Linus Torvalds's avatar
Linus Torvalds committed
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64

#include <asm/mman.h>

/*
 * Shared mappings implemented 30.11.1994. It's not fully working yet,
 * though.
 *
 * Shared mappings now work. 15.8.1995  Bruno.
 *
 * finished 'unifying' the page and buffer cache and SMP-threaded the
 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
 *
 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
 */

/*
 * Lock ordering:
 *
65
 *  ->i_mmap_rwsem		(truncate_pagecache)
Linus Torvalds's avatar
Linus Torvalds committed
66
 *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
67 68
 *      ->swap_lock		(exclusive_swap_page, others)
 *        ->mapping->tree_lock
Linus Torvalds's avatar
Linus Torvalds committed
69
 *
70
 *  ->i_mutex
71
 *    ->i_mmap_rwsem		(truncate->unmap_mapping_range)
Linus Torvalds's avatar
Linus Torvalds committed
72 73
 *
 *  ->mmap_sem
74
 *    ->i_mmap_rwsem
75
 *      ->page_table_lock or pte_lock	(various, mainly in memory.c)
Linus Torvalds's avatar
Linus Torvalds committed
76 77 78 79 80
 *        ->mapping->tree_lock	(arch-dependent flush_dcache_mmap_lock)
 *
 *  ->mmap_sem
 *    ->lock_page		(access_process_vm)
 *
81
 *  ->i_mutex			(generic_perform_write)
82
 *    ->mmap_sem		(fault_in_pages_readable->do_page_fault)
Linus Torvalds's avatar
Linus Torvalds committed
83
 *
84
 *  bdi->wb.list_lock
85
 *    sb_lock			(fs/fs-writeback.c)
Linus Torvalds's avatar
Linus Torvalds committed
86 87
 *    ->mapping->tree_lock	(__sync_single_inode)
 *
88
 *  ->i_mmap_rwsem
Linus Torvalds's avatar
Linus Torvalds committed
89 90 91
 *    ->anon_vma.lock		(vma_adjust)
 *
 *  ->anon_vma.lock
92
 *    ->page_table_lock or pte_lock	(anon_vma_prepare and various)
Linus Torvalds's avatar
Linus Torvalds committed
93
 *
94
 *  ->page_table_lock or pte_lock
95
 *    ->swap_lock		(try_to_unmap_one)
Linus Torvalds's avatar
Linus Torvalds committed
96 97 98
 *    ->private_lock		(try_to_unmap_one)
 *    ->tree_lock		(try_to_unmap_one)
 *    ->zone.lru_lock		(follow_page->mark_page_accessed)
99
 *    ->zone.lru_lock		(check_pte_range->isolate_lru_page)
Linus Torvalds's avatar
Linus Torvalds committed
100 101
 *    ->private_lock		(page_remove_rmap->set_page_dirty)
 *    ->tree_lock		(page_remove_rmap->set_page_dirty)
102
 *    bdi.wb->list_lock		(page_remove_rmap->set_page_dirty)
103
 *    ->inode->i_lock		(page_remove_rmap->set_page_dirty)
104
 *    ->memcg->move_lock	(page_remove_rmap->lock_page_memcg)
105
 *    bdi.wb->list_lock		(zap_pte_range->set_page_dirty)
106
 *    ->inode->i_lock		(zap_pte_range->set_page_dirty)
Linus Torvalds's avatar
Linus Torvalds committed
107 108
 *    ->private_lock		(zap_pte_range->__set_page_dirty_buffers)
 *
109
 * ->i_mmap_rwsem
110
 *   ->tasklist_lock            (memory_failure, collect_procs_ao)
Linus Torvalds's avatar
Linus Torvalds committed
111 112
 */

113 114 115
static void page_cache_tree_delete(struct address_space *mapping,
				   struct page *page, void *shadow)
{
116
	struct radix_tree_node *node;
117

118 119
	VM_BUG_ON(!PageLocked(page));

120 121
	node = radix_tree_replace_clear_tags(&mapping->page_tree, page->index,
								shadow);
122 123

	if (shadow) {
124
		mapping->nrexceptional++;
125
		/*
126
		 * Make sure the nrexceptional update is committed before
127 128 129 130 131
		 * the nrpages update so that final truncate racing
		 * with reclaim does not see both counters 0 at the
		 * same time and miss a shadow entry.
		 */
		smp_wmb();
132
	}
133
	mapping->nrpages--;
134

135
	if (!node)
136 137 138 139 140 141 142 143 144 145
		return;

	workingset_node_pages_dec(node);
	if (shadow)
		workingset_node_shadows_inc(node);
	else
		if (__radix_tree_delete_node(&mapping->page_tree, node))
			return;

	/*
Jan Kara's avatar
Jan Kara committed
146 147 148
	 * Track node that only contains shadow entries. DAX mappings contain
	 * no shadow entries and may contain other exceptional entries so skip
	 * those.
149 150 151 152 153
	 *
	 * Avoid acquiring the list_lru lock if already tracked.  The
	 * list_empty() test is safe as node->private_list is
	 * protected by mapping->tree_lock.
	 */
Jan Kara's avatar
Jan Kara committed
154
	if (!dax_mapping(mapping) && !workingset_node_pages(node) &&
155 156 157 158
	    list_empty(&node->private_list)) {
		node->private_data = mapping;
		list_lru_add(&workingset_shadow_nodes, &node->private_list);
	}
159 160
}

Linus Torvalds's avatar
Linus Torvalds committed
161
/*
162
 * Delete a page from the page cache and free it. Caller has to make
Linus Torvalds's avatar
Linus Torvalds committed
163
 * sure the page is locked and that nobody else uses it - or that usage
164
 * is safe.  The caller must hold the mapping's tree_lock.
Linus Torvalds's avatar
Linus Torvalds committed
165
 */
166
void __delete_from_page_cache(struct page *page, void *shadow)
Linus Torvalds's avatar
Linus Torvalds committed
167 168 169
{
	struct address_space *mapping = page->mapping;

170
	trace_mm_filemap_delete_from_page_cache(page);
171 172 173 174 175 176 177 178
	/*
	 * if we're uptodate, flush out into the cleancache, otherwise
	 * invalidate any existing cleancache entries.  We can't leave
	 * stale data around in the cleancache once our page is gone
	 */
	if (PageUptodate(page) && PageMappedToDisk(page))
		cleancache_put_page(page);
	else
179
		cleancache_invalidate_page(mapping, page);
180

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
	VM_BUG_ON_PAGE(page_mapped(page), page);
	if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
		int mapcount;

		pr_alert("BUG: Bad page cache in process %s  pfn:%05lx\n",
			 current->comm, page_to_pfn(page));
		dump_page(page, "still mapped when deleted");
		dump_stack();
		add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);

		mapcount = page_mapcount(page);
		if (mapping_exiting(mapping) &&
		    page_count(page) >= mapcount + 2) {
			/*
			 * All vmas have already been torn down, so it's
			 * a good bet that actually the page is unmapped,
			 * and we'd prefer not to leak it: if we're wrong,
			 * some other bad page check should catch it later.
			 */
			page_mapcount_reset(page);
201
			page_ref_sub(page, mapcount);
202 203 204
		}
	}

205 206
	page_cache_tree_delete(mapping, page, shadow);

Linus Torvalds's avatar
Linus Torvalds committed
207
	page->mapping = NULL;
208
	/* Leave page->index set: truncation lookup relies upon it */
209

210 211 212
	/* hugetlb pages do not participate in page cache accounting. */
	if (!PageHuge(page))
		__dec_zone_page_state(page, NR_FILE_PAGES);
213 214
	if (PageSwapBacked(page))
		__dec_zone_page_state(page, NR_SHMEM);
215 216

	/*
217 218
	 * At this point page must be either written or cleaned by truncate.
	 * Dirty page here signals a bug and loss of unwritten data.
219
	 *
220 221 222
	 * This fixes dirty accounting after removing the page entirely but
	 * leaves PageDirty set: it has no effect for truncated page and
	 * anyway will be cleared before returning page into buddy allocator.
223
	 */
224
	if (WARN_ON_ONCE(PageDirty(page)))
225
		account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
Linus Torvalds's avatar
Linus Torvalds committed
226 227
}

228 229 230 231 232 233 234 235 236
/**
 * delete_from_page_cache - delete page from page cache
 * @page: the page which the kernel is trying to remove from page cache
 *
 * This must be called only on pages that have been verified to be in the page
 * cache and locked.  It will never put the page into the free list, the caller
 * has a reference on the page.
 */
void delete_from_page_cache(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
237 238
{
	struct address_space *mapping = page->mapping;
239 240
	unsigned long flags;

241
	void (*freepage)(struct page *);
Linus Torvalds's avatar
Linus Torvalds committed
242

243
	BUG_ON(!PageLocked(page));
Linus Torvalds's avatar
Linus Torvalds committed
244

245
	freepage = mapping->a_ops->freepage;
246 247

	spin_lock_irqsave(&mapping->tree_lock, flags);
248
	__delete_from_page_cache(page, NULL);
249
	spin_unlock_irqrestore(&mapping->tree_lock, flags);
250 251 252

	if (freepage)
		freepage(page);
253
	put_page(page);
254 255 256
}
EXPORT_SYMBOL(delete_from_page_cache);

257 258 259 260
static int filemap_check_errors(struct address_space *mapping)
{
	int ret = 0;
	/* Check for outstanding write errors */
261 262
	if (test_bit(AS_ENOSPC, &mapping->flags) &&
	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))
263
		ret = -ENOSPC;
264 265
	if (test_bit(AS_EIO, &mapping->flags) &&
	    test_and_clear_bit(AS_EIO, &mapping->flags))
266 267 268 269
		ret = -EIO;
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
270
/**
271
 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
272 273
 * @mapping:	address space structure to write
 * @start:	offset in bytes where the range starts
274
 * @end:	offset in bytes where the range ends (inclusive)
275
 * @sync_mode:	enable synchronous operation
Linus Torvalds's avatar
Linus Torvalds committed
276
 *
277 278 279
 * Start writeback against all of a mapping's dirty pages that lie
 * within the byte offsets <start, end> inclusive.
 *
Linus Torvalds's avatar
Linus Torvalds committed
280
 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
281
 * opposed to a regular memory cleansing writeback.  The difference between
Linus Torvalds's avatar
Linus Torvalds committed
282 283 284
 * these two operations is that if a dirty page/buffer is encountered, it must
 * be waited upon, and not just skipped over.
 */
285 286
int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
				loff_t end, int sync_mode)
Linus Torvalds's avatar
Linus Torvalds committed
287 288 289 290
{
	int ret;
	struct writeback_control wbc = {
		.sync_mode = sync_mode,
291
		.nr_to_write = LONG_MAX,
292 293
		.range_start = start,
		.range_end = end,
Linus Torvalds's avatar
Linus Torvalds committed
294 295 296 297 298
	};

	if (!mapping_cap_writeback_dirty(mapping))
		return 0;

299
	wbc_attach_fdatawrite_inode(&wbc, mapping->host);
Linus Torvalds's avatar
Linus Torvalds committed
300
	ret = do_writepages(mapping, &wbc);
301
	wbc_detach_inode(&wbc);
Linus Torvalds's avatar
Linus Torvalds committed
302 303 304 305 306 307
	return ret;
}

static inline int __filemap_fdatawrite(struct address_space *mapping,
	int sync_mode)
{
308
	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
Linus Torvalds's avatar
Linus Torvalds committed
309 310 311 312 313 314 315 316
}

int filemap_fdatawrite(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite);

317
int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
318
				loff_t end)
Linus Torvalds's avatar
Linus Torvalds committed
319 320 321
{
	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
}
322
EXPORT_SYMBOL(filemap_fdatawrite_range);
Linus Torvalds's avatar
Linus Torvalds committed
323

324 325 326 327
/**
 * filemap_flush - mostly a non-blocking flush
 * @mapping:	target address_space
 *
Linus Torvalds's avatar
Linus Torvalds committed
328 329 330 331 332 333 334 335 336
 * This is a mostly non-blocking flush.  Not suitable for data-integrity
 * purposes - I/O may not be started against all dirty pages.
 */
int filemap_flush(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
}
EXPORT_SYMBOL(filemap_flush);

337 338
static int __filemap_fdatawait_range(struct address_space *mapping,
				     loff_t start_byte, loff_t end_byte)
Linus Torvalds's avatar
Linus Torvalds committed
339
{
340 341
	pgoff_t index = start_byte >> PAGE_SHIFT;
	pgoff_t end = end_byte >> PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
342 343
	struct pagevec pvec;
	int nr_pages;
344
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
345

346
	if (end_byte < start_byte)
347
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363

	pagevec_init(&pvec, 0);
	while ((index <= end) &&
			(nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
			PAGECACHE_TAG_WRITEBACK,
			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
		unsigned i;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/* until radix tree lookup accepts end_index */
			if (page->index > end)
				continue;

			wait_on_page_writeback(page);
364
			if (TestClearPageError(page))
Linus Torvalds's avatar
Linus Torvalds committed
365 366 367 368 369
				ret = -EIO;
		}
		pagevec_release(&pvec);
		cond_resched();
	}
370
out:
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
	return ret;
}

/**
 * filemap_fdatawait_range - wait for writeback to complete
 * @mapping:		address space structure to wait for
 * @start_byte:		offset in bytes where the range starts
 * @end_byte:		offset in bytes where the range ends (inclusive)
 *
 * Walk the list of under-writeback pages of the given address space
 * in the given range and wait for all of them.  Check error status of
 * the address space and return it.
 *
 * Since the error status of the address space is cleared by this function,
 * callers are responsible for checking the return value and handling and/or
 * reporting the error.
 */
int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
			    loff_t end_byte)
{
	int ret, ret2;

	ret = __filemap_fdatawait_range(mapping, start_byte, end_byte);
394 395 396
	ret2 = filemap_check_errors(mapping);
	if (!ret)
		ret = ret2;
Linus Torvalds's avatar
Linus Torvalds committed
397 398 399

	return ret;
}
400 401
EXPORT_SYMBOL(filemap_fdatawait_range);

402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
/**
 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
 * @mapping: address space structure to wait for
 *
 * Walk the list of under-writeback pages of the given address space
 * and wait for all of them.  Unlike filemap_fdatawait(), this function
 * does not clear error status of the address space.
 *
 * Use this function if callers don't handle errors themselves.  Expected
 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
 * fsfreeze(8)
 */
void filemap_fdatawait_keep_errors(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return;

	__filemap_fdatawait_range(mapping, 0, i_size - 1);
}

Linus Torvalds's avatar
Linus Torvalds committed
424
/**
425
 * filemap_fdatawait - wait for all under-writeback pages to complete
Linus Torvalds's avatar
Linus Torvalds committed
426
 * @mapping: address space structure to wait for
427 428
 *
 * Walk the list of under-writeback pages of the given address space
429 430 431 432 433 434
 * and wait for all of them.  Check error status of the address space
 * and return it.
 *
 * Since the error status of the address space is cleared by this function,
 * callers are responsible for checking the return value and handling and/or
 * reporting the error.
Linus Torvalds's avatar
Linus Torvalds committed
435 436 437 438 439 440 441 442
 */
int filemap_fdatawait(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return 0;

443
	return filemap_fdatawait_range(mapping, 0, i_size - 1);
Linus Torvalds's avatar
Linus Torvalds committed
444 445 446 447 448
}
EXPORT_SYMBOL(filemap_fdatawait);

int filemap_write_and_wait(struct address_space *mapping)
{
449
	int err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
450

451 452
	if ((!dax_mapping(mapping) && mapping->nrpages) ||
	    (dax_mapping(mapping) && mapping->nrexceptional)) {
453 454 455 456 457 458 459 460 461 462 463 464
		err = filemap_fdatawrite(mapping);
		/*
		 * Even if the above returned error, the pages may be
		 * written partially (e.g. -ENOSPC), so we wait for it.
		 * But the -EIO is special case, it may indicate the worst
		 * thing (e.g. bug) happened, so we avoid waiting for it.
		 */
		if (err != -EIO) {
			int err2 = filemap_fdatawait(mapping);
			if (!err)
				err = err2;
		}
465 466
	} else {
		err = filemap_check_errors(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
467
	}
468
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
469
}
470
EXPORT_SYMBOL(filemap_write_and_wait);
Linus Torvalds's avatar
Linus Torvalds committed
471

472 473 474 475 476 477
/**
 * filemap_write_and_wait_range - write out & wait on a file range
 * @mapping:	the address_space for the pages
 * @lstart:	offset in bytes where the range starts
 * @lend:	offset in bytes where the range ends (inclusive)
 *
478 479 480 481 482
 * Write out and wait upon file offsets lstart->lend, inclusive.
 *
 * Note that `lend' is inclusive (describes the last byte to be written) so
 * that this function can be used to write to the very end-of-file (end = -1).
 */
Linus Torvalds's avatar
Linus Torvalds committed
483 484 485
int filemap_write_and_wait_range(struct address_space *mapping,
				 loff_t lstart, loff_t lend)
{
486
	int err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
487

488 489
	if ((!dax_mapping(mapping) && mapping->nrpages) ||
	    (dax_mapping(mapping) && mapping->nrexceptional)) {
490 491 492 493
		err = __filemap_fdatawrite_range(mapping, lstart, lend,
						 WB_SYNC_ALL);
		/* See comment of filemap_write_and_wait() */
		if (err != -EIO) {
494 495
			int err2 = filemap_fdatawait_range(mapping,
						lstart, lend);
496 497 498
			if (!err)
				err = err2;
		}
499 500
	} else {
		err = filemap_check_errors(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
501
	}
502
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
503
}
504
EXPORT_SYMBOL(filemap_write_and_wait_range);
Linus Torvalds's avatar
Linus Torvalds committed
505

506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
/**
 * replace_page_cache_page - replace a pagecache page with a new one
 * @old:	page to be replaced
 * @new:	page to replace with
 * @gfp_mask:	allocation mode
 *
 * This function replaces a page in the pagecache with a new one.  On
 * success it acquires the pagecache reference for the new page and
 * drops it for the old page.  Both the old and new pages must be
 * locked.  This function does not add the new page to the LRU, the
 * caller must do that.
 *
 * The remove + add is atomic.  The only way this function can fail is
 * memory allocation failure.
 */
int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
{
	int error;

525 526 527
	VM_BUG_ON_PAGE(!PageLocked(old), old);
	VM_BUG_ON_PAGE(!PageLocked(new), new);
	VM_BUG_ON_PAGE(new->mapping, new);
528 529 530 531 532

	error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
	if (!error) {
		struct address_space *mapping = old->mapping;
		void (*freepage)(struct page *);
533
		unsigned long flags;
534 535 536 537

		pgoff_t offset = old->index;
		freepage = mapping->a_ops->freepage;

538
		get_page(new);
539 540 541
		new->mapping = mapping;
		new->index = offset;

542
		spin_lock_irqsave(&mapping->tree_lock, flags);
543
		__delete_from_page_cache(old, NULL);
544 545 546
		error = radix_tree_insert(&mapping->page_tree, offset, new);
		BUG_ON(error);
		mapping->nrpages++;
547 548 549 550 551 552

		/*
		 * hugetlb pages do not participate in page cache accounting.
		 */
		if (!PageHuge(new))
			__inc_zone_page_state(new, NR_FILE_PAGES);
553 554
		if (PageSwapBacked(new))
			__inc_zone_page_state(new, NR_SHMEM);
555
		spin_unlock_irqrestore(&mapping->tree_lock, flags);
556
		mem_cgroup_migrate(old, new);
557 558 559
		radix_tree_preload_end();
		if (freepage)
			freepage(old);
560
		put_page(old);
561 562 563 564 565 566
	}

	return error;
}
EXPORT_SYMBOL_GPL(replace_page_cache_page);

567
static int page_cache_tree_insert(struct address_space *mapping,
568
				  struct page *page, void **shadowp)
569
{
570
	struct radix_tree_node *node;
571 572 573
	void **slot;
	int error;

574
	error = __radix_tree_create(&mapping->page_tree, page->index, 0,
575 576 577 578
				    &node, &slot);
	if (error)
		return error;
	if (*slot) {
579 580 581 582 583
		void *p;

		p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
		if (!radix_tree_exceptional_entry(p))
			return -EEXIST;
584 585

		mapping->nrexceptional--;
586 587 588 589 590 591 592 593 594 595 596 597 598
		if (!dax_mapping(mapping)) {
			if (shadowp)
				*shadowp = p;
			if (node)
				workingset_node_shadows_dec(node);
		} else {
			/* DAX can replace empty locked entry with a hole */
			WARN_ON_ONCE(p !=
				(void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
					 RADIX_DAX_ENTRY_LOCK));
			/* DAX accounts exceptional entries as normal pages */
			if (node)
				workingset_node_pages_dec(node);
Jan Kara's avatar
Jan Kara committed
599 600 601
			/* Wakeup waiters for exceptional entry lock */
			dax_wake_mapping_entry_waiter(mapping, page->index,
						      false);
602
		}
603
	}
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
	radix_tree_replace_slot(slot, page);
	mapping->nrpages++;
	if (node) {
		workingset_node_pages_inc(node);
		/*
		 * Don't track node that contains actual pages.
		 *
		 * Avoid acquiring the list_lru lock if already
		 * untracked.  The list_empty() test is safe as
		 * node->private_list is protected by
		 * mapping->tree_lock.
		 */
		if (!list_empty(&node->private_list))
			list_lru_del(&workingset_shadow_nodes,
				     &node->private_list);
	}
	return 0;
621 622
}

623 624 625 626
static int __add_to_page_cache_locked(struct page *page,
				      struct address_space *mapping,
				      pgoff_t offset, gfp_t gfp_mask,
				      void **shadowp)
Linus Torvalds's avatar
Linus Torvalds committed
627
{
628 629
	int huge = PageHuge(page);
	struct mem_cgroup *memcg;
630 631
	int error;

632 633
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(PageSwapBacked(page), page);
634

635 636
	if (!huge) {
		error = mem_cgroup_try_charge(page, current->mm,
637
					      gfp_mask, &memcg, false);
638 639 640
		if (error)
			return error;
	}
Linus Torvalds's avatar
Linus Torvalds committed
641

642
	error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
643
	if (error) {
644
		if (!huge)
645
			mem_cgroup_cancel_charge(page, memcg, false);
646 647 648
		return error;
	}

649
	get_page(page);
650 651 652 653
	page->mapping = mapping;
	page->index = offset;

	spin_lock_irq(&mapping->tree_lock);
654
	error = page_cache_tree_insert(mapping, page, shadowp);
655 656 657
	radix_tree_preload_end();
	if (unlikely(error))
		goto err_insert;
658 659 660 661

	/* hugetlb pages do not participate in page cache accounting. */
	if (!huge)
		__inc_zone_page_state(page, NR_FILE_PAGES);
662
	spin_unlock_irq(&mapping->tree_lock);
663
	if (!huge)
664
		mem_cgroup_commit_charge(page, memcg, false, false);
665 666 667 668 669 670
	trace_mm_filemap_add_to_page_cache(page);
	return 0;
err_insert:
	page->mapping = NULL;
	/* Leave page->index set: truncation relies upon it */
	spin_unlock_irq(&mapping->tree_lock);
671
	if (!huge)
672
		mem_cgroup_cancel_charge(page, memcg, false);
673
	put_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
674 675
	return error;
}
676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692

/**
 * add_to_page_cache_locked - add a locked page to the pagecache
 * @page:	page to add
 * @mapping:	the page's address_space
 * @offset:	page index
 * @gfp_mask:	page allocation mode
 *
 * This function is used to add a page to the pagecache. It must be locked.
 * This function does not add the page to the LRU.  The caller must do that.
 */
int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
		pgoff_t offset, gfp_t gfp_mask)
{
	return __add_to_page_cache_locked(page, mapping, offset,
					  gfp_mask, NULL);
}
693
EXPORT_SYMBOL(add_to_page_cache_locked);
Linus Torvalds's avatar
Linus Torvalds committed
694 695

int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
696
				pgoff_t offset, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
697
{
698
	void *shadow = NULL;
699 700
	int ret;

701
	__SetPageLocked(page);
702 703 704
	ret = __add_to_page_cache_locked(page, mapping, offset,
					 gfp_mask, &shadow);
	if (unlikely(ret))
705
		__ClearPageLocked(page);
706 707 708 709 710
	else {
		/*
		 * The page might have been evicted from cache only
		 * recently, in which case it should be activated like
		 * any other repeatedly accessed page.
711 712 713
		 * The exception is pages getting rewritten; evicting other
		 * data from the working set, only to cache data that will
		 * get overwritten with something else, is a waste of memory.
714
		 */
715 716
		if (!(gfp_mask & __GFP_WRITE) &&
		    shadow && workingset_refault(shadow)) {
717 718 719 720 721 722
			SetPageActive(page);
			workingset_activation(page);
		} else
			ClearPageActive(page);
		lru_cache_add(page);
	}
Linus Torvalds's avatar
Linus Torvalds committed
723 724
	return ret;
}
725
EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
Linus Torvalds's avatar
Linus Torvalds committed
726

727
#ifdef CONFIG_NUMA
728
struct page *__page_cache_alloc(gfp_t gfp)
729
{
730 731 732
	int n;
	struct page *page;

733
	if (cpuset_do_page_mem_spread()) {
734 735
		unsigned int cpuset_mems_cookie;
		do {
736
			cpuset_mems_cookie = read_mems_allowed_begin();
737
			n = cpuset_mem_spread_node();
738
			page = __alloc_pages_node(n, gfp, 0);
739
		} while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
740

741
		return page;
742
	}
743
	return alloc_pages(gfp, 0);
744
}
745
EXPORT_SYMBOL(__page_cache_alloc);
746 747
#endif

Linus Torvalds's avatar
Linus Torvalds committed
748 749 750 751 752 753 754 755 756 757
/*
 * In order to wait for pages to become available there must be
 * waitqueues associated with pages. By using a hash table of
 * waitqueues where the bucket discipline is to maintain all
 * waiters on the same queue and wake all when any of the pages
 * become available, and for the woken contexts to check to be
 * sure the appropriate page became available, this saves space
 * at a cost of "thundering herd" phenomena during rare hash
 * collisions.
 */
758
wait_queue_head_t *page_waitqueue(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
759 760 761 762 763
{
	const struct zone *zone = page_zone(page);

	return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
}
764
EXPORT_SYMBOL(page_waitqueue);
Linus Torvalds's avatar
Linus Torvalds committed
765

766
void wait_on_page_bit(struct page *page, int bit_nr)
Linus Torvalds's avatar
Linus Torvalds committed
767 768 769 770
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (test_bit(bit_nr, &page->flags))
771
		__wait_on_bit(page_waitqueue(page), &wait, bit_wait_io,
Linus Torvalds's avatar
Linus Torvalds committed
772 773 774 775
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(wait_on_page_bit);

776 777 778 779 780 781 782 783
int wait_on_page_bit_killable(struct page *page, int bit_nr)
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (!test_bit(bit_nr, &page->flags))
		return 0;

	return __wait_on_bit(page_waitqueue(page), &wait,
784
			     bit_wait_io, TASK_KILLABLE);
785 786
}

787 788 789 790 791 792 793 794 795 796 797 798 799
int wait_on_page_bit_killable_timeout(struct page *page,
				       int bit_nr, unsigned long timeout)
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	wait.key.timeout = jiffies + timeout;
	if (!test_bit(bit_nr, &page->flags))
		return 0;
	return __wait_on_bit(page_waitqueue(page), &wait,
			     bit_wait_io_timeout, TASK_KILLABLE);
}
EXPORT_SYMBOL_GPL(wait_on_page_bit_killable_timeout);

800 801
/**
 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
802 803
 * @page: Page defining the wait queue of interest
 * @waiter: Waiter to add to the queue
804 805 806 807 808 809 810 811 812 813 814 815 816 817
 *
 * Add an arbitrary @waiter to the wait queue for the nominated @page.
 */
void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
{
	wait_queue_head_t *q = page_waitqueue(page);
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, waiter);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(add_page_wait_queue);

Linus Torvalds's avatar
Linus Torvalds committed
818
/**
819
 * unlock_page - unlock a locked page
Linus Torvalds's avatar
Linus Torvalds committed
820 821 822 823
 * @page: the page
 *
 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
824
 * mechanism between PageLocked pages and PageWriteback pages is shared.
Linus Torvalds's avatar
Linus Torvalds committed
825 826
 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
 *
827 828
 * The mb is necessary to enforce ordering between the clear_bit and the read
 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
Linus Torvalds's avatar
Linus Torvalds committed
829
 */
830
void unlock_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
831
{
832
	page = compound_head(page);
833
	VM_BUG_ON_PAGE(!PageLocked(page), page);
834
	clear_bit_unlock(PG_locked, &page->flags);
835
	smp_mb__after_atomic();
Linus Torvalds's avatar
Linus Torvalds committed
836 837 838 839
	wake_up_page(page, PG_locked);
}
EXPORT_SYMBOL(unlock_page);

840 841 842
/**
 * end_page_writeback - end writeback against a page
 * @page: the page
Linus Torvalds's avatar
Linus Torvalds committed
843 844 845
 */
void end_page_writeback(struct page *page)
{
846 847 848 849 850 851 852 853 854
	/*
	 * TestClearPageReclaim could be used here but it is an atomic
	 * operation and overkill in this particular case. Failing to
	 * shuffle a page marked for immediate reclaim is too mild to
	 * justify taking an atomic operation penalty at the end of
	 * ever page writeback.
	 */
	if (PageReclaim(page)) {
		ClearPageReclaim(page);
855
		rotate_reclaimable_page(page);
856
	}
857 858 859 860

	if (!test_clear_page_writeback(page))
		BUG();

861
	smp_mb__after_atomic();
Linus Torvalds's avatar
Linus Torvalds committed
862 863 864 865
	wake_up_page(page, PG_writeback);
}
EXPORT_SYMBOL(end_page_writeback);

866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
/*
 * After completing I/O on a page, call this routine to update the page
 * flags appropriately
 */
void page_endio(struct page *page, int rw, int err)
{
	if (rw == READ) {
		if (!err) {
			SetPageUptodate(page);
		} else {
			ClearPageUptodate(page);
			SetPageError(page);
		}
		unlock_page(page);
	} else { /* rw == WRITE */
		if (err) {
			SetPageError(page);
			if (page->mapping)
				mapping_set_error(page->mapping, err);
		}
		end_page_writeback(page);
	}
}
EXPORT_SYMBOL_GPL(page_endio);

891 892 893
/**
 * __lock_page - get a lock on the page, assuming we need to sleep to get it
 * @page: the page to lock
Linus Torvalds's avatar
Linus Torvalds committed
894
 */
895
void __lock_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
896
{
897 898
	struct page *page_head = compound_head(page);
	DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
Linus Torvalds's avatar
Linus Torvalds committed
899

900
	__wait_on_bit_lock(page_waitqueue(page_head), &wait, bit_wait_io,
Linus Torvalds's avatar
Linus Torvalds committed
901 902 903 904
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__lock_page);

905
int __lock_page_killable(struct page *page)
Matthew Wilcox's avatar
Matthew Wilcox committed
906
{
907 908
	struct page *page_head = compound_head(page);
	DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
Matthew Wilcox's avatar
Matthew Wilcox committed
909

910
	return __wait_on_bit_lock(page_waitqueue(page_head), &wait,
911
					bit_wait_io, TASK_KILLABLE);
Matthew Wilcox's avatar
Matthew Wilcox committed
912
}
913
EXPORT_SYMBOL_GPL(__lock_page_killable);
Matthew Wilcox's avatar
Matthew Wilcox committed
914

915 916 917 918 919 920 921 922 923 924 925
/*
 * Return values:
 * 1 - page is locked; mmap_sem is still held.
 * 0 - page is not locked.
 *     mmap_sem has been released (up_read()), unless flags had both
 *     FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
 *     which case mmap_sem is still held.
 *
 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
 * with the page locked and the mmap_sem unperturbed.
 */
926 927 928
int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
			 unsigned int flags)
{
929 930 931 932 933 934 935 936 937 938 939 940
	if (flags & FAULT_FLAG_ALLOW_RETRY) {
		/*
		 * CAUTION! In this case, mmap_sem is not released
		 * even though return 0.
		 */
		if (flags & FAULT_FLAG_RETRY_NOWAIT)
			return 0;

		up_read(&mm->mmap_sem);
		if (flags & FAULT_FLAG_KILLABLE)
			wait_on_page_locked_killable(page);
		else
941
			wait_on_page_locked(page);
942
		return 0;
943 944 945 946 947 948 949 950 951 952 953 954
	} else {
		if (flags & FAULT_FLAG_KILLABLE) {
			int ret;

			ret = __lock_page_killable(page);
			if (ret) {
				up_read(&mm->mmap_sem);
				return 0;
			}
		} else
			__lock_page(page);
		return 1;
955 956 957
	}
}

958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
/**
 * page_cache_next_hole - find the next hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
 * lowest indexed hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'return - index >=
 * max_scan' will be true). In rare cases of index wrap-around, 0 will
 * be returned.
 *
 * page_cache_next_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 5, then subsequently a hole is created at
 * index 10, page_cache_next_hole covering both indexes may return 10
 * if called under rcu_read_lock.
 */
pgoff_t page_cache_next_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
985 986 987 988
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
			break;
		index++;
		if (index == 0)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_next_hole);

/**
 * page_cache_prev_hole - find the prev hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search backwards in the range [max(index-max_scan+1, 0), index] for
 * the first hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'index - return >=
 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
 * will be returned.
 *
 * page_cache_prev_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 10, then subsequently a hole is created at
 * index 5, page_cache_prev_hole covering both indexes may return 5 if
 * called under rcu_read_lock.
 */
pgoff_t page_cache_prev_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
1026 1027 1028 1029
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
			break;
		index--;
		if (index == ULONG_MAX)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_prev_hole);

1040
/**
1041
 * find_get_entry - find and get a page cache entry
1042
 * @mapping: the address_space to search
1043 1044 1045 1046
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned with an increased refcount.
1047
 *
1048 1049
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
1050 1051
 *
 * Otherwise, %NULL is returned.
Linus Torvalds's avatar
Linus Torvalds committed
1052
 */
1053
struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds's avatar
Linus Torvalds committed
1054
{
Nick Piggin's avatar
Nick Piggin committed
1055
	void **pagep;
Linus Torvalds's avatar
Linus Torvalds committed
1056 1057
	struct page *page;

Nick Piggin's avatar
Nick Piggin committed
1058 1059 1060 1061 1062 1063
	rcu_read_lock();
repeat:
	page = NULL;
	pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
	if (pagep) {
		page = radix_tree_deref_slot(pagep);
Nick Piggin's avatar
Nick Piggin committed
1064 1065
		if (unlikely(!page))
			goto out;
1066
		if (radix_tree_exception(page)) {
1067 1068 1069
			if (radix_tree_deref_retry(page))
				goto repeat;
			/*
1070 1071 1072
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Return
			 * it without attempting to raise page count.
1073 1074
			 */
			goto out;
1075
		}
Nick Piggin's avatar
Nick Piggin committed
1076 1077 1078 1079 1080 1081 1082 1083 1084
		if (!page_cache_get_speculative(page))
			goto repeat;

		/*
		 * Has the page moved?
		 * This is part of the lockless pagecache protocol. See
		 * include/linux/pagemap.h for details.
		 */
		if (unlikely(page != *pagep)) {
1085
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1086 1087 1088
			goto repeat;
		}
	}
Nick Piggin's avatar
Nick Piggin committed
1089
out:
Nick Piggin's avatar
Nick Piggin committed
1090 1091
	rcu_read_unlock();

Linus Torvalds's avatar
Linus Torvalds committed
1092 1093
	return page;
}
1094
EXPORT_SYMBOL(find_get_entry);
Linus Torvalds's avatar
Linus Torvalds committed
1095

1096 1097 1098 1099 1100 1101 1102 1103 1104
/**
 * find_lock_entry - locate, pin and lock a page cache entry
 * @mapping: the address_space to search
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned locked and with an increased
 * refcount.
 *
1105 1106
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
1107 1108 1109 1110 1111 1112
 *
 * Otherwise, %NULL is returned.
 *
 * find_lock_entry() may sleep.
 */
struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds's avatar
Linus Torvalds committed
1113 1114 1115 1116
{
	struct page *page;

repeat:
1117
	page = find_get_entry(mapping, offset);
1118
	if (page && !radix_tree_exception(page)) {
Nick Piggin's avatar
Nick Piggin committed
1119 1120 1121 1122
		lock_page(page);
		/* Has the page been truncated? */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
1123
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1124
			goto repeat;
Linus Torvalds's avatar
Linus Torvalds committed
1125
		}
1126
		VM_BUG_ON_PAGE(page->index != offset, page);
Linus Torvalds's avatar
Linus Torvalds committed
1127 1128 1129
	}
	return page;
}
1130 1131 1132
EXPORT_SYMBOL(find_lock_entry);

/**
1133
 * pagecache_get_page - find and get a page reference
1134 1135
 * @mapping: the address_space to search
 * @offset: the page index
1136
 * @fgp_flags: PCG flags
1137
 * @gfp_mask: gfp mask to use for the page cache data page allocation
1138
 *
1139
 * Looks up the page cache slot at @mapping & @offset.
Linus Torvalds's avatar
Linus Torvalds committed
1140
 *
1141
 * PCG flags modify how the page is returned.
1142
 *
1143 1144 1145
 * FGP_ACCESSED: the page will be marked accessed
 * FGP_LOCK: Page is return locked
 * FGP_CREAT: If page is not present then a new page is allocated using
1146 1147 1148
 *		@gfp_mask and added to the page cache and the VM's LRU
 *		list. The page is returned locked and with an increased
 *		refcount. Otherwise, %NULL is returned.
Linus Torvalds's avatar
Linus Torvalds committed
1149
 *
1150 1151
 * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
 * if the GFP flags specified for FGP_CREAT are atomic.
Linus Torvalds's avatar
Linus Torvalds committed
1152
 *
1153
 * If there is a page cache page, it is returned with an increased refcount.
Linus Torvalds's avatar
Linus Torvalds committed
1154
 */
1155
struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
1156
	int fgp_flags, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1157
{
Nick Piggin's avatar
Nick Piggin committed
1158
	struct page *page;
1159

Linus Torvalds's avatar
Linus Torvalds committed
1160
repeat:
1161 1162 1163 1164 1165 1166 1167 1168 1169
	page = find_get_entry(mapping, offset);
	if (radix_tree_exceptional_entry(page))
		page = NULL;
	if (!page)
		goto no_page;

	if (fgp_flags & FGP_LOCK) {
		if (fgp_flags & FGP_NOWAIT) {
			if (!trylock_page(page)) {
1170
				put_page(page);
1171 1172 1173 1174 1175 1176 1177 1178 1179
				return NULL;
			}
		} else {
			lock_page(page);
		}

		/* Has the page been truncated? */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
1180
			put_page(page);
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
			goto repeat;
		}
		VM_BUG_ON_PAGE(page->index != offset, page);
	}

	if (page && (fgp_flags & FGP_ACCESSED))
		mark_page_accessed(page);

no_page:
	if (!page && (fgp_flags & FGP_CREAT)) {
		int err;
		if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
1193 1194 1195
			gfp_mask |= __GFP_WRITE;
		if (fgp_flags & FGP_NOFS)
			gfp_mask &= ~__GFP_FS;
1196

1197
		page = __page_cache_alloc(gfp_mask);
Nick Piggin's avatar
Nick Piggin committed
1198 1199
		if (!page)
			return NULL;
1200 1201 1202 1203

		if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
			fgp_flags |= FGP_LOCK;

1204
		/* Init accessed so avoid atomic mark_page_accessed later */
1205
		if (fgp_flags & FGP_ACCESSED)
1206
			__SetPageReferenced(page);
1207

1208 1209
		err = add_to_page_cache_lru(page, mapping, offset,
				gfp_mask & GFP_RECLAIM_MASK);
Nick Piggin's avatar
Nick Piggin committed
1210
		if (unlikely(err)) {
1211
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1212 1213 1214
			page = NULL;
			if (err == -EEXIST)
				goto repeat;
Linus Torvalds's avatar
Linus Torvalds committed
1215 1216
		}
	}
1217

Linus Torvalds's avatar
Linus Torvalds committed
1218 1219
	return page;
}
1220
EXPORT_SYMBOL(pagecache_get_page);
Linus Torvalds's avatar
Linus Torvalds committed
1221

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
/**
 * find_get_entries - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page cache index
 * @nr_entries:	The maximum number of entries
 * @entries:	Where the resulting entries are placed
 * @indices:	The cache indices corresponding to the entries in @entries
 *
 * find_get_entries() will search for and return a group of up to
 * @nr_entries entries in the mapping.  The entries are placed at
 * @entries.  find_get_entries() takes a reference against any actual
 * pages it returns.
 *
 * The search returns a group of mapping-contiguous page cache entries
 * with ascending indexes.  There may be holes in the indices due to
 * not-present pages.
 *
1239 1240
 * Any shadow entries of evicted pages, or swap entries from
 * shmem/tmpfs, are included in the returned array.
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263
 *
 * find_get_entries() returns the number of pages and shadow entries
 * which were found.
 */
unsigned find_get_entries(struct address_space *mapping,
			  pgoff_t start, unsigned int nr_entries,
			  struct page **entries, pgoff_t *indices)
{
	void **slot;
	unsigned int ret = 0;
	struct radix_tree_iter iter;

	if (!nr_entries)
		return 0;

	rcu_read_lock();
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
		struct page *page;
repeat:
		page = radix_tree_deref_slot(slot);
		if (unlikely(!page))
			continue;
		if (radix_tree_exception(page)) {
1264 1265 1266 1267
			if (radix_tree_deref_retry(page)) {
				slot = radix_tree_iter_retry(&iter);
				continue;
			}
1268
			/*
1269 1270 1271
			 * A shadow entry of a recently evicted page, a swap
			 * entry from shmem/tmpfs or a DAX entry.  Return it
			 * without attempting to raise page count.
1272 1273 1274 1275 1276 1277 1278 1279
			 */
			goto export;
		}
		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
		if (unlikely(page != *slot)) {
1280
			put_page(page);
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
			goto repeat;
		}
export:
		indices[ret] = iter.index;
		entries[ret] = page;
		if (++ret == nr_entries)
			break;
	}
	rcu_read_unlock();
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
/**
 * find_get_pages - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages() will search for and return a group of up to
 * @nr_pages pages in the mapping.  The pages are placed at @pages.
 * find_get_pages() takes a reference against the returned pages.
 *
 * The search returns a group of mapping-contiguous pages with ascending
 * indexes.  There may be holes in the indices due to not-present pages.
 *
 * find_get_pages() returns the number of pages which were found.
 */
unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
			    unsigned int nr_pages, struct page **pages)
{
1312 1313 1314 1315 1316 1317
	struct radix_tree_iter iter;
	void **slot;
	unsigned ret = 0;

	if (unlikely(!nr_pages))
		return 0;
Nick Piggin's avatar
Nick Piggin committed
1318 1319

	rcu_read_lock();
1320
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
Nick Piggin's avatar
Nick Piggin committed
1321 1322
		struct page *page;
repeat:
1323
		page = radix_tree_deref_slot(slot);
Nick Piggin's avatar
Nick Piggin committed
1324 1325
		if (unlikely(!page))
			continue;
1326

1327
		if (radix_tree_exception(page)) {
1328
			if (radix_tree_deref_retry(page)) {
1329 1330
				slot = radix_tree_iter_retry(&iter);
				continue;
1331
			}
1332
			/*
1333 1334 1335
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Skip
			 * over it.
1336
			 */
1337
			continue;
Nick Piggin's avatar
Nick Piggin committed
1338
		}
Nick Piggin's avatar
Nick Piggin committed
1339 1340 1341 1342 1343

		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
1344
		if (unlikely(page != *slot)) {
1345
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1346 1347
			goto repeat;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1348

Nick Piggin's avatar
Nick Piggin committed
1349
		pages[ret] = page;
1350 1351
		if (++ret == nr_pages)
			break;
Nick Piggin's avatar
Nick Piggin committed
1352
	}
1353

Nick Piggin's avatar
Nick Piggin committed
1354
	rcu_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
1355 1356 1357
	return ret;
}

1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
/**
 * find_get_pages_contig - gang contiguous pagecache lookup
 * @mapping:	The address_space to search
 * @index:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages_contig() works exactly like find_get_pages(), except
 * that the returned number of pages are guaranteed to be contiguous.
 *
 * find_get_pages_contig() returns the number of pages which were found.
 */
unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
			       unsigned int nr_pages, struct page **pages)
{
1373 1374 1375 1376 1377 1378
	struct radix_tree_iter iter;
	void **slot;
	unsigned int ret = 0;

	if (unlikely(!nr_pages))
		return 0;
Nick Piggin's avatar
Nick Piggin committed
1379 1380

	rcu_read_lock();
1381
	radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
Nick Piggin's avatar
Nick Piggin committed
1382 1383
		struct page *page;
repeat:
1384 1385
		page = radix_tree_deref_slot(slot);
		/* The hole, there no reason to continue */
Nick Piggin's avatar
Nick Piggin committed
1386
		if (unlikely(!page))
1387
			break;
1388

1389
		if (radix_tree_exception(page)) {
1390
			if (radix_tree_deref_retry(page)) {
1391 1392
				slot = radix_tree_iter_retry(&iter);
				continue;
1393
			}
1394
			/*
1395 1396 1397
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Stop
			 * looking for contiguous pages.
1398
			 */
1399
			break;
1400
		}
1401

Nick Piggin's avatar
Nick Piggin committed
1402 1403 1404 1405
		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
1406
		if (unlikely(page != *slot)) {
1407
			put_page(page);
Nick Piggin's avatar
Nick Piggin committed
1408 1409 1410
			goto repeat;
		}

1411 1412 1413 1414 1415
		/*
		 * must check mapping and index after taking the ref.
		 * otherwise we can get both false positives and false
		 * negatives, which is just confusing to the caller.
		 */
1416
		if (page->mapping == NULL || page->index != iter.index) {
1417
			put_page(page);