inode.c 40.6 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 * linux/fs/inode.c
 *
 * (C) 1997 Linus Torvalds
 */

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/dcache.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/writeback.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/wait.h>
Nick Piggin's avatar
Nick Piggin committed
16
#include <linux/rwsem.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
18
19
20
21
22
#include <linux/hash.h>
#include <linux/swap.h>
#include <linux/security.h>
#include <linux/pagemap.h>
#include <linux/cdev.h>
#include <linux/bootmem.h>
23
#include <linux/fsnotify.h>
24
#include <linux/mount.h>
25
#include <linux/async.h>
Al Viro's avatar
Al Viro committed
26
#include <linux/posix_acl.h>
Linus Torvalds's avatar
Linus Torvalds committed
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

/*
 * This is needed for the following functions:
 *  - inode_has_buffers
 *  - invalidate_inode_buffers
 *  - invalidate_bdev
 *
 * FIXME: remove all knowledge of the buffer layer from this file
 */
#include <linux/buffer_head.h>

/*
 * New inode.c implementation.
 *
 * This implementation has the basic premise of trying
 * to be extremely low-overhead and SMP-safe, yet be
 * simple enough to be "obviously correct".
 *
 * Famous last words.
 */

/* inode dynamic allocation 1999, Andrea Arcangeli <andrea@suse.de> */

/* #define INODE_PARANOIA 1 */
/* #define INODE_DEBUG 1 */

/*
 * Inode lookup is no longer as critical as it used to be:
 * most of the lookups are going to be through the dcache.
 */
#define I_HASHBITS	i_hash_shift
#define I_HASHMASK	i_hash_mask

60
61
static unsigned int i_hash_mask __read_mostly;
static unsigned int i_hash_shift __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

/*
 * Each inode can be on two separate lists. One is
 * the hash list of the inode, used for lookups. The
 * other linked list is the "type" list:
 *  "in_use" - valid inode, i_count > 0, i_nlink > 0
 *  "dirty"  - as "in_use" but also dirty
 *  "unused" - valid inode, i_count = 0
 *
 * A "dirty" list is maintained for each super block,
 * allowing for low-overhead inode sync() operations.
 */

LIST_HEAD(inode_in_use);
LIST_HEAD(inode_unused);
77
static struct hlist_head *inode_hashtable __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
78
79
80
81
82
83
84
85
86
87

/*
 * A simple spinlock to protect the list manipulations.
 *
 * NOTE! You also have to own the lock if you change
 * the i_state of an inode while it is in use..
 */
DEFINE_SPINLOCK(inode_lock);

/*
Nick Piggin's avatar
Nick Piggin committed
88
 * iprune_sem provides exclusion between the kswapd or try_to_free_pages
Linus Torvalds's avatar
Linus Torvalds committed
89
90
91
92
93
 * icache shrinking path, and the umount path.  Without this exclusion,
 * by the time prune_icache calls iput for the inode whose pages it has
 * been invalidating, or by the time it calls clear_inode & destroy_inode
 * from its final dispose_list, the struct super_block they refer to
 * (for inode->i_sb->s_op) may already have been freed and reused.
Nick Piggin's avatar
Nick Piggin committed
94
95
96
97
 *
 * We make this an rwsem because the fastpath is icache shrinking. In
 * some cases a filesystem may be doing a significant amount of work in
 * its inode reclaim code, so this should improve parallelism.
Linus Torvalds's avatar
Linus Torvalds committed
98
 */
Nick Piggin's avatar
Nick Piggin committed
99
static DECLARE_RWSEM(iprune_sem);
Linus Torvalds's avatar
Linus Torvalds committed
100
101
102
103
104
105

/*
 * Statistics gathering..
 */
struct inodes_stat_t inodes_stat;

106
static struct kmem_cache *inode_cachep __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
107

Joern Engel's avatar
Joern Engel committed
108
109
110
111
112
113
static void wake_up_inode(struct inode *inode)
{
	/*
	 * Prevent speculative execution through spin_unlock(&inode_lock);
	 */
	smp_mb();
Christoph Hellwig's avatar
Christoph Hellwig committed
114
	wake_up_bit(&inode->i_state, __I_NEW);
Joern Engel's avatar
Joern Engel committed
115
116
}

117
118
/**
 * inode_init_always - perform inode structure intialisation
119
120
 * @sb: superblock inode belongs to
 * @inode: inode to initialise
121
122
123
124
 *
 * These are initializations that need to be done on every inode
 * allocation as the fields are not initialised by slab allocation.
 */
125
int inode_init_always(struct super_block *sb, struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
126
{
127
	static const struct address_space_operations empty_aops;
128
	static const struct inode_operations empty_iops;
129
	static const struct file_operations empty_fops;
130
	struct address_space *const mapping = &inode->i_data;
131
132
133
134
135
136
137
138

	inode->i_sb = sb;
	inode->i_blkbits = sb->s_blocksize_bits;
	inode->i_flags = 0;
	atomic_set(&inode->i_count, 1);
	inode->i_op = &empty_iops;
	inode->i_fop = &empty_fops;
	inode->i_nlink = 1;
Al Viro's avatar
Al Viro committed
139
140
	inode->i_uid = 0;
	inode->i_gid = 0;
141
142
143
144
145
	atomic_set(&inode->i_writecount, 0);
	inode->i_size = 0;
	inode->i_blocks = 0;
	inode->i_bytes = 0;
	inode->i_generation = 0;
Linus Torvalds's avatar
Linus Torvalds committed
146
#ifdef CONFIG_QUOTA
147
	memset(&inode->i_dquot, 0, sizeof(inode->i_dquot));
Linus Torvalds's avatar
Linus Torvalds committed
148
#endif
149
150
151
152
153
	inode->i_pipe = NULL;
	inode->i_bdev = NULL;
	inode->i_cdev = NULL;
	inode->i_rdev = 0;
	inode->dirtied_when = 0;
Mimi Zohar's avatar
Mimi Zohar committed
154
155

	if (security_inode_alloc(inode))
156
		goto out;
157
158
159
160
161
162
163
164
165
166
167
168
	spin_lock_init(&inode->i_lock);
	lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);

	mutex_init(&inode->i_mutex);
	lockdep_set_class(&inode->i_mutex, &sb->s_type->i_mutex_key);

	init_rwsem(&inode->i_alloc_sem);
	lockdep_set_class(&inode->i_alloc_sem, &sb->s_type->i_alloc_sem_key);

	mapping->a_ops = &empty_aops;
	mapping->host = inode;
	mapping->flags = 0;
169
	mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
170
171
172
173
174
175
176
177
178
179
180
181
	mapping->assoc_mapping = NULL;
	mapping->backing_dev_info = &default_backing_dev_info;
	mapping->writeback_index = 0;

	/*
	 * If the block_device provides a backing_dev_info for client
	 * inodes then use that.  Otherwise the inode share the bdev's
	 * backing_dev_info.
	 */
	if (sb->s_bdev) {
		struct backing_dev_info *bdi;

182
		bdi = sb->s_bdev->bd_inode->i_mapping->backing_dev_info;
183
184
185
186
		mapping->backing_dev_info = bdi;
	}
	inode->i_private = NULL;
	inode->i_mapping = mapping;
Al Viro's avatar
Al Viro committed
187
188
189
#ifdef CONFIG_FS_POSIX_ACL
	inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
#endif
190

191
192
193
194
#ifdef CONFIG_FSNOTIFY
	inode->i_fsnotify_mask = 0;
#endif

195
196
197
	return 0;
out:
	return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
198
}
199
200
201
202
203
204
205
206
207
208
209
EXPORT_SYMBOL(inode_init_always);

static struct inode *alloc_inode(struct super_block *sb)
{
	struct inode *inode;

	if (sb->s_op->alloc_inode)
		inode = sb->s_op->alloc_inode(sb);
	else
		inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);

210
211
212
213
214
215
216
217
218
219
220
221
	if (!inode)
		return NULL;

	if (unlikely(inode_init_always(sb, inode))) {
		if (inode->i_sb->s_op->destroy_inode)
			inode->i_sb->s_op->destroy_inode(inode);
		else
			kmem_cache_free(inode_cachep, inode);
		return NULL;
	}

	return inode;
222
}
Linus Torvalds's avatar
Linus Torvalds committed
223

Christoph Hellwig's avatar
Christoph Hellwig committed
224
void __destroy_inode(struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
225
{
226
	BUG_ON(inode_has_buffers(inode));
Linus Torvalds's avatar
Linus Torvalds committed
227
	security_inode_free(inode);
228
	fsnotify_inode_delete(inode);
Al Viro's avatar
Al Viro committed
229
230
231
232
233
234
#ifdef CONFIG_FS_POSIX_ACL
	if (inode->i_acl && inode->i_acl != ACL_NOT_CACHED)
		posix_acl_release(inode->i_acl);
	if (inode->i_default_acl && inode->i_default_acl != ACL_NOT_CACHED)
		posix_acl_release(inode->i_default_acl);
#endif
Christoph Hellwig's avatar
Christoph Hellwig committed
235
236
237
}
EXPORT_SYMBOL(__destroy_inode);

238
static void destroy_inode(struct inode *inode)
Christoph Hellwig's avatar
Christoph Hellwig committed
239
240
{
	__destroy_inode(inode);
Linus Torvalds's avatar
Linus Torvalds committed
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
	if (inode->i_sb->s_op->destroy_inode)
		inode->i_sb->s_op->destroy_inode(inode);
	else
		kmem_cache_free(inode_cachep, (inode));
}

/*
 * These are initializations that only need to be done
 * once, because the fields are idempotent across use
 * of the inode, so let the slab aware of that.
 */
void inode_init_once(struct inode *inode)
{
	memset(inode, 0, sizeof(*inode));
	INIT_HLIST_NODE(&inode->i_hash);
	INIT_LIST_HEAD(&inode->i_dentry);
	INIT_LIST_HEAD(&inode->i_devices);
	INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC);
Nick Piggin's avatar
Nick Piggin committed
259
	spin_lock_init(&inode->i_data.tree_lock);
Linus Torvalds's avatar
Linus Torvalds committed
260
261
262
263
264
265
	spin_lock_init(&inode->i_data.i_mmap_lock);
	INIT_LIST_HEAD(&inode->i_data.private_list);
	spin_lock_init(&inode->i_data.private_lock);
	INIT_RAW_PRIO_TREE_ROOT(&inode->i_data.i_mmap);
	INIT_LIST_HEAD(&inode->i_data.i_mmap_nonlinear);
	i_size_ordered_init(inode);
266
#ifdef CONFIG_FSNOTIFY
267
	INIT_HLIST_HEAD(&inode->i_fsnotify_marks);
268
#endif
Linus Torvalds's avatar
Linus Torvalds committed
269
270
271
}
EXPORT_SYMBOL(inode_init_once);

272
static void init_once(void *foo)
Linus Torvalds's avatar
Linus Torvalds committed
273
{
274
	struct inode *inode = (struct inode *) foo;
Linus Torvalds's avatar
Linus Torvalds committed
275

276
	inode_init_once(inode);
Linus Torvalds's avatar
Linus Torvalds committed
277
278
279
280
281
}

/*
 * inode_lock must be held
 */
282
void __iget(struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
283
{
284
	if (atomic_inc_return(&inode->i_count) != 1)
Linus Torvalds's avatar
Linus Torvalds committed
285
		return;
286

Joern Engel's avatar
Joern Engel committed
287
	if (!(inode->i_state & (I_DIRTY|I_SYNC)))
Linus Torvalds's avatar
Linus Torvalds committed
288
289
290
291
		list_move(&inode->i_list, &inode_in_use);
	inodes_stat.nr_unused--;
}

Al Viro's avatar
Al Viro committed
292
293
294
295
296
297
298
299
300
301
302
303
void end_writeback(struct inode *inode)
{
	might_sleep();
	BUG_ON(inode->i_data.nrpages);
	BUG_ON(!list_empty(&inode->i_data.private_list));
	BUG_ON(!(inode->i_state & I_FREEING));
	BUG_ON(inode->i_state & I_CLEAR);
	inode_sync_wait(inode);
	inode->i_state = I_FREEING | I_CLEAR;
}
EXPORT_SYMBOL(end_writeback);

304
static void evict(struct inode *inode)
305
306
307
{
	const struct super_operations *op = inode->i_sb->s_op;

Al Viro's avatar
Al Viro committed
308
309
	if (op->evict_inode) {
		op->evict_inode(inode);
310
311
312
	} else {
		if (inode->i_data.nrpages)
			truncate_inode_pages(&inode->i_data, 0);
Al Viro's avatar
Al Viro committed
313
		end_writeback(inode);
314
	}
315
316
317
318
	if (S_ISBLK(inode->i_mode) && inode->i_bdev)
		bd_forget(inode);
	if (S_ISCHR(inode->i_mode) && inode->i_cdev)
		cd_forget(inode);
319
320
}

Linus Torvalds's avatar
Linus Torvalds committed
321
322
323
324
325
326
327
328
329
330
331
332
333
334
/*
 * dispose_list - dispose of the contents of a local list
 * @head: the head of the list to free
 *
 * Dispose-list gets a local list with local inodes in it, so it doesn't
 * need to worry about list corruption and SMP locks.
 */
static void dispose_list(struct list_head *head)
{
	int nr_disposed = 0;

	while (!list_empty(head)) {
		struct inode *inode;

335
		inode = list_first_entry(head, struct inode, i_list);
Linus Torvalds's avatar
Linus Torvalds committed
336
337
		list_del(&inode->i_list);

338
		evict(inode);
339
340
341
342
343
344
345

		spin_lock(&inode_lock);
		hlist_del_init(&inode->i_hash);
		list_del_init(&inode->i_sb_list);
		spin_unlock(&inode_lock);

		wake_up_inode(inode);
Linus Torvalds's avatar
Linus Torvalds committed
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
		destroy_inode(inode);
		nr_disposed++;
	}
	spin_lock(&inode_lock);
	inodes_stat.nr_inodes -= nr_disposed;
	spin_unlock(&inode_lock);
}

/*
 * Invalidate all inodes for a device.
 */
static int invalidate_list(struct list_head *head, struct list_head *dispose)
{
	struct list_head *next;
	int busy = 0, count = 0;

	next = head->next;
	for (;;) {
364
365
		struct list_head *tmp = next;
		struct inode *inode;
Linus Torvalds's avatar
Linus Torvalds committed
366
367
368
369

		/*
		 * We can reschedule here without worrying about the list's
		 * consistency because the per-sb list of inodes must not
Nick Piggin's avatar
Nick Piggin committed
370
		 * change during umount anymore, and because iprune_sem keeps
Linus Torvalds's avatar
Linus Torvalds committed
371
372
373
374
375
376
377
378
		 * shrink_icache_memory() away.
		 */
		cond_resched_lock(&inode_lock);

		next = next->next;
		if (tmp == head)
			break;
		inode = list_entry(tmp, struct inode, i_sb_list);
Nick Piggin's avatar
Nick Piggin committed
379
380
		if (inode->i_state & I_NEW)
			continue;
Linus Torvalds's avatar
Linus Torvalds committed
381
382
383
		invalidate_inode_buffers(inode);
		if (!atomic_read(&inode->i_count)) {
			list_move(&inode->i_list, dispose);
384
			WARN_ON(inode->i_state & I_NEW);
Linus Torvalds's avatar
Linus Torvalds committed
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
			inode->i_state |= I_FREEING;
			count++;
			continue;
		}
		busy = 1;
	}
	/* only unused inodes may be cached with i_count zero */
	inodes_stat.nr_unused -= count;
	return busy;
}

/**
 *	invalidate_inodes	- discard the inodes on a device
 *	@sb: superblock
 *
 *	Discard all of the inodes for a given superblock. If the discard
 *	fails because there are busy inodes then a non zero value is returned.
 *	If the discard is successful all the inodes have been discarded.
 */
404
int invalidate_inodes(struct super_block *sb)
Linus Torvalds's avatar
Linus Torvalds committed
405
406
407
408
{
	int busy;
	LIST_HEAD(throw_away);

Nick Piggin's avatar
Nick Piggin committed
409
	down_write(&iprune_sem);
Linus Torvalds's avatar
Linus Torvalds committed
410
	spin_lock(&inode_lock);
411
	fsnotify_unmount_inodes(&sb->s_inodes);
Linus Torvalds's avatar
Linus Torvalds committed
412
413
414
415
	busy = invalidate_list(&sb->s_inodes, &throw_away);
	spin_unlock(&inode_lock);

	dispose_list(&throw_away);
Nick Piggin's avatar
Nick Piggin committed
416
	up_write(&iprune_sem);
Linus Torvalds's avatar
Linus Torvalds committed
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453

	return busy;
}

static int can_unuse(struct inode *inode)
{
	if (inode->i_state)
		return 0;
	if (inode_has_buffers(inode))
		return 0;
	if (atomic_read(&inode->i_count))
		return 0;
	if (inode->i_data.nrpages)
		return 0;
	return 1;
}

/*
 * Scan `goal' inodes on the unused list for freeable ones. They are moved to
 * a temporary list and then are freed outside inode_lock by dispose_list().
 *
 * Any inodes which are pinned purely because of attached pagecache have their
 * pagecache removed.  We expect the final iput() on that inode to add it to
 * the front of the inode_unused list.  So look for it there and if the
 * inode is still freeable, proceed.  The right inode is found 99.9% of the
 * time in testing on a 4-way.
 *
 * If the inode has metadata buffers attached to mapping->private_list then
 * try to remove them.
 */
static void prune_icache(int nr_to_scan)
{
	LIST_HEAD(freeable);
	int nr_pruned = 0;
	int nr_scanned;
	unsigned long reap = 0;

Nick Piggin's avatar
Nick Piggin committed
454
	down_read(&iprune_sem);
Linus Torvalds's avatar
Linus Torvalds committed
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
	spin_lock(&inode_lock);
	for (nr_scanned = 0; nr_scanned < nr_to_scan; nr_scanned++) {
		struct inode *inode;

		if (list_empty(&inode_unused))
			break;

		inode = list_entry(inode_unused.prev, struct inode, i_list);

		if (inode->i_state || atomic_read(&inode->i_count)) {
			list_move(&inode->i_list, &inode_unused);
			continue;
		}
		if (inode_has_buffers(inode) || inode->i_data.nrpages) {
			__iget(inode);
			spin_unlock(&inode_lock);
			if (remove_inode_buffers(inode))
472
473
				reap += invalidate_mapping_pages(&inode->i_data,
								0, -1);
Linus Torvalds's avatar
Linus Torvalds committed
474
475
476
477
478
479
480
481
482
483
			iput(inode);
			spin_lock(&inode_lock);

			if (inode != list_entry(inode_unused.next,
						struct inode, i_list))
				continue;	/* wrong inode or list_empty */
			if (!can_unuse(inode))
				continue;
		}
		list_move(&inode->i_list, &freeable);
484
		WARN_ON(inode->i_state & I_NEW);
Linus Torvalds's avatar
Linus Torvalds committed
485
486
487
488
		inode->i_state |= I_FREEING;
		nr_pruned++;
	}
	inodes_stat.nr_unused -= nr_pruned;
489
490
491
492
	if (current_is_kswapd())
		__count_vm_events(KSWAPD_INODESTEAL, reap);
	else
		__count_vm_events(PGINODESTEAL, reap);
Linus Torvalds's avatar
Linus Torvalds committed
493
494
495
	spin_unlock(&inode_lock);

	dispose_list(&freeable);
Nick Piggin's avatar
Nick Piggin committed
496
	up_read(&iprune_sem);
Linus Torvalds's avatar
Linus Torvalds committed
497
498
499
500
501
502
503
504
505
506
507
}

/*
 * shrink_icache_memory() will attempt to reclaim some unused inodes.  Here,
 * "unused" means that no dentries are referring to the inodes: the files are
 * not open and the dcache references to those inodes have already been
 * reclaimed.
 *
 * This function is passed the number of inodes to scan, and it returns the
 * total number of remaining possibly-reclaimable inodes.
 */
508
static int shrink_icache_memory(struct shrinker *shrink, int nr, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
509
510
511
512
513
514
{
	if (nr) {
		/*
		 * Nasty deadlock avoidance.  We may hold various FS locks,
		 * and we don't want to recurse into the FS that called us
		 * in clear_inode() and friends..
515
		 */
Linus Torvalds's avatar
Linus Torvalds committed
516
517
518
519
520
521
522
		if (!(gfp_mask & __GFP_FS))
			return -1;
		prune_icache(nr);
	}
	return (inodes_stat.nr_unused / 100) * sysctl_vfs_cache_pressure;
}

523
524
525
526
527
static struct shrinker icache_shrinker = {
	.shrink = shrink_icache_memory,
	.seeks = DEFAULT_SEEKS,
};

Linus Torvalds's avatar
Linus Torvalds committed
528
529
530
531
532
533
534
static void __wait_on_freeing_inode(struct inode *inode);
/*
 * Called with the inode lock held.
 * NOTE: we are not increasing the inode-refcount, you must call __iget()
 * by hand after calling find_inode now! This simplifies iunique and won't
 * add any additional branch in the common code.
 */
535
536
537
538
static struct inode *find_inode(struct super_block *sb,
				struct hlist_head *head,
				int (*test)(struct inode *, void *),
				void *data)
Linus Torvalds's avatar
Linus Torvalds committed
539
540
{
	struct hlist_node *node;
541
	struct inode *inode = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
542
543

repeat:
544
	hlist_for_each_entry(inode, node, head, i_hash) {
Linus Torvalds's avatar
Linus Torvalds committed
545
546
547
548
		if (inode->i_sb != sb)
			continue;
		if (!test(inode, data))
			continue;
549
		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
Linus Torvalds's avatar
Linus Torvalds committed
550
551
552
553
554
555
556
557
558
559
560
561
			__wait_on_freeing_inode(inode);
			goto repeat;
		}
		break;
	}
	return node ? inode : NULL;
}

/*
 * find_inode_fast is the fast path version of find_inode, see the comment at
 * iget_locked for details.
 */
562
563
static struct inode *find_inode_fast(struct super_block *sb,
				struct hlist_head *head, unsigned long ino)
Linus Torvalds's avatar
Linus Torvalds committed
564
565
{
	struct hlist_node *node;
566
	struct inode *inode = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
567
568

repeat:
569
	hlist_for_each_entry(inode, node, head, i_hash) {
Linus Torvalds's avatar
Linus Torvalds committed
570
571
572
573
		if (inode->i_ino != ino)
			continue;
		if (inode->i_sb != sb)
			continue;
574
		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
Linus Torvalds's avatar
Linus Torvalds committed
575
576
577
578
579
580
581
582
			__wait_on_freeing_inode(inode);
			goto repeat;
		}
		break;
	}
	return node ? inode : NULL;
}

583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
static unsigned long hash(struct super_block *sb, unsigned long hashval)
{
	unsigned long tmp;

	tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
			L1_CACHE_BYTES;
	tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> I_HASHBITS);
	return tmp & I_HASHMASK;
}

static inline void
__inode_add_to_lists(struct super_block *sb, struct hlist_head *head,
			struct inode *inode)
{
	inodes_stat.nr_inodes++;
	list_add(&inode->i_list, &inode_in_use);
	list_add(&inode->i_sb_list, &sb->s_inodes);
	if (head)
		hlist_add_head(&inode->i_hash, head);
}

/**
 * inode_add_to_lists - add a new inode to relevant lists
606
607
 * @sb: superblock inode belongs to
 * @inode: inode to mark in use
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
 *
 * When an inode is allocated it needs to be accounted for, added to the in use
 * list, the owning superblock and the inode hash. This needs to be done under
 * the inode_lock, so export a function to do this rather than the inode lock
 * itself. We calculate the hash list to add to here so it is all internal
 * which requires the caller to have already set up the inode number in the
 * inode to add.
 */
void inode_add_to_lists(struct super_block *sb, struct inode *inode)
{
	struct hlist_head *head = inode_hashtable + hash(sb, inode->i_ino);

	spin_lock(&inode_lock);
	__inode_add_to_lists(sb, head, inode);
	spin_unlock(&inode_lock);
}
EXPORT_SYMBOL_GPL(inode_add_to_lists);

Linus Torvalds's avatar
Linus Torvalds committed
626
627
628
629
/**
 *	new_inode 	- obtain an inode
 *	@sb: superblock
 *
630
 *	Allocates a new inode for given superblock. The default gfp_mask
631
 *	for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE.
632
633
634
635
636
 *	If HIGHMEM pages are unsuitable or it is known that pages allocated
 *	for the page cache are not reclaimable or migratable,
 *	mapping_set_gfp_mask() must be called with suitable flags on the
 *	newly created inode's mapping
 *
Linus Torvalds's avatar
Linus Torvalds committed
637
638
639
 */
struct inode *new_inode(struct super_block *sb)
{
640
641
642
643
644
645
	/*
	 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
	 * error if st_ino won't fit in target struct field. Use 32bit counter
	 * here to attempt to avoid that.
	 */
	static unsigned int last_ino;
646
	struct inode *inode;
Linus Torvalds's avatar
Linus Torvalds committed
647
648

	spin_lock_prefetch(&inode_lock);
649

Linus Torvalds's avatar
Linus Torvalds committed
650
651
652
	inode = alloc_inode(sb);
	if (inode) {
		spin_lock(&inode_lock);
653
		__inode_add_to_lists(sb, NULL, inode);
Linus Torvalds's avatar
Linus Torvalds committed
654
655
656
657
658
659
660
661
662
663
		inode->i_ino = ++last_ino;
		inode->i_state = 0;
		spin_unlock(&inode_lock);
	}
	return inode;
}
EXPORT_SYMBOL(new_inode);

void unlock_new_inode(struct inode *inode)
{
664
#ifdef CONFIG_DEBUG_LOCK_ALLOC
665
	if (S_ISDIR(inode->i_mode)) {
666
667
		struct file_system_type *type = inode->i_sb->s_type;

668
669
670
671
672
673
674
675
676
677
678
		/* Set new key only if filesystem hasn't already changed it */
		if (!lockdep_match_class(&inode->i_mutex,
		    &type->i_mutex_key)) {
			/*
			 * ensure nobody is actually holding i_mutex
			 */
			mutex_destroy(&inode->i_mutex);
			mutex_init(&inode->i_mutex);
			lockdep_set_class(&inode->i_mutex,
					  &type->i_mutex_dir_key);
		}
679
	}
680
#endif
Linus Torvalds's avatar
Linus Torvalds committed
681
	/*
Christoph Hellwig's avatar
Christoph Hellwig committed
682
	 * This is special!  We do not need the spinlock when clearing I_NEW,
683
684
	 * because we're guaranteed that nobody else tries to do anything about
	 * the state of the inode when it is locked, as we just created it (so
Christoph Hellwig's avatar
Christoph Hellwig committed
685
	 * there can be no old holders that haven't tested I_NEW).
686
	 * However we must emit the memory barrier so that other CPUs reliably
Christoph Hellwig's avatar
Christoph Hellwig committed
687
	 * see the clearing of I_NEW after the other inode initialisation has
688
	 * completed.
Linus Torvalds's avatar
Linus Torvalds committed
689
	 */
690
	smp_mb();
Christoph Hellwig's avatar
Christoph Hellwig committed
691
692
	WARN_ON(!(inode->i_state & I_NEW));
	inode->i_state &= ~I_NEW;
Linus Torvalds's avatar
Linus Torvalds committed
693
694
695
696
697
698
699
700
701
702
	wake_up_inode(inode);
}
EXPORT_SYMBOL(unlock_new_inode);

/*
 * This is called without the inode lock held.. Be careful.
 *
 * We no longer cache the sb_flags in i_flags - see fs.h
 *	-- rmk@arm.uk.linux.org
 */
703
704
705
706
707
static struct inode *get_new_inode(struct super_block *sb,
				struct hlist_head *head,
				int (*test)(struct inode *, void *),
				int (*set)(struct inode *, void *),
				void *data)
Linus Torvalds's avatar
Linus Torvalds committed
708
{
709
	struct inode *inode;
Linus Torvalds's avatar
Linus Torvalds committed
710
711
712

	inode = alloc_inode(sb);
	if (inode) {
713
		struct inode *old;
Linus Torvalds's avatar
Linus Torvalds committed
714
715
716
717
718
719
720
721

		spin_lock(&inode_lock);
		/* We released the lock, so.. */
		old = find_inode(sb, head, test, data);
		if (!old) {
			if (set(inode, data))
				goto set_failed;

722
			__inode_add_to_lists(sb, head, inode);
Christoph Hellwig's avatar
Christoph Hellwig committed
723
			inode->i_state = I_NEW;
Linus Torvalds's avatar
Linus Torvalds committed
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
			spin_unlock(&inode_lock);

			/* Return the locked inode with I_NEW set, the
			 * caller is responsible for filling in the contents
			 */
			return inode;
		}

		/*
		 * Uhhuh, somebody else created the same inode under
		 * us. Use the old inode instead of the one we just
		 * allocated.
		 */
		__iget(old);
		spin_unlock(&inode_lock);
		destroy_inode(inode);
		inode = old;
		wait_on_inode(inode);
	}
	return inode;

set_failed:
	spin_unlock(&inode_lock);
	destroy_inode(inode);
	return NULL;
}

/*
 * get_new_inode_fast is the fast path version of get_new_inode, see the
 * comment at iget_locked for details.
 */
755
756
static struct inode *get_new_inode_fast(struct super_block *sb,
				struct hlist_head *head, unsigned long ino)
Linus Torvalds's avatar
Linus Torvalds committed
757
{
758
	struct inode *inode;
Linus Torvalds's avatar
Linus Torvalds committed
759
760
761

	inode = alloc_inode(sb);
	if (inode) {
762
		struct inode *old;
Linus Torvalds's avatar
Linus Torvalds committed
763
764
765
766
767
768

		spin_lock(&inode_lock);
		/* We released the lock, so.. */
		old = find_inode_fast(sb, head, ino);
		if (!old) {
			inode->i_ino = ino;
769
			__inode_add_to_lists(sb, head, inode);
Christoph Hellwig's avatar
Christoph Hellwig committed
770
			inode->i_state = I_NEW;
Linus Torvalds's avatar
Linus Torvalds committed
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
			spin_unlock(&inode_lock);

			/* Return the locked inode with I_NEW set, the
			 * caller is responsible for filling in the contents
			 */
			return inode;
		}

		/*
		 * Uhhuh, somebody else created the same inode under
		 * us. Use the old inode instead of the one we just
		 * allocated.
		 */
		__iget(old);
		spin_unlock(&inode_lock);
		destroy_inode(inode);
		inode = old;
		wait_on_inode(inode);
	}
	return inode;
}

/**
 *	iunique - get a unique inode number
 *	@sb: superblock
 *	@max_reserved: highest reserved inode number
 *
 *	Obtain an inode number that is unique on the system for a given
 *	superblock. This is used by file systems that have no natural
 *	permanent inode numbering system. An inode number is returned that
 *	is higher than the reserved limit but unique.
 *
 *	BUGS:
 *	With a large number of inodes live on the file system this function
 *	currently becomes quite slow.
 */
ino_t iunique(struct super_block *sb, ino_t max_reserved)
{
809
810
811
812
813
814
	/*
	 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
	 * error if st_ino won't fit in target struct field. Use 32bit counter
	 * here to attempt to avoid that.
	 */
	static unsigned int counter;
Linus Torvalds's avatar
Linus Torvalds committed
815
	struct inode *inode;
816
	struct hlist_head *head;
Linus Torvalds's avatar
Linus Torvalds committed
817
	ino_t res;
818

Linus Torvalds's avatar
Linus Torvalds committed
819
	spin_lock(&inode_lock);
820
821
822
	do {
		if (counter <= max_reserved)
			counter = max_reserved + 1;
Linus Torvalds's avatar
Linus Torvalds committed
823
		res = counter++;
824
		head = inode_hashtable + hash(sb, res);
Linus Torvalds's avatar
Linus Torvalds committed
825
		inode = find_inode_fast(sb, head, res);
826
827
	} while (inode != NULL);
	spin_unlock(&inode_lock);
Linus Torvalds's avatar
Linus Torvalds committed
828

829
830
	return res;
}
Linus Torvalds's avatar
Linus Torvalds committed
831
832
833
834
835
EXPORT_SYMBOL(iunique);

struct inode *igrab(struct inode *inode)
{
	spin_lock(&inode_lock);
836
	if (!(inode->i_state & (I_FREEING|I_WILL_FREE)))
Linus Torvalds's avatar
Linus Torvalds committed
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
		__iget(inode);
	else
		/*
		 * Handle the case where s_op->clear_inode is not been
		 * called yet, and somebody is calling igrab
		 * while the inode is getting freed.
		 */
		inode = NULL;
	spin_unlock(&inode_lock);
	return inode;
}
EXPORT_SYMBOL(igrab);

/**
 * ifind - internal function, you want ilookup5() or iget5().
 * @sb:		super block of file system to search
 * @head:       the head of the list to search
 * @test:	callback used for comparisons between inodes
 * @data:	opaque data pointer to pass to @test
856
 * @wait:	if true wait for the inode to be unlocked, if false do not
Linus Torvalds's avatar
Linus Torvalds committed
857
858
859
860
861
862
863
864
865
866
867
868
 *
 * ifind() searches for the inode specified by @data in the inode
 * cache. This is a generalized version of ifind_fast() for file systems where
 * the inode number is not sufficient for unique identification of an inode.
 *
 * If the inode is in the cache, the inode is returned with an incremented
 * reference count.
 *
 * Otherwise NULL is returned.
 *
 * Note, @test is called with the inode_lock held, so can't sleep.
 */
869
static struct inode *ifind(struct super_block *sb,
Linus Torvalds's avatar
Linus Torvalds committed
870
		struct hlist_head *head, int (*test)(struct inode *, void *),
871
		void *data, const int wait)
Linus Torvalds's avatar
Linus Torvalds committed
872
873
874
875
876
877
878
879
{
	struct inode *inode;

	spin_lock(&inode_lock);
	inode = find_inode(sb, head, test, data);
	if (inode) {
		__iget(inode);
		spin_unlock(&inode_lock);
880
881
		if (likely(wait))
			wait_on_inode(inode);
Linus Torvalds's avatar
Linus Torvalds committed
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
		return inode;
	}
	spin_unlock(&inode_lock);
	return NULL;
}

/**
 * ifind_fast - internal function, you want ilookup() or iget().
 * @sb:		super block of file system to search
 * @head:       head of the list to search
 * @ino:	inode number to search for
 *
 * ifind_fast() searches for the inode @ino in the inode cache. This is for
 * file systems where the inode number is sufficient for unique identification
 * of an inode.
 *
 * If the inode is in the cache, the inode is returned with an incremented
 * reference count.
 *
 * Otherwise NULL is returned.
 */
903
static struct inode *ifind_fast(struct super_block *sb,
Linus Torvalds's avatar
Linus Torvalds committed
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
		struct hlist_head *head, unsigned long ino)
{
	struct inode *inode;

	spin_lock(&inode_lock);
	inode = find_inode_fast(sb, head, ino);
	if (inode) {
		__iget(inode);
		spin_unlock(&inode_lock);
		wait_on_inode(inode);
		return inode;
	}
	spin_unlock(&inode_lock);
	return NULL;
}

/**
921
 * ilookup5_nowait - search for an inode in the inode cache
Linus Torvalds's avatar
Linus Torvalds committed
922
923
924
925
926
927
928
929
930
931
932
 * @sb:		super block of file system to search
 * @hashval:	hash value (usually inode number) to search for
 * @test:	callback used for comparisons between inodes
 * @data:	opaque data pointer to pass to @test
 *
 * ilookup5() uses ifind() to search for the inode specified by @hashval and
 * @data in the inode cache. This is a generalized version of ilookup() for
 * file systems where the inode number is not sufficient for unique
 * identification of an inode.
 *
 * If the inode is in the cache, the inode is returned with an incremented
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
 * reference count.  Note, the inode lock is not waited upon so you have to be
 * very careful what you do with the returned inode.  You probably should be
 * using ilookup5() instead.
 *
 * Otherwise NULL is returned.
 *
 * Note, @test is called with the inode_lock held, so can't sleep.
 */
struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
		int (*test)(struct inode *, void *), void *data)
{
	struct hlist_head *head = inode_hashtable + hash(sb, hashval);

	return ifind(sb, head, test, data, 0);
}
EXPORT_SYMBOL(ilookup5_nowait);

/**
 * ilookup5 - search for an inode in the inode cache
 * @sb:		super block of file system to search
 * @hashval:	hash value (usually inode number) to search for
 * @test:	callback used for comparisons between inodes
 * @data:	opaque data pointer to pass to @test
 *
 * ilookup5() uses ifind() to search for the inode specified by @hashval and
 * @data in the inode cache. This is a generalized version of ilookup() for
 * file systems where the inode number is not sufficient for unique
 * identification of an inode.
 *
 * If the inode is in the cache, the inode lock is waited upon and the inode is
 * returned with an incremented reference count.
Linus Torvalds's avatar
Linus Torvalds committed
964
965
966
967
968
969
970
971
972
973
 *
 * Otherwise NULL is returned.
 *
 * Note, @test is called with the inode_lock held, so can't sleep.
 */
struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
		int (*test)(struct inode *, void *), void *data)
{
	struct hlist_head *head = inode_hashtable + hash(sb, hashval);

974
	return ifind(sb, head, test, data, 1);
Linus Torvalds's avatar
Linus Torvalds committed
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
}
EXPORT_SYMBOL(ilookup5);

/**
 * ilookup - search for an inode in the inode cache
 * @sb:		super block of file system to search
 * @ino:	inode number to search for
 *
 * ilookup() uses ifind_fast() to search for the inode @ino in the inode cache.
 * This is for file systems where the inode number is sufficient for unique
 * identification of an inode.
 *
 * If the inode is in the cache, the inode is returned with an incremented
 * reference count.
 *
 * Otherwise NULL is returned.
 */
struct inode *ilookup(struct super_block *sb, unsigned long ino)
{
	struct hlist_head *head = inode_hashtable + hash(sb, ino);

	return ifind_fast(sb, head, ino);
}
EXPORT_SYMBOL(ilookup);

/**
 * iget5_locked - obtain an inode from a mounted file system
 * @sb:		super block of file system
 * @hashval:	hash value (usually inode number) to get
 * @test:	callback used for comparisons between inodes
 * @set:	callback used to initialize a new struct inode
 * @data:	opaque data pointer to pass to @test and @set
 *
 * iget5_locked() uses ifind() to search for the inode specified by @hashval
 * and @data in the inode cache and if present it is returned with an increased
 * reference count. This is a generalized version of iget_locked() for file
 * systems where the inode number is not sufficient for unique identification
 * of an inode.
 *
 * If the inode is not in cache, get_new_inode() is called to allocate a new
 * inode and this is returned locked, hashed, and with the I_NEW flag set. The
 * file system gets to fill it in before unlocking it via unlock_new_inode().
 *
 * Note both @test and @set are called with the inode_lock held, so can't sleep.
 */
struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
		int (*test)(struct inode *, void *),
		int (*set)(struct inode *, void *), void *data)
{
	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
	struct inode *inode;

1027
	inode = ifind(sb, head, test, data, 1);
Linus Torvalds's avatar
Linus Torvalds committed
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
	if (inode)
		return inode;
	/*
	 * get_new_inode() will do the right thing, re-trying the search
	 * in case it had to block at any point.
	 */
	return get_new_inode(sb, head, test, set, data);
}
EXPORT_SYMBOL(iget5_locked);

/**
 * iget_locked - obtain an inode from a mounted file system
 * @sb:		super block of file system
 * @ino:	inode number to get
 *
 * iget_locked() uses ifind_fast() to search for the inode specified by @ino in
 * the inode cache and if present it is returned with an increased reference
 * count. This is for file systems where the inode number is sufficient for
 * unique identification of an inode.
 *
 * If the inode is not in cache, get_new_inode_fast() is called to allocate a
 * new inode and this is returned locked, hashed, and with the I_NEW flag set.
 * The file system gets to fill it in before unlocking it via
 * unlock_new_inode().
 */
struct inode *iget_locked(struct super_block *sb, unsigned long ino)
{
	struct hlist_head *head = inode_hashtable + hash(sb, ino);
	struct inode *inode;

	inode = ifind_fast(sb, head, ino);
	if (inode)
		return inode;
	/*
	 * get_new_inode_fast() will do the right thing, re-trying the search
	 * in case it had to block at any point.
	 */
	return get_new_inode_fast(sb, head, ino);
}
EXPORT_SYMBOL(iget_locked);

1069
1070
1071
1072
1073
1074
int insert_inode_locked(struct inode *inode)
{
	struct super_block *sb = inode->i_sb;
	ino_t ino = inode->i_ino;
	struct hlist_head *head = inode_hashtable + hash(sb, ino);

Christoph Hellwig's avatar
Christoph Hellwig committed
1075
	inode->i_state |= I_NEW;
1076
	while (1) {
1077
1078
		struct hlist_node *node;
		struct inode *old = NULL;
1079
		spin_lock(&inode_lock);
1080
1081
1082
1083
1084
		hlist_for_each_entry(old, node, head, i_hash) {
			if (old->i_ino != ino)
				continue;
			if (old->i_sb != sb)
				continue;
1085
			if (old->i_state & (I_FREEING|I_WILL_FREE))
1086
1087
1088
1089
				continue;
			break;
		}
		if (likely(!node)) {
1090
1091
1092
1093
1094
1095
1096
			hlist_add_head(&inode->i_hash, head);
			spin_unlock(&inode_lock);
			return 0;
		}
		__iget(old);
		spin_unlock(&inode_lock);
		wait_on_inode(old);
Al Viro's avatar
Al Viro committed
1097
		if (unlikely(!inode_unhashed(old))) {
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
			iput(old);
			return -EBUSY;
		}
		iput(old);
	}
}
EXPORT_SYMBOL(insert_inode_locked);

int insert_inode_locked4(struct inode *inode, unsigned long hashval,
		int (*test)(struct inode *, void *), void *data)
{
	struct super_block *sb = inode->i_sb;
	struct hlist_head *head = inode_hashtable + hash(sb, hashval);

Christoph Hellwig's avatar
Christoph Hellwig committed
1112
	inode->i_state |= I_NEW;
1113
1114

	while (1) {
1115
1116
1117
		struct hlist_node *node;
		struct inode *old = NULL;

1118
		spin_lock(&inode_lock);
1119
1120
1121
1122
1123
		hlist_for_each_entry(old, node, head, i_hash) {
			if (old->i_sb != sb)
				continue;
			if (!test(old, data))
				continue;
1124
			if (old->i_state & (I_FREEING|I_WILL_FREE))
1125
1126
1127
1128
				continue;
			break;
		}
		if (likely(!node)) {
1129
1130
1131
1132
1133
1134
1135
			hlist_add_head(&inode->i_hash, head);
			spin_unlock(&inode_lock);
			return 0;
		}
		__iget(old);
		spin_unlock(&inode_lock);
		wait_on_inode(old);
Al Viro's avatar
Al Viro committed
1136
		if (unlikely(!inode_unhashed(old))) {
1137
1138
1139
1140
1141
1142
1143
1144
			iput(old);
			return -EBUSY;
		}
		iput(old);
	}
}
EXPORT_SYMBOL(insert_inode_locked4);

Linus Torvalds's avatar
Linus Torvalds committed
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
/**
 *	__insert_inode_hash - hash an inode
 *	@inode: unhashed inode
 *	@hashval: unsigned long value used to locate this object in the
 *		inode_hashtable.
 *
 *	Add an inode to the inode hash for this superblock.
 */
void __insert_inode_hash(struct inode *inode, unsigned long hashval)
{
	struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
	spin_lock(&inode_lock);
	hlist_add_head(&inode->i_hash, head);
	spin_unlock(&inode_lock);
}
EXPORT_SYMBOL(__insert_inode_hash);

/**
 *	remove_inode_hash - remove an inode from the hash
 *	@inode: inode to unhash
 *
 *	Remove an inode from the superblock.
 */
void remove_inode_hash(struct inode *inode)
{
	spin_lock(&inode_lock);
	hlist_del_init(&inode->i_hash);
	spin_unlock(&inode_lock);
}
EXPORT_SYMBOL(remove_inode_hash);

1176
1177
1178
1179
1180
1181
int generic_delete_inode(struct inode *inode)
{
	return 1;
}
EXPORT_SYMBOL(generic_delete_inode);

Linus Torvalds's avatar
Linus Torvalds committed
1182
/*
1183
1184
1185
 * Normal UNIX filesystem behaviour: delete the
 * inode when the usage count drops to zero, and
 * i_nlink is zero.
Linus Torvalds's avatar
Linus Torvalds committed
1186
 */
1187
int generic_drop_inode(struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
1188
{
Al Viro's avatar
Al Viro committed
1189
	return !inode->i_nlink || inode_unhashed(inode);
Linus Torvalds's avatar
Linus Torvalds committed
1190
}
1191
EXPORT_SYMBOL_GPL(generic_drop_inode);
Linus Torvalds's avatar
Linus Torvalds committed
1192

1193
1194
1195
/*
 * Called when we're dropping the last reference
 * to an inode.
1196
 *
1197
1198
1199
1200
1201
 * Call the FS "drop_inode()" function, defaulting to
 * the legacy UNIX filesystem behaviour.  If it tells
 * us to evict inode, do so.  Otherwise, retain inode
 * in cache if fs is alive, sync and evict if fs is
 * shutting down.
1202
 */
1203
static void iput_final(struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
1204
1205
{
	struct super_block *sb = inode->i_sb;
1206
1207
1208
1209
1210
1211
1212
	const struct super_operations *op = inode->i_sb->s_op;
	int drop;

	if (op && op->drop_inode)
		drop = op->drop_inode(inode);
	else
		drop = generic_drop_inode(inode);
Linus Torvalds's avatar
Linus Torvalds committed
1213

1214
	if (!drop) {
Joern Engel's avatar
Joern Engel committed
1215
		if (!(inode->i_state & (I_DIRTY|I_SYNC)))
Linus Torvalds's avatar
Linus Torvalds committed
1216
1217
			list_move(&inode->i_list, &inode_unused);
		inodes_stat.nr_unused++;
1218
		if (sb->s_flags & MS_ACTIVE) {
1219
			spin_unlock(&inode_lock);
1220
			return;
1221
		}
1222
		WARN_ON(inode->i_state & I_NEW);
1223
1224
		inode->i_state |= I_WILL_FREE;
		spin_unlock(&inode_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1225
1226
		write_inode_now(inode, 1);
		spin_lock(&inode_lock);
1227
		WARN_ON(inode->i_state & I_NEW);
1228
		inode->i_state &= ~I_WILL_FREE;
Linus Torvalds's avatar
Linus Torvalds committed
1229
1230
1231
1232
1233
		inodes_stat.nr_unused--;
		hlist_del_init(&inode->i_hash);
	}
	list_del_init(&inode->i_list);
	list_del_init(&inode->i_sb_list);
1234
	WARN_ON(inode->i_state & I_NEW);
1235
	inode->i_state |= I_FREEING;
Linus Torvalds's avatar
Linus Torvalds committed
1236
1237
	inodes_stat.nr_inodes--;
	spin_unlock(&inode_lock);
1238
	evict(inode);
1239
1240
1241
	spin_lock(&inode_lock);
	hlist_del_init(&inode->i_hash);
	spin_unlock(&inode_lock);
1242
	wake_up_inode(inode);
1243
	BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
Linus Torvalds's avatar
Linus Torvalds committed
1244
1245
1246
1247
	destroy_inode(inode);
}

/**
1248
 *	iput	- put an inode
Linus Torvalds's avatar
Linus Torvalds committed
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
 *	@inode: inode to put
 *
 *	Puts an inode, dropping its usage count. If the inode use count hits
 *	zero, the inode is then freed and may also be destroyed.
 *
 *	Consequently, iput() can sleep.
 */
void iput(struct inode *inode)
{
	if (inode) {
1259
		BUG_ON(inode->i_state & I_CLEAR);
Linus Torvalds's avatar
Linus Torvalds committed
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274

		if (atomic_dec_and_lock(&inode->i_count, &inode_lock))
			iput_final(inode);
	}
}
EXPORT_SYMBOL(iput);

/**
 *	bmap	- find a block number in a file
 *	@inode: inode of file
 *	@block: block to find
 *
 *	Returns the block number on the device holding the inode that
 *	is the disk block number for the block of the file requested.
 *	That is, asked for block 4 of inode 1 the function will return the
1275
 *	disk block relative to the disk start that holds that block of the
Linus Torvalds's avatar
Linus Torvalds committed
1276
1277
 *	file.
 */
1278
sector_t bmap(struct inode *inode, sector_t block)
Linus Torvalds's avatar
Linus Torvalds committed
1279
1280
1281
1282
1283
1284
1285
1286
{
	sector_t res = 0;
	if (inode->i_mapping->a_ops->bmap)
		res = inode->i_mapping->a_ops->bmap(inode->i_mapping, block);
	return res;
}
EXPORT_SYMBOL(bmap);

1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
/*
 * With relative atime, only update atime if the previous atime is
 * earlier than either the ctime or mtime or if at least a day has
 * passed since the last atime update.
 */
static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
			     struct timespec now)
{

	if (!(mnt->mnt_flags & MNT_RELATIME))
		return 1;
	/*
	 * Is mtime younger than atime? If yes, update atime:
	 */
	if (timespec_compare(&inode->i_mtime, &inode->i_atime) >= 0)
		return 1;
	/*
	 * Is ctime younger than atime? If yes, update atime:
	 */
	if (timespec_compare(&inode->i_ctime, &inode->i_atime) >= 0)
		return 1;

	/*
	 * Is the previous atime value older than a day? If yes,
	 * update atime:
	 */
	if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
		return 1;
	/*
	 * Good, we can skip the atime update:
	 */
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
1321
/**
1322
1323
 *	touch_atime	-	update the access time
 *	@mnt: mount the inode is accessed on
1324
 *	@dentry: dentry accessed
Linus Torvalds's avatar
Linus Torvalds committed
1325
1326
1327
1328
1329
 *
 *	Update the accessed time on an inode and mark it for writeback.
 *	This function automatically handles read only file systems and media,
 *	as well as the "noatime" flag and inode specific "noatime" markers.
 */
1330
void touch_atime(struct vfsmount *mnt, struct dentry *dentry)
Linus Torvalds's avatar
Linus Torvalds committed
1331
{
1332
	struct inode *inode = dentry->d_inode;
Linus Torvalds's avatar
Linus Torvalds committed
1333
1334
	struct timespec now;

1335
	if (inode->i_flags & S_NOATIME)
1336
		return;
1337
	if (IS_NOATIME(inode))
1338
		return;
Andrew Morton's avatar
Andrew Morton committed
1339
	if ((inode->i_sb->s_flags & MS_NODIRATIME) && S_ISDIR(inode->i_mode))
1340
		return;
Valerie Henson's avatar
Valerie Henson committed
1341

1342
	if (mnt->mnt_flags & MNT_NOATIME)
1343
		return;
1344
	if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1345
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1346
1347

	now = current_fs_time(inode->i_sb);
1348
1349

	if (!relatime_need_update(mnt, inode, now))
1350
		return;
1351

Valerie Henson's avatar
Valerie Henson committed
1352
	if (timespec_equal(&inode->i_atime, &now))
1353
1354
1355
1356
		return;

	if (mnt_want_write(mnt))
		return;
Valerie Henson's avatar
Valerie Henson committed
1357
1358
1359

	inode->i_atime = now;
	mark_inode_dirty_sync(inode);
1360
	mnt_drop_write(mnt);
Linus Torvalds's avatar
Linus Torvalds committed
1361
}
1362
EXPORT_SYMBOL(touch_atime);
Linus Torvalds's avatar
Linus Torvalds committed
1363
1364

/**
1365
1366
 *	file_update_time	-	update mtime and ctime time
 *	@file: file accessed
Linus Torvalds's avatar
Linus Torvalds committed
1367
 *
1368
1369
1370
1371
 *	Update the mtime and ctime members of an inode and mark the inode
 *	for writeback.  Note that this function is meant exclusively for
 *	usage in the file write path of filesystems, and filesystems may
 *	choose to explicitly ignore update via this function with the
1372
 *	S_NOCMTIME inode flag, e.g. for network filesystem where these
1373
 *	timestamps are handled by the server.
Linus Torvalds's avatar
Linus Torvalds committed
1374
1375
 */

1376
void file_update_time(struct file *file)
Linus Torvalds's avatar
Linus Torvalds committed
1377
{
1378
	struct inode *inode = file->f_path.dentry->d_inode;
Linus Torvalds's avatar
Linus Torvalds committed
1379
	struct timespec now;
Andi Kleen's avatar
Andi Kleen committed
1380
	enum { S_MTIME = 1, S_CTIME = 2, S_VERSION = 4 } sync_it = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1381

Andi Kleen's avatar
Andi Kleen committed
1382
	/* First try to exhaust all avenues to not sync */
Linus Torvalds's avatar
Linus Torvalds committed
1383
1384
	if (IS_NOCMTIME(inode))
		return;
1385

Linus Torvalds's avatar
Linus Torvalds committed
1386
	now = current_fs_time(inode->i_sb);
Andi Kleen's avatar
Andi Kleen committed
1387
1388
	if (!timespec_equal(&inode->i_mtime, &now))
		sync_it = S_MTIME;
Linus Torvalds's avatar
Linus Torvalds committed
1389

Andi Kleen's avatar
Andi Kleen committed
1390
1391
	if (!timespec_equal(&inode->i_ctime, &now))
		sync_it |= S_CTIME;
1392

Andi Kleen's avatar
Andi Kleen committed
1393
1394
	if (IS_I_VERSION(inode))
		sync_it |= S_VERSION;
1395

Andi Kleen's avatar
Andi Kleen committed
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
	if (!sync_it)
		return;

	/* Finally allowed to write? Takes lock. */
	if (mnt_want_write_file(file))
		return;

	/* Only change inode inside the lock region */
	if (sync_it & S_VERSION)
		inode_inc_iversion(inode);
	if (sync_it & S_CTIME)
		inode->i_ctime = now;
	if (sync_it & S_MTIME)
		inode->i_mtime = now;
	mark_inode_dirty_sync(inode);
1411
	mnt_drop_write(file->f_path.mnt);
Linus Torvalds's avatar
Linus Torvalds committed
1412
}
1413
EXPORT_SYMBOL(file_update_time);
Linus Torvalds's avatar
Linus Torvalds committed
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429

int inode_needs_sync(struct inode *inode)
{
	if (IS_SYNC(inode))
		return 1;
	if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
		return 1;
	return 0;
}
EXPORT_SYMBOL(inode_needs_sync);

int inode_wait(void *word)
{
	schedule();
	return 0;
}
1430
EXPORT_SYMBOL(inode_wait);
Linus Torvalds's avatar
Linus Torvalds committed
1431
1432

/*
1433
1434
1435
1436
1437
1438
 * If we try to find an inode in the inode hash while it is being
 * deleted, we have to wait until the filesystem completes its
 * deletion before reporting that it isn't found.  This function waits
 * until the deletion _might_ have completed.  Callers are responsible
 * to recheck inode state.
 *
Christoph Hellwig's avatar
Christoph Hellwig committed
1439
 * It doesn't matter if I_NEW is not set initially, a call to
1440
1441
 * wake_up_inode() after removing from the hash list will DTRT.
 *
Linus Torvalds's avatar
Linus Torvalds committed
1442
1443
1444
1445
1446
 * This is called with inode_lock held.
 */
static void __wait_on_freeing_inode(struct inode *inode)
{
	wait_queue_head_t *wq;
Christoph Hellwig's avatar
Christoph Hellwig committed
1447
1448
	DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
	wq = bit_waitqueue(&inode->i_state, __I_NEW);
Linus Torvalds's avatar
Linus Torvalds committed
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
	prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
	spin_unlock(&inode_lock);
	schedule();
	finish_wait(wq, &wait.wait);
	spin_lock(&inode_lock);
}

static __initdata unsigned long ihash_entries;
static int __init set_ihash_entries(char *str)
{
	if (!str)
		return 0;
	ihash_entries = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("ihash_entries=", set_ihash_entries);

/*
 * Initialize the waitqueues and inode hash table.
 */
void __init inode_init_early(void)
{
	int loop;

	/* If hashes are distributed across NUMA nodes, defer
	 * hash allocation until vmalloc space is available.
	 */
	if (hashdist)
		return;

	inode_hashtable =
		alloc_large_system_hash("Inode-cache",
					sizeof(struct hlist_head),
					ihash_entries,
					14,
					HASH_EARLY,
					&i_hash_shift,
					&i_hash_mask,
					0);

	for (loop = 0; loop < (1 << i_hash_shift); loop++)
		INIT_HLIST_HEAD(&inode_hashtable[loop]);
}