Commit fa0d7e3d authored by Nick Piggin's avatar Nick Piggin
Browse files

fs: icache RCU free inodes



RCU free the struct inode. This will allow:

- Subsequent store-free path walking patch. The inode must be consulted for
  permissions when walking, so an RCU inode reference is a must.
- sb_inode_list_lock to be moved inside i_lock because sb list walkers who want
  to take i_lock no longer need to take sb_inode_list_lock to walk the list in
  the first place. This will simplify and optimize locking.
- Could remove some nested trylock loops in dcache code
- Could potentially simplify things a bit in VM land. Do not need to take the
  page lock to follow page->mapping.

The downsides of this is the performance cost of using RCU. In a simple
creat/unlink microbenchmark, performance drops by about 10% due to inability to
reuse cache-hot slab objects. As iterations increase and RCU freeing starts
kicking over, this increases to about 20%.

In cases where inode lifetimes are longer (ie. many inodes may be allocated
during the average life span of a single inode), a lot of this cache reuse is
not applicable, so the regression caused by this patch is smaller.

The cache-hot regression could largely be avoided by using SLAB_DESTROY_BY_RCU,
however this adds some complexity to list walking and store-free path walking,
so I prefer to implement this at a later date, if it is shown to be a win in
real situations. I haven't found a regression in any non-micro benchmark so I
doubt it will be a problem.
Signed-off-by: default avatarNick Piggin <npiggin@kernel.dk>
parent 77812a1e
......@@ -346,3 +346,17 @@ look at examples of other filesystems) for guidance.
for details of what locks to replace dcache_lock with in order to protect
particular things. Most of the time, a filesystem only needs ->d_lock, which
protects *all* the dcache state of a given dentry.
--
[mandatory]
Filesystems must RCU-free their inodes, if they can have been accessed
via rcu-walk path walk (basically, if the file can have had a path name in the
vfs namespace).
i_dentry and i_rcu share storage in a union, and the vfs expects
i_dentry to be reinitialized before it is freed, so an:
INIT_LIST_HEAD(&inode->i_dentry);
must be done in the RCU callback.
......@@ -71,12 +71,18 @@ spufs_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void
spufs_destroy_inode(struct inode *inode)
static void spufs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(spufs_inode_cache, SPUFS_I(inode));
}
static void spufs_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, spufs_i_callback);
}
static void
spufs_init_once(void *p)
{
......
......@@ -826,6 +826,13 @@ const struct address_space_operations pohmelfs_aops = {
.set_page_dirty = __set_page_dirty_nobuffers,
};
static void pohmelfs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(pohmelfs_inode_cache, POHMELFS_I(inode));
}
/*
* ->detroy_inode() callback. Deletes inode from the caches
* and frees private data.
......@@ -842,8 +849,8 @@ static void pohmelfs_destroy_inode(struct inode *inode)
dprintk("%s: pi: %p, inode: %p, ino: %llu.\n",
__func__, pi, &pi->vfs_inode, pi->ino);
kmem_cache_free(pohmelfs_inode_cache, pi);
atomic_long_dec(&psb->total_inodes);
call_rcu(&inode->i_rcu, pohmelfs_i_callback);
}
/*
......
......@@ -62,11 +62,18 @@ static struct inode *smb_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void smb_destroy_inode(struct inode *inode)
static void smb_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(smb_inode_cachep, SMB_I(inode));
}
static void smb_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, smb_i_callback);
}
static void init_once(void *foo)
{
struct smb_inode_info *ei = (struct smb_inode_info *) foo;
......
......@@ -237,10 +237,17 @@ struct inode *v9fs_alloc_inode(struct super_block *sb)
*
*/
void v9fs_destroy_inode(struct inode *inode)
static void v9fs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(vcookie_cache, v9fs_inode2cookie(inode));
}
void v9fs_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, v9fs_i_callback);
}
#endif
/**
......
......@@ -240,11 +240,18 @@ static struct inode *adfs_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void adfs_destroy_inode(struct inode *inode)
static void adfs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(adfs_inode_cachep, ADFS_I(inode));
}
static void adfs_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, adfs_i_callback);
}
static void init_once(void *foo)
{
struct adfs_inode_info *ei = (struct adfs_inode_info *) foo;
......
......@@ -95,11 +95,18 @@ static struct inode *affs_alloc_inode(struct super_block *sb)
return &i->vfs_inode;
}
static void affs_destroy_inode(struct inode *inode)
static void affs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(affs_inode_cachep, AFFS_I(inode));
}
static void affs_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, affs_i_callback);
}
static void init_once(void *foo)
{
struct affs_inode_info *ei = (struct affs_inode_info *) foo;
......
......@@ -498,6 +498,14 @@ static struct inode *afs_alloc_inode(struct super_block *sb)
return &vnode->vfs_inode;
}
static void afs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
struct afs_vnode *vnode = AFS_FS_I(inode);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(afs_inode_cachep, vnode);
}
/*
* destroy an AFS inode struct
*/
......@@ -511,7 +519,7 @@ static void afs_destroy_inode(struct inode *inode)
ASSERTCMP(vnode->server, ==, NULL);
kmem_cache_free(afs_inode_cachep, vnode);
call_rcu(&inode->i_rcu, afs_i_callback);
atomic_dec(&afs_count_active_inodes);
}
......
......@@ -284,12 +284,18 @@ befs_alloc_inode(struct super_block *sb)
return &bi->vfs_inode;
}
static void
befs_destroy_inode(struct inode *inode)
static void befs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(befs_inode_cachep, BEFS_I(inode));
}
static void befs_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, befs_i_callback);
}
static void init_once(void *foo)
{
struct befs_inode_info *bi = (struct befs_inode_info *) foo;
......
......@@ -248,11 +248,18 @@ static struct inode *bfs_alloc_inode(struct super_block *sb)
return &bi->vfs_inode;
}
static void bfs_destroy_inode(struct inode *inode)
static void bfs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(bfs_inode_cachep, BFS_I(inode));
}
static void bfs_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, bfs_i_callback);
}
static void init_once(void *foo)
{
struct bfs_inode_info *bi = foo;
......
......@@ -409,13 +409,20 @@ static struct inode *bdev_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void bdev_destroy_inode(struct inode *inode)
static void bdev_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
struct bdev_inode *bdi = BDEV_I(inode);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(bdev_cachep, bdi);
}
static void bdev_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, bdev_i_callback);
}
static void init_once(void *foo)
{
struct bdev_inode *ei = (struct bdev_inode *) foo;
......
......@@ -6495,6 +6495,13 @@ struct inode *btrfs_alloc_inode(struct super_block *sb)
return inode;
}
static void btrfs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
void btrfs_destroy_inode(struct inode *inode)
{
struct btrfs_ordered_extent *ordered;
......@@ -6564,7 +6571,7 @@ void btrfs_destroy_inode(struct inode *inode)
inode_tree_del(inode);
btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
free:
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
call_rcu(&inode->i_rcu, btrfs_i_callback);
}
int btrfs_drop_inode(struct inode *inode)
......
......@@ -368,6 +368,15 @@ struct inode *ceph_alloc_inode(struct super_block *sb)
return &ci->vfs_inode;
}
static void ceph_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
struct ceph_inode_info *ci = ceph_inode(inode);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(ceph_inode_cachep, ci);
}
void ceph_destroy_inode(struct inode *inode)
{
struct ceph_inode_info *ci = ceph_inode(inode);
......@@ -407,7 +416,7 @@ void ceph_destroy_inode(struct inode *inode)
if (ci->i_xattrs.prealloc_blob)
ceph_buffer_put(ci->i_xattrs.prealloc_blob);
kmem_cache_free(ceph_inode_cachep, ci);
call_rcu(&inode->i_rcu, ceph_i_callback);
}
......
......@@ -334,10 +334,17 @@ cifs_alloc_inode(struct super_block *sb)
return &cifs_inode->vfs_inode;
}
static void cifs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(cifs_inode_cachep, CIFS_I(inode));
}
static void
cifs_destroy_inode(struct inode *inode)
{
kmem_cache_free(cifs_inode_cachep, CIFS_I(inode));
call_rcu(&inode->i_rcu, cifs_i_callback);
}
static void
......
......@@ -56,11 +56,18 @@ static struct inode *coda_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void coda_destroy_inode(struct inode *inode)
static void coda_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(coda_inode_cachep, ITOC(inode));
}
static void coda_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, coda_i_callback);
}
static void init_once(void *foo)
{
struct coda_inode_info *ei = (struct coda_inode_info *) foo;
......
......@@ -62,6 +62,16 @@ out:
return inode;
}
static void ecryptfs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
struct ecryptfs_inode_info *inode_info;
inode_info = ecryptfs_inode_to_private(inode);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(ecryptfs_inode_info_cache, inode_info);
}
/**
* ecryptfs_destroy_inode
* @inode: The ecryptfs inode
......@@ -88,7 +98,7 @@ static void ecryptfs_destroy_inode(struct inode *inode)
}
}
ecryptfs_destroy_crypt_stat(&inode_info->crypt_stat);
kmem_cache_free(ecryptfs_inode_info_cache, inode_info);
call_rcu(&inode->i_rcu, ecryptfs_i_callback);
}
/**
......
......@@ -65,11 +65,18 @@ static struct inode *efs_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void efs_destroy_inode(struct inode *inode)
static void efs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(efs_inode_cachep, INODE_INFO(inode));
}
static void efs_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, efs_i_callback);
}
static void init_once(void *foo)
{
struct efs_inode_info *ei = (struct efs_inode_info *) foo;
......
......@@ -150,12 +150,19 @@ static struct inode *exofs_alloc_inode(struct super_block *sb)
return &oi->vfs_inode;
}
static void exofs_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(exofs_inode_cachep, exofs_i(inode));
}
/*
* Remove an inode from the cache
*/
static void exofs_destroy_inode(struct inode *inode)
{
kmem_cache_free(exofs_inode_cachep, exofs_i(inode));
call_rcu(&inode->i_rcu, exofs_i_callback);
}
/*
......
......@@ -161,11 +161,18 @@ static struct inode *ext2_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void ext2_destroy_inode(struct inode *inode)
static void ext2_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(ext2_inode_cachep, EXT2_I(inode));
}
static void ext2_destroy_inode(struct inode *inode)
{
call_rcu(&inode->i_rcu, ext2_i_callback);
}
static void init_once(void *foo)
{
struct ext2_inode_info *ei = (struct ext2_inode_info *) foo;
......
......@@ -479,6 +479,13 @@ static struct inode *ext3_alloc_inode(struct super_block *sb)
return &ei->vfs_inode;
}
static void ext3_i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
INIT_LIST_HEAD(&inode->i_dentry);
kmem_cache_free(ext3_inode_cachep, EXT3_I(inode));
}
static void ext3_destroy_inode(struct inode *inode)
{
if (!list_empty(&(EXT3_I(inode)->i_orphan))) {
......@@ -489,7 +496,7 @@ static void ext3_destroy_inode(struct inode *inode)
false);
dump_stack();
}
kmem_cache_free(ext3_inode_cachep, EXT3_I(inode));
call_rcu(&inode->i_rcu, ext3_i_callback);
}
static void init_once(void *foo)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment