Skip to content

Commit fa0d7e3

Browse files
author
Nick Piggin
committed
fs: icache RCU free inodes
RCU free the struct inode. This will allow: - Subsequent store-free path walking patch. The inode must be consulted for permissions when walking, so an RCU inode reference is a must. - sb_inode_list_lock to be moved inside i_lock because sb list walkers who want to take i_lock no longer need to take sb_inode_list_lock to walk the list in the first place. This will simplify and optimize locking. - Could remove some nested trylock loops in dcache code - Could potentially simplify things a bit in VM land. Do not need to take the page lock to follow page->mapping. The downsides of this is the performance cost of using RCU. In a simple creat/unlink microbenchmark, performance drops by about 10% due to inability to reuse cache-hot slab objects. As iterations increase and RCU freeing starts kicking over, this increases to about 20%. In cases where inode lifetimes are longer (ie. many inodes may be allocated during the average life span of a single inode), a lot of this cache reuse is not applicable, so the regression caused by this patch is smaller. The cache-hot regression could largely be avoided by using SLAB_DESTROY_BY_RCU, however this adds some complexity to list walking and store-free path walking, so I prefer to implement this at a later date, if it is shown to be a win in real situations. I haven't found a regression in any non-micro benchmark so I doubt it will be a problem. Signed-off-by: Nick Piggin <[email protected]>
1 parent 77812a1 commit fa0d7e3

File tree

60 files changed

+490
-68
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+490
-68
lines changed

‎Documentation/filesystems/porting

+14
Original file line numberDiff line numberDiff line change
@@ -346,3 +346,17 @@ look at examples of other filesystems) for guidance.
346346
for details of what locks to replace dcache_lock with in order to protect
347347
particular things. Most of the time, a filesystem only needs ->d_lock, which
348348
protects *all* the dcache state of a given dentry.
349+
350+
--
351+
[mandatory]
352+
353+
Filesystems must RCU-free their inodes, if they can have been accessed
354+
via rcu-walk path walk (basically, if the file can have had a path name in the
355+
vfs namespace).
356+
357+
i_dentry and i_rcu share storage in a union, and the vfs expects
358+
i_dentry to be reinitialized before it is freed, so an:
359+
360+
INIT_LIST_HEAD(&inode->i_dentry);
361+
362+
must be done in the RCU callback.

‎arch/powerpc/platforms/cell/spufs/inode.c

+8-2
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,18 @@ spufs_alloc_inode(struct super_block *sb)
7171
return &ei->vfs_inode;
7272
}
7373

74-
static void
75-
spufs_destroy_inode(struct inode *inode)
74+
static void spufs_i_callback(struct rcu_head *head)
7675
{
76+
struct inode *inode = container_of(head, struct inode, i_rcu);
77+
INIT_LIST_HEAD(&inode->i_dentry);
7778
kmem_cache_free(spufs_inode_cache, SPUFS_I(inode));
7879
}
7980

81+
static void spufs_destroy_inode(struct inode *inode)
82+
{
83+
call_rcu(&inode->i_rcu, spufs_i_callback);
84+
}
85+
8086
static void
8187
spufs_init_once(void *p)
8288
{

‎drivers/staging/pohmelfs/inode.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -826,6 +826,13 @@ const struct address_space_operations pohmelfs_aops = {
826826
.set_page_dirty = __set_page_dirty_nobuffers,
827827
};
828828

829+
static void pohmelfs_i_callback(struct rcu_head *head)
830+
{
831+
struct inode *inode = container_of(head, struct inode, i_rcu);
832+
INIT_LIST_HEAD(&inode->i_dentry);
833+
kmem_cache_free(pohmelfs_inode_cache, POHMELFS_I(inode));
834+
}
835+
829836
/*
830837
* ->detroy_inode() callback. Deletes inode from the caches
831838
* and frees private data.
@@ -842,8 +849,8 @@ static void pohmelfs_destroy_inode(struct inode *inode)
842849

843850
dprintk("%s: pi: %p, inode: %p, ino: %llu.\n",
844851
__func__, pi, &pi->vfs_inode, pi->ino);
845-
kmem_cache_free(pohmelfs_inode_cache, pi);
846852
atomic_long_dec(&psb->total_inodes);
853+
call_rcu(&inode->i_rcu, pohmelfs_i_callback);
847854
}
848855

849856
/*

‎drivers/staging/smbfs/inode.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,18 @@ static struct inode *smb_alloc_inode(struct super_block *sb)
6262
return &ei->vfs_inode;
6363
}
6464

65-
static void smb_destroy_inode(struct inode *inode)
65+
static void smb_i_callback(struct rcu_head *head)
6666
{
67+
struct inode *inode = container_of(head, struct inode, i_rcu);
68+
INIT_LIST_HEAD(&inode->i_dentry);
6769
kmem_cache_free(smb_inode_cachep, SMB_I(inode));
6870
}
6971

72+
static void smb_destroy_inode(struct inode *inode)
73+
{
74+
call_rcu(&inode->i_rcu, smb_i_callback);
75+
}
76+
7077
static void init_once(void *foo)
7178
{
7279
struct smb_inode_info *ei = (struct smb_inode_info *) foo;

‎fs/9p/vfs_inode.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -237,10 +237,17 @@ struct inode *v9fs_alloc_inode(struct super_block *sb)
237237
*
238238
*/
239239

240-
void v9fs_destroy_inode(struct inode *inode)
240+
static void v9fs_i_callback(struct rcu_head *head)
241241
{
242+
struct inode *inode = container_of(head, struct inode, i_rcu);
243+
INIT_LIST_HEAD(&inode->i_dentry);
242244
kmem_cache_free(vcookie_cache, v9fs_inode2cookie(inode));
243245
}
246+
247+
void v9fs_destroy_inode(struct inode *inode)
248+
{
249+
call_rcu(&inode->i_rcu, v9fs_i_callback);
250+
}
244251
#endif
245252

246253
/**

‎fs/adfs/super.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -240,11 +240,18 @@ static struct inode *adfs_alloc_inode(struct super_block *sb)
240240
return &ei->vfs_inode;
241241
}
242242

243-
static void adfs_destroy_inode(struct inode *inode)
243+
static void adfs_i_callback(struct rcu_head *head)
244244
{
245+
struct inode *inode = container_of(head, struct inode, i_rcu);
246+
INIT_LIST_HEAD(&inode->i_dentry);
245247
kmem_cache_free(adfs_inode_cachep, ADFS_I(inode));
246248
}
247249

250+
static void adfs_destroy_inode(struct inode *inode)
251+
{
252+
call_rcu(&inode->i_rcu, adfs_i_callback);
253+
}
254+
248255
static void init_once(void *foo)
249256
{
250257
struct adfs_inode_info *ei = (struct adfs_inode_info *) foo;

‎fs/affs/super.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,18 @@ static struct inode *affs_alloc_inode(struct super_block *sb)
9595
return &i->vfs_inode;
9696
}
9797

98-
static void affs_destroy_inode(struct inode *inode)
98+
static void affs_i_callback(struct rcu_head *head)
9999
{
100+
struct inode *inode = container_of(head, struct inode, i_rcu);
101+
INIT_LIST_HEAD(&inode->i_dentry);
100102
kmem_cache_free(affs_inode_cachep, AFFS_I(inode));
101103
}
102104

105+
static void affs_destroy_inode(struct inode *inode)
106+
{
107+
call_rcu(&inode->i_rcu, affs_i_callback);
108+
}
109+
103110
static void init_once(void *foo)
104111
{
105112
struct affs_inode_info *ei = (struct affs_inode_info *) foo;

‎fs/afs/super.c

+9-1
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,14 @@ static struct inode *afs_alloc_inode(struct super_block *sb)
498498
return &vnode->vfs_inode;
499499
}
500500

501+
static void afs_i_callback(struct rcu_head *head)
502+
{
503+
struct inode *inode = container_of(head, struct inode, i_rcu);
504+
struct afs_vnode *vnode = AFS_FS_I(inode);
505+
INIT_LIST_HEAD(&inode->i_dentry);
506+
kmem_cache_free(afs_inode_cachep, vnode);
507+
}
508+
501509
/*
502510
* destroy an AFS inode struct
503511
*/
@@ -511,7 +519,7 @@ static void afs_destroy_inode(struct inode *inode)
511519

512520
ASSERTCMP(vnode->server, ==, NULL);
513521

514-
kmem_cache_free(afs_inode_cachep, vnode);
522+
call_rcu(&inode->i_rcu, afs_i_callback);
515523
atomic_dec(&afs_count_active_inodes);
516524
}
517525

‎fs/befs/linuxvfs.c

+8-2
Original file line numberDiff line numberDiff line change
@@ -284,12 +284,18 @@ befs_alloc_inode(struct super_block *sb)
284284
return &bi->vfs_inode;
285285
}
286286

287-
static void
288-
befs_destroy_inode(struct inode *inode)
287+
static void befs_i_callback(struct rcu_head *head)
289288
{
289+
struct inode *inode = container_of(head, struct inode, i_rcu);
290+
INIT_LIST_HEAD(&inode->i_dentry);
290291
kmem_cache_free(befs_inode_cachep, BEFS_I(inode));
291292
}
292293

294+
static void befs_destroy_inode(struct inode *inode)
295+
{
296+
call_rcu(&inode->i_rcu, befs_i_callback);
297+
}
298+
293299
static void init_once(void *foo)
294300
{
295301
struct befs_inode_info *bi = (struct befs_inode_info *) foo;

‎fs/bfs/inode.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -248,11 +248,18 @@ static struct inode *bfs_alloc_inode(struct super_block *sb)
248248
return &bi->vfs_inode;
249249
}
250250

251-
static void bfs_destroy_inode(struct inode *inode)
251+
static void bfs_i_callback(struct rcu_head *head)
252252
{
253+
struct inode *inode = container_of(head, struct inode, i_rcu);
254+
INIT_LIST_HEAD(&inode->i_dentry);
253255
kmem_cache_free(bfs_inode_cachep, BFS_I(inode));
254256
}
255257

258+
static void bfs_destroy_inode(struct inode *inode)
259+
{
260+
call_rcu(&inode->i_rcu, bfs_i_callback);
261+
}
262+
256263
static void init_once(void *foo)
257264
{
258265
struct bfs_inode_info *bi = foo;

‎fs/block_dev.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -409,13 +409,20 @@ static struct inode *bdev_alloc_inode(struct super_block *sb)
409409
return &ei->vfs_inode;
410410
}
411411

412-
static void bdev_destroy_inode(struct inode *inode)
412+
static void bdev_i_callback(struct rcu_head *head)
413413
{
414+
struct inode *inode = container_of(head, struct inode, i_rcu);
414415
struct bdev_inode *bdi = BDEV_I(inode);
415416

417+
INIT_LIST_HEAD(&inode->i_dentry);
416418
kmem_cache_free(bdev_cachep, bdi);
417419
}
418420

421+
static void bdev_destroy_inode(struct inode *inode)
422+
{
423+
call_rcu(&inode->i_rcu, bdev_i_callback);
424+
}
425+
419426
static void init_once(void *foo)
420427
{
421428
struct bdev_inode *ei = (struct bdev_inode *) foo;

‎fs/btrfs/inode.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -6495,6 +6495,13 @@ struct inode *btrfs_alloc_inode(struct super_block *sb)
64956495
return inode;
64966496
}
64976497

6498+
static void btrfs_i_callback(struct rcu_head *head)
6499+
{
6500+
struct inode *inode = container_of(head, struct inode, i_rcu);
6501+
INIT_LIST_HEAD(&inode->i_dentry);
6502+
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
6503+
}
6504+
64986505
void btrfs_destroy_inode(struct inode *inode)
64996506
{
65006507
struct btrfs_ordered_extent *ordered;
@@ -6564,7 +6571,7 @@ void btrfs_destroy_inode(struct inode *inode)
65646571
inode_tree_del(inode);
65656572
btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
65666573
free:
6567-
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
6574+
call_rcu(&inode->i_rcu, btrfs_i_callback);
65686575
}
65696576

65706577
int btrfs_drop_inode(struct inode *inode)

‎fs/ceph/inode.c

+10-1
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,15 @@ struct inode *ceph_alloc_inode(struct super_block *sb)
368368
return &ci->vfs_inode;
369369
}
370370

371+
static void ceph_i_callback(struct rcu_head *head)
372+
{
373+
struct inode *inode = container_of(head, struct inode, i_rcu);
374+
struct ceph_inode_info *ci = ceph_inode(inode);
375+
376+
INIT_LIST_HEAD(&inode->i_dentry);
377+
kmem_cache_free(ceph_inode_cachep, ci);
378+
}
379+
371380
void ceph_destroy_inode(struct inode *inode)
372381
{
373382
struct ceph_inode_info *ci = ceph_inode(inode);
@@ -407,7 +416,7 @@ void ceph_destroy_inode(struct inode *inode)
407416
if (ci->i_xattrs.prealloc_blob)
408417
ceph_buffer_put(ci->i_xattrs.prealloc_blob);
409418

410-
kmem_cache_free(ceph_inode_cachep, ci);
419+
call_rcu(&inode->i_rcu, ceph_i_callback);
411420
}
412421

413422

‎fs/cifs/cifsfs.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -334,10 +334,17 @@ cifs_alloc_inode(struct super_block *sb)
334334
return &cifs_inode->vfs_inode;
335335
}
336336

337+
static void cifs_i_callback(struct rcu_head *head)
338+
{
339+
struct inode *inode = container_of(head, struct inode, i_rcu);
340+
INIT_LIST_HEAD(&inode->i_dentry);
341+
kmem_cache_free(cifs_inode_cachep, CIFS_I(inode));
342+
}
343+
337344
static void
338345
cifs_destroy_inode(struct inode *inode)
339346
{
340-
kmem_cache_free(cifs_inode_cachep, CIFS_I(inode));
347+
call_rcu(&inode->i_rcu, cifs_i_callback);
341348
}
342349

343350
static void

‎fs/coda/inode.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,18 @@ static struct inode *coda_alloc_inode(struct super_block *sb)
5656
return &ei->vfs_inode;
5757
}
5858

59-
static void coda_destroy_inode(struct inode *inode)
59+
static void coda_i_callback(struct rcu_head *head)
6060
{
61+
struct inode *inode = container_of(head, struct inode, i_rcu);
62+
INIT_LIST_HEAD(&inode->i_dentry);
6163
kmem_cache_free(coda_inode_cachep, ITOC(inode));
6264
}
6365

66+
static void coda_destroy_inode(struct inode *inode)
67+
{
68+
call_rcu(&inode->i_rcu, coda_i_callback);
69+
}
70+
6471
static void init_once(void *foo)
6572
{
6673
struct coda_inode_info *ei = (struct coda_inode_info *) foo;

‎fs/ecryptfs/super.c

+11-1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,16 @@ static struct inode *ecryptfs_alloc_inode(struct super_block *sb)
6262
return inode;
6363
}
6464

65+
static void ecryptfs_i_callback(struct rcu_head *head)
66+
{
67+
struct inode *inode = container_of(head, struct inode, i_rcu);
68+
struct ecryptfs_inode_info *inode_info;
69+
inode_info = ecryptfs_inode_to_private(inode);
70+
71+
INIT_LIST_HEAD(&inode->i_dentry);
72+
kmem_cache_free(ecryptfs_inode_info_cache, inode_info);
73+
}
74+
6575
/**
6676
* ecryptfs_destroy_inode
6777
* @inode: The ecryptfs inode
@@ -88,7 +98,7 @@ static void ecryptfs_destroy_inode(struct inode *inode)
8898
}
8999
}
90100
ecryptfs_destroy_crypt_stat(&inode_info->crypt_stat);
91-
kmem_cache_free(ecryptfs_inode_info_cache, inode_info);
101+
call_rcu(&inode->i_rcu, ecryptfs_i_callback);
92102
}
93103

94104
/**

‎fs/efs/super.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,18 @@ static struct inode *efs_alloc_inode(struct super_block *sb)
6565
return &ei->vfs_inode;
6666
}
6767

68-
static void efs_destroy_inode(struct inode *inode)
68+
static void efs_i_callback(struct rcu_head *head)
6969
{
70+
struct inode *inode = container_of(head, struct inode, i_rcu);
71+
INIT_LIST_HEAD(&inode->i_dentry);
7072
kmem_cache_free(efs_inode_cachep, INODE_INFO(inode));
7173
}
7274

75+
static void efs_destroy_inode(struct inode *inode)
76+
{
77+
call_rcu(&inode->i_rcu, efs_i_callback);
78+
}
79+
7380
static void init_once(void *foo)
7481
{
7582
struct efs_inode_info *ei = (struct efs_inode_info *) foo;

‎fs/exofs/super.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,19 @@ static struct inode *exofs_alloc_inode(struct super_block *sb)
150150
return &oi->vfs_inode;
151151
}
152152

153+
static void exofs_i_callback(struct rcu_head *head)
154+
{
155+
struct inode *inode = container_of(head, struct inode, i_rcu);
156+
INIT_LIST_HEAD(&inode->i_dentry);
157+
kmem_cache_free(exofs_inode_cachep, exofs_i(inode));
158+
}
159+
153160
/*
154161
* Remove an inode from the cache
155162
*/
156163
static void exofs_destroy_inode(struct inode *inode)
157164
{
158-
kmem_cache_free(exofs_inode_cachep, exofs_i(inode));
165+
call_rcu(&inode->i_rcu, exofs_i_callback);
159166
}
160167

161168
/*

0 commit comments

Comments
 (0)