[patch 26/52] fs: icache lock s_inodes list

Previous message: [thread] [date] [author]
Next message: [thread] [date] [author]
From: npiggin
Date: Wednesday, June 23, 2010 - 8:02 pm

Protect sb->s_inodes with a new lock, sb_inode_list_lock.

Signed-off-by: Nick Piggin <npiggin@suse.de>
---
 fs/drop_caches.c            |    4 ++++
 fs/fs-writeback.c           |    4 ++++
 fs/inode.c                  |   12 ++++++++++++
 fs/notify/inode_mark.c      |    2 ++
 fs/notify/inotify/inotify.c |    2 ++
 fs/quota/dquot.c            |    6 ++++++
 include/linux/writeback.h   |    1 +
 7 files changed, 31 insertions(+)

Index: linux-2.6/fs/drop_caches.c
===================================================================
--- linux-2.6.orig/fs/drop_caches.c
+++ linux-2.6/fs/drop_caches.c
@@ -17,18 +17,22 @@ static void drop_pagecache_sb(struct sup
 	struct inode *inode, *toput_inode = NULL;
 
 	spin_lock(&inode_lock);
+	spin_lock(&sb_inode_list_lock);
 	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
 		if (inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE|I_NEW))
 			continue;
 		if (inode->i_mapping->nrpages == 0)
 			continue;
 		__iget(inode);
+		spin_unlock(&sb_inode_list_lock);
 		spin_unlock(&inode_lock);
 		invalidate_mapping_pages(inode->i_mapping, 0, -1);
 		iput(toput_inode);
 		toput_inode = inode;
 		spin_lock(&inode_lock);
+		spin_lock(&sb_inode_list_lock);
 	}
+	spin_unlock(&sb_inode_list_lock);
 	spin_unlock(&inode_lock);
 	iput(toput_inode);
 }
Index: linux-2.6/fs/fs-writeback.c
===================================================================
--- linux-2.6.orig/fs/fs-writeback.c
+++ linux-2.6/fs/fs-writeback.c
@@ -1166,6 +1166,7 @@ static void wait_sb_inodes(struct super_
 	WARN_ON(!rwsem_is_locked(&sb->s_umount));
 
 	spin_lock(&inode_lock);
+	spin_lock(&sb_inode_list_lock);
 
 	/*
 	 * Data integrity sync. Must wait for all pages under writeback,
@@ -1183,6 +1184,7 @@ static void wait_sb_inodes(struct super_
 		if (mapping->nrpages == 0)
 			continue;
 		__iget(inode);
+		spin_unlock(&sb_inode_list_lock);
 		spin_unlock(&inode_lock);
 		/*
 		 * We hold a reference to 'inode' so it couldn't have
@@ -1200,7 +1202,9 @@ static void wait_sb_inodes(struct super_
 		cond_resched();
 
 		spin_lock(&inode_lock);
+		spin_lock(&sb_inode_list_lock);
 	}
+	spin_unlock(&sb_inode_list_lock);
 	spin_unlock(&inode_lock);
 	iput(old_inode);
 }
Index: linux-2.6/fs/inode.c
===================================================================
--- linux-2.6.orig/fs/inode.c
+++ linux-2.6/fs/inode.c
@@ -27,6 +27,15 @@
 #include <linux/posix_acl.h>
 
 /*
+ * Usage:
+ * sb_inode_list_lock protects:
+ *   s_inodes, i_sb_list
+ *
+ * Ordering:
+ * inode_lock
+ *   sb_inode_list_lock
+ */
+/*
  * This is needed for the following functions:
  *  - inode_has_buffers
  *  - invalidate_inode_buffers
@@ -84,6 +93,7 @@ static struct hlist_head *inode_hashtabl
  * the i_state of an inode while it is in use..
  */
 DEFINE_SPINLOCK(inode_lock);
+DEFINE_SPINLOCK(sb_inode_list_lock);
 
 /*
  * iprune_sem provides exclusion between the kswapd or try_to_free_pages
@@ -344,7 +354,9 @@ static void dispose_list(struct list_hea
 
 		spin_lock(&inode_lock);
 		hlist_del_init(&inode->i_hash);
+		spin_lock(&sb_inode_list_lock);
 		list_del_init(&inode->i_sb_list);
+		spin_unlock(&sb_inode_list_lock);
 		spin_unlock(&inode_lock);
 
 		wake_up_inode(inode);
@@ -376,6 +388,7 @@ static int invalidate_list(struct list_h
 		 * shrink_icache_memory() away.
 		 */
 		cond_resched_lock(&inode_lock);
+		cond_resched_lock(&sb_inode_list_lock);
 
 		next = next->next;
 		if (tmp == head)
@@ -413,9 +426,11 @@ int invalidate_inodes(struct super_block
 
 	down_write(&iprune_sem);
 	spin_lock(&inode_lock);
+	spin_lock(&sb_inode_list_lock);
 	inotify_unmount_inodes(&sb->s_inodes);
 	fsnotify_unmount_inodes(&sb->s_inodes);
 	busy = invalidate_list(&sb->s_inodes, &throw_away);
+	spin_unlock(&sb_inode_list_lock);
 	spin_unlock(&inode_lock);
 
 	dispose_list(&throw_away);
@@ -603,7 +618,9 @@ __inode_add_to_lists(struct super_block
 {
 	inodes_stat.nr_inodes++;
 	list_add(&inode->i_list, &inode_in_use);
+	spin_lock(&sb_inode_list_lock);
 	list_add(&inode->i_sb_list, &sb->s_inodes);
+	spin_unlock(&sb_inode_list_lock);
 	if (head)
 		hlist_add_head(&inode->i_hash, head);
 }
@@ -1197,7 +1214,9 @@ void generic_delete_inode(struct inode *
 	const struct super_operations *op = inode->i_sb->s_op;
 
 	list_del_init(&inode->i_list);
+	spin_lock(&sb_inode_list_lock);
 	list_del_init(&inode->i_sb_list);
+	spin_unlock(&sb_inode_list_lock);
 	WARN_ON(inode->i_state & I_NEW);
 	inode->i_state |= I_FREEING;
 	inodes_stat.nr_inodes--;
@@ -1255,7 +1274,9 @@ int generic_detach_inode(struct inode *i
 		hlist_del_init(&inode->i_hash);
 	}
 	list_del_init(&inode->i_list);
+	spin_lock(&sb_inode_list_lock);
 	list_del_init(&inode->i_sb_list);
+	spin_unlock(&sb_inode_list_lock);
 	WARN_ON(inode->i_state & I_NEW);
 	inode->i_state |= I_FREEING;
 	inodes_stat.nr_inodes--;
Index: linux-2.6/fs/notify/inotify/inotify.c
===================================================================
--- linux-2.6.orig/fs/notify/inotify/inotify.c
+++ linux-2.6/fs/notify/inotify/inotify.c
@@ -429,6 +429,7 @@ void inotify_unmount_inodes(struct list_
 		 * will be added since the umount has begun.  Finally,
 		 * iprune_mutex keeps shrink_icache_memory() away.
 		 */
+		spin_unlock(&sb_inode_list_lock);
 		spin_unlock(&inode_lock);
 
 		if (need_iput_tmp)
@@ -451,6 +452,7 @@ void inotify_unmount_inodes(struct list_
 		iput(inode);		
 
 		spin_lock(&inode_lock);
+		spin_lock(&sb_inode_list_lock);
 	}
 }
 EXPORT_SYMBOL_GPL(inotify_unmount_inodes);
Index: linux-2.6/fs/quota/dquot.c
===================================================================
--- linux-2.6.orig/fs/quota/dquot.c
+++ linux-2.6/fs/quota/dquot.c
@@ -884,6 +884,7 @@ static void add_dquot_ref(struct super_b
 #endif
 
 	spin_lock(&inode_lock);
+	spin_lock(&sb_inode_list_lock);
 	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
 		if (inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE|I_NEW))
 			continue;
@@ -897,6 +898,7 @@ static void add_dquot_ref(struct super_b
 			continue;
 
 		__iget(inode);
+		spin_unlock(&sb_inode_list_lock);
 		spin_unlock(&inode_lock);
 
 		iput(old_inode);
@@ -908,7 +910,9 @@ static void add_dquot_ref(struct super_b
 		 * keep the reference and iput it later. */
 		old_inode = inode;
 		spin_lock(&inode_lock);
+		spin_lock(&sb_inode_list_lock);
 	}
+	spin_unlock(&sb_inode_list_lock);
 	spin_unlock(&inode_lock);
 	iput(old_inode);
 
@@ -988,6 +992,7 @@ static void remove_dquot_ref(struct supe
 	struct inode *inode;
 
 	spin_lock(&inode_lock);
+	spin_lock(&sb_inode_list_lock);
 	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
 		/*
 		 *  We have to scan also I_NEW inodes because they can already
@@ -998,6 +1003,7 @@ static void remove_dquot_ref(struct supe
 		if (!IS_NOQUOTA(inode))
 			remove_inode_dquot_ref(inode, type, tofree_head);
 	}
+	spin_unlock(&sb_inode_list_lock);
 	spin_unlock(&inode_lock);
 }
 
Index: linux-2.6/include/linux/writeback.h
===================================================================
--- linux-2.6.orig/include/linux/writeback.h
+++ linux-2.6/include/linux/writeback.h
@@ -10,6 +10,7 @@
 struct backing_dev_info;
 
 extern spinlock_t inode_lock;
+extern spinlock_t sb_inode_list_lock;
 extern struct list_head inode_in_use;
 extern struct list_head inode_unused;
 
Index: linux-2.6/fs/notify/inode_mark.c
===================================================================
--- linux-2.6.orig/fs/notify/inode_mark.c
+++ linux-2.6/fs/notify/inode_mark.c
@@ -408,6 +408,7 @@ void fsnotify_unmount_inodes(struct list
 		 * will be added since the umount has begun.  Finally,
 		 * iprune_mutex keeps shrink_icache_memory() away.
 		 */
+		spin_unlock(&sb_inode_list_lock);
 		spin_unlock(&inode_lock);
 
 		if (need_iput_tmp)
@@ -421,5 +422,6 @@ void fsnotify_unmount_inodes(struct list
 		iput(inode);
 
 		spin_lock(&inode_lock);
+		spin_lock(&sb_inode_list_lock);
 	}
 }


--
Previous message: [thread] [date] [author]
Next message: [thread] [date] [author]

Messages in current thread:
[patch 00/52] vfs scalability patches updated, npiggin, (Wed Jun 23, 8:02 pm)
[patch 01/52] kernel: add bl_list, npiggin, (Wed Jun 23, 8:02 pm)
[patch 02/52] fs: fix superblock iteration race, npiggin, (Wed Jun 23, 8:02 pm)
[patch 03/52] fs: fs_struct rwlock to spinlock, npiggin, (Wed Jun 23, 8:02 pm)
[patch 04/52] fs: cleanup files_lock, npiggin, (Wed Jun 23, 8:02 pm)
[patch 06/52] fs: scale files_lock, npiggin, (Wed Jun 23, 8:02 pm)
[patch 07/52] fs: brlock vfsmount_lock, npiggin, (Wed Jun 23, 8:02 pm)
[patch 08/52] fs: scale mntget/mntput, npiggin, (Wed Jun 23, 8:02 pm)
[patch 09/52] fs: dcache scale hash, npiggin, (Wed Jun 23, 8:02 pm)
[patch 10/52] fs: dcache scale lru, npiggin, (Wed Jun 23, 8:02 pm)
[patch 11/52] fs: dcache scale nr_dentry, npiggin, (Wed Jun 23, 8:02 pm)
[patch 12/52] fs: dcache scale dentry refcount, npiggin, (Wed Jun 23, 8:02 pm)
[patch 13/52] fs: dcache scale d_unhashed, npiggin, (Wed Jun 23, 8:02 pm)
[patch 14/52] fs: dcache scale subdirs, npiggin, (Wed Jun 23, 8:02 pm)
[patch 15/52] fs: dcache scale inode alias list, npiggin, (Wed Jun 23, 8:02 pm)
[patch 17/52] fs: dcache remove dcache_lock, npiggin, (Wed Jun 23, 8:02 pm)
[patch 18/52] fs: dcache reduce dput locking, npiggin, (Wed Jun 23, 8:02 pm)
[patch 23/52] fs: dcache percpu nr_dentry, npiggin, (Wed Jun 23, 8:02 pm)
[patch 24/52] fs: dcache reduce d_parent locking, npiggin, (Wed Jun 23, 8:02 pm)
[patch 26/52] fs: icache lock s_inodes list, npiggin, (Wed Jun 23, 8:02 pm)
[patch 27/52] fs: icache lock inode hash, npiggin, (Wed Jun 23, 8:02 pm)
[patch 28/52] fs: icache lock i_state, npiggin, (Wed Jun 23, 8:02 pm)
[patch 29/52] fs: icache lock i_count, npiggin, (Wed Jun 23, 8:02 pm)
[patch 31/52] fs: icache atomic inodes_stat, npiggin, (Wed Jun 23, 8:02 pm)
[patch 32/52] fs: icache protect inode state, npiggin, (Wed Jun 23, 8:02 pm)
[patch 34/52] fs: icache remove inode_lock, npiggin, (Wed Jun 23, 8:02 pm)
[patch 37/52] fs: icache lazy lru, npiggin, (Wed Jun 23, 8:02 pm)
[patch 38/52] fs: icache RCU free inodes, npiggin, (Wed Jun 23, 8:02 pm)
[patch 39/52] fs: icache rcu walk for i_sb_list, npiggin, (Wed Jun 23, 8:02 pm)
[patch 41/52] fs: icache reduce atomics, npiggin, (Wed Jun 23, 8:02 pm)
[patch 45/52] fs: icache RCU hash lookups, npiggin, (Wed Jun 23, 8:02 pm)
[patch 46/52] fs: icache reduce locking, npiggin, (Wed Jun 23, 8:02 pm)
[patch 47/52] fs: keep inode with backing-dev, npiggin, (Wed Jun 23, 8:02 pm)
[patch 48/52] fs: icache split IO and LRU lists, npiggin, (Wed Jun 23, 8:03 pm)
[patch 50/52] mm: implement per-zone shrinker, npiggin, (Wed Jun 23, 8:03 pm)
[patch 51/52] fs: per-zone dentry and inode LRU, npiggin, (Wed Jun 23, 8:03 pm)
[patch 52/52] fs: icache less I_FREEING time, npiggin, (Wed Jun 23, 8:03 pm)
Re: [patch 01/52] kernel: add bl_list, Eric Dumazet, (Wed Jun 23, 11:04 pm)
Re: [patch 06/52] fs: scale files_lock, Peter Zijlstra, (Thu Jun 24, 12:52 am)
Re: [patch 14/52] fs: dcache scale subdirs, Peter Zijlstra, (Thu Jun 24, 12:56 am)
Re: [patch 16/52] fs: dcache RCU for multi-step operaitons, Peter Zijlstra, (Thu Jun 24, 12:58 am)
Re: [patch 24/52] fs: dcache reduce d_parent locking, Peter Zijlstra, (Thu Jun 24, 1:44 am)
Re: [patch 30/52] fs: icache lock lru/writeback lists, Peter Zijlstra, (Thu Jun 24, 1:58 am)
Re: [patch 14/52] fs: dcache scale subdirs, Andi Kleen, (Thu Jun 24, 2:50 am)
Re: [patch 37/52] fs: icache lazy lru, Andi Kleen, (Thu Jun 24, 2:52 am)
Re: [patch 50/52] mm: implement per-zone shrinker, Andi Kleen, (Thu Jun 24, 3:06 am)
Re: [patch 01/52] kernel: add bl_list, Nick Piggin, (Thu Jun 24, 7:42 am)
Re: [patch 06/52] fs: scale files_lock, Nick Piggin, (Thu Jun 24, 8:00 am)
Re: [patch 24/52] fs: dcache reduce d_parent locking, Nick Piggin, (Thu Jun 24, 8:07 am)
Re: [patch 30/52] fs: icache lock lru/writeback lists, Nick Piggin, (Thu Jun 24, 8:09 am)
Re: [patch 30/52] fs: icache lock lru/writeback lists, Peter Zijlstra, (Thu Jun 24, 8:13 am)
Re: [patch 24/52] fs: dcache reduce d_parent locking, Paul E. McKenney, (Thu Jun 24, 8:32 am)
Re: [patch 14/52] fs: dcache scale subdirs, Nick Piggin, (Thu Jun 24, 8:53 am)
Re: [patch 37/52] fs: icache lazy lru, Nick Piggin, (Thu Jun 24, 8:59 am)
Re: [patch 50/52] mm: implement per-zone shrinker, Nick Piggin, (Thu Jun 24, 9:00 am)
Re: [patch 01/52] kernel: add bl_list, Eric Dumazet, (Thu Jun 24, 9:01 am)
Re: [patch 24/52] fs: dcache reduce d_parent locking, Nick Piggin, (Thu Jun 24, 9:05 am)
Re: [patch 50/52] mm: implement per-zone shrinker, Andi Kleen, (Thu Jun 24, 9:27 am)
Re: [patch 50/52] mm: implement per-zone shrinker, Andi Kleen, (Thu Jun 24, 9:32 am)
Re: [patch 50/52] mm: implement per-zone shrinker, Andi Kleen, (Thu Jun 24, 9:37 am)
Re: [patch 24/52] fs: dcache reduce d_parent locking, Paul E. McKenney, (Thu Jun 24, 9:41 am)
Re: [patch 00/52] vfs scalability patches updated, Christoph Hellwig, (Fri Jun 25, 12:12 am)
Re: [patch 00/52] vfs scalability patches updated, Nick Piggin, (Fri Jun 25, 1:05 am)
Re: [patch 01/52] kernel: add bl_list, Paul E. McKenney, (Mon Jun 28, 2:37 pm)
Re: [patch 24/52] fs: dcache reduce d_parent locking, Paul E. McKenney, (Mon Jun 28, 2:50 pm)
Re: [patch 01/52] kernel: add bl_list, Nick Piggin, (Mon Jun 28, 11:30 pm)
Re: [patch 02/52] fs: fix superblock iteration race, Christoph Hellwig, (Tue Jun 29, 6:02 am)
Re: [patch 02/52] fs: fix superblock iteration race, Nick Piggin, (Tue Jun 29, 7:56 am)
Re: [patch 02/52] fs: fix superblock iteration race, Linus Torvalds, (Tue Jun 29, 10:35 am)
Re: [patch 02/52] fs: fix superblock iteration race, Nick Piggin, (Tue Jun 29, 10:41 am)
Re: [patch 02/52] fs: fix superblock iteration race, Linus Torvalds, (Tue Jun 29, 10:52 am)
Re: [patch 02/52] fs: fix superblock iteration race, Linus Torvalds, (Tue Jun 29, 10:58 am)
Re: [patch 02/52] fs: fix superblock iteration race, Nick Piggin, (Tue Jun 29, 1:14 pm)
Re: [patch 02/52] fs: fix superblock iteration race, Chris Clayton, (Tue Jun 29, 1:38 pm)
Re: [patch 50/52] mm: implement per-zone shrinker, Dave Chinner, (Tue Jun 29, 11:28 pm)
Re: [patch 02/52] fs: fix superblock iteration race, Chris Clayton, (Wed Jun 30, 12:13 am)
Re: [patch 29/52] fs: icache lock i_count, Dave Chinner, (Wed Jun 30, 12:27 am)
Re: [patch 37/52] fs: icache lazy lru, Dave Chinner, (Wed Jun 30, 1:38 am)
Re: [patch 38/52] fs: icache RCU free inodes, Dave Chinner, (Wed Jun 30, 1:57 am)
Re: [patch 52/52] fs: icache less I_FREEING time, Dave Chinner, (Wed Jun 30, 3:13 am)
Re: [patch 00/52] vfs scalability patches updated, Dave Chinner, (Wed Jun 30, 4:30 am)
Re: [patch 50/52] mm: implement per-zone shrinker, Nick Piggin, (Wed Jun 30, 5:03 am)
Re: [patch 37/52] fs: icache lazy lru, Nick Piggin, (Wed Jun 30, 5:06 am)
Re: [patch 38/52] fs: icache RCU free inodes, Nick Piggin, (Wed Jun 30, 5:07 am)
Re: [patch 51/52] fs: per-zone dentry and inode LRU, Nick Piggin, (Wed Jun 30, 5:13 am)
Re: [patch 52/52] fs: icache less I_FREEING time, Nick Piggin, (Wed Jun 30, 5:14 am)
Re: [patch 00/52] vfs scalability patches updated, Nick Piggin, (Wed Jun 30, 5:40 am)
Re: [patch 00/52] vfs scalability patches updated, Frank Mayhar, (Wed Jun 30, 10:08 am)
Re: [patch 29/52] fs: icache lock i_count, Dave Chinner, (Wed Jun 30, 7:36 pm)
Re: [patch 37/52] fs: icache lazy lru, Dave Chinner, (Wed Jun 30, 7:46 pm)
Re: [patch 52/52] fs: icache less I_FREEING time, Dave Chinner, (Wed Jun 30, 8:33 pm)
Re: [patch 00/52] vfs scalability patches updated, Dave Chinner, (Wed Jun 30, 8:56 pm)
Re: [patch 29/52] fs: icache lock i_count, Nick Piggin, (Thu Jul 1, 12:54 am)
Re: [patch 37/52] fs: icache lazy lru, Nick Piggin, (Thu Jul 1, 12:57 am)
Re: [patch 52/52] fs: icache less I_FREEING time, Nick Piggin, (Thu Jul 1, 1:06 am)
Re: [patch 00/52] vfs scalability patches updated, Nick Piggin, (Thu Jul 1, 1:20 am)
Re: [patch 29/52] fs: icache lock i_count, Nick Piggin, (Thu Jul 1, 2:36 am)
Re: [patch 29/52] fs: icache lock i_count, Frank Mayhar, (Thu Jul 1, 9:21 am)
Re: [patch 00/52] vfs scalability patches updated, Nick Piggin, (Thu Jul 1, 10:23 am)
Re: [patch 00/52] vfs scalability patches updated, Andi Kleen, (Thu Jul 1, 10:28 am)
Re: [patch 00/52] vfs scalability patches updated, Linus Torvalds, (Thu Jul 1, 10:35 am)
Re: [patch 00/52] vfs scalability patches updated, Andi Kleen, (Thu Jul 1, 10:36 am)
Re: [patch 00/52] vfs scalability patches updated, Nick Piggin, (Thu Jul 1, 10:52 am)
Re: [patch 00/52] vfs scalability patches updated, Paul E. McKenney, (Thu Jul 1, 9:01 pm)
Re: [patch 29/52] fs: icache lock i_count, Andrew Morton, (Fri Jul 2, 7:03 pm)
Re: [patch 29/52] fs: icache lock i_count, Nick Piggin, (Fri Jul 2, 8:41 pm)
Re: [patch 29/52] fs: icache lock i_count, Andrew Morton, (Fri Jul 2, 9:31 pm)
Re: [patch 29/52] fs: icache lock i_count, Nick Piggin, (Fri Jul 2, 10:06 pm)
Re: [patch 29/52] fs: icache lock i_count, Nick Piggin, (Fri Jul 2, 10:18 pm)
Re: [patch 29/52] fs: icache lock i_count, Dave Chinner, (Mon Jul 5, 3:41 pm)
Re: [patch 29/52] fs: icache lock i_count, Theodore Tso, (Tue Jul 6, 3:38 am)
Re: [patch 29/52] fs: icache lock i_count, Nick Piggin, (Tue Jul 6, 6:04 am)
Re: [patch 00/52] vfs scalability patches updated, Nick Piggin, (Tue Jul 6, 10:49 am)
Re: [patch 24/52] fs: dcache reduce d_parent locking, Nick Piggin, (Wed Jul 7, 7:35 am)
Re: [patch 29/52] fs: icache lock i_count, Frank Mayhar, (Wed Jul 7, 10:00 am)