Commit 451ea25d authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds
Browse files

mm: perform non-atomic test-clear of PG_mlocked on free



By the time PG_mlocked is cleared in the page freeing path, nobody else is
looking at our page->flags anymore.

It is thus safe to make the test-and-clear non-atomic and thereby removing
an unnecessary and expensive operation from a hotpath.
Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Reviewed-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Reviewed-by: default avatarKOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent bf88c8c8
...@@ -158,6 +158,9 @@ static inline int TestSetPage##uname(struct page *page) \ ...@@ -158,6 +158,9 @@ static inline int TestSetPage##uname(struct page *page) \
static inline int TestClearPage##uname(struct page *page) \ static inline int TestClearPage##uname(struct page *page) \
{ return test_and_clear_bit(PG_##lname, &page->flags); } { return test_and_clear_bit(PG_##lname, &page->flags); }
#define __TESTCLEARFLAG(uname, lname) \
static inline int __TestClearPage##uname(struct page *page) \
{ return __test_and_clear_bit(PG_##lname, &page->flags); }
#define PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \ #define PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \
SETPAGEFLAG(uname, lname) CLEARPAGEFLAG(uname, lname) SETPAGEFLAG(uname, lname) CLEARPAGEFLAG(uname, lname)
...@@ -184,6 +187,9 @@ static inline void __ClearPage##uname(struct page *page) { } ...@@ -184,6 +187,9 @@ static inline void __ClearPage##uname(struct page *page) { }
#define TESTCLEARFLAG_FALSE(uname) \ #define TESTCLEARFLAG_FALSE(uname) \
static inline int TestClearPage##uname(struct page *page) { return 0; } static inline int TestClearPage##uname(struct page *page) { return 0; }
#define __TESTCLEARFLAG_FALSE(uname) \
static inline int __TestClearPage##uname(struct page *page) { return 0; }
struct page; /* forward declaration */ struct page; /* forward declaration */
TESTPAGEFLAG(Locked, locked) TESTSETFLAG(Locked, locked) TESTPAGEFLAG(Locked, locked) TESTSETFLAG(Locked, locked)
...@@ -250,11 +256,11 @@ PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable) ...@@ -250,11 +256,11 @@ PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable)
#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT #ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
#define MLOCK_PAGES 1 #define MLOCK_PAGES 1
PAGEFLAG(Mlocked, mlocked) __CLEARPAGEFLAG(Mlocked, mlocked) PAGEFLAG(Mlocked, mlocked) __CLEARPAGEFLAG(Mlocked, mlocked)
TESTSCFLAG(Mlocked, mlocked) TESTSCFLAG(Mlocked, mlocked) __TESTCLEARFLAG(Mlocked, mlocked)
#else #else
#define MLOCK_PAGES 0 #define MLOCK_PAGES 0
PAGEFLAG_FALSE(Mlocked) PAGEFLAG_FALSE(Mlocked) SETPAGEFLAG_NOOP(Mlocked)
SETPAGEFLAG_NOOP(Mlocked) TESTCLEARFLAG_FALSE(Mlocked) TESTCLEARFLAG_FALSE(Mlocked) __TESTCLEARFLAG_FALSE(Mlocked)
#endif #endif
#ifdef CONFIG_ARCH_USES_PG_UNCACHED #ifdef CONFIG_ARCH_USES_PG_UNCACHED
......
...@@ -557,7 +557,7 @@ static void __free_pages_ok(struct page *page, unsigned int order) ...@@ -557,7 +557,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
unsigned long flags; unsigned long flags;
int i; int i;
int bad = 0; int bad = 0;
int wasMlocked = TestClearPageMlocked(page); int wasMlocked = __TestClearPageMlocked(page);
kmemcheck_free_shadow(page, order); kmemcheck_free_shadow(page, order);
...@@ -1026,7 +1026,7 @@ static void free_hot_cold_page(struct page *page, int cold) ...@@ -1026,7 +1026,7 @@ static void free_hot_cold_page(struct page *page, int cold)
struct zone *zone = page_zone(page); struct zone *zone = page_zone(page);
struct per_cpu_pages *pcp; struct per_cpu_pages *pcp;
unsigned long flags; unsigned long flags;
int wasMlocked = TestClearPageMlocked(page); int wasMlocked = __TestClearPageMlocked(page);
kmemcheck_free_shadow(page, 0); kmemcheck_free_shadow(page, 0);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment