[PATCH] splice: fix page stealing LRU handling.

Originally from Nick Piggin, just adapted to the newer branch.

You can't check PageLRU without holding zone->lru_lock.  The page
release code can get away with it only because the page refcount is 0 at
that point. Also, you can't reliably remove pages from the LRU unless
the refcount is 0. Ever.

Signed-off-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Jens Axboe <axboe@suse.de>

3 files changed, 18 insertions, 19 deletions

diff --git a/fs/pipe.c b/fs/pipe.c
index 5093408..795df98 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -95,6 +95,8 @@ static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buff
 {
 	struct page *page = buf->page;
 
+	buf->flags &= ~PIPE_BUF_FLAG_STOLEN;
+
 	/*
 	 * If nobody else uses this page, and we don't already have a
 	 * temporary page, let's keep track of it as a one-deep
@@ -124,6 +126,7 @@ static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer
 static int anon_pipe_buf_steal(struct pipe_inode_info *info,
 			       struct pipe_buffer *buf)
 {
+	buf->flags |= PIPE_BUF_FLAG_STOLEN;
 	return 0;
 }
 
diff --git a/fs/splice.c b/fs/splice.c
index b5fb2f3..bfa42a2 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -67,16 +67,7 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
 	if (!remove_mapping(mapping, page))
 		return 1;
 
-	if (PageLRU(page)) {
-		struct zone *zone = page_zone(page);
-
-		spin_lock_irq(&zone->lru_lock);
-		BUG_ON(!PageLRU(page));
-		__ClearPageLRU(page);
-		del_page_from_lru(zone, page);
-		spin_unlock_irq(&zone->lru_lock);
-	}
-
+	buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
 	return 0;
 }
 
@@ -85,6 +76,7 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
 {
 	page_cache_release(buf->page);
 	buf->page = NULL;
+	buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
 }
 
 static void *page_cache_pipe_buf_map(struct file *file,
@@ -414,11 +406,12 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
 {
 	struct file *file = sd->file;
 	struct address_space *mapping = file->f_mapping;
+	gfp_t gfp_mask = mapping_gfp_mask(mapping);
 	unsigned int offset;
 	struct page *page;
 	pgoff_t index;
 	char *src;
-	int ret, stolen;
+	int ret;
 
 	/*
 	 * after this, page will be locked and unmapped
@@ -429,7 +422,6 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
 
 	index = sd->pos >> PAGE_CACHE_SHIFT;
 	offset = sd->pos & ~PAGE_CACHE_MASK;
-	stolen = 0;
 
 	/*
 	 * reuse buf page, if SPLICE_F_MOVE is set
@@ -443,15 +435,15 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
 			goto find_page;
 
 		page = buf->page;
-		stolen = 1;
-		if (add_to_page_cache_lru(page, mapping, index,
-						mapping_gfp_mask(mapping)))
+		if (add_to_page_cache(page, mapping, index, gfp_mask))
 			goto find_page;
+
+		if (!(buf->flags & PIPE_BUF_FLAG_LRU))
+			lru_cache_add(page);
 	} else {
 find_page:
 		ret = -ENOMEM;
-		page = find_or_create_page(mapping, index,
-						mapping_gfp_mask(mapping));
+		page = find_or_create_page(mapping, index, gfp_mask);
 		if (!page)
 			goto out;
 
@@ -494,7 +486,7 @@ find_page:
 	} else if (ret)
 		goto out;
 
-	if (!stolen) {
+	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
 		char *dst = kmap_atomic(page, KM_USER0);
 
 		memcpy(dst + offset, src + buf->offset, sd->len);
@@ -511,7 +503,7 @@ find_page:
 
 	balance_dirty_pages_ratelimited(mapping);
 out:
-	if (!stolen) {
+	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
 		page_cache_release(page);
 		unlock_page(page);
 	}
diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h
index 70ae933..ec38495 100644
--- a/include/linux/pipe_fs_i.h
+++ b/include/linux/pipe_fs_i.h
@@ -5,10 +5,14 @@
 
 #define PIPE_BUFFERS (16)
 
+#define PIPE_BUF_FLAG_STOLEN	0x01
+#define PIPE_BUF_FLAG_LRU	0x02
+
 struct pipe_buffer {
 	struct page *page;
 	unsigned int offset, len;
 	struct pipe_buf_operations *ops;
+	unsigned int flags;
 };
 
 struct pipe_buf_operations {