[PATCH] splice: speedup __generic_file_splice_read

Using find_get_page() is a lot faster than find_or_create_page(). This
gets splice a lot closer to sendfile() for fd -> socket transfers.

Signed-off-by: Jens Axboe <axboe@suse.de>

1 files changed, 63 insertions, 11 deletions

diff --git a/fs/splice.c b/fs/splice.c
index c47b561..e30743c 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -240,7 +240,7 @@ __generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
 	struct page *pages[PIPE_BUFFERS];
 	struct page *page;
 	pgoff_t index;
-	int i;
+	int i, error;
 
 	index = in->f_pos >> PAGE_CACHE_SHIFT;
 	offset = in->f_pos & ~PAGE_CACHE_MASK;
@@ -260,32 +260,84 @@ __generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
 	/*
 	 * now fill in the holes
 	 */
+	error = 0;
 	for (i = 0; i < nr_pages; i++, index++) {
+find_page:
 		/*
-		 * no page there, look one up / create it
+		 * lookup the page for this index
 		 */
-		page = find_or_create_page(mapping, index,
-						   mapping_gfp_mask(mapping));
-		if (!page)
-			break;
+		page = find_get_page(mapping, index);
+		if (!page) {
+			/*
+			 * If in nonblock mode then dont block on
+			 * readpage (we've kicked readahead so there
+			 * will be asynchronous progress):
+			 */
+			if (flags & SPLICE_F_NONBLOCK)
+				break;
+
+			/*
+			 * page didn't exist, allocate one
+			 */
+			page = page_cache_alloc_cold(mapping);
+			if (!page)
+				break;
+
+			error = add_to_page_cache_lru(page, mapping, index,
+						mapping_gfp_mask(mapping));
+			if (unlikely(error)) {
+				page_cache_release(page);
+				break;
+			}
 
-		if (PageUptodate(page))
-			unlock_page(page);
-		else {
-			int error = mapping->a_ops->readpage(in, page);
+			goto readpage;
+		}
+
+		/*
+		 * If the page isn't uptodate, we may need to start io on it
+		 */
+		if (!PageUptodate(page)) {
+			lock_page(page);
+
+			/*
+			 * page was truncated, stop here. if this isn't the
+			 * first page, we'll just complete what we already
+			 * added
+			 */
+			if (!page->mapping) {
+				unlock_page(page);
+				page_cache_release(page);
+				break;
+			}
+			/*
+			 * page was already under io and is now done, great
+			 */
+			if (PageUptodate(page)) {
+				unlock_page(page);
+				goto fill_it;
+			}
+
+readpage:
+			/*
+			 * need to read in the page
+			 */
+			error = mapping->a_ops->readpage(in, page);
 
 			if (unlikely(error)) {
 				page_cache_release(page);
+				if (error == AOP_TRUNCATED_PAGE)
+					goto find_page;
 				break;
 			}
 		}
+fill_it:
 		pages[i] = page;
 	}
 
 	if (i)
 		return move_to_pipe(pipe, pages, i, offset, len, flags);
 
-	return 0;
+	return error;
 }
 
 /**