From c7f21e4f5a3d4e378e4d453b2be209dcfd1bb964 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 10 Apr 2006 09:01:01 +0200 Subject: [PATCH] splice: mark the io page as accessed We should do that, since we do the LRU manipulation ourselves now. Suggested by Nick Piggin. Signed-off-by: Jens Axboe --- fs/splice.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/splice.c b/fs/splice.c index bfa42a2..b450acd 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -501,6 +501,7 @@ find_page: } else if (ret) goto out; + mark_page_accessed(page); balance_dirty_pages_ratelimited(mapping); out: if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { -- cgit v1.1 From 9aefe431f5a000884db7ae74ac208de814fe5913 Mon Sep 17 00:00:00 2001 From: Dave Jones Date: Mon, 10 Apr 2006 09:02:40 +0200 Subject: [PATCH] splice: potential !page dereference We can get to out: with a NULL page, which we probably don't want to be calling page_cache_release() on. Signed-off-by: Dave Jones Signed-off-by: Jens Axboe --- fs/splice.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/splice.c b/fs/splice.c index b450acd..26f5f7e 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -445,7 +445,7 @@ find_page: ret = -ENOMEM; page = find_or_create_page(mapping, index, gfp_mask); if (!page) - goto out; + goto out_nomem; /* * If the page is uptodate, it is also locked. If it isn't @@ -508,6 +508,7 @@ out: page_cache_release(page); unlock_page(page); } +out_nomem: buf->ops->unmap(info, buf); return ret; } -- cgit v1.1 From c0bd1f650bd06a43435808d44f1e9520ea806206 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 10 Apr 2006 09:03:32 +0200 Subject: [PATCH] splice: only call wake_up_interruptible() when we really have to __wake_up_common() is pretty heavy in the kernel profiles, this brings it down to a more acceptable level. Signed-off-by: Jens Axboe --- fs/splice.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 26f5f7e..9f796b1 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -187,7 +187,9 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages, } if (do_wakeup) { - wake_up_interruptible_sync(PIPE_WAIT(*inode)); + smp_mb(); + if (waitqueue_active(PIPE_WAIT(*inode))) + wake_up_interruptible_sync(PIPE_WAIT(*inode)); kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN); do_wakeup = 0; @@ -201,7 +203,9 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages, mutex_unlock(PIPE_MUTEX(*inode)); if (do_wakeup) { - wake_up_interruptible(PIPE_WAIT(*inode)); + smp_mb(); + if (waitqueue_active(PIPE_WAIT(*inode))) + wake_up_interruptible(PIPE_WAIT(*inode)); kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN); } @@ -600,7 +604,9 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, } if (do_wakeup) { - wake_up_interruptible_sync(PIPE_WAIT(*inode)); + smp_mb(); + if (waitqueue_active(PIPE_WAIT(*inode))) + wake_up_interruptible_sync(PIPE_WAIT(*inode)); kill_fasync(PIPE_FASYNC_WRITERS(*inode),SIGIO,POLL_OUT); do_wakeup = 0; } @@ -611,7 +617,9 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, mutex_unlock(PIPE_MUTEX(*inode)); if (do_wakeup) { - wake_up_interruptible(PIPE_WAIT(*inode)); + smp_mb(); + if (waitqueue_active(PIPE_WAIT(*inode))) + wake_up_interruptible(PIPE_WAIT(*inode)); kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT); } -- cgit v1.1 From 16c523ddabcce5d3d817f4a2491d628f84dfaaa1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 10 Apr 2006 09:03:58 +0200 Subject: [PATCH] splice: cleanup __generic_file_splice_read() The whole shadow/pages logic got overly complex, and this simpler approach is actually faster in testing. Signed-off-by: Jens Axboe --- fs/splice.c | 59 ++++++++++------------------------------------------------- 1 file changed, 10 insertions(+), 49 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 9f796b1..8b5efcc 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -220,10 +220,10 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe, { struct address_space *mapping = in->f_mapping; unsigned int offset, nr_pages; - struct page *pages[PIPE_BUFFERS], *shadow[PIPE_BUFFERS]; + struct page *pages[PIPE_BUFFERS]; struct page *page; - pgoff_t index, pidx; - int i, j; + pgoff_t index; + int i; index = in->f_pos >> PAGE_CACHE_SHIFT; offset = in->f_pos & ~PAGE_CACHE_MASK; @@ -238,41 +238,13 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe, do_page_cache_readahead(mapping, in, index, nr_pages); /* - * Get as many pages from the page cache as possible.. - * Start IO on the page cache entries we create (we - * can assume that any pre-existing ones we find have - * already had IO started on them). - */ - i = find_get_pages(mapping, index, nr_pages, pages); - - /* - * common case - we found all pages and they are contiguous, - * kick them off - */ - if (i && (pages[i - 1]->index == index + i - 1)) - goto splice_them; - - /* - * fill shadow[] with pages at the right locations, so we only - * have to fill holes - */ - memset(shadow, 0, nr_pages * sizeof(struct page *)); - for (j = 0; j < i; j++) - shadow[pages[j]->index - index] = pages[j]; - - /* * now fill in the holes */ - for (i = 0, pidx = index; i < nr_pages; pidx++, i++) { - int error; - - if (shadow[i]) - continue; - + for (i = 0; i < nr_pages; i++, index++) { /* * no page there, look one up / create it */ - page = find_or_create_page(mapping, pidx, + page = find_or_create_page(mapping, index, mapping_gfp_mask(mapping)); if (!page) break; @@ -280,31 +252,20 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe, if (PageUptodate(page)) unlock_page(page); else { - error = mapping->a_ops->readpage(in, page); + int error = mapping->a_ops->readpage(in, page); if (unlikely(error)) { page_cache_release(page); break; } } - shadow[i] = page; - } - - if (!i) { - for (i = 0; i < nr_pages; i++) { - if (shadow[i]) - page_cache_release(shadow[i]); - } - return 0; + pages[i] = page; } - memcpy(pages, shadow, i * sizeof(struct page *)); + if (i) + return move_to_pipe(pipe, pages, i, offset, len, flags); - /* - * Now we splice them into the pipe.. - */ -splice_them: - return move_to_pipe(pipe, pages, i, offset, len, flags); + return 0; } /** -- cgit v1.1 From 49d0b21be21efc07526d637e0ae935019667e532 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 10 Apr 2006 09:04:41 +0200 Subject: [PATCH] splice: optimize the splice buffer mapping We don't really need to lock down the pages, just make sure they are uptodate. Signed-off-by: Jens Axboe --- fs/splice.c | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 8b5efcc..50c43a1 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -84,26 +84,43 @@ static void *page_cache_pipe_buf_map(struct file *file, struct pipe_buffer *buf) { struct page *page = buf->page; - - lock_page(page); + int err; if (!PageUptodate(page)) { - unlock_page(page); - return ERR_PTR(-EIO); - } + lock_page(page); + + /* + * Page got truncated/unhashed. This will cause a 0-byte + * splice, if this is the first page + */ + if (!page->mapping) { + err = -ENODATA; + goto error; + } + + /* + * uh oh, read-error from disk + */ + if (!PageUptodate(page)) { + err = -EIO; + goto error; + } - if (!page->mapping) { + /* + * page is ok afterall, fall through to mapping + */ unlock_page(page); - return ERR_PTR(-ENODATA); } - return kmap(buf->page); + return kmap(page); +error: + unlock_page(page); + return ERR_PTR(err); } static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer *buf) { - unlock_page(buf->page); kunmap(buf->page); } @@ -379,7 +396,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, int ret; /* - * after this, page will be locked and unmapped + * make sure the data in this buffer is uptodate */ src = buf->ops->map(file, info, buf); if (IS_ERR(src)) @@ -399,6 +416,9 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, if (buf->ops->steal(info, buf)) goto find_page; + /* + * this will also set the page locked + */ page = buf->page; if (add_to_page_cache(page, mapping, index, gfp_mask)) goto find_page; -- cgit v1.1 From 0b749ce3802428007a37870eb51ba3c0bdf90857 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 10 Apr 2006 09:05:04 +0200 Subject: [PATCH] splice: be smarter about calling do_page_cache_readahead() We don't want to call into the read-ahead logic unless we are at the start of a page, _or_ we have multiple pages to read. Signed-off-by: Jens Axboe --- fs/splice.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 50c43a1..9bfd6af 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -250,9 +250,12 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe, nr_pages = PIPE_BUFFERS; /* - * initiate read-ahead on this page range + * initiate read-ahead on this page range. however, don't call into + * read-ahead if this is a non-zero offset (we are likely doing small + * chunk splice and the page is already there) for a single page. */ - do_page_cache_readahead(mapping, in, index, nr_pages); + if (!offset || nr_pages > 1) + do_page_cache_readahead(mapping, in, index, nr_pages); /* * now fill in the holes -- cgit v1.1 From 3a326a2ce88e71d00ac0d133e314a3342a7709f8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 10 Apr 2006 15:18:35 +0200 Subject: [PATCH] introduce a "kernel-internal pipe object" abstraction separate out the 'internal pipe object' abstraction, and make it usable to splice. This cleans up and fixes several aspects of the internal splice APIs and the pipe code: - pipes: the allocation and freeing of pipe_inode_info is now more symmetric and more streamlined with existing kernel practices. - splice: small micro-optimization: less pointer dereferencing in splice methods Signed-off-by: Ingo Molnar Update XFS for the ->splice_read/->splice_write changes. Signed-off-by: Jens Axboe --- fs/fifo.c | 12 +++-- fs/pipe.c | 51 +++++++++--------- fs/splice.c | 122 ++++++++++++++++++++++--------------------- fs/xfs/linux-2.6/xfs_file.c | 8 +-- fs/xfs/linux-2.6/xfs_lrw.c | 4 +- fs/xfs/linux-2.6/xfs_lrw.h | 4 +- fs/xfs/linux-2.6/xfs_vnode.h | 4 +- include/linux/fs.h | 8 +-- include/linux/pipe_fs_i.h | 7 +-- 9 files changed, 114 insertions(+), 106 deletions(-) diff --git a/fs/fifo.c b/fs/fifo.c index 889f722..b16e2f5 100644 --- a/fs/fifo.c +++ b/fs/fifo.c @@ -15,12 +15,13 @@ #include #include -static void wait_for_partner(struct inode* inode, unsigned int* cnt) +static void wait_for_partner(struct inode* inode, unsigned int *cnt) { int cur = *cnt; - while(cur == *cnt) { - pipe_wait(inode); - if(signal_pending(current)) + + while (cur == *cnt) { + pipe_wait(inode->i_pipe); + if (signal_pending(current)) break; } } @@ -37,7 +38,8 @@ static int fifo_open(struct inode *inode, struct file *filp) mutex_lock(PIPE_MUTEX(*inode)); if (!inode->i_pipe) { ret = -ENOMEM; - if(!pipe_new(inode)) + inode->i_pipe = alloc_pipe_info(inode); + if (!inode->i_pipe) goto err_nocleanup; } filp->f_version = 0; diff --git a/fs/pipe.c b/fs/pipe.c index 795df98..705b486 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -36,7 +36,7 @@ */ /* Drop the inode semaphore and wait for a pipe event, atomically */ -void pipe_wait(struct inode * inode) +void pipe_wait(struct pipe_inode_info *pipe) { DEFINE_WAIT(wait); @@ -44,11 +44,13 @@ void pipe_wait(struct inode * inode) * Pipes are system-local resources, so sleeping on them * is considered a noninteractive wait: */ - prepare_to_wait(PIPE_WAIT(*inode), &wait, TASK_INTERRUPTIBLE|TASK_NONINTERACTIVE); - mutex_unlock(PIPE_MUTEX(*inode)); + prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE|TASK_NONINTERACTIVE); + if (pipe->inode) + mutex_unlock(&pipe->inode->i_mutex); schedule(); - finish_wait(PIPE_WAIT(*inode), &wait); - mutex_lock(PIPE_MUTEX(*inode)); + finish_wait(&pipe->wait, &wait); + if (pipe->inode) + mutex_lock(&pipe->inode->i_mutex); } static int @@ -223,7 +225,7 @@ pipe_readv(struct file *filp, const struct iovec *_iov, wake_up_interruptible_sync(PIPE_WAIT(*inode)); kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT); } - pipe_wait(inode); + pipe_wait(inode->i_pipe); } mutex_unlock(PIPE_MUTEX(*inode)); /* Signal writers asynchronously that there is more room. */ @@ -370,7 +372,7 @@ pipe_writev(struct file *filp, const struct iovec *_iov, do_wakeup = 0; } PIPE_WAITING_WRITERS(*inode)++; - pipe_wait(inode); + pipe_wait(inode->i_pipe); PIPE_WAITING_WRITERS(*inode)--; } out: @@ -675,6 +677,20 @@ static struct file_operations rdwr_pipe_fops = { .fasync = pipe_rdwr_fasync, }; +struct pipe_inode_info * alloc_pipe_info(struct inode *inode) +{ + struct pipe_inode_info *info; + + info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); + if (info) { + init_waitqueue_head(&info->wait); + info->r_counter = info->w_counter = 1; + info->inode = inode; + } + + return info; +} + void free_pipe_info(struct inode *inode) { int i; @@ -691,23 +707,6 @@ void free_pipe_info(struct inode *inode) kfree(info); } -struct inode* pipe_new(struct inode* inode) -{ - struct pipe_inode_info *info; - - info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); - if (!info) - goto fail_page; - inode->i_pipe = info; - - init_waitqueue_head(PIPE_WAIT(*inode)); - PIPE_RCOUNTER(*inode) = PIPE_WCOUNTER(*inode) = 1; - - return inode; -fail_page: - return NULL; -} - static struct vfsmount *pipe_mnt __read_mostly; static int pipefs_delete_dentry(struct dentry *dentry) { @@ -724,8 +723,10 @@ static struct inode * get_pipe_inode(void) if (!inode) goto fail_inode; - if(!pipe_new(inode)) + inode->i_pipe = alloc_pipe_info(inode); + if (!inode->i_pipe) goto fail_iput; + PIPE_READERS(*inode) = PIPE_WRITERS(*inode) = 1; inode->i_fop = &rdwr_pipe_fops; diff --git a/fs/splice.c b/fs/splice.c index 9bfd6af..ed91a62 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -136,34 +136,33 @@ static struct pipe_buf_operations page_cache_pipe_buf_ops = { * Pipe output worker. This sets up our pipe format with the page cache * pipe buffer operations. Otherwise very similar to the regular pipe_writev(). */ -static ssize_t move_to_pipe(struct inode *inode, struct page **pages, +static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, int nr_pages, unsigned long offset, unsigned long len, unsigned int flags) { - struct pipe_inode_info *info; int ret, do_wakeup, i; ret = 0; do_wakeup = 0; i = 0; - mutex_lock(PIPE_MUTEX(*inode)); + if (pipe->inode) + mutex_lock(&pipe->inode->i_mutex); - info = inode->i_pipe; for (;;) { int bufs; - if (!PIPE_READERS(*inode)) { + if (!pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } - bufs = info->nrbufs; + bufs = pipe->nrbufs; if (bufs < PIPE_BUFFERS) { - int newbuf = (info->curbuf + bufs) & (PIPE_BUFFERS - 1); - struct pipe_buffer *buf = info->bufs + newbuf; + int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS - 1); + struct pipe_buffer *buf = pipe->bufs + newbuf; struct page *page = pages[i++]; unsigned long this_len; @@ -175,7 +174,7 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages, buf->offset = offset; buf->len = this_len; buf->ops = &page_cache_pipe_buf_ops; - info->nrbufs = ++bufs; + pipe->nrbufs = ++bufs; do_wakeup = 1; ret += this_len; @@ -205,25 +204,25 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages, if (do_wakeup) { smp_mb(); - if (waitqueue_active(PIPE_WAIT(*inode))) - wake_up_interruptible_sync(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, - POLL_IN); + if (waitqueue_active(&pipe->wait)) + wake_up_interruptible_sync(&pipe->wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); do_wakeup = 0; } - PIPE_WAITING_WRITERS(*inode)++; - pipe_wait(inode); - PIPE_WAITING_WRITERS(*inode)--; + pipe->waiting_writers++; + pipe_wait(pipe); + pipe->waiting_writers--; } - mutex_unlock(PIPE_MUTEX(*inode)); + if (pipe->inode) + mutex_unlock(&pipe->inode->i_mutex); if (do_wakeup) { smp_mb(); - if (waitqueue_active(PIPE_WAIT(*inode))) - wake_up_interruptible(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN); + if (waitqueue_active(&pipe->wait)) + wake_up_interruptible(&pipe->wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } while (i < nr_pages) @@ -232,8 +231,9 @@ static ssize_t move_to_pipe(struct inode *inode, struct page **pages, return ret; } -static int __generic_file_splice_read(struct file *in, struct inode *pipe, - size_t len, unsigned int flags) +static int +__generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, + size_t len, unsigned int flags) { struct address_space *mapping = in->f_mapping; unsigned int offset, nr_pages; @@ -298,7 +298,7 @@ static int __generic_file_splice_read(struct file *in, struct inode *pipe, * Will read pages from given file and fill them into a pipe. * */ -ssize_t generic_file_splice_read(struct file *in, struct inode *pipe, +ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, size_t len, unsigned int flags) { ssize_t spliced; @@ -306,6 +306,7 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe, ret = 0; spliced = 0; + while (len) { ret = __generic_file_splice_read(in, pipe, len, flags); @@ -509,11 +510,10 @@ typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *, * key here is the 'actor' worker passed in that actually moves the data * to the wanted destination. See pipe_to_file/pipe_to_sendpage above. */ -static ssize_t move_from_pipe(struct inode *inode, struct file *out, +static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, size_t len, unsigned int flags, splice_actor *actor) { - struct pipe_inode_info *info; int ret, do_wakeup, err; struct splice_desc sd; @@ -525,22 +525,22 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, sd.file = out; sd.pos = out->f_pos; - mutex_lock(PIPE_MUTEX(*inode)); + if (pipe->inode) + mutex_lock(&pipe->inode->i_mutex); - info = inode->i_pipe; for (;;) { - int bufs = info->nrbufs; + int bufs = pipe->nrbufs; if (bufs) { - int curbuf = info->curbuf; - struct pipe_buffer *buf = info->bufs + curbuf; + int curbuf = pipe->curbuf; + struct pipe_buffer *buf = pipe->bufs + curbuf; struct pipe_buf_operations *ops = buf->ops; sd.len = buf->len; if (sd.len > sd.total_len) sd.len = sd.total_len; - err = actor(info, buf, &sd); + err = actor(pipe, buf, &sd); if (err) { if (!ret && err != -ENODATA) ret = err; @@ -553,10 +553,10 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, buf->len -= sd.len; if (!buf->len) { buf->ops = NULL; - ops->release(info, buf); + ops->release(pipe, buf); curbuf = (curbuf + 1) & (PIPE_BUFFERS - 1); - info->curbuf = curbuf; - info->nrbufs = --bufs; + pipe->curbuf = curbuf; + pipe->nrbufs = --bufs; do_wakeup = 1; } @@ -568,9 +568,9 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, if (bufs) continue; - if (!PIPE_WRITERS(*inode)) + if (!pipe->writers) break; - if (!PIPE_WAITING_WRITERS(*inode)) { + if (!pipe->waiting_writers) { if (ret) break; } @@ -589,22 +589,23 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, if (do_wakeup) { smp_mb(); - if (waitqueue_active(PIPE_WAIT(*inode))) - wake_up_interruptible_sync(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_WRITERS(*inode),SIGIO,POLL_OUT); + if (waitqueue_active(&pipe->wait)) + wake_up_interruptible_sync(&pipe->wait); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); do_wakeup = 0; } - pipe_wait(inode); + pipe_wait(pipe); } - mutex_unlock(PIPE_MUTEX(*inode)); + if (pipe->inode) + mutex_unlock(&pipe->inode->i_mutex); if (do_wakeup) { smp_mb(); - if (waitqueue_active(PIPE_WAIT(*inode))) - wake_up_interruptible(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT); + if (waitqueue_active(&pipe->wait)) + wake_up_interruptible(&pipe->wait); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } mutex_lock(&out->f_mapping->host->i_mutex); @@ -616,7 +617,7 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, /** * generic_file_splice_write - splice data from a pipe to a file - * @inode: pipe inode + * @pipe: pipe info * @out: file to write to * @len: number of bytes to splice * @flags: splice modifier flags @@ -625,11 +626,14 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, * the given pipe inode to the given file. * */ -ssize_t generic_file_splice_write(struct inode *inode, struct file *out, - size_t len, unsigned int flags) +ssize_t +generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, + size_t len, unsigned int flags) { struct address_space *mapping = out->f_mapping; - ssize_t ret = move_from_pipe(inode, out, len, flags, pipe_to_file); + ssize_t ret; + + ret = move_from_pipe(pipe, out, len, flags, pipe_to_file); /* * if file or inode is SYNC and we actually wrote some data, sync it @@ -664,10 +668,10 @@ EXPORT_SYMBOL(generic_file_splice_write); * is involved. * */ -ssize_t generic_splice_sendpage(struct inode *inode, struct file *out, +ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, size_t len, unsigned int flags) { - return move_from_pipe(inode, out, len, flags, pipe_to_sendpage); + return move_from_pipe(pipe, out, len, flags, pipe_to_sendpage); } EXPORT_SYMBOL(generic_splice_sendpage); @@ -675,8 +679,8 @@ EXPORT_SYMBOL(generic_splice_sendpage); /* * Attempt to initiate a splice from pipe to file. */ -static long do_splice_from(struct inode *pipe, struct file *out, size_t len, - unsigned int flags) +static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, + size_t len, unsigned int flags) { loff_t pos; int ret; @@ -698,8 +702,8 @@ static long do_splice_from(struct inode *pipe, struct file *out, size_t len, /* * Attempt to initiate a splice from a file to a pipe. */ -static long do_splice_to(struct file *in, struct inode *pipe, size_t len, - unsigned int flags) +static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, + size_t len, unsigned int flags) { loff_t pos, isize, left; int ret; @@ -732,14 +736,14 @@ static long do_splice_to(struct file *in, struct inode *pipe, size_t len, static long do_splice(struct file *in, struct file *out, size_t len, unsigned int flags) { - struct inode *pipe; + struct pipe_inode_info *pipe; - pipe = in->f_dentry->d_inode; - if (pipe->i_pipe) + pipe = in->f_dentry->d_inode->i_pipe; + if (pipe) return do_splice_from(pipe, out, len, flags); - pipe = out->f_dentry->d_inode; - if (pipe->i_pipe) + pipe = out->f_dentry->d_inode->i_pipe; + if (pipe) return do_splice_to(in, pipe, len, flags); return -EINVAL; diff --git a/fs/xfs/linux-2.6/xfs_file.c b/fs/xfs/linux-2.6/xfs_file.c index ae4c475..269721a 100644 --- a/fs/xfs/linux-2.6/xfs_file.c +++ b/fs/xfs/linux-2.6/xfs_file.c @@ -252,7 +252,7 @@ xfs_file_sendfile_invis( STATIC ssize_t xfs_file_splice_read( struct file *infilp, - struct inode *pipe, + struct pipe_inode_info *pipe, size_t len, unsigned int flags) { @@ -266,7 +266,7 @@ xfs_file_splice_read( STATIC ssize_t xfs_file_splice_read_invis( struct file *infilp, - struct inode *pipe, + struct pipe_inode_info *pipe, size_t len, unsigned int flags) { @@ -279,7 +279,7 @@ xfs_file_splice_read_invis( STATIC ssize_t xfs_file_splice_write( - struct inode *pipe, + struct pipe_inode_info *pipe, struct file *outfilp, size_t len, unsigned int flags) @@ -293,7 +293,7 @@ xfs_file_splice_write( STATIC ssize_t xfs_file_splice_write_invis( - struct inode *pipe, + struct pipe_inode_info *pipe, struct file *outfilp, size_t len, unsigned int flags) diff --git a/fs/xfs/linux-2.6/xfs_lrw.c b/fs/xfs/linux-2.6/xfs_lrw.c index 90cd314..74a52937 100644 --- a/fs/xfs/linux-2.6/xfs_lrw.c +++ b/fs/xfs/linux-2.6/xfs_lrw.c @@ -338,7 +338,7 @@ ssize_t xfs_splice_read( bhv_desc_t *bdp, struct file *infilp, - struct inode *pipe, + struct pipe_inode_info *pipe, size_t count, int flags, int ioflags, @@ -380,7 +380,7 @@ xfs_splice_read( ssize_t xfs_splice_write( bhv_desc_t *bdp, - struct inode *pipe, + struct pipe_inode_info *pipe, struct file *outfilp, size_t count, int flags, diff --git a/fs/xfs/linux-2.6/xfs_lrw.h b/fs/xfs/linux-2.6/xfs_lrw.h index eaa5659..55c689a 100644 --- a/fs/xfs/linux-2.6/xfs_lrw.h +++ b/fs/xfs/linux-2.6/xfs_lrw.h @@ -94,9 +94,9 @@ extern ssize_t xfs_sendfile(struct bhv_desc *, struct file *, loff_t *, int, size_t, read_actor_t, void *, struct cred *); extern ssize_t xfs_splice_read(struct bhv_desc *, struct file *, - struct inode *, size_t, int, int, + struct pipe_inode_info *, size_t, int, int, struct cred *); -extern ssize_t xfs_splice_write(struct bhv_desc *, struct inode *, +extern ssize_t xfs_splice_write(struct bhv_desc *, struct pipe_inode_info *, struct file *, size_t, int, int, struct cred *); diff --git a/fs/xfs/linux-2.6/xfs_vnode.h b/fs/xfs/linux-2.6/xfs_vnode.h index 6f1c79a..88b09f1 100644 --- a/fs/xfs/linux-2.6/xfs_vnode.h +++ b/fs/xfs/linux-2.6/xfs_vnode.h @@ -174,9 +174,9 @@ typedef ssize_t (*vop_sendfile_t)(bhv_desc_t *, struct file *, loff_t *, int, size_t, read_actor_t, void *, struct cred *); typedef ssize_t (*vop_splice_read_t)(bhv_desc_t *, struct file *, - struct inode *, size_t, int, int, + struct pipe_inode_info *, size_t, int, int, struct cred *); -typedef ssize_t (*vop_splice_write_t)(bhv_desc_t *, struct inode *, +typedef ssize_t (*vop_splice_write_t)(bhv_desc_t *, struct pipe_inode_info *, struct file *, size_t, int, int, struct cred *); typedef int (*vop_ioctl_t)(bhv_desc_t *, struct inode *, struct file *, diff --git a/include/linux/fs.h b/include/linux/fs.h index 1e9ebab..7e64544 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1039,8 +1039,8 @@ struct file_operations { int (*check_flags)(int); int (*dir_notify)(struct file *filp, unsigned long arg); int (*flock) (struct file *, int, struct file_lock *); - ssize_t (*splice_write)(struct inode *, struct file *, size_t, unsigned int); - ssize_t (*splice_read)(struct file *, struct inode *, size_t, unsigned int); + ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, size_t, unsigned int); + ssize_t (*splice_read)(struct file *, struct pipe_inode_info *, size_t, unsigned int); }; struct inode_operations { @@ -1611,8 +1611,8 @@ extern ssize_t generic_file_sendfile(struct file *, loff_t *, size_t, read_actor extern void do_generic_mapping_read(struct address_space *mapping, struct file_ra_state *, struct file *, loff_t *, read_descriptor_t *, read_actor_t); -extern ssize_t generic_file_splice_read(struct file *, struct inode *, size_t, unsigned int); -extern ssize_t generic_file_splice_write(struct inode *, struct file *, size_t, unsigned int); +extern ssize_t generic_file_splice_read(struct file *, struct pipe_inode_info *, size_t, unsigned int); +extern ssize_t generic_file_splice_write(struct pipe_inode_info *, struct file *, size_t, unsigned int); extern void file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping); extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index ec38495..9cf99cb 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -36,6 +36,7 @@ struct pipe_inode_info { unsigned int w_counter; struct fasync_struct *fasync_readers; struct fasync_struct *fasync_writers; + struct inode *inode; }; /* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual @@ -53,10 +54,10 @@ struct pipe_inode_info { #define PIPE_FASYNC_WRITERS(inode) (&((inode).i_pipe->fasync_writers)) /* Drop the inode semaphore and wait for a pipe event, atomically */ -void pipe_wait(struct inode * inode); +void pipe_wait(struct pipe_inode_info *pipe); -struct inode* pipe_new(struct inode* inode); -void free_pipe_info(struct inode* inode); +struct pipe_inode_info * alloc_pipe_info(struct inode * inode); +void free_pipe_info(struct inode * inode); /* * splice is tied to pipes as a transport (at least for now), so we'll just -- cgit v1.1 From 529565dcb1581c9a1e3f6df1c1763ca3e0f0d512 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 10 Apr 2006 15:18:58 +0200 Subject: [PATCH] splice: add optional input and output offsets add optional input and output offsets to sys_splice(), for seekable file descriptors: asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, int fd_out, loff_t __user *off_out, size_t len, unsigned int flags); semantics are straightforward: f_pos will be updated with the offset provided by user-space, before the splice transfer is about to begin. Providing a NULL offset pointer means the existing f_pos will be used (and updated in situ). Providing an offset for a pipe results in -ESPIPE. Providing an invalid offset pointer results in -EFAULT. Signed-off-by: Ingo Molnar Signed-off-by: Jens Axboe --- fs/splice.c | 54 ++++++++++++++++++++++++++++++++++++------------ include/linux/syscalls.h | 7 +++++-- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index ed91a62..a532612 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -680,7 +680,8 @@ EXPORT_SYMBOL(generic_splice_sendpage); * Attempt to initiate a splice from pipe to file. */ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, - size_t len, unsigned int flags) + loff_t __user *off_out, size_t len, + unsigned int flags) { loff_t pos; int ret; @@ -691,7 +692,11 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, if (!(out->f_mode & FMODE_WRITE)) return -EBADF; + if (off_out && copy_from_user(&out->f_pos, off_out, sizeof(loff_t))) + return -EFAULT; + pos = out->f_pos; + ret = rw_verify_area(WRITE, out, &pos, len); if (unlikely(ret < 0)) return ret; @@ -702,8 +707,9 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, /* * Attempt to initiate a splice from a file to a pipe. */ -static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, - size_t len, unsigned int flags) +static long do_splice_to(struct file *in, loff_t __user *off_in, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) { loff_t pos, isize, left; int ret; @@ -714,7 +720,11 @@ static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, if (!(in->f_mode & FMODE_READ)) return -EBADF; + if (off_in && copy_from_user(&in->f_pos, off_in, sizeof(loff_t))) + return -EFAULT; + pos = in->f_pos; + ret = rw_verify_area(READ, in, &pos, len); if (unlikely(ret < 0)) return ret; @@ -733,23 +743,39 @@ static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, /* * Determine where to splice to/from. */ -static long do_splice(struct file *in, struct file *out, size_t len, - unsigned int flags) +static long do_splice(struct file *in, loff_t __user *off_in, + struct file *out, loff_t __user *off_out, + size_t len, unsigned int flags) { struct pipe_inode_info *pipe; + if (off_out && out->f_op->llseek == no_llseek) + return -EINVAL; + if (off_in && in->f_op->llseek == no_llseek) + return -EINVAL; + pipe = in->f_dentry->d_inode->i_pipe; - if (pipe) - return do_splice_from(pipe, out, len, flags); + if (pipe) { + if (off_in) + return -ESPIPE; + + return do_splice_from(pipe, out, off_out, len, flags); + } pipe = out->f_dentry->d_inode->i_pipe; - if (pipe) - return do_splice_to(in, pipe, len, flags); + if (pipe) { + if (off_out) + return -ESPIPE; + + return do_splice_to(in, off_in, pipe, len, flags); + } return -EINVAL; } -asmlinkage long sys_splice(int fdin, int fdout, size_t len, unsigned int flags) +asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, + int fd_out, loff_t __user *off_out, + size_t len, unsigned int flags) { long error; struct file *in, *out; @@ -759,13 +785,15 @@ asmlinkage long sys_splice(int fdin, int fdout, size_t len, unsigned int flags) return 0; error = -EBADF; - in = fget_light(fdin, &fput_in); + in = fget_light(fd_in, &fput_in); if (in) { if (in->f_mode & FMODE_READ) { - out = fget_light(fdout, &fput_out); + out = fget_light(fd_out, &fput_out); if (out) { if (out->f_mode & FMODE_WRITE) - error = do_splice(in, out, len, flags); + error = do_splice(in, off_in, + out, off_out, + len, flags); fput_light(out, fput_out); } } diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 5717147..4c292fa 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -569,8 +569,11 @@ asmlinkage long compat_sys_newfstatat(unsigned int dfd, char __user * filename, asmlinkage long compat_sys_openat(unsigned int dfd, const char __user *filename, int flags, int mode); asmlinkage long sys_unshare(unsigned long unshare_flags); -asmlinkage long sys_splice(int fdin, int fdout, size_t len, - unsigned int flags); + +asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, + int fd_out, loff_t __user *off_out, + size_t len, unsigned int flags); + asmlinkage long sys_sync_file_range(int fd, loff_t offset, loff_t nbytes, int flags); -- cgit v1.1 From b92ce55893745e011edae70830b8bc863be881f9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Apr 2006 13:52:07 +0200 Subject: [PATCH] splice: add direct fd <-> fd splicing support It's more efficient for sendfile() emulation. Basically we cache an internal private pipe and just use that as the intermediate area for pages. Direct splicing is not available from sys_splice(), it is only meant to be used for sendfile() emulation. Additional patch from Ingo Molnar to avoid the PIPE_BUFFERS loop at exit for the normal fast path. Signed-off-by: Jens Axboe --- fs/pipe.c | 10 +++- fs/splice.c | 148 ++++++++++++++++++++++++++++++++++++++++------ include/linux/fs.h | 2 + include/linux/pipe_fs_i.h | 1 + include/linux/sched.h | 6 ++ kernel/exit.c | 4 ++ 6 files changed, 150 insertions(+), 21 deletions(-) diff --git a/fs/pipe.c b/fs/pipe.c index 705b486..036536f 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -691,12 +691,10 @@ struct pipe_inode_info * alloc_pipe_info(struct inode *inode) return info; } -void free_pipe_info(struct inode *inode) +void __free_pipe_info(struct pipe_inode_info *info) { int i; - struct pipe_inode_info *info = inode->i_pipe; - inode->i_pipe = NULL; for (i = 0; i < PIPE_BUFFERS; i++) { struct pipe_buffer *buf = info->bufs + i; if (buf->ops) @@ -707,6 +705,12 @@ void free_pipe_info(struct inode *inode) kfree(info); } +void free_pipe_info(struct inode *inode) +{ + __free_pipe_info(inode->i_pipe); + inode->i_pipe = NULL; +} + static struct vfsmount *pipe_mnt __read_mostly; static int pipefs_delete_dentry(struct dentry *dentry) { diff --git a/fs/splice.c b/fs/splice.c index a532612..c47b561 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -680,8 +680,7 @@ EXPORT_SYMBOL(generic_splice_sendpage); * Attempt to initiate a splice from pipe to file. */ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, - loff_t __user *off_out, size_t len, - unsigned int flags) + size_t len, unsigned int flags) { loff_t pos; int ret; @@ -692,9 +691,6 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, if (!(out->f_mode & FMODE_WRITE)) return -EBADF; - if (off_out && copy_from_user(&out->f_pos, off_out, sizeof(loff_t))) - return -EFAULT; - pos = out->f_pos; ret = rw_verify_area(WRITE, out, &pos, len); @@ -707,9 +703,8 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, /* * Attempt to initiate a splice from a file to a pipe. */ -static long do_splice_to(struct file *in, loff_t __user *off_in, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags) +static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, + size_t len, unsigned int flags) { loff_t pos, isize, left; int ret; @@ -720,9 +715,6 @@ static long do_splice_to(struct file *in, loff_t __user *off_in, if (!(in->f_mode & FMODE_READ)) return -EBADF; - if (off_in && copy_from_user(&in->f_pos, off_in, sizeof(loff_t))) - return -EFAULT; - pos = in->f_pos; ret = rw_verify_area(READ, in, &pos, len); @@ -740,6 +732,118 @@ static long do_splice_to(struct file *in, loff_t __user *off_in, return in->f_op->splice_read(in, pipe, len, flags); } +long do_splice_direct(struct file *in, struct file *out, size_t len, + unsigned int flags) +{ + struct pipe_inode_info *pipe; + long ret, bytes; + umode_t i_mode; + int i; + + /* + * We require the input being a regular file, as we don't want to + * randomly drop data for eg socket -> socket splicing. Use the + * piped splicing for that! + */ + i_mode = in->f_dentry->d_inode->i_mode; + if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) + return -EINVAL; + + /* + * neither in nor out is a pipe, setup an internal pipe attached to + * 'out' and transfer the wanted data from 'in' to 'out' through that + */ + pipe = current->splice_pipe; + if (!pipe) { + pipe = alloc_pipe_info(NULL); + if (!pipe) + return -ENOMEM; + + /* + * We don't have an immediate reader, but we'll read the stuff + * out of the pipe right after the move_to_pipe(). So set + * PIPE_READERS appropriately. + */ + pipe->readers = 1; + + current->splice_pipe = pipe; + } + + /* + * do the splice + */ + ret = 0; + bytes = 0; + + while (len) { + size_t read_len, max_read_len; + + /* + * Do at most PIPE_BUFFERS pages worth of transfer: + */ + max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE)); + + ret = do_splice_to(in, pipe, max_read_len, flags); + if (unlikely(ret < 0)) + goto out_release; + + read_len = ret; + + /* + * NOTE: nonblocking mode only applies to the input. We + * must not do the output in nonblocking mode as then we + * could get stuck data in the internal pipe: + */ + ret = do_splice_from(pipe, out, read_len, + flags & ~SPLICE_F_NONBLOCK); + if (unlikely(ret < 0)) + goto out_release; + + bytes += ret; + len -= ret; + + /* + * In nonblocking mode, if we got back a short read then + * that was due to either an IO error or due to the + * pagecache entry not being there. In the IO error case + * the _next_ splice attempt will produce a clean IO error + * return value (not a short read), so in both cases it's + * correct to break out of the loop here: + */ + if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len)) + break; + } + + pipe->nrbufs = pipe->curbuf = 0; + + return bytes; + +out_release: + /* + * If we did an incomplete transfer we must release + * the pipe buffers in question: + */ + for (i = 0; i < PIPE_BUFFERS; i++) { + struct pipe_buffer *buf = pipe->bufs + i; + + if (buf->ops) { + buf->ops->release(pipe, buf); + buf->ops = NULL; + } + } + pipe->nrbufs = pipe->curbuf = 0; + + /* + * If we transferred some data, return the number of bytes: + */ + if (bytes > 0) + return bytes; + + return ret; +} + +EXPORT_SYMBOL(do_splice_direct); + /* * Determine where to splice to/from. */ @@ -749,25 +853,33 @@ static long do_splice(struct file *in, loff_t __user *off_in, { struct pipe_inode_info *pipe; - if (off_out && out->f_op->llseek == no_llseek) - return -EINVAL; - if (off_in && in->f_op->llseek == no_llseek) - return -EINVAL; - pipe = in->f_dentry->d_inode->i_pipe; if (pipe) { if (off_in) return -ESPIPE; + if (off_out) { + if (out->f_op->llseek == no_llseek) + return -EINVAL; + if (copy_from_user(&out->f_pos, off_out, + sizeof(loff_t))) + return -EFAULT; + } - return do_splice_from(pipe, out, off_out, len, flags); + return do_splice_from(pipe, out, len, flags); } pipe = out->f_dentry->d_inode->i_pipe; if (pipe) { if (off_out) return -ESPIPE; + if (off_in) { + if (in->f_op->llseek == no_llseek) + return -EINVAL; + if (copy_from_user(&in->f_pos, off_in, sizeof(loff_t))) + return -EFAULT; + } - return do_splice_to(in, off_in, pipe, len, flags); + return do_splice_to(in, pipe, len, flags); } return -EINVAL; diff --git a/include/linux/fs.h b/include/linux/fs.h index 7e64544..9e8e2ee 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1613,6 +1613,8 @@ extern void do_generic_mapping_read(struct address_space *mapping, loff_t *, read_descriptor_t *, read_actor_t); extern ssize_t generic_file_splice_read(struct file *, struct pipe_inode_info *, size_t, unsigned int); extern ssize_t generic_file_splice_write(struct pipe_inode_info *, struct file *, size_t, unsigned int); +extern long do_splice_direct(struct file *in, struct file *out, + size_t len, unsigned int flags); extern void file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping); extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 9cf99cb..660e9d8 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -58,6 +58,7 @@ void pipe_wait(struct pipe_inode_info *pipe); struct pipe_inode_info * alloc_pipe_info(struct inode * inode); void free_pipe_info(struct inode * inode); +void __free_pipe_info(struct pipe_inode_info *); /* * splice is tied to pipes as a transport (at least for now), so we'll just diff --git a/include/linux/sched.h b/include/linux/sched.h index 541f482..e194ec7 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -684,6 +684,7 @@ static inline void prefetch_stack(struct task_struct *t) { } struct audit_context; /* See audit.c */ struct mempolicy; +struct pipe_inode_info; enum sleep_type { SLEEP_NORMAL, @@ -882,6 +883,11 @@ struct task_struct { atomic_t fs_excl; /* holding fs exclusive resources */ struct rcu_head rcu; + + /* + * cache last used pipe for splice + */ + struct pipe_inode_info *splice_pipe; }; static inline pid_t process_group(struct task_struct *tsk) diff --git a/kernel/exit.c b/kernel/exit.c index 6c2eeb8..1a9787a 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -941,6 +942,9 @@ fastcall NORET_TYPE void do_exit(long code) if (tsk->io_context) exit_io_context(); + if (tsk->splice_pipe) + __free_pipe_info(tsk->splice_pipe); + /* PF_DEAD causes final put_task_struct after we schedule. */ preempt_disable(); BUG_ON(tsk->flags & PF_DEAD); -- cgit v1.1 From 7480a90435673b4c717b6caf1350ec577d5f1adf Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Apr 2006 13:52:47 +0200 Subject: [PATCH] splice: speedup __generic_file_splice_read Using find_get_page() is a lot faster than find_or_create_page(). This gets splice a lot closer to sendfile() for fd -> socket transfers. Signed-off-by: Jens Axboe --- fs/splice.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index c47b561..e30743c 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -240,7 +240,7 @@ __generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, struct page *pages[PIPE_BUFFERS]; struct page *page; pgoff_t index; - int i; + int i, error; index = in->f_pos >> PAGE_CACHE_SHIFT; offset = in->f_pos & ~PAGE_CACHE_MASK; @@ -260,32 +260,84 @@ __generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, /* * now fill in the holes */ + error = 0; for (i = 0; i < nr_pages; i++, index++) { +find_page: /* - * no page there, look one up / create it + * lookup the page for this index */ - page = find_or_create_page(mapping, index, - mapping_gfp_mask(mapping)); - if (!page) - break; + page = find_get_page(mapping, index); + if (!page) { + /* + * If in nonblock mode then dont block on + * readpage (we've kicked readahead so there + * will be asynchronous progress): + */ + if (flags & SPLICE_F_NONBLOCK) + break; + + /* + * page didn't exist, allocate one + */ + page = page_cache_alloc_cold(mapping); + if (!page) + break; + + error = add_to_page_cache_lru(page, mapping, index, + mapping_gfp_mask(mapping)); + if (unlikely(error)) { + page_cache_release(page); + break; + } - if (PageUptodate(page)) - unlock_page(page); - else { - int error = mapping->a_ops->readpage(in, page); + goto readpage; + } + + /* + * If the page isn't uptodate, we may need to start io on it + */ + if (!PageUptodate(page)) { + lock_page(page); + + /* + * page was truncated, stop here. if this isn't the + * first page, we'll just complete what we already + * added + */ + if (!page->mapping) { + unlock_page(page); + page_cache_release(page); + break; + } + /* + * page was already under io and is now done, great + */ + if (PageUptodate(page)) { + unlock_page(page); + goto fill_it; + } + +readpage: + /* + * need to read in the page + */ + error = mapping->a_ops->readpage(in, page); if (unlikely(error)) { page_cache_release(page); + if (error == AOP_TRUNCATED_PAGE) + goto find_page; break; } } +fill_it: pages[i] = page; } if (i) return move_to_pipe(pipe, pages, i, offset, len, flags); - return 0; + return error; } /** -- cgit v1.1 From 9aeedfc4712ed58d9f7ae41596185c72b8dc97e8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 11 Apr 2006 13:53:10 +0200 Subject: [PATCH] get rid of the PIPE_*() macros get rid of the PIPE_*() macros. Scripted transformation. Signed-off-by: Ingo Molnar Signed-off-by: Jens Axboe --- fs/fifo.c | 48 ++++++++++----------- fs/pipe.c | 104 +++++++++++++++++++++++----------------------- include/linux/pipe_fs_i.h | 10 ----- 3 files changed, 76 insertions(+), 86 deletions(-) diff --git a/fs/fifo.c b/fs/fifo.c index b16e2f5..2c27f56 100644 --- a/fs/fifo.c +++ b/fs/fifo.c @@ -28,14 +28,14 @@ static void wait_for_partner(struct inode* inode, unsigned int *cnt) static void wake_up_partner(struct inode* inode) { - wake_up_interruptible(PIPE_WAIT(*inode)); + wake_up_interruptible(&inode->i_pipe->wait); } static int fifo_open(struct inode *inode, struct file *filp) { int ret; - mutex_lock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); if (!inode->i_pipe) { ret = -ENOMEM; inode->i_pipe = alloc_pipe_info(inode); @@ -55,18 +55,18 @@ static int fifo_open(struct inode *inode, struct file *filp) * opened, even when there is no process writing the FIFO. */ filp->f_op = &read_fifo_fops; - PIPE_RCOUNTER(*inode)++; - if (PIPE_READERS(*inode)++ == 0) + inode->i_pipe->r_counter++; + if (inode->i_pipe->readers++ == 0) wake_up_partner(inode); - if (!PIPE_WRITERS(*inode)) { + if (!inode->i_pipe->writers) { if ((filp->f_flags & O_NONBLOCK)) { /* suppress POLLHUP until we have * seen a writer */ - filp->f_version = PIPE_WCOUNTER(*inode); + filp->f_version = inode->i_pipe->w_counter; } else { - wait_for_partner(inode, &PIPE_WCOUNTER(*inode)); + wait_for_partner(inode, &inode->i_pipe->w_counter); if(signal_pending(current)) goto err_rd; } @@ -80,16 +80,16 @@ static int fifo_open(struct inode *inode, struct file *filp) * errno=ENXIO when there is no process reading the FIFO. */ ret = -ENXIO; - if ((filp->f_flags & O_NONBLOCK) && !PIPE_READERS(*inode)) + if ((filp->f_flags & O_NONBLOCK) && !inode->i_pipe->readers) goto err; filp->f_op = &write_fifo_fops; - PIPE_WCOUNTER(*inode)++; - if (!PIPE_WRITERS(*inode)++) + inode->i_pipe->w_counter++; + if (!inode->i_pipe->writers++) wake_up_partner(inode); - if (!PIPE_READERS(*inode)) { - wait_for_partner(inode, &PIPE_RCOUNTER(*inode)); + if (!inode->i_pipe->readers) { + wait_for_partner(inode, &inode->i_pipe->r_counter); if (signal_pending(current)) goto err_wr; } @@ -104,11 +104,11 @@ static int fifo_open(struct inode *inode, struct file *filp) */ filp->f_op = &rdwr_fifo_fops; - PIPE_READERS(*inode)++; - PIPE_WRITERS(*inode)++; - PIPE_RCOUNTER(*inode)++; - PIPE_WCOUNTER(*inode)++; - if (PIPE_READERS(*inode) == 1 || PIPE_WRITERS(*inode) == 1) + inode->i_pipe->readers++; + inode->i_pipe->writers++; + inode->i_pipe->r_counter++; + inode->i_pipe->w_counter++; + if (inode->i_pipe->readers == 1 || inode->i_pipe->writers == 1) wake_up_partner(inode); break; @@ -118,27 +118,27 @@ static int fifo_open(struct inode *inode, struct file *filp) } /* Ok! */ - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_unlock(&inode->i_mutex); return 0; err_rd: - if (!--PIPE_READERS(*inode)) - wake_up_interruptible(PIPE_WAIT(*inode)); + if (!--inode->i_pipe->readers) + wake_up_interruptible(&inode->i_pipe->wait); ret = -ERESTARTSYS; goto err; err_wr: - if (!--PIPE_WRITERS(*inode)) - wake_up_interruptible(PIPE_WAIT(*inode)); + if (!--inode->i_pipe->writers) + wake_up_interruptible(&inode->i_pipe->wait); ret = -ERESTARTSYS; goto err; err: - if (!PIPE_READERS(*inode) && !PIPE_WRITERS(*inode)) + if (!inode->i_pipe->readers && !inode->i_pipe->writers) free_pipe_info(inode); err_nocleanup: - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_unlock(&inode->i_mutex); return ret; } diff --git a/fs/pipe.c b/fs/pipe.c index 036536f..0602fc9 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -158,7 +158,7 @@ pipe_readv(struct file *filp, const struct iovec *_iov, do_wakeup = 0; ret = 0; - mutex_lock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); info = inode->i_pipe; for (;;) { int bufs = info->nrbufs; @@ -202,9 +202,9 @@ pipe_readv(struct file *filp, const struct iovec *_iov, } if (bufs) /* More to do? */ continue; - if (!PIPE_WRITERS(*inode)) + if (!inode->i_pipe->writers) break; - if (!PIPE_WAITING_WRITERS(*inode)) { + if (!inode->i_pipe->waiting_writers) { /* syscall merging: Usually we must not sleep * if O_NONBLOCK is set, or if we got some data. * But if a writer sleeps in kernel space, then @@ -222,16 +222,16 @@ pipe_readv(struct file *filp, const struct iovec *_iov, break; } if (do_wakeup) { - wake_up_interruptible_sync(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT); + wake_up_interruptible_sync(&inode->i_pipe->wait); + kill_fasync(&inode->i_pipe->fasync_writers, SIGIO, POLL_OUT); } pipe_wait(inode->i_pipe); } - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_unlock(&inode->i_mutex); /* Signal writers asynchronously that there is more room. */ if (do_wakeup) { - wake_up_interruptible(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT); + wake_up_interruptible(&inode->i_pipe->wait); + kill_fasync(&inode->i_pipe->fasync_writers, SIGIO, POLL_OUT); } if (ret > 0) file_accessed(filp); @@ -264,10 +264,10 @@ pipe_writev(struct file *filp, const struct iovec *_iov, do_wakeup = 0; ret = 0; - mutex_lock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); info = inode->i_pipe; - if (!PIPE_READERS(*inode)) { + if (!inode->i_pipe->readers) { send_sig(SIGPIPE, current, 0); ret = -EPIPE; goto out; @@ -306,7 +306,7 @@ pipe_writev(struct file *filp, const struct iovec *_iov, for (;;) { int bufs; - if (!PIPE_READERS(*inode)) { + if (!inode->i_pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; @@ -367,19 +367,19 @@ pipe_writev(struct file *filp, const struct iovec *_iov, break; } if (do_wakeup) { - wake_up_interruptible_sync(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN); + wake_up_interruptible_sync(&inode->i_pipe->wait); + kill_fasync(&inode->i_pipe->fasync_readers, SIGIO, POLL_IN); do_wakeup = 0; } - PIPE_WAITING_WRITERS(*inode)++; + inode->i_pipe->waiting_writers++; pipe_wait(inode->i_pipe); - PIPE_WAITING_WRITERS(*inode)--; + inode->i_pipe->waiting_writers--; } out: - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_unlock(&inode->i_mutex); if (do_wakeup) { - wake_up_interruptible(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN); + wake_up_interruptible(&inode->i_pipe->wait); + kill_fasync(&inode->i_pipe->fasync_readers, SIGIO, POLL_IN); } if (ret > 0) file_update_time(filp); @@ -416,7 +416,7 @@ pipe_ioctl(struct inode *pino, struct file *filp, switch (cmd) { case FIONREAD: - mutex_lock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); info = inode->i_pipe; count = 0; buf = info->curbuf; @@ -425,7 +425,7 @@ pipe_ioctl(struct inode *pino, struct file *filp, count += info->bufs[buf].len; buf = (buf+1) & (PIPE_BUFFERS-1); } - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_unlock(&inode->i_mutex); return put_user(count, (int __user *)arg); default: return -EINVAL; @@ -441,14 +441,14 @@ pipe_poll(struct file *filp, poll_table *wait) struct pipe_inode_info *info = inode->i_pipe; int nrbufs; - poll_wait(filp, PIPE_WAIT(*inode), wait); + poll_wait(filp, &inode->i_pipe->wait, wait); /* Reading only -- no need for acquiring the semaphore. */ nrbufs = info->nrbufs; mask = 0; if (filp->f_mode & FMODE_READ) { mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0; - if (!PIPE_WRITERS(*inode) && filp->f_version != PIPE_WCOUNTER(*inode)) + if (!inode->i_pipe->writers && filp->f_version != inode->i_pipe->w_counter) mask |= POLLHUP; } @@ -458,7 +458,7 @@ pipe_poll(struct file *filp, poll_table *wait) * Most Unices do not set POLLERR for FIFOs but on Linux they * behave exactly like pipes for poll(). */ - if (!PIPE_READERS(*inode)) + if (!inode->i_pipe->readers) mask |= POLLERR; } @@ -468,17 +468,17 @@ pipe_poll(struct file *filp, poll_table *wait) static int pipe_release(struct inode *inode, int decr, int decw) { - mutex_lock(PIPE_MUTEX(*inode)); - PIPE_READERS(*inode) -= decr; - PIPE_WRITERS(*inode) -= decw; - if (!PIPE_READERS(*inode) && !PIPE_WRITERS(*inode)) { + mutex_lock(&inode->i_mutex); + inode->i_pipe->readers -= decr; + inode->i_pipe->writers -= decw; + if (!inode->i_pipe->readers && !inode->i_pipe->writers) { free_pipe_info(inode); } else { - wake_up_interruptible(PIPE_WAIT(*inode)); - kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN); - kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT); + wake_up_interruptible(&inode->i_pipe->wait); + kill_fasync(&inode->i_pipe->fasync_readers, SIGIO, POLL_IN); + kill_fasync(&inode->i_pipe->fasync_writers, SIGIO, POLL_OUT); } - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_unlock(&inode->i_mutex); return 0; } @@ -489,9 +489,9 @@ pipe_read_fasync(int fd, struct file *filp, int on) struct inode *inode = filp->f_dentry->d_inode; int retval; - mutex_lock(PIPE_MUTEX(*inode)); - retval = fasync_helper(fd, filp, on, PIPE_FASYNC_READERS(*inode)); - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); + retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_readers); + mutex_unlock(&inode->i_mutex); if (retval < 0) return retval; @@ -506,9 +506,9 @@ pipe_write_fasync(int fd, struct file *filp, int on) struct inode *inode = filp->f_dentry->d_inode; int retval; - mutex_lock(PIPE_MUTEX(*inode)); - retval = fasync_helper(fd, filp, on, PIPE_FASYNC_WRITERS(*inode)); - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); + retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_writers); + mutex_unlock(&inode->i_mutex); if (retval < 0) return retval; @@ -523,14 +523,14 @@ pipe_rdwr_fasync(int fd, struct file *filp, int on) struct inode *inode = filp->f_dentry->d_inode; int retval; - mutex_lock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); - retval = fasync_helper(fd, filp, on, PIPE_FASYNC_READERS(*inode)); + retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_readers); if (retval >= 0) - retval = fasync_helper(fd, filp, on, PIPE_FASYNC_WRITERS(*inode)); + retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_writers); - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_unlock(&inode->i_mutex); if (retval < 0) return retval; @@ -569,9 +569,9 @@ pipe_read_open(struct inode *inode, struct file *filp) { /* We could have perhaps used atomic_t, but this and friends below are the only places. So it doesn't seem worthwhile. */ - mutex_lock(PIPE_MUTEX(*inode)); - PIPE_READERS(*inode)++; - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); + inode->i_pipe->readers++; + mutex_unlock(&inode->i_mutex); return 0; } @@ -579,9 +579,9 @@ pipe_read_open(struct inode *inode, struct file *filp) static int pipe_write_open(struct inode *inode, struct file *filp) { - mutex_lock(PIPE_MUTEX(*inode)); - PIPE_WRITERS(*inode)++; - mutex_unlock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); + inode->i_pipe->writers++; + mutex_unlock(&inode->i_mutex); return 0; } @@ -589,12 +589,12 @@ pipe_write_open(struct inode *inode, struct file *filp) static int pipe_rdwr_open(struct inode *inode, struct file *filp) { - mutex_lock(PIPE_MUTEX(*inode)); + mutex_lock(&inode->i_mutex); if (filp->f_mode & FMODE_READ) - PIPE_READERS(*inode)++; + inode->i_pipe->readers++; if (filp->f_mode & FMODE_WRITE) - PIPE_WRITERS(*inode)++; - mutex_unlock(PIPE_MUTEX(*inode)); + inode->i_pipe->writers++; + mutex_unlock(&inode->i_mutex); return 0; } @@ -731,7 +731,7 @@ static struct inode * get_pipe_inode(void) if (!inode->i_pipe) goto fail_iput; - PIPE_READERS(*inode) = PIPE_WRITERS(*inode) = 1; + inode->i_pipe->readers = inode->i_pipe->writers = 1; inode->i_fop = &rdwr_pipe_fops; /* diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 660e9d8..123a7c2 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -43,16 +43,6 @@ struct pipe_inode_info { memory allocation, whereas PIPE_BUF makes atomicity guarantees. */ #define PIPE_SIZE PAGE_SIZE -#define PIPE_MUTEX(inode) (&(inode).i_mutex) -#define PIPE_WAIT(inode) (&(inode).i_pipe->wait) -#define PIPE_READERS(inode) ((inode).i_pipe->readers) -#define PIPE_WRITERS(inode) ((inode).i_pipe->writers) -#define PIPE_WAITING_WRITERS(inode) ((inode).i_pipe->waiting_writers) -#define PIPE_RCOUNTER(inode) ((inode).i_pipe->r_counter) -#define PIPE_WCOUNTER(inode) ((inode).i_pipe->w_counter) -#define PIPE_FASYNC_READERS(inode) (&((inode).i_pipe->fasync_readers)) -#define PIPE_FASYNC_WRITERS(inode) (&((inode).i_pipe->fasync_writers)) - /* Drop the inode semaphore and wait for a pipe event, atomically */ void pipe_wait(struct pipe_inode_info *pipe); -- cgit v1.1 From 923f4f23940d2361e8d5c4245982163a8e9d1c91 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 11 Apr 2006 13:53:33 +0200 Subject: [PATCH] pipe.c/fifo.c code cleanups more code cleanups after the macro conversion: - standardize on 'struct pipe_inode_info *pipe' variable names - introduce 'pipe' temporaries to reduce mass inode->i_pipe dereferencing Signed-off-by: Ingo Molnar Signed-off-by: Jens Axboe --- fs/fifo.c | 49 ++++++++++--------- fs/pipe.c | 163 ++++++++++++++++++++++++++++++++------------------------------ 2 files changed, 111 insertions(+), 101 deletions(-) diff --git a/fs/fifo.c b/fs/fifo.c index 2c27f56..49035b1 100644 --- a/fs/fifo.c +++ b/fs/fifo.c @@ -33,14 +33,17 @@ static void wake_up_partner(struct inode* inode) static int fifo_open(struct inode *inode, struct file *filp) { + struct pipe_inode_info *pipe; int ret; mutex_lock(&inode->i_mutex); - if (!inode->i_pipe) { + pipe = inode->i_pipe; + if (!pipe) { ret = -ENOMEM; - inode->i_pipe = alloc_pipe_info(inode); - if (!inode->i_pipe) + pipe = alloc_pipe_info(inode); + if (!pipe) goto err_nocleanup; + inode->i_pipe = pipe; } filp->f_version = 0; @@ -55,18 +58,18 @@ static int fifo_open(struct inode *inode, struct file *filp) * opened, even when there is no process writing the FIFO. */ filp->f_op = &read_fifo_fops; - inode->i_pipe->r_counter++; - if (inode->i_pipe->readers++ == 0) + pipe->r_counter++; + if (pipe->readers++ == 0) wake_up_partner(inode); - if (!inode->i_pipe->writers) { + if (!pipe->writers) { if ((filp->f_flags & O_NONBLOCK)) { /* suppress POLLHUP until we have * seen a writer */ - filp->f_version = inode->i_pipe->w_counter; + filp->f_version = pipe->w_counter; } else { - wait_for_partner(inode, &inode->i_pipe->w_counter); + wait_for_partner(inode, &pipe->w_counter); if(signal_pending(current)) goto err_rd; } @@ -80,16 +83,16 @@ static int fifo_open(struct inode *inode, struct file *filp) * errno=ENXIO when there is no process reading the FIFO. */ ret = -ENXIO; - if ((filp->f_flags & O_NONBLOCK) && !inode->i_pipe->readers) + if ((filp->f_flags & O_NONBLOCK) && !pipe->readers) goto err; filp->f_op = &write_fifo_fops; - inode->i_pipe->w_counter++; - if (!inode->i_pipe->writers++) + pipe->w_counter++; + if (!pipe->writers++) wake_up_partner(inode); - if (!inode->i_pipe->readers) { - wait_for_partner(inode, &inode->i_pipe->r_counter); + if (!pipe->readers) { + wait_for_partner(inode, &pipe->r_counter); if (signal_pending(current)) goto err_wr; } @@ -104,11 +107,11 @@ static int fifo_open(struct inode *inode, struct file *filp) */ filp->f_op = &rdwr_fifo_fops; - inode->i_pipe->readers++; - inode->i_pipe->writers++; - inode->i_pipe->r_counter++; - inode->i_pipe->w_counter++; - if (inode->i_pipe->readers == 1 || inode->i_pipe->writers == 1) + pipe->readers++; + pipe->writers++; + pipe->r_counter++; + pipe->w_counter++; + if (pipe->readers == 1 || pipe->writers == 1) wake_up_partner(inode); break; @@ -122,19 +125,19 @@ static int fifo_open(struct inode *inode, struct file *filp) return 0; err_rd: - if (!--inode->i_pipe->readers) - wake_up_interruptible(&inode->i_pipe->wait); + if (!--pipe->readers) + wake_up_interruptible(&pipe->wait); ret = -ERESTARTSYS; goto err; err_wr: - if (!--inode->i_pipe->writers) - wake_up_interruptible(&inode->i_pipe->wait); + if (!--pipe->writers) + wake_up_interruptible(&pipe->wait); ret = -ERESTARTSYS; goto err; err: - if (!inode->i_pipe->readers && !inode->i_pipe->writers) + if (!pipe->readers && !pipe->writers) free_pipe_info(inode); err_nocleanup: diff --git a/fs/pipe.c b/fs/pipe.c index 0602fc9..b941e19 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -93,7 +93,7 @@ pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len) return 0; } -static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buffer *buf) +static void anon_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct page *page = buf->page; @@ -104,8 +104,8 @@ static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buff * temporary page, let's keep track of it as a one-deep * allocation cache */ - if (page_count(page) == 1 && !info->tmp_page) { - info->tmp_page = page; + if (page_count(page) == 1 && !pipe->tmp_page) { + pipe->tmp_page = page; return; } @@ -115,17 +115,17 @@ static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buff page_cache_release(page); } -static void *anon_pipe_buf_map(struct file *file, struct pipe_inode_info *info, struct pipe_buffer *buf) +static void *anon_pipe_buf_map(struct file *file, struct pipe_inode_info *pipe, struct pipe_buffer *buf) { return kmap(buf->page); } -static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer *buf) +static void anon_pipe_buf_unmap(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { kunmap(buf->page); } -static int anon_pipe_buf_steal(struct pipe_inode_info *info, +static int anon_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { buf->flags |= PIPE_BUF_FLAG_STOLEN; @@ -145,7 +145,7 @@ pipe_readv(struct file *filp, const struct iovec *_iov, unsigned long nr_segs, loff_t *ppos) { struct inode *inode = filp->f_dentry->d_inode; - struct pipe_inode_info *info; + struct pipe_inode_info *pipe; int do_wakeup; ssize_t ret; struct iovec *iov = (struct iovec *)_iov; @@ -159,12 +159,12 @@ pipe_readv(struct file *filp, const struct iovec *_iov, do_wakeup = 0; ret = 0; mutex_lock(&inode->i_mutex); - info = inode->i_pipe; + pipe = inode->i_pipe; for (;;) { - int bufs = info->nrbufs; + int bufs = pipe->nrbufs; if (bufs) { - int curbuf = info->curbuf; - struct pipe_buffer *buf = info->bufs + curbuf; + int curbuf = pipe->curbuf; + struct pipe_buffer *buf = pipe->bufs + curbuf; struct pipe_buf_operations *ops = buf->ops; void *addr; size_t chars = buf->len; @@ -173,14 +173,14 @@ pipe_readv(struct file *filp, const struct iovec *_iov, if (chars > total_len) chars = total_len; - addr = ops->map(filp, info, buf); + addr = ops->map(filp, pipe, buf); if (IS_ERR(addr)) { if (!ret) ret = PTR_ERR(addr); break; } error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars); - ops->unmap(info, buf); + ops->unmap(pipe, buf); if (unlikely(error)) { if (!ret) ret = -EFAULT; break; @@ -190,10 +190,10 @@ pipe_readv(struct file *filp, const struct iovec *_iov, buf->len -= chars; if (!buf->len) { buf->ops = NULL; - ops->release(info, buf); + ops->release(pipe, buf); curbuf = (curbuf + 1) & (PIPE_BUFFERS-1); - info->curbuf = curbuf; - info->nrbufs = --bufs; + pipe->curbuf = curbuf; + pipe->nrbufs = --bufs; do_wakeup = 1; } total_len -= chars; @@ -202,9 +202,9 @@ pipe_readv(struct file *filp, const struct iovec *_iov, } if (bufs) /* More to do? */ continue; - if (!inode->i_pipe->writers) + if (!pipe->writers) break; - if (!inode->i_pipe->waiting_writers) { + if (!pipe->waiting_writers) { /* syscall merging: Usually we must not sleep * if O_NONBLOCK is set, or if we got some data. * But if a writer sleeps in kernel space, then @@ -222,16 +222,16 @@ pipe_readv(struct file *filp, const struct iovec *_iov, break; } if (do_wakeup) { - wake_up_interruptible_sync(&inode->i_pipe->wait); - kill_fasync(&inode->i_pipe->fasync_writers, SIGIO, POLL_OUT); + wake_up_interruptible_sync(&pipe->wait); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } - pipe_wait(inode->i_pipe); + pipe_wait(pipe); } mutex_unlock(&inode->i_mutex); /* Signal writers asynchronously that there is more room. */ if (do_wakeup) { - wake_up_interruptible(&inode->i_pipe->wait); - kill_fasync(&inode->i_pipe->fasync_writers, SIGIO, POLL_OUT); + wake_up_interruptible(&pipe->wait); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } if (ret > 0) file_accessed(filp); @@ -250,7 +250,7 @@ pipe_writev(struct file *filp, const struct iovec *_iov, unsigned long nr_segs, loff_t *ppos) { struct inode *inode = filp->f_dentry->d_inode; - struct pipe_inode_info *info; + struct pipe_inode_info *pipe; ssize_t ret; int do_wakeup; struct iovec *iov = (struct iovec *)_iov; @@ -265,9 +265,9 @@ pipe_writev(struct file *filp, const struct iovec *_iov, do_wakeup = 0; ret = 0; mutex_lock(&inode->i_mutex); - info = inode->i_pipe; + pipe = inode->i_pipe; - if (!inode->i_pipe->readers) { + if (!pipe->readers) { send_sig(SIGPIPE, current, 0); ret = -EPIPE; goto out; @@ -275,23 +275,23 @@ pipe_writev(struct file *filp, const struct iovec *_iov, /* We try to merge small writes */ chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */ - if (info->nrbufs && chars != 0) { - int lastbuf = (info->curbuf + info->nrbufs - 1) & (PIPE_BUFFERS-1); - struct pipe_buffer *buf = info->bufs + lastbuf; + if (pipe->nrbufs && chars != 0) { + int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & (PIPE_BUFFERS-1); + struct pipe_buffer *buf = pipe->bufs + lastbuf; struct pipe_buf_operations *ops = buf->ops; int offset = buf->offset + buf->len; if (ops->can_merge && offset + chars <= PAGE_SIZE) { void *addr; int error; - addr = ops->map(filp, info, buf); + addr = ops->map(filp, pipe, buf); if (IS_ERR(addr)) { error = PTR_ERR(addr); goto out; } error = pipe_iov_copy_from_user(offset + addr, iov, chars); - ops->unmap(info, buf); + ops->unmap(pipe, buf); ret = error; do_wakeup = 1; if (error) @@ -306,16 +306,16 @@ pipe_writev(struct file *filp, const struct iovec *_iov, for (;;) { int bufs; - if (!inode->i_pipe->readers) { + if (!pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } - bufs = info->nrbufs; + bufs = pipe->nrbufs; if (bufs < PIPE_BUFFERS) { - int newbuf = (info->curbuf + bufs) & (PIPE_BUFFERS-1); - struct pipe_buffer *buf = info->bufs + newbuf; - struct page *page = info->tmp_page; + int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1); + struct pipe_buffer *buf = pipe->bufs + newbuf; + struct page *page = pipe->tmp_page; int error; if (!page) { @@ -324,7 +324,7 @@ pipe_writev(struct file *filp, const struct iovec *_iov, ret = ret ? : -ENOMEM; break; } - info->tmp_page = page; + pipe->tmp_page = page; } /* Always wakeup, even if the copy fails. Otherwise * we lock up (O_NONBLOCK-)readers that sleep due to @@ -349,8 +349,8 @@ pipe_writev(struct file *filp, const struct iovec *_iov, buf->ops = &anon_pipe_buf_ops; buf->offset = 0; buf->len = chars; - info->nrbufs = ++bufs; - info->tmp_page = NULL; + pipe->nrbufs = ++bufs; + pipe->tmp_page = NULL; total_len -= chars; if (!total_len) @@ -367,19 +367,19 @@ pipe_writev(struct file *filp, const struct iovec *_iov, break; } if (do_wakeup) { - wake_up_interruptible_sync(&inode->i_pipe->wait); - kill_fasync(&inode->i_pipe->fasync_readers, SIGIO, POLL_IN); + wake_up_interruptible_sync(&pipe->wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); do_wakeup = 0; } - inode->i_pipe->waiting_writers++; - pipe_wait(inode->i_pipe); - inode->i_pipe->waiting_writers--; + pipe->waiting_writers++; + pipe_wait(pipe); + pipe->waiting_writers--; } out: mutex_unlock(&inode->i_mutex); if (do_wakeup) { - wake_up_interruptible(&inode->i_pipe->wait); - kill_fasync(&inode->i_pipe->fasync_readers, SIGIO, POLL_IN); + wake_up_interruptible(&pipe->wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } if (ret > 0) file_update_time(filp); @@ -411,21 +411,22 @@ pipe_ioctl(struct inode *pino, struct file *filp, unsigned int cmd, unsigned long arg) { struct inode *inode = filp->f_dentry->d_inode; - struct pipe_inode_info *info; + struct pipe_inode_info *pipe; int count, buf, nrbufs; switch (cmd) { case FIONREAD: mutex_lock(&inode->i_mutex); - info = inode->i_pipe; + pipe = inode->i_pipe; count = 0; - buf = info->curbuf; - nrbufs = info->nrbufs; + buf = pipe->curbuf; + nrbufs = pipe->nrbufs; while (--nrbufs >= 0) { - count += info->bufs[buf].len; + count += pipe->bufs[buf].len; buf = (buf+1) & (PIPE_BUFFERS-1); } mutex_unlock(&inode->i_mutex); + return put_user(count, (int __user *)arg); default: return -EINVAL; @@ -438,17 +439,17 @@ pipe_poll(struct file *filp, poll_table *wait) { unsigned int mask; struct inode *inode = filp->f_dentry->d_inode; - struct pipe_inode_info *info = inode->i_pipe; + struct pipe_inode_info *pipe = inode->i_pipe; int nrbufs; - poll_wait(filp, &inode->i_pipe->wait, wait); + poll_wait(filp, &pipe->wait, wait); /* Reading only -- no need for acquiring the semaphore. */ - nrbufs = info->nrbufs; + nrbufs = pipe->nrbufs; mask = 0; if (filp->f_mode & FMODE_READ) { mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0; - if (!inode->i_pipe->writers && filp->f_version != inode->i_pipe->w_counter) + if (!pipe->writers && filp->f_version != pipe->w_counter) mask |= POLLHUP; } @@ -458,7 +459,7 @@ pipe_poll(struct file *filp, poll_table *wait) * Most Unices do not set POLLERR for FIFOs but on Linux they * behave exactly like pipes for poll(). */ - if (!inode->i_pipe->readers) + if (!pipe->readers) mask |= POLLERR; } @@ -468,15 +469,18 @@ pipe_poll(struct file *filp, poll_table *wait) static int pipe_release(struct inode *inode, int decr, int decw) { + struct pipe_inode_info *pipe; + mutex_lock(&inode->i_mutex); - inode->i_pipe->readers -= decr; - inode->i_pipe->writers -= decw; - if (!inode->i_pipe->readers && !inode->i_pipe->writers) { + pipe = inode->i_pipe; + pipe->readers -= decr; + pipe->writers -= decw; + if (!pipe->readers && !pipe->writers) { free_pipe_info(inode); } else { - wake_up_interruptible(&inode->i_pipe->wait); - kill_fasync(&inode->i_pipe->fasync_readers, SIGIO, POLL_IN); - kill_fasync(&inode->i_pipe->fasync_writers, SIGIO, POLL_OUT); + wake_up_interruptible(&pipe->wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } mutex_unlock(&inode->i_mutex); @@ -679,30 +683,30 @@ static struct file_operations rdwr_pipe_fops = { struct pipe_inode_info * alloc_pipe_info(struct inode *inode) { - struct pipe_inode_info *info; + struct pipe_inode_info *pipe; - info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); - if (info) { - init_waitqueue_head(&info->wait); - info->r_counter = info->w_counter = 1; - info->inode = inode; + pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); + if (pipe) { + init_waitqueue_head(&pipe->wait); + pipe->r_counter = pipe->w_counter = 1; + pipe->inode = inode; } - return info; + return pipe; } -void __free_pipe_info(struct pipe_inode_info *info) +void __free_pipe_info(struct pipe_inode_info *pipe) { int i; for (i = 0; i < PIPE_BUFFERS; i++) { - struct pipe_buffer *buf = info->bufs + i; + struct pipe_buffer *buf = pipe->bufs + i; if (buf->ops) - buf->ops->release(info, buf); + buf->ops->release(pipe, buf); } - if (info->tmp_page) - __free_page(info->tmp_page); - kfree(info); + if (pipe->tmp_page) + __free_page(pipe->tmp_page); + kfree(pipe); } void free_pipe_info(struct inode *inode) @@ -723,15 +727,17 @@ static struct dentry_operations pipefs_dentry_operations = { static struct inode * get_pipe_inode(void) { struct inode *inode = new_inode(pipe_mnt->mnt_sb); + struct pipe_inode_info *pipe; if (!inode) goto fail_inode; - inode->i_pipe = alloc_pipe_info(inode); - if (!inode->i_pipe) + pipe = alloc_pipe_info(inode); + if (!pipe) goto fail_iput; + inode->i_pipe = pipe; - inode->i_pipe->readers = inode->i_pipe->writers = 1; + pipe->readers = pipe->writers = 1; inode->i_fop = &rdwr_pipe_fops; /* @@ -746,6 +752,7 @@ static struct inode * get_pipe_inode(void) inode->i_gid = current->fsgid; inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; inode->i_blksize = PAGE_SIZE; + return inode; fail_iput: -- cgit v1.1 From 6f767b0425f5902e4817648632230b512e81c963 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Apr 2006 13:53:56 +0200 Subject: [PATCH] splice: speedups and optimizations - Kill the local variables that cache ->nrbufs, they just take up space. - Only set do_wakeup for a real pipe. This is a big win for direct splicing. - Kill i_mutex lock around ->f_pos update, regular io paths don't do this either. Signed-off-by: Jens Axboe --- fs/splice.c | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index e30743c..36bc262 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -150,8 +150,6 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, mutex_lock(&pipe->inode->i_mutex); for (;;) { - int bufs; - if (!pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) @@ -159,9 +157,8 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, break; } - bufs = pipe->nrbufs; - if (bufs < PIPE_BUFFERS) { - int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS - 1); + if (pipe->nrbufs < PIPE_BUFFERS) { + int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1); struct pipe_buffer *buf = pipe->bufs + newbuf; struct page *page = pages[i++]; unsigned long this_len; @@ -174,8 +171,9 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, buf->offset = offset; buf->len = this_len; buf->ops = &page_cache_pipe_buf_ops; - pipe->nrbufs = ++bufs; - do_wakeup = 1; + pipe->nrbufs++; + if (pipe->inode) + do_wakeup = 1; ret += this_len; len -= this_len; @@ -184,7 +182,7 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, break; if (!len) break; - if (bufs < PIPE_BUFFERS) + if (pipe->nrbufs < PIPE_BUFFERS) continue; break; @@ -581,11 +579,8 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, mutex_lock(&pipe->inode->i_mutex); for (;;) { - int bufs = pipe->nrbufs; - - if (bufs) { - int curbuf = pipe->curbuf; - struct pipe_buffer *buf = pipe->bufs + curbuf; + if (pipe->nrbufs) { + struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; struct pipe_buf_operations *ops = buf->ops; sd.len = buf->len; @@ -606,10 +601,10 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, if (!buf->len) { buf->ops = NULL; ops->release(pipe, buf); - curbuf = (curbuf + 1) & (PIPE_BUFFERS - 1); - pipe->curbuf = curbuf; - pipe->nrbufs = --bufs; - do_wakeup = 1; + pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1); + pipe->nrbufs--; + if (pipe->inode) + do_wakeup = 1; } sd.pos += sd.len; @@ -618,7 +613,7 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, break; } - if (bufs) + if (pipe->nrbufs) continue; if (!pipe->writers) break; @@ -660,9 +655,7 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } - mutex_lock(&out->f_mapping->host->i_mutex); out->f_pos = sd.pos; - mutex_unlock(&out->f_mapping->host->i_mutex); return ret; } -- cgit v1.1 From 49570e9b29a3d78950b5eba6b73bdcca955f0877 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Apr 2006 13:56:09 +0200 Subject: [PATCH] splice: unlikely() optimizations Also corrects a few comments. Patch mainly from Ingo, changes by me. Signed-off-by: Ingo Molnar Signed-off-by: Jens Axboe --- fs/read_write.c | 2 +- fs/splice.c | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/read_write.c b/fs/read_write.c index 6256ca8..5bc0e92 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -202,7 +202,7 @@ int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count goto Einval; inode = file->f_dentry->d_inode; - if (inode->i_flock && MANDATORY_LOCK(inode)) { + if (unlikely(inode->i_flock && MANDATORY_LOCK(inode))) { int retval = locks_mandatory_area( read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE, inode, file, pos, count); diff --git a/fs/splice.c b/fs/splice.c index 36bc262..77b026b 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -346,7 +346,6 @@ fill_it: * @flags: splice modifier flags * * Will read pages from given file and fill them into a pipe. - * */ ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, size_t len, unsigned int flags) @@ -690,7 +689,7 @@ generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, mutex_lock(&inode->i_mutex); err = generic_osync_inode(mapping->host, mapping, - OSYNC_METADATA|OSYNC_DATA); + OSYNC_METADATA|OSYNC_DATA); mutex_unlock(&inode->i_mutex); if (err) @@ -730,10 +729,10 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, loff_t pos; int ret; - if (!out->f_op || !out->f_op->splice_write) + if (unlikely(!out->f_op || !out->f_op->splice_write)) return -EINVAL; - if (!(out->f_mode & FMODE_WRITE)) + if (unlikely(!(out->f_mode & FMODE_WRITE))) return -EBADF; pos = out->f_pos; @@ -754,10 +753,10 @@ static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, loff_t pos, isize, left; int ret; - if (!in->f_op || !in->f_op->splice_read) + if (unlikely(!in->f_op || !in->f_op->splice_read)) return -EINVAL; - if (!(in->f_mode & FMODE_READ)) + if (unlikely(!(in->f_mode & FMODE_READ))) return -EBADF; pos = in->f_pos; @@ -771,7 +770,7 @@ static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, return 0; left = isize - in->f_pos; - if (left < len) + if (unlikely(left < len)) len = left; return in->f_op->splice_read(in, pipe, len, flags); @@ -799,7 +798,7 @@ long do_splice_direct(struct file *in, struct file *out, size_t len, * 'out' and transfer the wanted data from 'in' to 'out' through that */ pipe = current->splice_pipe; - if (!pipe) { + if (unlikely(!pipe)) { pipe = alloc_pipe_info(NULL); if (!pipe) return -ENOMEM; -- cgit v1.1 From c2058e0611c2fb4a00e63db61ef3a638cf760a98 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Apr 2006 13:56:34 +0200 Subject: [PATCH] splice: add Ingo as addition copyright holder The comment is also somewhat out of date, correct that as well. Signed-off-by: Jens Axboe --- fs/splice.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 77b026b..48b4382 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -9,11 +9,12 @@ * that transfers data buffers to or from a pipe buffer. * * Named by Larry McVoy, original implementation from Linus, extended by - * Jens to support splicing to files and fixing the initial implementation - * bugs. + * Jens to support splicing to files, network, direct splicing, etc and + * fixing lots of bugs. * - * Copyright (C) 2005 Jens Axboe - * Copyright (C) 2005 Linus Torvalds + * Copyright (C) 2005-2006 Jens Axboe + * Copyright (C) 2005-2006 Linus Torvalds + * Copyright (C) 2006 Ingo Molnar * */ #include -- cgit v1.1 From 73d62d83ec3627782ba6f55defc76f3ffbef46ee Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 11 Apr 2006 13:57:21 +0200 Subject: [PATCH] splice: comment styles - capitalize consistently - end sentences in one way or another - update comment text to match the implementation Signed-off-by: Ingo Molnar Signed-off-by: Jens Axboe --- fs/splice.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 48b4382..e50a460 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -92,7 +92,7 @@ static void *page_cache_pipe_buf_map(struct file *file, /* * Page got truncated/unhashed. This will cause a 0-byte - * splice, if this is the first page + * splice, if this is the first page. */ if (!page->mapping) { err = -ENODATA; @@ -100,7 +100,7 @@ static void *page_cache_pipe_buf_map(struct file *file, } /* - * uh oh, read-error from disk + * Uh oh, read-error from disk. */ if (!PageUptodate(page)) { err = -EIO; @@ -108,7 +108,7 @@ static void *page_cache_pipe_buf_map(struct file *file, } /* - * page is ok afterall, fall through to mapping + * Page is ok afterall, fall through to mapping. */ unlock_page(page); } @@ -249,7 +249,7 @@ __generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, nr_pages = PIPE_BUFFERS; /* - * initiate read-ahead on this page range. however, don't call into + * Initiate read-ahead on this page range. however, don't call into * read-ahead if this is a non-zero offset (we are likely doing small * chunk splice and the page is already there) for a single page. */ @@ -257,7 +257,7 @@ __generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, do_page_cache_readahead(mapping, in, index, nr_pages); /* - * now fill in the holes + * Now fill in the holes: */ error = 0; for (i = 0; i < nr_pages; i++, index++) { @@ -396,10 +396,10 @@ static int pipe_to_sendpage(struct pipe_inode_info *info, int more; /* - * sub-optimal, but we are limited by the pipe ->map. we don't + * Sub-optimal, but we are limited by the pipe ->map. We don't * need a kmap'ed buffer here, we just want to make sure we * have the page pinned if the pipe page originates from the - * page cache + * page cache. */ ptr = buf->ops->map(file, info, buf); if (IS_ERR(ptr)) @@ -460,7 +460,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, offset = sd->pos & ~PAGE_CACHE_MASK; /* - * reuse buf page, if SPLICE_F_MOVE is set + * Reuse buf page, if SPLICE_F_MOVE is set. */ if (sd->flags & SPLICE_F_MOVE) { /* @@ -501,7 +501,7 @@ find_page: if (!PageUptodate(page)) { /* - * page got invalidated, repeat + * Page got invalidated, repeat. */ if (!page->mapping) { unlock_page(page); @@ -598,6 +598,7 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, ret += sd.len; buf->offset += sd.len; buf->len -= sd.len; + if (!buf->len) { buf->ops = NULL; ops->release(pipe, buf); @@ -681,7 +682,7 @@ generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, ret = move_from_pipe(pipe, out, len, flags, pipe_to_file); /* - * if file or inode is SYNC and we actually wrote some data, sync it + * If file or inode is SYNC and we actually wrote some data, sync it. */ if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host)) && ret > 0) { @@ -815,7 +816,7 @@ long do_splice_direct(struct file *in, struct file *out, size_t len, } /* - * do the splice + * Do the splice. */ ret = 0; bytes = 0; -- cgit v1.1 From 341b446bc5aa36d1d5b8159c1e66716b5d89024d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 11 Apr 2006 13:57:45 +0200 Subject: [PATCH] another round of fs/pipe.c cleanups make pipe.c a bit more readable and hackable. Signed-off-by: Ingo Molnar Signed-off-by: Jens Axboe --- fs/pipe.c | 76 ++++++++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/fs/pipe.c b/fs/pipe.c index b941e19..e984beb 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -44,7 +44,8 @@ void pipe_wait(struct pipe_inode_info *pipe) * Pipes are system-local resources, so sleeping on them * is considered a noninteractive wait: */ - prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE|TASK_NONINTERACTIVE); + prepare_to_wait(&pipe->wait, &wait, + TASK_INTERRUPTIBLE | TASK_NONINTERACTIVE); if (pipe->inode) mutex_unlock(&pipe->inode->i_mutex); schedule(); @@ -93,7 +94,8 @@ pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len) return 0; } -static void anon_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +static void anon_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) { struct page *page = buf->page; @@ -102,25 +104,22 @@ static void anon_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buff /* * If nobody else uses this page, and we don't already have a * temporary page, let's keep track of it as a one-deep - * allocation cache + * allocation cache. (Otherwise just release our reference to it) */ - if (page_count(page) == 1 && !pipe->tmp_page) { + if (page_count(page) == 1 && !pipe->tmp_page) pipe->tmp_page = page; - return; - } - - /* - * Otherwise just release our reference to it - */ - page_cache_release(page); + else + page_cache_release(page); } -static void *anon_pipe_buf_map(struct file *file, struct pipe_inode_info *pipe, struct pipe_buffer *buf) +static void * anon_pipe_buf_map(struct file *file, struct pipe_inode_info *pipe, + struct pipe_buffer *buf) { return kmap(buf->page); } -static void anon_pipe_buf_unmap(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +static void anon_pipe_buf_unmap(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) { kunmap(buf->page); } @@ -182,7 +181,8 @@ pipe_readv(struct file *filp, const struct iovec *_iov, error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars); ops->unmap(pipe, buf); if (unlikely(error)) { - if (!ret) ret = -EFAULT; + if (!ret) + ret = -EFAULT; break; } ret += chars; @@ -218,7 +218,8 @@ pipe_readv(struct file *filp, const struct iovec *_iov, } } if (signal_pending(current)) { - if (!ret) ret = -ERESTARTSYS; + if (!ret) + ret = -ERESTARTSYS; break; } if (do_wakeup) { @@ -228,7 +229,8 @@ pipe_readv(struct file *filp, const struct iovec *_iov, pipe_wait(pipe); } mutex_unlock(&inode->i_mutex); - /* Signal writers asynchronously that there is more room. */ + + /* Signal writers asynchronously that there is more room. */ if (do_wakeup) { wake_up_interruptible(&pipe->wait); kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); @@ -242,6 +244,7 @@ static ssize_t pipe_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { struct iovec iov = { .iov_base = buf, .iov_len = count }; + return pipe_readv(filp, &iov, 1, ppos); } @@ -276,10 +279,12 @@ pipe_writev(struct file *filp, const struct iovec *_iov, /* We try to merge small writes */ chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */ if (pipe->nrbufs && chars != 0) { - int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & (PIPE_BUFFERS-1); + int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & + (PIPE_BUFFERS-1); struct pipe_buffer *buf = pipe->bufs + lastbuf; struct pipe_buf_operations *ops = buf->ops; int offset = buf->offset + buf->len; + if (ops->can_merge && offset + chars <= PAGE_SIZE) { void *addr; int error; @@ -306,9 +311,11 @@ pipe_writev(struct file *filp, const struct iovec *_iov, for (;;) { int bufs; + if (!pipe->readers) { send_sig(SIGPIPE, current, 0); - if (!ret) ret = -EPIPE; + if (!ret) + ret = -EPIPE; break; } bufs = pipe->nrbufs; @@ -326,7 +333,7 @@ pipe_writev(struct file *filp, const struct iovec *_iov, } pipe->tmp_page = page; } - /* Always wakeup, even if the copy fails. Otherwise + /* Always wake up, even if the copy fails. Otherwise * we lock up (O_NONBLOCK-)readers that sleep due to * syscall merging. * FIXME! Is this really true? @@ -339,7 +346,8 @@ pipe_writev(struct file *filp, const struct iovec *_iov, error = pipe_iov_copy_from_user(kmap(page), iov, chars); kunmap(page); if (unlikely(error)) { - if (!ret) ret = -EFAULT; + if (!ret) + ret = -EFAULT; break; } ret += chars; @@ -359,11 +367,13 @@ pipe_writev(struct file *filp, const struct iovec *_iov, if (bufs < PIPE_BUFFERS) continue; if (filp->f_flags & O_NONBLOCK) { - if (!ret) ret = -EAGAIN; + if (!ret) + ret = -EAGAIN; break; } if (signal_pending(current)) { - if (!ret) ret = -ERESTARTSYS; + if (!ret) + ret = -ERESTARTSYS; break; } if (do_wakeup) { @@ -391,6 +401,7 @@ pipe_write(struct file *filp, const char __user *buf, size_t count, loff_t *ppos) { struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = count }; + return pipe_writev(filp, &iov, 1, ppos); } @@ -401,7 +412,8 @@ bad_pipe_r(struct file *filp, char __user *buf, size_t count, loff_t *ppos) } static ssize_t -bad_pipe_w(struct file *filp, const char __user *buf, size_t count, loff_t *ppos) +bad_pipe_w(struct file *filp, const char __user *buf, size_t count, + loff_t *ppos) { return -EBADF; } @@ -475,6 +487,7 @@ pipe_release(struct inode *inode, int decr, int decw) pipe = inode->i_pipe; pipe->readers -= decr; pipe->writers -= decw; + if (!pipe->readers && !pipe->writers) { free_pipe_info(inode); } else { @@ -525,14 +538,15 @@ static int pipe_rdwr_fasync(int fd, struct file *filp, int on) { struct inode *inode = filp->f_dentry->d_inode; + struct pipe_inode_info *pipe = inode->i_pipe; int retval; mutex_lock(&inode->i_mutex); - retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_readers); + retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); if (retval >= 0) - retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_writers); + retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); mutex_unlock(&inode->i_mutex); @@ -720,6 +734,7 @@ static int pipefs_delete_dentry(struct dentry *dentry) { return 1; } + static struct dentry_operations pipefs_dentry_operations = { .d_delete = pipefs_delete_dentry, }; @@ -757,6 +772,7 @@ static struct inode * get_pipe_inode(void) fail_iput: iput(inode); + fail_inode: return NULL; } @@ -769,7 +785,7 @@ int do_pipe(int *fd) struct inode * inode; struct file *f1, *f2; int error; - int i,j; + int i, j; error = -ENFILE; f1 = get_empty_filp(); @@ -802,6 +818,7 @@ int do_pipe(int *fd) dentry = d_alloc(pipe_mnt->mnt_sb->s_root, &this); if (!dentry) goto close_f12_inode_i_j; + dentry->d_op = &pipefs_dentry_operations; d_add(dentry, inode); f1->f_vfsmnt = f2->f_vfsmnt = mntget(mntget(pipe_mnt)); @@ -825,6 +842,7 @@ int do_pipe(int *fd) fd_install(j, f2); fd[0] = i; fd[1] = j; + return 0; close_f12_inode_i_j: @@ -849,8 +867,9 @@ no_files: * d_name - pipe: will go nicely and kill the special-casing in procfs. */ -static struct super_block *pipefs_get_sb(struct file_system_type *fs_type, - int flags, const char *dev_name, void *data) +static struct super_block * +pipefs_get_sb(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) { return get_sb_pseudo(fs_type, "pipe:", NULL, PIPEFS_MAGIC); } @@ -864,6 +883,7 @@ static struct file_system_type pipe_fs_type = { static int __init init_pipe_fs(void) { int err = register_filesystem(&pipe_fs_type); + if (!err) { pipe_mnt = kern_mount(&pipe_fs_type); if (IS_ERR(pipe_mnt)) { -- cgit v1.1 From 88e6faefaeed40a162041d85689e94595756d4c3 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Tue, 11 Apr 2006 13:59:36 +0200 Subject: [PATCH] splice: warning fix From: Andrew Morton net/socket.c:148: warning: initialization from incompatible pointer type extern declarations in .c files! Bad boy. Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Jens Axboe --- include/linux/fs.h | 13 ++++++++++--- net/socket.c | 4 ---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/linux/fs.h b/include/linux/fs.h index 9e8e2ee..2a629fd 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1611,10 +1611,17 @@ extern ssize_t generic_file_sendfile(struct file *, loff_t *, size_t, read_actor extern void do_generic_mapping_read(struct address_space *mapping, struct file_ra_state *, struct file *, loff_t *, read_descriptor_t *, read_actor_t); -extern ssize_t generic_file_splice_read(struct file *, struct pipe_inode_info *, size_t, unsigned int); -extern ssize_t generic_file_splice_write(struct pipe_inode_info *, struct file *, size_t, unsigned int); + +/* fs/splice.c */ +extern ssize_t generic_file_splice_read(struct file *, + struct pipe_inode_info *, size_t, unsigned int); +extern ssize_t generic_file_splice_write(struct pipe_inode_info *, + struct file *, size_t, unsigned int); +extern ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, + struct file *out, size_t len, unsigned int flags); extern long do_splice_direct(struct file *in, struct file *out, - size_t len, unsigned int flags); + size_t len, unsigned int flags); + extern void file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping); extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, diff --git a/net/socket.c b/net/socket.c index b807f36..9ed9f65 100644 --- a/net/socket.c +++ b/net/socket.c @@ -119,10 +119,6 @@ static ssize_t sock_writev(struct file *file, const struct iovec *vector, static ssize_t sock_sendpage(struct file *file, struct page *page, int offset, size_t size, loff_t *ppos, int more); -extern ssize_t generic_splice_sendpage(struct inode *inode, struct file *out, - size_t len, unsigned int flags); - - /* * Socket files have a set of 'special' operations as well as the generic file ones. These don't appear * in the operation structures but are done directly via the socketcall() multiplexor. -- cgit v1.1 From 7519fdc90fe577cb966ab1ce2bf51ac639f05a0e Mon Sep 17 00:00:00 2001 From: OGAWA Hirofumi Date: Tue, 11 Apr 2006 14:00:04 +0200 Subject: [PATCH] Remove sys_ prefix of new syscalls from __NR_sys_* On i386, we don't use sys_ prefix for __NR_*. This patch removes it [FWIW, _syscall*() macros will generate foo() instead of sys_foo().] Signed-off-by: OGAWA Hirofumi Signed-off-by: Jens Axboe --- include/asm-i386/unistd.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/asm-i386/unistd.h b/include/asm-i386/unistd.h index 2e7f3e2..7b1ba84 100644 --- a/include/asm-i386/unistd.h +++ b/include/asm-i386/unistd.h @@ -318,8 +318,8 @@ #define __NR_unshare 310 #define __NR_set_robust_list 311 #define __NR_get_robust_list 312 -#define __NR_sys_splice 313 -#define __NR_sys_sync_file_range 314 +#define __NR_splice 313 +#define __NR_sync_file_range 314 #define NR_syscalls 315 -- cgit v1.1 From d1195c516a9acd767cb541f914be2c6ddcafcfc1 Mon Sep 17 00:00:00 2001 From: Pekka J Enberg Date: Tue, 11 Apr 2006 14:21:59 +0200 Subject: [PATCH] vfs: add splice_write and splice_read to documentation This patch adds the new splice_write and splice_read file operations to Documentation/filesystems/vfs.txt. Signed-off-by: Pekka Enberg Signed-off-by: Jens Axboe --- Documentation/filesystems/vfs.txt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Documentation/filesystems/vfs.txt b/Documentation/filesystems/vfs.txt index adaa899..3a2e552 100644 --- a/Documentation/filesystems/vfs.txt +++ b/Documentation/filesystems/vfs.txt @@ -694,7 +694,7 @@ struct file_operations ---------------------- This describes how the VFS can manipulate an open file. As of kernel -2.6.13, the following members are defined: +2.6.17, the following members are defined: struct file_operations { loff_t (*llseek) (struct file *, loff_t, int); @@ -723,6 +723,10 @@ struct file_operations { int (*check_flags)(int); int (*dir_notify)(struct file *filp, unsigned long arg); int (*flock) (struct file *, int, struct file_lock *); + ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, size_t, unsigned +int); + ssize_t (*splice_read)(struct file *, struct pipe_inode_info *, size_t, unsigned +int); }; Again, all methods are called without any locks being held, unless @@ -790,6 +794,12 @@ otherwise noted. flock: called by the flock(2) system call + splice_write: called by the VFS to splice data from a pipe to a file. This + method is used by the splice(2) system call + + splice_read: called by the VFS to splice data from file to a pipe. This + method is used by the splice(2) system call + Note that the file operations are implemented by the specific filesystem in which the inode resides. When opening a device node (character or block special) most filesystems will call special -- cgit v1.1