Make sendfile() a method in the struct fileops. Currently only

vnode backed file descriptors have this method implemented.

Reviewed by:	kib
Sponsored by:	Nginx, Inc.
Sponsored by:	Netflix

15 files changed, 112 insertions, 55 deletions

diff --git a/sys/compat/freebsd32/freebsd32_misc.c b/sys/compat/freebsd32/freebsd32_misc.c
index 4899e03..51175aa 100644
--- a/sys/compat/freebsd32/freebsd32_misc.c
+++ b/sys/compat/freebsd32/freebsd32_misc.c
@@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/bus.h>
+#include <sys/capability.h>
 #include <sys/clock.h>
 #include <sys/exec.h>
 #include <sys/fcntl.h>
@@ -1653,22 +1654,19 @@ static int
 freebsd32_do_sendfile(struct thread *td,
     struct freebsd32_sendfile_args *uap, int compat)
 {
-	struct sendfile_args ap;
 	struct sf_hdtr32 hdtr32;
 	struct sf_hdtr hdtr;
 	struct uio *hdr_uio, *trl_uio;
 	struct iovec32 *iov32;
+	struct file *fp;
+	off_t offset;
 	int error;
 
-	hdr_uio = trl_uio = NULL;
+	offset = PAIR32TO64(off_t, uap->offset);
+	if (offset < 0)
+		return (EINVAL);
 
-	ap.fd = uap->fd;
-	ap.s = uap->s;
-	ap.offset = PAIR32TO64(off_t,uap->offset);
-	ap.nbytes = uap->nbytes;
-	ap.hdtr = (struct sf_hdtr *)uap->hdtr;		/* XXX not used */
-	ap.sbytes = uap->sbytes;
-	ap.flags = uap->flags;
+	hdr_uio = trl_uio = NULL;
 
 	if (uap->hdtr != NULL) {
 		error = copyin(uap->hdtr, &hdtr32, sizeof(hdtr32));
@@ -1695,7 +1693,15 @@ freebsd32_do_sendfile(struct thread *td,
 		}
 	}
 
-	error = kern_sendfile(td, &ap, hdr_uio, trl_uio, compat);
+	AUDIT_ARG_FD(uap->fd);
+
+	if ((error = fget_read(td, uap->fd, CAP_PREAD, &fp)) != 0)
+		goto out;
+
+	error = fo_sendfile(fp, uap->s, hdr_uio, trl_uio, offset,
+	    uap->nbytes, uap->sbytes, uap->flags, compat ? SFK_COMPAT : 0, td);
+	fdrop(fp, td);
+
 out:
 	if (hdr_uio)
 		free(hdr_uio, M_IOV);
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index fb264ba..d0de6b9 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -3887,6 +3887,15 @@ badfo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
 	return (EBADF);
 }
 
+static int
+badfo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
+    struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
+    int kflags, struct thread *td)
+{
+
+	return (EBADF);
+}
+
 struct fileops badfileops = {
 	.fo_read = badfo_readwrite,
 	.fo_write = badfo_readwrite,
@@ -3898,6 +3907,7 @@ struct fileops badfileops = {
 	.fo_close = badfo_close,
 	.fo_chmod = badfo_chmod,
 	.fo_chown = badfo_chown,
+	.fo_sendfile = badfo_sendfile,
 };
 
 int
@@ -3916,6 +3926,15 @@ invfo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
 	return (EINVAL);
 }
 
+int
+invfo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
+    struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
+    int kflags, struct thread *td)
+{
+
+	return (EINVAL);
+}
+
 /*-------------------------------------------------------------------*/
 
 /*
diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c
index 48e0905..d75ba22 100644
--- a/sys/kern/kern_event.c
+++ b/sys/kern/kern_event.c
@@ -127,6 +127,7 @@ static struct fileops kqueueops = {
 	.fo_close = kqueue_close,
 	.fo_chmod = invfo_chmod,
 	.fo_chown = invfo_chown,
+	.fo_sendfile = invfo_sendfile,
 };
 
 static int 	knote_attach(struct knote *kn, struct kqueue *kq);
diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c
index 493fee5e..76c295e 100644
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@@ -164,6 +164,7 @@ struct fileops pipeops = {
 	.fo_close = pipe_close,
 	.fo_chmod = pipe_chmod,
 	.fo_chown = pipe_chown,
+	.fo_sendfile = invfo_sendfile,
 	.fo_flags = DFLAG_PASSABLE
 };
 
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index cd6f655..6a766af 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -66,6 +66,7 @@ struct fileops	socketops = {
 	.fo_close = soo_close,
 	.fo_chmod = invfo_chmod,
 	.fo_chown = invfo_chown,
+	.fo_sendfile = invfo_sendfile,
 	.fo_flags = DFLAG_PASSABLE
 };
 
diff --git a/sys/kern/tty_pts.c b/sys/kern/tty_pts.c
index d6a2477..8d2ac03 100644
--- a/sys/kern/tty_pts.c
+++ b/sys/kern/tty_pts.c
@@ -599,6 +599,7 @@ static struct fileops ptsdev_ops = {
 	.fo_close	= ptsdev_close,
 	.fo_chmod	= invfo_chmod,
 	.fo_chown	= invfo_chown,
+	.fo_sendfile	= invfo_sendfile,
 	.fo_flags	= DFLAG_PASSABLE,
 };
 
diff --git a/sys/kern/uipc_mqueue.c b/sys/kern/uipc_mqueue.c
index 11acedb..5a1a414 100644
--- a/sys/kern/uipc_mqueue.c
+++ b/sys/kern/uipc_mqueue.c
@@ -2597,7 +2597,8 @@ static struct fileops mqueueops = {
 	.fo_stat		= mqf_stat,
 	.fo_chmod		= mqf_chmod,
 	.fo_chown		= mqf_chown,
-	.fo_close		= mqf_close
+	.fo_close		= mqf_close,
+	.fo_sendfile		= invfo_sendfile,
 };
 
 static struct vop_vector mqfs_vnodeops = {
diff --git a/sys/kern/uipc_sem.c b/sys/kern/uipc_sem.c
index a9f60f1..8c86cc4 100644
--- a/sys/kern/uipc_sem.c
+++ b/sys/kern/uipc_sem.c
@@ -149,6 +149,7 @@ static struct fileops ksem_ops = {
 	.fo_close = ksem_closef,
 	.fo_chmod = ksem_chmod,
 	.fo_chown = ksem_chown,
+	.fo_sendfile = invfo_sendfile,
 	.fo_flags = DFLAG_PASSABLE
 };
 
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index b8ad19b..64f5477 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -132,6 +132,7 @@ static struct fileops shm_ops = {
 	.fo_close = shm_close,
 	.fo_chmod = shm_chmod,
 	.fo_chown = shm_chown,
+	.fo_sendfile = invfo_sendfile,
 	.fo_flags = DFLAG_PASSABLE
 };
 
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 7be7033..10658de 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -157,6 +157,9 @@ sfstat_sysctl(SYSCTL_HANDLER_ARGS)
 }
 SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE | CTLFLAG_RW,
     NULL, 0, sfstat_sysctl, "I", "sendfile statistics");
+
+fo_sendfile_t vn_sendfile;
+
 /*
  * Convert a user file descriptor to a kernel file entry and check if required
  * capability rights are present.
@@ -1904,8 +1907,12 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
 {
 	struct sf_hdtr hdtr;
 	struct uio *hdr_uio, *trl_uio;
+	struct file *fp;
 	int error;
 
+	if (uap->offset < 0)
+		return (EINVAL);
+
 	hdr_uio = trl_uio = NULL;
 
 	if (uap->hdtr != NULL) {
@@ -1925,7 +1932,19 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
 		}
 	}
 
-	error = kern_sendfile(td, uap, hdr_uio, trl_uio, compat);
+	AUDIT_ARG_FD(uap->fd);
+
+	/*
+	 * sendfile(2) can start at any offset within a file so we require
+	 * CAP_READ+CAP_SEEK = CAP_PREAD.
+	 */
+	if ((error = fget_read(td, uap->fd, CAP_PREAD, &fp)) != 0)
+		goto out;
+
+	error = fo_sendfile(fp, uap->s, hdr_uio, trl_uio, uap->offset,
+	    uap->nbytes, uap->sbytes, uap->flags, compat ? SFK_COMPAT : 0, td);
+	fdrop(fp, td);
+
 out:
 	if (hdr_uio)
 		free(hdr_uio, M_IOV);
@@ -1953,11 +1972,12 @@ freebsd4_sendfile(struct thread *td, struct freebsd4_sendfile_args *uap)
 #endif /* COMPAT_FREEBSD4 */
 
 int
-kern_sendfile(struct thread *td, struct sendfile_args *uap,
-    struct uio *hdr_uio, struct uio *trl_uio, int compat)
+vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
+    struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
+    int kflags, struct thread *td)
 {
+	struct vnode *vp = fp->f_vnode;
 	struct file *sock_fp;
-	struct vnode *vp;
 	struct vm_object *obj = NULL;
 	struct socket *so = NULL;
 	struct mbuf *m = NULL;
@@ -1969,23 +1989,10 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 	int bsize;
 	struct sendfile_sync *sfs = NULL;
 
-	/*
-	 * The file descriptor must be a regular file and have a
-	 * backing VM object.
-	 * File offset must be positive.  If it goes beyond EOF
-	 * we send only the header/trailer and no payload data.
-	 */
-	AUDIT_ARG_FD(uap->fd);
-	/*
-	 * sendfile(2) can start at any offset within a file so we require
-	 * CAP_READ+CAP_SEEK = CAP_PREAD.
-	 */
-	if ((error = fgetvp_read(td, uap->fd, CAP_PREAD, &vp)) != 0)
-		goto out;
 	vn_lock(vp, LK_SHARED | LK_RETRY);
 	if (vp->v_type == VREG) {
 		bsize = vp->v_mount->mnt_stat.f_iosize;
-		if (uap->nbytes == 0) {
+		if (nbytes == 0) {
 			error = VOP_GETATTR(vp, &va, td->td_ucred);
 			if (error != 0) {
 				VOP_UNLOCK(vp, 0);
@@ -1994,7 +2001,7 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 			}
 			rem = va.va_size;
 		} else
-			rem = uap->nbytes;
+			rem = nbytes;
 		obj = vp->v_object;
 		if (obj != NULL) {
 			/*
@@ -2019,16 +2026,12 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 		error = EINVAL;
 		goto out;
 	}
-	if (uap->offset < 0) {
-		error = EINVAL;
-		goto out;
-	}
 
 	/*
 	 * The socket must be a stream socket and connected.
 	 * Remember if it a blocking or non-blocking socket.
 	 */
-	if ((error = getsock_cap(td->td_proc->p_fd, uap->s, CAP_SEND,
+	if ((error = getsock_cap(td->td_proc->p_fd, sockfd, CAP_SEND,
 	    &sock_fp, NULL)) != 0)
 		goto out;
 	so = sock_fp->f_data;
@@ -2045,10 +2048,10 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 	 * caller to retry later.
 	 * XXX: Experimental.
 	 */
-	if (uap->flags & SF_MNOWAIT)
+	if (flags & SF_MNOWAIT)
 		mnw = 1;
 
-	if (uap->flags & SF_SYNC) {
+	if (flags & SF_SYNC) {
 		sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK | M_ZERO);
 		mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF);
 		cv_init(&sfs->cv, "sendfile");
@@ -2070,11 +2073,11 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 			 * the header.  If compat is specified subtract the
 			 * header size from nbytes.
 			 */
-			if (compat) {
-				if (uap->nbytes > hdr_uio->uio_resid)
-					uap->nbytes -= hdr_uio->uio_resid;
+			if (kflags & SFK_COMPAT) {
+				if (nbytes > hdr_uio->uio_resid)
+					nbytes -= hdr_uio->uio_resid;
 				else
-					uap->nbytes = 0;
+					nbytes = 0;
 			}
 			m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK),
 			    0, 0, 0);
@@ -2105,14 +2108,14 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 	 * The outer loop checks the state and available space of the socket
 	 * and takes care of the overall progress.
 	 */
-	for (off = uap->offset; ; ) {
+	for (off = offset; ; ) {
 		struct mbuf *mtail;
 		int loopbytes;
 		int space;
 		int done;
 
-		if ((uap->nbytes != 0 && uap->nbytes == fsbytes) ||
-		    (uap->nbytes == 0 && va.va_size == fsbytes))
+		if ((nbytes != 0 && nbytes == fsbytes) ||
+		    (nbytes == 0 && va.va_size == fsbytes))
 			break;
 
 		mtail = NULL;
@@ -2210,11 +2213,11 @@ retry_space:
 			 * or the passed in nbytes.
 			 */
 			pgoff = (vm_offset_t)(off & PAGE_MASK);
-			if (uap->nbytes)
-				rem = (uap->nbytes - fsbytes - loopbytes);
+			if (nbytes)
+				rem = (nbytes - fsbytes - loopbytes);
 			else
 				rem = va.va_size -
-				    uap->offset - fsbytes - loopbytes;
+				    offset - fsbytes - loopbytes;
 			xfsize = omin(PAGE_SIZE - pgoff, rem);
 			xfsize = omin(space - loopbytes, xfsize);
 			if (xfsize <= 0) {
@@ -2242,7 +2245,7 @@ retry_space:
 				VM_OBJECT_WUNLOCK(obj);
 			else if (m != NULL)
 				error = EAGAIN;	/* send what we already got */
-			else if (uap->flags & SF_NODISKIO)
+			else if (flags & SF_NODISKIO)
 				error = EBUSY;
 			else {
 				ssize_t resid;
@@ -2299,7 +2302,7 @@ retry_space:
 				vm_page_lock(pg);
 				vm_page_unwire(pg, 0);
 				KASSERT(pg->object != NULL,
-				    ("kern_sendfile: object disappeared"));
+				    ("%s: object disappeared", __func__));
 				vm_page_unlock(pg);
 				if (m == NULL)
 					error = (mnw ? EAGAIN : EINTR);
@@ -2399,7 +2402,7 @@ retry_space:
 	 */
 	if (trl_uio != NULL) {
 		sbunlock(&so->so_snd);
-		error = kern_writev(td, uap->s, trl_uio);
+		error = kern_writev(td, sockfd, trl_uio);
 		if (error == 0)
 			sbytes += td->td_retval[0];
 		goto out;
@@ -2415,13 +2418,11 @@ out:
 	if (error == 0) {
 		td->td_retval[0] = 0;
 	}
-	if (uap->sbytes != NULL) {
-		copyout(&sbytes, uap->sbytes, sizeof(off_t));
+	if (sent != NULL) {
+		copyout(&sbytes, sent, sizeof(off_t));
 	}
 	if (obj != NULL)
 		vm_object_deallocate(obj);
-	if (vp != NULL)
-		vrele(vp);
 	if (so)
 		fdrop(sock_fp, td);
 	if (m)
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 06e59f9..2fcbf97 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -88,6 +88,7 @@ static fo_poll_t	vn_poll;
 static fo_kqfilter_t	vn_kqfilter;
 static fo_stat_t	vn_statfile;
 static fo_close_t	vn_closefile;
+extern fo_sendfile_t	vn_sendfile;
 
 struct 	fileops vnops = {
 	.fo_read = vn_io_fault,
@@ -100,6 +101,7 @@ struct 	fileops vnops = {
 	.fo_close = vn_closefile,
 	.fo_chmod = vn_chmod,
 	.fo_chown = vn_chown,
+	.fo_sendfile = vn_sendfile,
 	.fo_flags = DFLAG_PASSABLE | DFLAG_SEEKABLE
 };
 
diff --git a/sys/ofed/include/linux/linux_compat.c b/sys/ofed/include/linux/linux_compat.c
index 95bd6c8..4dbdad9 100644
--- a/sys/ofed/include/linux/linux_compat.c
+++ b/sys/ofed/include/linux/linux_compat.c
@@ -565,6 +565,7 @@ struct fileops linuxfileops = {
 	.fo_ioctl = linux_file_ioctl,
 	.fo_chmod = invfo_chmod,
 	.fo_chown = invfo_chown,
+	.fo_sendfile = invfo_sendfile,
 };
 
 /*
diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c
index 2bc0e1f..44bfa5c 100644
--- a/sys/opencrypto/cryptodev.c
+++ b/sys/opencrypto/cryptodev.c
@@ -304,6 +304,7 @@ static struct fileops cryptofops = {
     .fo_close = cryptof_close,
     .fo_chmod = invfo_chmod,
     .fo_chown = invfo_chown,
+    .fo_sendfile = invfo_sendfile,
 };
 
 static struct csession *csefind(struct fcrypt *, u_int);
diff --git a/sys/sys/file.h b/sys/sys/file.h
index cfdc1d8..63a7e8f 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -105,6 +105,9 @@ typedef	int fo_chmod_t(struct file *fp, mode_t mode,
 		    struct ucred *active_cred, struct thread *td);
 typedef	int fo_chown_t(struct file *fp, uid_t uid, gid_t gid,
 		    struct ucred *active_cred, struct thread *td);
+typedef int fo_sendfile_t(struct file *fp, int sockfd, struct uio *hdr_uio,
+		    struct uio *trl_uio, off_t offset, size_t nbytes,
+		    off_t *sent, int flags, int kflags, struct thread *td);
 typedef	int fo_flags_t;
 
 struct fileops {
@@ -118,6 +121,7 @@ struct fileops {
 	fo_close_t	*fo_close;
 	fo_chmod_t	*fo_chmod;
 	fo_chown_t	*fo_chown;
+	fo_sendfile_t	*fo_sendfile;
 	fo_flags_t	fo_flags;	/* DFLAG_* below */
 };
 
@@ -235,6 +239,7 @@ fo_close_t	soo_close;
 
 fo_chmod_t	invfo_chmod;
 fo_chown_t	invfo_chown;
+fo_sendfile_t	invfo_sendfile;
 
 void finit(struct file *, u_int, short, void *, struct fileops *);
 int fgetvp(struct thread *td, int fd, cap_rights_t rights, struct vnode **vpp);
@@ -273,6 +278,7 @@ static __inline fo_stat_t	fo_stat;
 static __inline fo_close_t	fo_close;
 static __inline fo_chmod_t	fo_chmod;
 static __inline fo_chown_t	fo_chown;
+static __inline fo_sendfile_t	fo_sendfile;
 
 static __inline int
 fo_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
@@ -352,6 +358,16 @@ fo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
 	return ((*fp->f_ops->fo_chown)(fp, uid, gid, active_cred, td));
 }
 
+static __inline int
+fo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
+    struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
+    int kflags, struct thread *td)
+{
+
+	return ((*fp->f_ops->fo_sendfile)(fp, sockfd, hdr_uio, trl_uio, offset,
+	    nbytes, sent, flags, kflags, td));
+}
+
 #endif /* _KERNEL */
 
 #endif /* !SYS_FILE_H */
diff --git a/sys/sys/socket.h b/sys/sys/socket.h
index 084c1a4..014c916 100644
--- a/sys/sys/socket.h
+++ b/sys/sys/socket.h
@@ -628,7 +628,11 @@ struct sf_hdtr {
 #define	SF_NODISKIO     0x00000001
 #define	SF_MNOWAIT	0x00000002
 #define	SF_SYNC		0x00000004
-#endif
+
+#ifdef _KERNEL
+#define	SFK_COMPAT	0x00000001
+#endif /* _KERNEL */
+#endif /* __BSD_VISIBLE */
 
 #ifndef	_KERNEL