- Add 32-bit compat system calls for VFS_AIO. The system calls live in the

  aio code and are registered via the recently added SYSCALL32_*() helpers.
- Since the aio code likes to invoke fuword and suword a lot down in the
  "bowels" of system calls, add a structure holding a set of operations for
  things like storing errors, copying in the aiocb structure, storing
  status, etc.  The 32-bit system calls use a separate operations vector to
  handle fuword32 vs fuword, etc.  Also, the oldsigevent handling is now
  done by having seperate operation vectors with different aiocb copyin
  routines.
- Split out kern_foo() functions for the various AIO system calls so the
  32-bit front ends can manage things like copying in and converting
  timespec structures, etc.
- For both the native and 32-bit aio_suspend() and lio_listio() calls,
  just use copyin() to read the array of aiocb pointers instead of using
  a for loop that iterated over fuword/fuword32.  The error handling in
  the old case was incomplete (lio_listio() just ignored any aiocb's that
  it got an EFAULT trying to read rather than reporting an error), and
  possibly slower.

MFC after:	1 month

1 files changed, 755 insertions, 126 deletions

diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index e86ae2c..14e980d 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -21,6 +21,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_compat.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
@@ -121,6 +123,8 @@ static uint64_t jobseqno;
 
 FEATURE(aio, "Asynchronous I/O");
 
+static MALLOC_DEFINE(M_LIO, "lio", "listio aio control block list");
+
 static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "Async IO management");
 
 static int max_aio_procs = MAX_AIO_PROCS;
@@ -308,6 +312,20 @@ struct kaioinfo {
 #define KAIO_RUNDOWN	0x1	/* process is being run down */
 #define KAIO_WAKEUP	0x2	/* wakeup process when there is a significant event */
 
+/*
+ * Operations used to interact with userland aio control blocks.
+ * Different ABIs provide their own operations.
+ */
+struct aiocb_ops {
+	int	(*copyin)(struct aiocb *ujob, struct aiocb *kjob);
+	long	(*fetch_status)(struct aiocb *ujob);
+	long	(*fetch_error)(struct aiocb *ujob);
+	int	(*store_status)(struct aiocb *ujob, long status);
+	int	(*store_error)(struct aiocb *ujob, long error);
+	int	(*store_kernelinfo)(struct aiocb *ujob, long jobref);
+	int	(*store_aiocb)(struct aiocb **ujobp, struct aiocb *ujob);
+};
+
 static TAILQ_HEAD(,aiothreadlist) aio_freeproc;		/* (c) Idle daemons */
 static struct sema aio_newproc_sem;
 static struct mtx aio_job_mtx;
@@ -321,7 +339,7 @@ static int	aio_free_entry(struct aiocblist *aiocbe);
 static void	aio_process(struct aiocblist *aiocbe);
 static int	aio_newproc(int *);
 int		aio_aqueue(struct thread *td, struct aiocb *job,
-			struct aioliojob *lio, int type, int osigev);
+			struct aioliojob *lio, int type, struct aiocb_ops *ops);
 static void	aio_physwakeup(struct buf *bp);
 static void	aio_proc_rundown(void *arg, struct proc *p);
 static void	aio_proc_rundown_exec(void *arg, struct proc *p, struct image_params *imgp);
@@ -333,7 +351,6 @@ static int	aio_unload(void);
 static void	aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type);
 #define DONE_BUF	1
 #define DONE_QUEUE	2
-static int	do_lio_listio(struct thread *td, struct lio_listio_args *uap, int oldsigev);
 static int	aio_kick(struct proc *userp);
 static void	aio_kick_nowait(struct proc *userp);
 static void	aio_kick_helper(void *context, int pending);
@@ -1322,13 +1339,122 @@ aio_swake_cb(struct socket *so, struct sockbuf *sb)
 	SOCKBUF_UNLOCK(sb);
 }
 
+static int
+convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig)
+{
+
+	/*
+	 * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
+	 * supported by AIO with the old sigevent structure.
+	 */
+	nsig->sigev_notify = osig->sigev_notify;
+	switch (nsig->sigev_notify) {
+	case SIGEV_NONE:
+		break;
+	case SIGEV_SIGNAL:
+		nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
+		break;
+	case SIGEV_KEVENT:
+		nsig->sigev_notify_kqueue =
+		    osig->__sigev_u.__sigev_notify_kqueue;
+		nsig->sigev_value.sival_ptr = osig->sigev_value.sival_ptr;
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (0);
+}
+
+static int
+aiocb_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob)
+{
+	struct oaiocb *ojob;
+	int error;
+
+	bzero(kjob, sizeof(struct aiocb));
+	error = copyin(ujob, kjob, sizeof(struct oaiocb));
+	if (error)
+		return (error);
+	ojob = (struct oaiocb *)kjob;
+	return (convert_old_sigevent(&ojob->aio_sigevent, &kjob->aio_sigevent));
+}
+
+static int
+aiocb_copyin(struct aiocb *ujob, struct aiocb *kjob)
+{
+
+	return (copyin(ujob, kjob, sizeof(struct aiocb)));
+}
+
+static long
+aiocb_fetch_status(struct aiocb *ujob)
+{
+
+	return (fuword(&ujob->_aiocb_private.status));
+}
+
+static long
+aiocb_fetch_error(struct aiocb *ujob)
+{
+
+	return (fuword(&ujob->_aiocb_private.error));
+}
+
+static int
+aiocb_store_status(struct aiocb *ujob, long status)
+{
+
+	return (suword(&ujob->_aiocb_private.status, status));
+}
+
+static int
+aiocb_store_error(struct aiocb *ujob, long error)
+{
+
+	return (suword(&ujob->_aiocb_private.error, error));
+}
+
+static int
+aiocb_store_kernelinfo(struct aiocb *ujob, long jobref)
+{
+
+	return (suword(&ujob->_aiocb_private.kernelinfo, jobref));
+}
+
+static int
+aiocb_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
+{
+
+	return (suword(ujobp, (long)ujob));
+}
+
+static struct aiocb_ops aiocb_ops = {
+	.copyin = aiocb_copyin,
+	.fetch_status = aiocb_fetch_status,
+	.fetch_error = aiocb_fetch_error,
+	.store_status = aiocb_store_status,
+	.store_error = aiocb_store_error,
+	.store_kernelinfo = aiocb_store_kernelinfo,
+	.store_aiocb = aiocb_store_aiocb,
+};
+
+static struct aiocb_ops aiocb_ops_osigevent = {
+	.copyin = aiocb_copyin_old_sigevent,
+	.fetch_status = aiocb_fetch_status,
+	.fetch_error = aiocb_fetch_error,
+	.store_status = aiocb_store_status,
+	.store_error = aiocb_store_error,
+	.store_kernelinfo = aiocb_store_kernelinfo,
+	.store_aiocb = aiocb_store_aiocb,
+};
+
 /*
  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
  * technique is done in this code.
  */
 int
 aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
-	int type, int oldsigev)
+	int type, struct aiocb_ops *ops)
 {
 	struct proc *p = td->td_proc;
 	struct file *fp;
@@ -1347,13 +1473,13 @@ aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
 
 	ki = p->p_aioinfo;
 
-	suword(&job->_aiocb_private.status, -1);
-	suword(&job->_aiocb_private.error, 0);
-	suword(&job->_aiocb_private.kernelinfo, -1);
+	ops->store_status(job, -1);
+	ops->store_error(job, 0);
+	ops->store_kernelinfo(job, -1);
 
 	if (num_queue_count >= max_queue_count ||
 	    ki->kaio_count >= ki->kaio_qallowed_count) {
-		suword(&job->_aiocb_private.error, EAGAIN);
+		ops->store_error(job, EAGAIN);
 		return (EAGAIN);
 	}
 
@@ -1362,16 +1488,9 @@ aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
 	aiocbe->outputcharge = 0;
 	knlist_init(&aiocbe->klist, AIO_MTX(ki), NULL, NULL, NULL);
 
-	if (oldsigev) {
-		bzero(&aiocbe->uaiocb, sizeof(struct aiocb));
-		error = copyin(job, &aiocbe->uaiocb, sizeof(struct oaiocb));
-		bcopy(&aiocbe->uaiocb.__spare__, &aiocbe->uaiocb.aio_sigevent,
-			sizeof(struct osigevent));
-	} else {
-		error = copyin(job, &aiocbe->uaiocb, sizeof(struct aiocb));
-	}
+	error = ops->copyin(job, &aiocbe->uaiocb);
 	if (error) {
-		suword(&job->_aiocb_private.error, error);
+		ops->store_error(job, error);
 		uma_zfree(aiocb_zone, aiocbe);
 		return (error);
 	}
@@ -1380,11 +1499,11 @@ aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
 	    aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL &&
 	    aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID &&
 	    aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) {
-		suword(&job->_aiocb_private.error, EINVAL);
+		ops->store_error(job, EINVAL);
 		uma_zfree(aiocb_zone, aiocbe);
 		return (EINVAL);
 	}
-	
+
 	if ((aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL ||
 	     aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) &&
 		!_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) {
@@ -1416,7 +1535,7 @@ aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
 	}
 	if (error) {
 		uma_zfree(aiocb_zone, aiocbe);
-		suword(&job->_aiocb_private.error, error);
+		ops->store_error(job, error);
 		return (error);
 	}
 
@@ -1436,7 +1555,7 @@ aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
 	jid = jobrefid++;
 	aiocbe->seqno = jobseqno++;
 	mtx_unlock(&aio_job_mtx);
-	error = suword(&job->_aiocb_private.kernelinfo, jid);
+	error = ops->store_kernelinfo(job, jid);
 	if (error) {
 		error = EINVAL;
 		goto aqueue_fail;
@@ -1467,12 +1586,12 @@ aqueue_fail:
 	if (error) {
 		fdrop(fp, td);
 		uma_zfree(aiocb_zone, aiocbe);
-		suword(&job->_aiocb_private.error, error);
+		ops->store_error(job, error);
 		goto done;
 	}
 no_kqueue:
 
-	suword(&job->_aiocb_private.error, EINPROGRESS);
+	ops->store_error(job, EINPROGRESS);
 	aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
 	aiocbe->userproc = p;
 	aiocbe->cred = crhold(td->td_ucred);
@@ -1528,7 +1647,7 @@ no_kqueue:
 #if 0
 	if (error > 0) {
 		aiocbe->uaiocb._aiocb_private.error = error;
-		suword(&job->_aiocb_private.error, error);
+		ops->store_error(job, error);
 		goto done;
 	}
 #endif
@@ -1643,19 +1762,17 @@ aio_kick_helper(void *context, int pending)
  * Support the aio_return system call, as a side-effect, kernel resources are
  * released.
  */
-int
-aio_return(struct thread *td, struct aio_return_args *uap)
+static int
+kern_aio_return(struct thread *td, struct aiocb *uaiocb, struct aiocb_ops *ops)
 {
 	struct proc *p = td->td_proc;
 	struct aiocblist *cb;
-	struct aiocb *uaiocb;
 	struct kaioinfo *ki;
 	int status, error;
 
 	ki = p->p_aioinfo;
 	if (ki == NULL)
 		return (EINVAL);
-	uaiocb = uap->aiocbp;
 	AIO_LOCK(ki);
 	TAILQ_FOREACH(cb, &ki->kaio_done, plist) {
 		if (cb->uuaiocb == uaiocb)
@@ -1675,8 +1792,8 @@ aio_return(struct thread *td, struct aio_return_args *uap)
 		}
 		aio_free_entry(cb);
 		AIO_UNLOCK(ki);
-		suword(&uaiocb->_aiocb_private.error, error);
-		suword(&uaiocb->_aiocb_private.status, status);
+		ops->store_error(uaiocb, error);
+		ops->store_status(uaiocb, status);
 	} else {
 		error = EINVAL;
 		AIO_UNLOCK(ki);
@@ -1684,37 +1801,32 @@ aio_return(struct thread *td, struct aio_return_args *uap)
 	return (error);
 }
 
+int
+aio_return(struct thread *td, struct aio_return_args *uap)
+{
+
+	return (kern_aio_return(td, uap->aiocbp, &aiocb_ops));
+}
+
 /*
  * Allow a process to wakeup when any of the I/O requests are completed.
  */
-int
-aio_suspend(struct thread *td, struct aio_suspend_args *uap)
+static int
+kern_aio_suspend(struct thread *td, int njoblist, struct aiocb **ujoblist,
+    struct timespec *ts)
 {
 	struct proc *p = td->td_proc;
 	struct timeval atv;
-	struct timespec ts;
-	struct aiocb *const *cbptr, *cbp;
 	struct kaioinfo *ki;
 	struct aiocblist *cb, *cbfirst;
-	struct aiocb **ujoblist;
-	int njoblist;
-	int error;
-	int timo;
-	int i;
-
-	if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
-		return (EINVAL);
+	int error, i, timo;
 
 	timo = 0;
-	if (uap->timeout) {
-		/* Get timespec struct. */
-		if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
-			return (error);
-
-		if (ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000)
+	if (ts) {
+		if (ts->tv_nsec < 0 || ts->tv_nsec >= 1000000000)
 			return (EINVAL);
 
-		TIMESPEC_TO_TIMEVAL(&atv, &ts);
+		TIMESPEC_TO_TIMEVAL(&atv, ts);
 		if (itimerfix(&atv))
 			return (EINVAL);
 		timo = tvtohz(&atv);
@@ -1724,22 +1836,8 @@ aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 	if (ki == NULL)
 		return (EAGAIN);
 
-	njoblist = 0;
-	ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
-	cbptr = uap->aiocbp;
-
-	for (i = 0; i < uap->nent; i++) {
-		cbp = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
-		if (cbp == 0)
-			continue;
-		ujoblist[njoblist] = cbp;
-		njoblist++;
-	}
-
-	if (njoblist == 0) {
-		uma_zfree(aiol_zone, ujoblist);
+	if (njoblist == 0)
 		return (0);
-	}
 
 	AIO_LOCK(ki);
 	for (;;) {
@@ -1769,6 +1867,31 @@ aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 	}
 RETURN:
 	AIO_UNLOCK(ki);
+	return (error);
+}
+
+int
+aio_suspend(struct thread *td, struct aio_suspend_args *uap)
+{
+	struct timespec ts, *tsp;
+	struct aiocb **ujoblist;
+	int error;
+
+	if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
+		return (EINVAL);
+
+	if (uap->timeout) {
+		/* Get timespec struct. */
+		if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
+			return (error);
+		tsp = &ts;
+	} else
+		tsp = NULL;
+
+	ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
+	error = copyin(uap->aiocbp, ujoblist, uap->nent * sizeof(ujoblist[0]));
+	if (error == 0)
+		error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
 	uma_zfree(aiol_zone, ujoblist);
 	return (error);
 }
@@ -1876,8 +1999,8 @@ done:
  * only.  For a user mode async implementation, it would be best to do it in
  * a userland subroutine.
  */
-int
-aio_error(struct thread *td, struct aio_error_args *uap)
+static int
+kern_aio_error(struct thread *td, struct aiocb *aiocbp, struct aiocb_ops *ops)
 {
 	struct proc *p = td->td_proc;
 	struct aiocblist *cb;
@@ -1892,7 +2015,7 @@ aio_error(struct thread *td, struct aio_error_args *uap)
 
 	AIO_LOCK(ki);
 	TAILQ_FOREACH(cb, &ki->kaio_all, allist) {
-		if (cb->uuaiocb == uap->aiocbp) {
+		if (cb->uuaiocb == aiocbp) {
 			if (cb->jobstate == JOBST_JOBFINISHED)
 				td->td_retval[0] =
 					cb->uaiocb._aiocb_private.error;
@@ -1907,9 +2030,9 @@ aio_error(struct thread *td, struct aio_error_args *uap)
 	/*
 	 * Hack for failure of aio_aqueue.
 	 */
-	status = fuword(&uap->aiocbp->_aiocb_private.status);
+	status = ops->fetch_status(aiocbp);
 	if (status == -1) {
-		td->td_retval[0] = fuword(&uap->aiocbp->_aiocb_private.error);
+		td->td_retval[0] = ops->fetch_error(aiocbp);
 		return (0);
 	}
 
@@ -1917,19 +2040,27 @@ aio_error(struct thread *td, struct aio_error_args *uap)
 	return (0);
 }
 
+int
+aio_error(struct thread *td, struct aio_error_args *uap)
+{
+
+	return (kern_aio_error(td, uap->aiocbp, &aiocb_ops));
+}
+
 /* syscall - asynchronous read from a file (REALTIME) */
 int
 oaio_read(struct thread *td, struct oaio_read_args *uap)
 {
 
-	return aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ, 1);
+	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
+	    &aiocb_ops_osigevent));
 }
 
 int
 aio_read(struct thread *td, struct aio_read_args *uap)
 {
 
-	return aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, 0);
+	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, &aiocb_ops));
 }
 
 /* syscall - asynchronous write to a file (REALTIME) */
@@ -1937,47 +2068,34 @@ int
 oaio_write(struct thread *td, struct oaio_write_args *uap)
 {
 
-	return aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE, 1);
+	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
+	    &aiocb_ops_osigevent));
 }
 
 int
 aio_write(struct thread *td, struct aio_write_args *uap)
 {
 
-	return aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, 0);
-}
-
-/* syscall - list directed I/O (REALTIME) */
-int
-olio_listio(struct thread *td, struct olio_listio_args *uap)
-{
-	return do_lio_listio(td, (struct lio_listio_args *)uap, 1);
-}
-
-/* syscall - list directed I/O (REALTIME) */
-int
-lio_listio(struct thread *td, struct lio_listio_args *uap)
-{
-	return do_lio_listio(td, uap, 0);
+	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops));
 }
 
 static int
-do_lio_listio(struct thread *td, struct lio_listio_args *uap, int oldsigev)
+kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list,
+    struct aiocb **acb_list, int nent, struct sigevent *sig,
+    struct aiocb_ops *ops)
 {
 	struct proc *p = td->td_proc;
-	struct aiocb *iocb, * const *cbptr;
+	struct aiocb *iocb;
 	struct kaioinfo *ki;
 	struct aioliojob *lj;
 	struct kevent kev;
-	int nent;
 	int error;
 	int nerror;
 	int i;
 
-	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
+	if ((mode != LIO_NOWAIT) && (mode != LIO_WAIT))
 		return (EINVAL);
 
-	nent = uap->nent;
 	if (nent < 0 || nent > AIO_LISTIO_MAX)
 		return (EINVAL);
 
@@ -1996,21 +2114,13 @@ do_lio_listio(struct thread *td, struct lio_listio_args *uap, int oldsigev)
 	/*
 	 * Setup signal.
 	 */
-	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
-		bzero(&lj->lioj_signal, sizeof(&lj->lioj_signal));
-		error = copyin(uap->sig, &lj->lioj_signal,
-				oldsigev ? sizeof(struct osigevent) :
-					   sizeof(struct sigevent));
-		if (error) {
-			uma_zfree(aiolio_zone, lj);
-			return (error);
-		}
-
+	if (sig && (mode == LIO_NOWAIT)) {
+		bcopy(sig, &lj->lioj_signal, sizeof(lj->lioj_signal));
 		if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
 			/* Assume only new style KEVENT */
 			kev.filter = EVFILT_LIO;
 			kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
-			kev.ident = (uintptr_t)uap->acb_list; /* something unique */
+			kev.ident = (uintptr_t)uacb_list; /* something unique */
 			kev.data = (intptr_t)lj;
 			/* pass user defined sigval data */
 			kev.udata = lj->lioj_signal.sigev_value.sival_ptr;
@@ -2050,11 +2160,10 @@ do_lio_listio(struct thread *td, struct lio_listio_args *uap, int oldsigev)
 	 * Get pointers to the list of I/O requests.
 	 */
 	nerror = 0;
-	cbptr = uap->acb_list;
-	for (i = 0; i < uap->nent; i++) {
-		iocb = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
-		if (((intptr_t)iocb != -1) && ((intptr_t)iocb != 0)) {
-			error = aio_aqueue(td, iocb, lj, LIO_NOP, oldsigev);
+	for (i = 0; i < nent; i++) {
+		iocb = acb_list[i];
+		if (iocb != NULL) {
+			error = aio_aqueue(td, iocb, lj, LIO_NOP, ops);
 			if (error != 0)
 				nerror++;
 		}
@@ -2062,7 +2171,7 @@ do_lio_listio(struct thread *td, struct lio_listio_args *uap, int oldsigev)
 
 	error = 0;
 	AIO_LOCK(ki);
-	if (uap->mode == LIO_WAIT) {
+	if (mode == LIO_WAIT) {
 		while (lj->lioj_count - 1 != lj->lioj_finished_count) {
 			ki->kaio_flags |= KAIO_WAKEUP;
 			error = msleep(&p->p_aioinfo, AIO_MTX(ki),
@@ -2105,6 +2214,75 @@ do_lio_listio(struct thread *td, struct lio_listio_args *uap, int oldsigev)
 	return (error);
 }
 
+/* syscall - list directed I/O (REALTIME) */
+int
+olio_listio(struct thread *td, struct olio_listio_args *uap)
+{
+	struct aiocb **acb_list;
+	struct sigevent *sigp, sig;
+	struct osigevent osig;
+	int error, nent;
+
+	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
+		return (EINVAL);
+
+	nent = uap->nent;
+	if (nent < 0 || nent > AIO_LISTIO_MAX)
+		return (EINVAL);
+
+	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
+		error = copyin(uap->sig, &osig, sizeof(osig));
+		if (error)
+			return (error);
+		error = convert_old_sigevent(&osig, &sig);
+		if (error)
+			return (error);
+		sigp = &sig;
+	} else
+		sigp = NULL;
+
+	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
+	error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
+	if (error == 0)
+		error = kern_lio_listio(td, uap->mode,
+		    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
+		    &aiocb_ops_osigevent);
+	free(acb_list, M_LIO);
+	return (error);
+}
+
+/* syscall - list directed I/O (REALTIME) */
+int
+lio_listio(struct thread *td, struct lio_listio_args *uap)
+{
+	struct aiocb **acb_list;
+	struct sigevent *sigp, sig;
+	int error, nent;
+
+	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
+		return (EINVAL);
+
+	nent = uap->nent;
+	if (nent < 0 || nent > AIO_LISTIO_MAX)
+		return (EINVAL);
+
+	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
+		error = copyin(uap->sig, &sig, sizeof(sig));
+		if (error)
+			return (error);
+		sigp = &sig;
+	} else
+		sigp = NULL;
+
+	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
+	error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
+	if (error == 0)
+		error = kern_lio_listio(td, uap->mode, uap->acb_list, acb_list,
+		    nent, sigp, &aiocb_ops);
+	free(acb_list, M_LIO);
+	return (error);
+}
+
 /*
  * Called from interrupt thread for physio, we should return as fast
  * as possible, so we schedule a biohelper task.
@@ -2156,30 +2334,25 @@ biohelper(void *context, int pending)
 }
 
 /* syscall - wait for the next completion of an aio request */
-int
-aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
+static int
+kern_aio_waitcomplete(struct thread *td, struct aiocb **aiocbp,
+    struct timespec *ts, struct aiocb_ops *ops)
 {
 	struct proc *p = td->td_proc;
 	struct timeval atv;
-	struct timespec ts;
 	struct kaioinfo *ki;
 	struct aiocblist *cb;
 	struct aiocb *uuaiocb;
 	int error, status, timo;
 
-	suword(uap->aiocbp, (long)NULL);
+	ops->store_aiocb(aiocbp, NULL);
 
 	timo = 0;
-	if (uap->timeout) {
-		/* Get timespec struct. */
-		error = copyin(uap->timeout, &ts, sizeof(ts));
-		if (error)
-			return (error);
-
-		if ((ts.tv_nsec < 0) || (ts.tv_nsec >= 1000000000))
+	if (ts) {
+		if ((ts->tv_nsec < 0) || (ts->tv_nsec >= 1000000000))
 			return (EINVAL);
 
-		TIMESPEC_TO_TIMEVAL(&atv, &ts);
+		TIMESPEC_TO_TIMEVAL(&atv, ts);
 		if (itimerfix(&atv))
 			return (EINVAL);
 		timo = tvtohz(&atv);
@@ -2217,9 +2390,9 @@ aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 		}
 		aio_free_entry(cb);
 		AIO_UNLOCK(ki);
-		suword(uap->aiocbp, (long)uuaiocb);
-		suword(&uuaiocb->_aiocb_private.error, error);
-		suword(&uuaiocb->_aiocb_private.status, status);
+		ops->store_aiocb(aiocbp, uuaiocb);
+		ops->store_error(uuaiocb, error);
+		ops->store_status(uuaiocb, status);
 	} else
 		AIO_UNLOCK(ki);
 
@@ -2227,17 +2400,43 @@ aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 }
 
 int
-aio_fsync(struct thread *td, struct aio_fsync_args *uap)
+aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
+{
+	struct timespec ts, *tsp;
+	int error;
+
+	if (uap->timeout) {
+		/* Get timespec struct. */
+		error = copyin(uap->timeout, &ts, sizeof(ts));
+		if (error)
+			return (error);
+		tsp = &ts;
+	} else
+		tsp = NULL;
+
+	return (kern_aio_waitcomplete(td, uap->aiocbp, tsp, &aiocb_ops));
+}
+
+static int
+kern_aio_fsync(struct thread *td, int op, struct aiocb *aiocbp,
+    struct aiocb_ops *ops)
 {
 	struct proc *p = td->td_proc;
 	struct kaioinfo *ki;
 
-	if (uap->op != O_SYNC) /* XXX lack of O_DSYNC */
+	if (op != O_SYNC) /* XXX lack of O_DSYNC */
 		return (EINVAL);
 	ki = p->p_aioinfo;
 	if (ki == NULL)
 		aio_init_aioinfo(p);
-	return aio_aqueue(td, uap->aiocbp, NULL, LIO_SYNC, 0);
+	return (aio_aqueue(td, aiocbp, NULL, LIO_SYNC, ops));
+}
+
+int
+aio_fsync(struct thread *td, struct aio_fsync_args *uap)
+{
+
+	return (kern_aio_fsync(td, uap->op, uap->aiocbp, &aiocb_ops));
 }
 
 /* kqueue attach function */
@@ -2325,3 +2524,433 @@ filt_lio(struct knote *kn, long hint)
 
 	return (lj->lioj_flags & LIOJ_KEVENT_POSTED);
 }
+
+#ifdef COMPAT_IA32
+#include <sys/mount.h>
+#include <sys/socket.h>
+#include <compat/freebsd32/freebsd32.h>
+#include <compat/freebsd32/freebsd32_proto.h>
+#include <compat/freebsd32/freebsd32_signal.h>
+#include <compat/freebsd32/freebsd32_syscall.h>
+#include <compat/freebsd32/freebsd32_util.h>
+
+struct __aiocb_private32 {
+	int32_t	status;
+	int32_t	error;
+	uint32_t kernelinfo;
+};
+
+typedef struct oaiocb32 {
+	int	aio_fildes;		/* File descriptor */
+	uint64_t aio_offset __packed;	/* File offset for I/O */
+	uint32_t aio_buf;		/* I/O buffer in process space */
+	uint32_t aio_nbytes;		/* Number of bytes for I/O */
+	struct	osigevent32 aio_sigevent; /* Signal to deliver */
+	int	aio_lio_opcode;		/* LIO opcode */
+	int	aio_reqprio;		/* Request priority -- ignored */
+	struct	__aiocb_private32 _aiocb_private;
+} oaiocb32_t;
+
+typedef struct aiocb32 {
+	int32_t	aio_fildes;		/* File descriptor */
+	uint64_t aio_offset __packed;	/* File offset for I/O */
+	uint32_t aio_buf;		/* I/O buffer in process space */
+	uint32_t aio_nbytes;		/* Number of bytes for I/O */
+	int	__spare__[2];
+	uint32_t __spare2__;
+	int	aio_lio_opcode;		/* LIO opcode */
+	int	aio_reqprio;		/* Request priority -- ignored */
+	struct __aiocb_private32 _aiocb_private;
+	struct sigevent32 aio_sigevent;	/* Signal to deliver */
+} aiocb32_t;
+
+static int
+convert_old_sigevent32(struct osigevent32 *osig, struct sigevent *nsig)
+{
+
+	/*
+	 * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
+	 * supported by AIO with the old sigevent structure.
+	 */
+	CP(*osig, *nsig, sigev_notify);
+	switch (nsig->sigev_notify) {
+	case SIGEV_NONE:
+		break;
+	case SIGEV_SIGNAL:
+		nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
+		break;
+	case SIGEV_KEVENT:
+		nsig->sigev_notify_kqueue =
+		    osig->__sigev_u.__sigev_notify_kqueue;
+		PTRIN_CP(*osig, *nsig, sigev_value.sival_ptr);
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (0);
+}
+
+static int
+aiocb32_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob)
+{
+	struct oaiocb32 job32;
+	int error;
+
+	bzero(kjob, sizeof(struct aiocb));
+	error = copyin(ujob, &job32, sizeof(job32));
+	if (error)
+		return (error);
+
+	CP(job32, *kjob, aio_fildes);
+	CP(job32, *kjob, aio_offset);
+	PTRIN_CP(job32, *kjob, aio_buf);
+	CP(job32, *kjob, aio_nbytes);
+	CP(job32, *kjob, aio_lio_opcode);
+	CP(job32, *kjob, aio_reqprio);
+	CP(job32, *kjob, _aiocb_private.status);
+	CP(job32, *kjob, _aiocb_private.error);
+	PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo);
+	return (convert_old_sigevent32(&job32.aio_sigevent,
+	    &kjob->aio_sigevent));
+}
+
+static int
+convert_sigevent32(struct sigevent32 *sig32, struct sigevent *sig)
+{
+
+	CP(*sig32, *sig, sigev_notify);
+	switch (sig->sigev_notify) {
+	case SIGEV_NONE:
+		break;
+	case SIGEV_THREAD_ID:
+		CP(*sig32, *sig, sigev_notify_thread_id);
+		/* FALLTHROUGH */
+	case SIGEV_SIGNAL:
+		CP(*sig32, *sig, sigev_signo);
+		break;
+	case SIGEV_KEVENT:
+		CP(*sig32, *sig, sigev_notify_kqueue);
+		PTRIN_CP(*sig32, *sig, sigev_value.sival_ptr);
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (0);
+}
+
+static int
+aiocb32_copyin(struct aiocb *ujob, struct aiocb *kjob)
+{
+	struct aiocb32 job32;
+	int error;
+
+	error = copyin(ujob, &job32, sizeof(job32));
+	if (error)
+		return (error);
+	CP(job32, *kjob, aio_fildes);
+	CP(job32, *kjob, aio_offset);
+	PTRIN_CP(job32, *kjob, aio_buf);
+	CP(job32, *kjob, aio_nbytes);
+	CP(job32, *kjob, aio_lio_opcode);
+	CP(job32, *kjob, aio_reqprio);
+	CP(job32, *kjob, _aiocb_private.status);
+	CP(job32, *kjob, _aiocb_private.error);
+	PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo);
+	return (convert_sigevent32(&job32.aio_sigevent, &kjob->aio_sigevent));
+}
+
+static long
+aiocb32_fetch_status(struct aiocb *ujob)
+{
+	struct aiocb32 *ujob32;
+
+	ujob32 = (struct aiocb32 *)ujob;
+	return (fuword32(&ujob32->_aiocb_private.status));
+}
+
+static long
+aiocb32_fetch_error(struct aiocb *ujob)
+{
+	struct aiocb32 *ujob32;
+
+	ujob32 = (struct aiocb32 *)ujob;
+	return (fuword32(&ujob32->_aiocb_private.error));
+}
+
+static int
+aiocb32_store_status(struct aiocb *ujob, long status)
+{
+	struct aiocb32 *ujob32;
+
+	ujob32 = (struct aiocb32 *)ujob;
+	return (suword32(&ujob32->_aiocb_private.status, status));
+}
+
+static int
+aiocb32_store_error(struct aiocb *ujob, long error)
+{
+	struct aiocb32 *ujob32;
+
+	ujob32 = (struct aiocb32 *)ujob;
+	return (suword32(&ujob32->_aiocb_private.error, error));
+}
+
+static int
+aiocb32_store_kernelinfo(struct aiocb *ujob, long jobref)
+{
+	struct aiocb32 *ujob32;
+
+	ujob32 = (struct aiocb32 *)ujob;
+	return (suword32(&ujob32->_aiocb_private.kernelinfo, jobref));
+}
+
+static int
+aiocb32_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
+{
+
+	return (suword32(ujobp, (long)ujob));
+}
+
+static struct aiocb_ops aiocb32_ops = {
+	.copyin = aiocb32_copyin,
+	.fetch_status = aiocb32_fetch_status,
+	.fetch_error = aiocb32_fetch_error,
+	.store_status = aiocb32_store_status,
+	.store_error = aiocb32_store_error,
+	.store_kernelinfo = aiocb32_store_kernelinfo,
+	.store_aiocb = aiocb32_store_aiocb,
+};
+
+static struct aiocb_ops aiocb32_ops_osigevent = {
+	.copyin = aiocb32_copyin_old_sigevent,
+	.fetch_status = aiocb32_fetch_status,
+	.fetch_error = aiocb32_fetch_error,
+	.store_status = aiocb32_store_status,
+	.store_error = aiocb32_store_error,
+	.store_kernelinfo = aiocb32_store_kernelinfo,
+	.store_aiocb = aiocb32_store_aiocb,
+};
+
+int
+freebsd32_aio_return(struct thread *td, struct freebsd32_aio_return_args *uap)
+{
+
+	return (kern_aio_return(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
+}
+
+int
+freebsd32_aio_suspend(struct thread *td, struct freebsd32_aio_suspend_args *uap)
+{
+	struct timespec32 ts32;
+	struct timespec ts, *tsp;
+	struct aiocb **ujoblist;
+	uint32_t *ujoblist32;
+	int error, i;
+
+	if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
+		return (EINVAL);
+
+	if (uap->timeout) {
+		/* Get timespec struct. */
+		if ((error = copyin(uap->timeout, &ts32, sizeof(ts32))) != 0)
+			return (error);
+		CP(ts32, ts, tv_sec);
+		CP(ts32, ts, tv_nsec);
+		tsp = &ts;
+	} else
+		tsp = NULL;
+
+	ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
+	ujoblist32 = (uint32_t *)ujoblist;
+	error = copyin(uap->aiocbp, ujoblist32, uap->nent *
+	    sizeof(ujoblist32[0]));
+	if (error == 0) {
+		for (i = uap->nent; i > 0; i--)
+			ujoblist[i] = PTRIN(ujoblist32[i]);
+
+		error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
+	}
+	uma_zfree(aiol_zone, ujoblist);
+	return (error);
+}
+
+int
+freebsd32_aio_cancel(struct thread *td, struct freebsd32_aio_cancel_args *uap)
+{
+
+	return (aio_cancel(td, (struct aio_cancel_args *)uap));
+}
+
+int
+freebsd32_aio_error(struct thread *td, struct freebsd32_aio_error_args *uap)
+{
+
+	return (kern_aio_error(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
+}
+
+int
+freebsd32_oaio_read(struct thread *td, struct freebsd32_oaio_read_args *uap)
+{
+
+	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
+	    &aiocb32_ops_osigevent));
+}
+
+int
+freebsd32_aio_read(struct thread *td, struct freebsd32_aio_read_args *uap)
+{
+
+	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
+	    &aiocb32_ops));
+}
+
+int
+freebsd32_oaio_write(struct thread *td, struct freebsd32_oaio_write_args *uap)
+{
+
+	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
+	    &aiocb32_ops_osigevent));
+}
+
+int
+freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap)
+{
+
+	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
+	    &aiocb32_ops));
+}
+
+int
+freebsd32_aio_waitcomplete(struct thread *td,
+    struct freebsd32_aio_waitcomplete_args *uap)
+{
+	struct timespec ts32;
+	struct timespec ts, *tsp;
+	int error;
+
+	if (uap->timeout) {
+		/* Get timespec struct. */
+		error = copyin(uap->timeout, &ts32, sizeof(ts32));
+		if (error)
+			return (error);
+		CP(ts32, ts, tv_sec);
+		CP(ts32, ts, tv_nsec);
+		tsp = &ts;
+	} else
+		tsp = NULL;
+
+	return (kern_aio_waitcomplete(td, (struct aiocb **)uap->aiocbp, tsp,
+	    &aiocb32_ops));
+}
+
+int
+freebsd32_aio_fsync(struct thread *td, struct freebsd32_aio_fsync_args *uap)
+{
+
+	return (kern_aio_fsync(td, uap->op, (struct aiocb *)uap->aiocbp,
+	    &aiocb32_ops));
+}
+
+int
+freebsd32_olio_listio(struct thread *td, struct freebsd32_olio_listio_args *uap)
+{
+	struct aiocb **acb_list;
+	struct sigevent *sigp, sig;
+	struct osigevent32 osig;
+	uint32_t *acb_list32;
+	int error, i, nent;
+
+	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
+		return (EINVAL);
+
+	nent = uap->nent;
+	if (nent < 0 || nent > AIO_LISTIO_MAX)
+		return (EINVAL);
+
+	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
+		error = copyin(uap->sig, &osig, sizeof(osig));
+		if (error)
+			return (error);
+		error = convert_old_sigevent32(&osig, &sig);
+		if (error)
+			return (error);
+		sigp = &sig;
+	} else
+		sigp = NULL;
+
+	acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
+	error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
+	if (error) {
+		free(acb_list32, M_LIO);
+		return (error);
+	}
+	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
+	for (i = 0; i < nent; i++)
+		acb_list[i] = PTRIN(acb_list32[i]);
+	free(acb_list32, M_LIO);
+
+	error = kern_lio_listio(td, uap->mode,
+	    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
+	    &aiocb32_ops_osigevent);
+	free(acb_list, M_LIO);
+	return (error);
+}
+
+int
+freebsd32_lio_listio(struct thread *td, struct freebsd32_lio_listio_args *uap)
+{
+	struct aiocb **acb_list;
+	struct sigevent *sigp, sig;
+	struct sigevent32 sig32;
+	uint32_t *acb_list32;
+	int error, i, nent;
+
+	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
+		return (EINVAL);
+
+	nent = uap->nent;
+	if (nent < 0 || nent > AIO_LISTIO_MAX)
+		return (EINVAL);
+
+	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
+		error = copyin(uap->sig, &sig32, sizeof(sig32));
+		if (error)
+			return (error);
+		error = convert_sigevent32(&sig32, &sig);
+		if (error)
+			return (error);
+		sigp = &sig;
+	} else
+		sigp = NULL;
+
+	acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
+	error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
+	if (error) {
+		free(acb_list32, M_LIO);
+		return (error);
+	}
+	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
+	for (i = 0; i < nent; i++)
+		acb_list[i] = PTRIN(acb_list32[i]);
+	free(acb_list32, M_LIO);
+
+	error = kern_lio_listio(td, uap->mode,
+	    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
+	    &aiocb32_ops);
+	free(acb_list, M_LIO);
+	return (error);
+}
+
+SYSCALL32_MODULE_HELPER(freebsd32_aio_return);
+SYSCALL32_MODULE_HELPER(freebsd32_aio_suspend);
+SYSCALL32_MODULE_HELPER(freebsd32_aio_cancel);
+SYSCALL32_MODULE_HELPER(freebsd32_aio_error);
+SYSCALL32_MODULE_HELPER(freebsd32_aio_fsync);
+SYSCALL32_MODULE_HELPER(freebsd32_aio_read);
+SYSCALL32_MODULE_HELPER(freebsd32_aio_write);
+SYSCALL32_MODULE_HELPER(freebsd32_aio_waitcomplete);
+SYSCALL32_MODULE_HELPER(freebsd32_lio_listio);
+SYSCALL32_MODULE_HELPER(freebsd32_oaio_read);
+SYSCALL32_MODULE_HELPER(freebsd32_oaio_write);
+SYSCALL32_MODULE_HELPER(freebsd32_olio_listio);
+#endif