Make AIO a loadable module.

Remove the explicit call to aio_proc_rundown() from exit1(), instead AIO
will use at_exit(9).

Add functions at_exec(9), rm_at_exec(9) which function nearly the
same as at_exec(9) and rm_at_exec(9), these functions are called
on behalf of modules at the time of execve(2) after the image
activator has run.

Use a modified version of tegge's suggestion via at_exec(9) to close
an exploitable race in AIO.

Fix SYSCALL_MODULE_HELPER such that it's archetecuterally neutral,
the problem was that one had to pass it a paramater indicating the
number of arguments which were actually the number of "int".  Fix
it by using an inline version of the AS macro against the syscall
arguments.  (AS should be available globally but we'll get to that
later.)

Add a primative system for dynamically adding kqueue ops, it's really
not as sophisticated as it should be, but I'll discuss with jlemon when
he's around.

18 files changed, 242 insertions, 112 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 72d070d..d0ddd23 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -856,7 +856,7 @@ kern/uipc_socket.c	standard
 kern/uipc_socket2.c	standard
 kern/uipc_syscalls.c	standard
 kern/uipc_usrreq.c	standard
-kern/vfs_aio.c		standard
+kern/vfs_aio.c		optional vfs_aio
 kern/vfs_bio.c		standard
 kern/vfs_cache.c	standard
 kern/vfs_cluster.c	standard
diff --git a/sys/kern/init_sysent.c b/sys/kern/init_sysent.c
index 872fc11..82bb4b3 100644
--- a/sys/kern/init_sysent.c
+++ b/sys/kern/init_sysent.c
@@ -336,13 +336,13 @@ struct sysent sysent[] = {
 	{ SYF_MPSAFE | AS(setresuid_args), (sy_call_t *)setresuid },	/* 311 = setresuid */
 	{ SYF_MPSAFE | AS(setresgid_args), (sy_call_t *)setresgid },	/* 312 = setresgid */
 	{ 0, (sy_call_t *)nosys },			/* 313 = obsolete signanosleep */
-	{ AS(aio_return_args), (sy_call_t *)aio_return },	/* 314 = aio_return */
-	{ AS(aio_suspend_args), (sy_call_t *)aio_suspend },	/* 315 = aio_suspend */
-	{ AS(aio_cancel_args), (sy_call_t *)aio_cancel },	/* 316 = aio_cancel */
-	{ AS(aio_error_args), (sy_call_t *)aio_error },	/* 317 = aio_error */
-	{ AS(aio_read_args), (sy_call_t *)aio_read },	/* 318 = aio_read */
-	{ AS(aio_write_args), (sy_call_t *)aio_write },	/* 319 = aio_write */
-	{ AS(lio_listio_args), (sy_call_t *)lio_listio },	/* 320 = lio_listio */
+	{ AS(aio_return_args), (sy_call_t *)lkmressys },	/* 314 = aio_return */
+	{ AS(aio_suspend_args), (sy_call_t *)lkmressys },	/* 315 = aio_suspend */
+	{ AS(aio_cancel_args), (sy_call_t *)lkmressys },	/* 316 = aio_cancel */
+	{ AS(aio_error_args), (sy_call_t *)lkmressys },	/* 317 = aio_error */
+	{ AS(aio_read_args), (sy_call_t *)lkmressys },	/* 318 = aio_read */
+	{ AS(aio_write_args), (sy_call_t *)lkmressys },	/* 319 = aio_write */
+	{ AS(lio_listio_args), (sy_call_t *)lkmressys },	/* 320 = lio_listio */
 	{ SYF_MPSAFE | 0, (sy_call_t *)yield },		/* 321 = yield */
 	{ 0, (sy_call_t *)nosys },			/* 322 = obsolete thr_sleep */
 	{ 0, (sy_call_t *)nosys },			/* 323 = obsolete thr_wakeup */
@@ -381,7 +381,7 @@ struct sysent sysent[] = {
 	{ AS(extattr_set_file_args), (sy_call_t *)extattr_set_file },	/* 356 = extattr_set_file */
 	{ AS(extattr_get_file_args), (sy_call_t *)extattr_get_file },	/* 357 = extattr_get_file */
 	{ AS(extattr_delete_file_args), (sy_call_t *)extattr_delete_file },	/* 358 = extattr_delete_file */
-	{ AS(aio_waitcomplete_args), (sy_call_t *)aio_waitcomplete },	/* 359 = aio_waitcomplete */
+	{ AS(aio_waitcomplete_args), (sy_call_t *)lkmressys },	/* 359 = aio_waitcomplete */
 	{ SYF_MPSAFE | AS(getresuid_args), (sy_call_t *)getresuid },	/* 360 = getresuid */
 	{ SYF_MPSAFE | AS(getresgid_args), (sy_call_t *)getresgid },	/* 361 = getresgid */
 	{ SYF_MPSAFE | 0, (sy_call_t *)kqueue },	/* 362 = kqueue */
diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c
index 5b4445c..6bec056 100644
--- a/sys/kern/kern_event.c
+++ b/sys/kern/kern_event.c
@@ -122,7 +122,16 @@ SYSCTL_INT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
 #define	KN_HASHSIZE		64		/* XXX should be tunable */
 #define KN_HASH(val, mask)	(((val) ^ (val >> 8)) & (mask))
 
-extern struct filterops aio_filtops;
+static int
+filt_nullattach(struct knote *kn)
+{
+
+	return (ENXIO);
+};
+
+struct filterops null_filtops =
+	{ 0, filt_nullattach, NULL, NULL };
+
 extern struct filterops sig_filtops;
 
 /*
@@ -131,7 +140,7 @@ extern struct filterops sig_filtops;
 static struct filterops *sysfilt_ops[] = {
 	&file_filtops,			/* EVFILT_READ */
 	&file_filtops,			/* EVFILT_WRITE */
-	&aio_filtops,			/* EVFILT_AIO */
+	&null_filtops,			/* EVFILT_AIO */
 	&file_filtops,			/* EVFILT_VNODE */
 	&proc_filtops,			/* EVFILT_PROC */
 	&sig_filtops,			/* EVFILT_SIGNAL */
@@ -460,6 +469,36 @@ done:
 }
 
 int
+kqueue_add_filteropts(int filt, struct filterops *filtops)
+{
+
+	if (filt > 0)
+		panic("filt(%d) > 0", filt);
+	if (filt + EVFILT_SYSCOUNT < 0)
+		panic("filt(%d) + EVFILT_SYSCOUNT(%d) == %d < 0",
+		    filt, EVFILT_SYSCOUNT, filt + EVFILT_SYSCOUNT);
+	if (sysfilt_ops[~filt] != &null_filtops)
+		panic("sysfilt_ops[~filt(%d)] != &null_filtops", filt);
+	sysfilt_ops[~filt] = filtops;
+	return (0);
+}
+
+int
+kqueue_del_filteropts(int filt)
+{
+
+	if (filt > 0)
+		panic("filt(%d) > 0", filt);
+	if (filt + EVFILT_SYSCOUNT < 0)
+		panic("filt(%d) + EVFILT_SYSCOUNT(%d) == %d < 0",
+		    filt, EVFILT_SYSCOUNT, filt + EVFILT_SYSCOUNT);
+	if (sysfilt_ops[~filt] == &null_filtops)
+		panic("sysfilt_ops[~filt(%d)] != &null_filtops", filt);
+	sysfilt_ops[~filt] = &null_filtops;
+	return (0);
+}
+
+int
 kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td)
 {
 	struct filedesc *fdp = kq->kq_fdp;
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 41ff7d8..670d5dd 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -65,6 +65,19 @@
 
 MALLOC_DEFINE(M_PARGS, "proc-args", "Process arguments");
 
+static MALLOC_DEFINE(M_ATEXEC, "atexec", "atexec callback");
+
+/*
+ * callout list for things to do at exec time
+ */
+struct execlist {
+	execlist_fn function;
+	TAILQ_ENTRY(execlist) next;
+};
+
+TAILQ_HEAD(exec_list_head, execlist);
+static struct exec_list_head exec_list = TAILQ_HEAD_INITIALIZER(exec_list);
+
 static register_t *exec_copyout_strings __P((struct image_params *));
 
 /* XXX This should be vm_size_t. */
@@ -115,6 +128,7 @@ execve(td, uap)
 	struct vattr attr;
 	int (*img_first) __P((struct image_params *));
 	struct pargs *pa;
+	struct execlist *ep;
 
 	imgp = &image_params;
 
@@ -244,6 +258,9 @@ interpret:
 		goto interpret;
 	}
 
+	TAILQ_FOREACH(ep, &exec_list, next)
+		(*ep->function)(p);
+
 	/*
 	 * Copy out strings (args and env) and initialize stack base
 	 */
@@ -926,3 +943,44 @@ exec_unregister(execsw_arg)
 	execsw = newexecsw;
 	return 0;
 }
+
+int
+at_exec(function)
+	execlist_fn function;
+{
+	struct execlist *ep;
+
+#ifdef INVARIANTS
+	/* Be noisy if the programmer has lost track of things */
+	if (rm_at_exec(function)) 
+		printf("WARNING: exec callout entry (%p) already present\n",
+		    function);
+#endif
+	ep = malloc(sizeof(*ep), M_ATEXEC, M_NOWAIT);
+	if (ep == NULL)
+		return (ENOMEM);
+	ep->function = function;
+	TAILQ_INSERT_TAIL(&exec_list, ep, next);
+	return (0);
+}
+
+/*
+ * Scan the exec callout list for the given item and remove it.
+ * Returns the number of items removed (0 or 1)
+ */
+int
+rm_at_exec(function)
+	execlist_fn function;
+{
+	struct execlist *ep;
+
+	TAILQ_FOREACH(ep, &exec_list, next) {
+		if (ep->function == function) {
+			TAILQ_REMOVE(&exec_list, ep, next);
+			free(ep, M_ATEXEC);
+			return(1);
+		}
+	}	
+	return (0);
+}
+
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index e13ac15..c41fec6 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -63,7 +63,6 @@
 #include <sys/filedesc.h>
 #include <sys/shm.h>
 #include <sys/sem.h>
-#include <sys/aio.h>
 #include <sys/jail.h>
 
 #include <vm/vm.h>
@@ -140,8 +139,6 @@ exit1(td, rv)
 /* XXXXKSE */
 /* MUST abort all other threads before proceeding past this point */
 
-	aio_proc_rundown(p);
-
 	/* are we a task leader? */
 	PROC_LOCK(p);
 	if(p == p->p_leader) {
diff --git a/sys/kern/syscalls.master b/sys/kern/syscalls.master
index 050adb3..7eb38c8 100644
--- a/sys/kern/syscalls.master
+++ b/sys/kern/syscalls.master
@@ -454,13 +454,13 @@
 311	MSTD	BSD	{ int setresuid(uid_t ruid, uid_t euid, uid_t suid); }
 312	MSTD	BSD	{ int setresgid(gid_t rgid, gid_t egid, gid_t sgid); }
 313	OBSOL	NOHIDE	signanosleep
-314     STD     BSD     { int aio_return(struct aiocb *aiocbp); }
-315     STD     BSD     { int aio_suspend(struct aiocb * const * aiocbp, int nent, const struct timespec *timeout); }
-316     STD     BSD     { int aio_cancel(int fd, struct aiocb *aiocbp); }
-317     STD     BSD     { int aio_error(struct aiocb *aiocbp); }
-318     STD     BSD     { int aio_read(struct aiocb *aiocbp); }
-319     STD     BSD     { int aio_write(struct aiocb *aiocbp); }
-320     STD     BSD     { int lio_listio(int mode, struct aiocb * const *acb_list, int nent, struct sigevent *sig); }
+314     NOSTD   BSD     { int aio_return(struct aiocb *aiocbp); }
+315     NOSTD   BSD     { int aio_suspend(struct aiocb * const * aiocbp, int nent, const struct timespec *timeout); }
+316     NOSTD   BSD     { int aio_cancel(int fd, struct aiocb *aiocbp); }
+317     NOSTD   BSD     { int aio_error(struct aiocb *aiocbp); }
+318     NOSTD   BSD     { int aio_read(struct aiocb *aiocbp); }
+319     NOSTD   BSD     { int aio_write(struct aiocb *aiocbp); }
+320     NOSTD   BSD     { int lio_listio(int mode, struct aiocb * const *acb_list, int nent, struct sigevent *sig); }
 321     MSTD	BSD     { int yield(void); }
 322	OBSOL	NOHIDE	thr_sleep
 323	OBSOL	NOHIDE	thr_wakeup
@@ -519,7 +519,7 @@
 			    struct iovec *iovp, unsigned iovcnt); }
 358	STD	BSD	{ int extattr_delete_file(const char *path, \
 			    int attrnamespace, const char *attrname); }
-359	STD	BSD	{ int aio_waitcomplete(struct aiocb **aiocbp, struct timespec *timeout); }
+359	NOSTD	BSD	{ int aio_waitcomplete(struct aiocb **aiocbp, struct timespec *timeout); }
 360	MSTD	BSD	{ int getresuid(uid_t *ruid, uid_t *euid, uid_t *suid); }
 361	MSTD	BSD	{ int getresgid(gid_t *rgid, gid_t *egid, gid_t *sgid); }
 362	MSTD	BSD	{ int kqueue(void); }
diff --git a/sys/kern/sysv_msg.c b/sys/kern/sysv_msg.c
index a8262f7..2c176dd 100644
--- a/sys/kern/sysv_msg.c
+++ b/sys/kern/sysv_msg.c
@@ -258,11 +258,11 @@ static moduledata_t sysvmsg_mod = {
 	NULL
 };
 
-SYSCALL_MODULE_HELPER(msgsys, 6);
-SYSCALL_MODULE_HELPER(msgctl, 3);
-SYSCALL_MODULE_HELPER(msgget, 2);
-SYSCALL_MODULE_HELPER(msgsnd, 4);
-SYSCALL_MODULE_HELPER(msgrcv, 5);
+SYSCALL_MODULE_HELPER(msgsys);
+SYSCALL_MODULE_HELPER(msgctl);
+SYSCALL_MODULE_HELPER(msgget);
+SYSCALL_MODULE_HELPER(msgsnd);
+SYSCALL_MODULE_HELPER(msgrcv);
 
 DECLARE_MODULE(sysvmsg, sysvmsg_mod,
 	SI_SUB_SYSV_MSG, SI_ORDER_FIRST);
diff --git a/sys/kern/sysv_sem.c b/sys/kern/sysv_sem.c
index fef5756..ed99f57 100644
--- a/sys/kern/sysv_sem.c
+++ b/sys/kern/sysv_sem.c
@@ -246,10 +246,10 @@ static moduledata_t sysvsem_mod = {
 	NULL
 };
 
-SYSCALL_MODULE_HELPER(semsys, 5);
-SYSCALL_MODULE_HELPER(__semctl, 4);
-SYSCALL_MODULE_HELPER(semget, 3);
-SYSCALL_MODULE_HELPER(semop, 3);
+SYSCALL_MODULE_HELPER(semsys);
+SYSCALL_MODULE_HELPER(__semctl);
+SYSCALL_MODULE_HELPER(semget);
+SYSCALL_MODULE_HELPER(semop);
 
 DECLARE_MODULE(sysvsem, sysvsem_mod,
 	SI_SUB_SYSV_SEM, SI_ORDER_FIRST);
diff --git a/sys/kern/sysv_shm.c b/sys/kern/sysv_shm.c
index dc4fa3b..0ec9ad4 100644
--- a/sys/kern/sysv_shm.c
+++ b/sys/kern/sysv_shm.c
@@ -894,11 +894,11 @@ static moduledata_t sysvshm_mod = {
 	NULL
 };
 
-SYSCALL_MODULE_HELPER(shmsys, 4);
-SYSCALL_MODULE_HELPER(shmat, 3);
-SYSCALL_MODULE_HELPER(shmctl, 3);
-SYSCALL_MODULE_HELPER(shmdt, 1);
-SYSCALL_MODULE_HELPER(shmget, 3);
+SYSCALL_MODULE_HELPER(shmsys);
+SYSCALL_MODULE_HELPER(shmat);
+SYSCALL_MODULE_HELPER(shmctl);
+SYSCALL_MODULE_HELPER(shmdt);
+SYSCALL_MODULE_HELPER(shmget);
 
 DECLARE_MODULE(sysvshm, sysvshm_mod,
 	SI_SUB_SYSV_SHM, SI_ORDER_FIRST);
diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index 0c379d9..dc1f4cb 100644
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -57,6 +57,8 @@
 
 int	maxsockets;
 
+void (*aio_swake)(struct socket *, struct sockbuf *);
+
 /*
  * Primitive routines for operating on sockets and socket buffers
  */
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index 0c379d9..dc1f4cb 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -57,6 +57,8 @@
 
 int	maxsockets;
 
+void (*aio_swake)(struct socket *, struct sockbuf *);
+
 /*
  * Primitive routines for operating on sockets and socket buffers
  */
diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index 7398d54..ec4eef7 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -38,6 +38,8 @@
 #include <sys/signalvar.h>
 #include <sys/protosw.h>
 #include <sys/socketvar.h>
+#include <sys/syscall.h>
+#include <sys/sysent.h>
 #include <sys/sysctl.h>
 #include <sys/vnode.h>
 #include <sys/conf.h>
@@ -54,8 +56,6 @@
 
 #include "opt_vfs_aio.h"
 
-#ifdef VFS_AIO
-
 static	long jobrefid;
 
 #define JOBST_NULL		0x0
@@ -107,6 +107,7 @@ static int num_buf_aio = 0;
 static int num_aio_resv_start = 0;
 static int aiod_timeout;
 static int aiod_lifetime;
+static int unloadable = 0;
 
 static int max_aio_per_proc = MAX_AIO_PER_PROC;
 static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
@@ -147,6 +148,9 @@ SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime,
 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout,
 	CTLFLAG_RW, &aiod_timeout, 0, "");
 
+SYSCTL_INT(_vfs_aio, OID_AUTO, unloadable, CTLFLAG_RW, &unloadable, 0,
+    "Allow unload of aio (not recommended)");
+
 /*
  * AIO process info
  */
@@ -206,28 +210,80 @@ static TAILQ_HEAD(,aiocblist) aio_jobs;			/* Async job list */
 static TAILQ_HEAD(,aiocblist) aio_bufjobs;		/* Phys I/O job list */
 
 static void	aio_init_aioinfo(struct proc *p);
-static void	aio_onceonly(void *);
+static void	aio_onceonly(void);
 static int	aio_free_entry(struct aiocblist *aiocbe);
 static void	aio_process(struct aiocblist *aiocbe);
 static int	aio_newproc(void);
 static int	aio_aqueue(struct thread *td, struct aiocb *job, int type);
 static void	aio_physwakeup(struct buf *bp);
+static void	aio_proc_rundown(struct proc *p);
 static int	aio_fphysio(struct proc *p, struct aiocblist *aiocbe);
 static int	aio_qphysio(struct proc *p, struct aiocblist *iocb);
 static void	aio_daemon(void *uproc);
+static int	aio_unload(void);
 static void	process_signal(void *aioj);
-
-SYSINIT(aio, SI_SUB_VFS, SI_ORDER_ANY, aio_onceonly, NULL);
+static int	filt_aioattach(struct knote *kn);
+static void	filt_aiodetach(struct knote *kn);
+static int	filt_aio(struct knote *kn, long hint);
 
 static vm_zone_t kaio_zone = 0, aiop_zone = 0, aiocb_zone = 0, aiol_zone = 0;
 static vm_zone_t aiolio_zone = 0;
 
+static struct filterops aio_filtops =
+	{ 0, filt_aioattach, filt_aiodetach, filt_aio };
+
+static int
+aio_modload(struct module *module, int cmd, void *arg)
+{
+	int error = 0;
+
+	switch (cmd) {
+	case MOD_LOAD:
+		aio_onceonly();
+		break;
+	case MOD_UNLOAD:
+		error = aio_unload();
+		break;
+	case MOD_SHUTDOWN:
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+	return (error);
+}
+
+static moduledata_t aio_mod = {
+	"aio",
+	&aio_modload,
+	NULL
+};
+
+SYSCALL_MODULE_HELPER(aio_return);
+SYSCALL_MODULE_HELPER(aio_suspend);
+SYSCALL_MODULE_HELPER(aio_cancel);
+SYSCALL_MODULE_HELPER(aio_error);
+SYSCALL_MODULE_HELPER(aio_read);
+SYSCALL_MODULE_HELPER(aio_write);
+SYSCALL_MODULE_HELPER(aio_waitcomplete);
+SYSCALL_MODULE_HELPER(lio_listio);
+
+DECLARE_MODULE(aio, aio_mod,
+	SI_SUB_VFS, SI_ORDER_ANY);
+MODULE_VERSION(aio, 1);
+
 /*
  * Startup initialization
  */
 static void
-aio_onceonly(void *na)
+aio_onceonly(void)
 {
+
+	/* XXX: should probably just use so->callback */
+	aio_swake = &aio_swake_cb;
+	at_exit(aio_proc_rundown);
+	at_exec(aio_proc_rundown);
+	kqueue_add_filteropts(EVFILT_AIO, &aio_filtops);
 	TAILQ_INIT(&aio_freeproc);
 	TAILQ_INIT(&aio_activeproc);
 	TAILQ_INIT(&aio_jobs);
@@ -243,6 +299,25 @@ aio_onceonly(void *na)
 	jobrefid = 1;
 }
 
+static int
+aio_unload(void)
+{
+
+	/*
+	 * XXX: no unloads by default, it's too dangerous.
+	 * perhaps we could do it if locked out callers and then
+	 * did an aio_proc_rundown() on each process.
+	 */
+	if (!unloadable)
+		return (EOPNOTSUPP);
+
+	aio_swake = NULL;
+	rm_at_exit(aio_proc_rundown);
+	rm_at_exec(aio_proc_rundown);
+	kqueue_del_filteropts(EVFILT_AIO);
+	return (0);
+}
+
 /*
  * Init the per-process aioinfo structure.  The aioinfo limits are set
  * per-process for user limit (resource) management.
@@ -384,17 +459,13 @@ aio_free_entry(struct aiocblist *aiocbe)
 	zfree(aiocb_zone, aiocbe);
 	return 0;
 }
-#endif /* VFS_AIO */
 
 /*
  * Rundown the jobs for a given process.  
  */
-void
+static void
 aio_proc_rundown(struct proc *p)
 {
-#ifndef VFS_AIO
-	return;
-#else
 	int s;
 	struct kaioinfo *ki;
 	struct aio_liojob *lj, *ljn;
@@ -519,10 +590,8 @@ restart4:
 
 	zfree(kaio_zone, ki);
 	p->p_aioinfo = NULL;
-#endif /* VFS_AIO */
 }
 
-#ifdef VFS_AIO
 /*
  * Select a job to run (called by an AIO daemon).
  */
@@ -1150,17 +1219,13 @@ aio_fphysio(struct proc *p, struct aiocblist *iocb)
 	relpbuf(bp, NULL);
 	return (error);
 }
-#endif /* VFS_AIO */
 
 /*
  * Wake up aio requests that may be serviceable now.
  */
 void
-aio_swake(struct socket *so, struct sockbuf *sb)
+aio_swake_cb(struct socket *so, struct sockbuf *sb)
 {
-#ifndef VFS_AIO
-	return;
-#else
 	struct aiocblist *cb,*cbn;
 	struct proc *p;
 	struct kaioinfo *ki = NULL;
@@ -1198,10 +1263,8 @@ aio_swake(struct socket *so, struct sockbuf *sb)
 			wakeup(aiop->aiothread);
 		}
 	}
-#endif /* VFS_AIO */
 }
 
-#ifdef VFS_AIO
 /*
  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
  * technique is done in this code.
@@ -1474,7 +1537,6 @@ aio_aqueue(struct thread *td, struct aiocb *job, int type)
 
 	return _aio_aqueue(td, job, NULL, type);
 }
-#endif /* VFS_AIO */
 
 /*
  * Support the aio_return system call, as a side-effect, kernel resources are
@@ -1483,9 +1545,6 @@ aio_aqueue(struct thread *td, struct aiocb *job, int type)
 int
 aio_return(struct thread *td, struct aio_return_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
 	struct proc *p = td->td_proc;
 	int s;
 	int jobref;
@@ -1547,7 +1606,6 @@ aio_return(struct thread *td, struct aio_return_args *uap)
 	splx(s);
 
 	return (EINVAL);
-#endif /* VFS_AIO */
 }
 
 /*
@@ -1556,9 +1614,6 @@ aio_return(struct thread *td, struct aio_return_args *uap)
 int
 aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
 	struct proc *p = td->td_proc;
 	struct timeval atv;
 	struct timespec ts;
@@ -1664,7 +1719,6 @@ aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 
 /* NOTREACHED */
 	return EINVAL;
-#endif /* VFS_AIO */
 }
 
 /*
@@ -1674,9 +1728,6 @@ aio_suspend(struct thread *td, struct aio_suspend_args *uap)
 int
 aio_cancel(struct thread *td, struct aio_cancel_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
 	struct proc *p = td->td_proc;
 	struct kaioinfo *ki;
 	struct aiocblist *cbe, *cbn;
@@ -1796,7 +1847,6 @@ aio_cancel(struct thread *td, struct aio_cancel_args *uap)
 	td->td_retval[0] = AIO_ALLDONE;
 
 	return 0;
-#endif /* VFS_AIO */
 }
 
 /*
@@ -1807,9 +1857,6 @@ aio_cancel(struct thread *td, struct aio_cancel_args *uap)
 int
 aio_error(struct thread *td, struct aio_error_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
 	struct proc *p = td->td_proc;
 	int s;
 	struct aiocblist *cb;
@@ -1887,35 +1934,25 @@ aio_error(struct thread *td, struct aio_error_args *uap)
 		return fuword(&uap->aiocbp->_aiocb_private.error);
 #endif
 	return EINVAL;
-#endif /* VFS_AIO */
 }
 
 int
 aio_read(struct thread *td, struct aio_read_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
+
 	return aio_aqueue(td, uap->aiocbp, LIO_READ);
-#endif /* VFS_AIO */
 }
 
 int
 aio_write(struct thread *td, struct aio_write_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
+
 	return aio_aqueue(td, uap->aiocbp, LIO_WRITE);
-#endif /* VFS_AIO */
 }
 
 int
 lio_listio(struct thread *td, struct lio_listio_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
 	struct proc *p = td->td_proc;
 	int nent, nentqueued;
 	struct aiocb *iocb, * const *cbptr;
@@ -2080,10 +2117,8 @@ lio_listio(struct thread *td, struct lio_listio_args *uap)
 	}
 
 	return runningcode;
-#endif /* VFS_AIO */
 }
 
-#ifdef VFS_AIO
 /*
  * This is a weird hack so that we can post a signal.  It is safe to do so from
  * a timeout routine, but *not* from an interrupt routine.
@@ -2179,14 +2214,10 @@ aio_physwakeup(struct buf *bp)
 				timeout(process_signal, aiocbe, 0);
 	}
 }
-#endif /* VFS_AIO */
 
 int
 aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 {
-#ifndef VFS_AIO
-	return ENOSYS;
-#else
 	struct proc *p = td->td_proc;
 	struct timeval atv;
 	struct timespec ts;
@@ -2258,22 +2289,8 @@ aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
 		else if (error == EWOULDBLOCK)
 			return EAGAIN;
 	}
-#endif /* VFS_AIO */
 }
 
-
-#ifndef VFS_AIO
-static int
-filt_aioattach(struct knote *kn)
-{
-
-	return (ENXIO);
-}
-
-struct filterops aio_filtops =
-	{ 0, filt_aioattach, NULL, NULL };
-
-#else
 static int
 filt_aioattach(struct knote *kn)
 {
@@ -2314,7 +2331,3 @@ filt_aio(struct knote *kn, long hint)
 	kn->kn_flags |= EV_EOF; 
 	return (1);
 }
-
-struct filterops aio_filtops =
-	{ 0, filt_aioattach, filt_aiodetach, filt_aio };
-#endif /* VFS_AIO */
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index 3ab64c5..1d558b3 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -9,6 +9,7 @@ SUBDIR=	3dfx \
 	accf_http \
 	agp \
 	aha \
+	aio \
 	amr \
 	an \
 	aue \
diff --git a/sys/modules/aio/Makefile b/sys/modules/aio/Makefile
new file mode 100644
index 0000000..4136135
--- /dev/null
+++ b/sys/modules/aio/Makefile
@@ -0,0 +1,8 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../kern
+
+KMOD=	aio
+SRCS=	vfs_aio.c opt_vfs_aio.h vnode_if.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/sys/aio.h b/sys/sys/aio.h
index 7dc5f05..b0c56b2 100644
--- a/sys/sys/aio.h
+++ b/sys/sys/aio.h
@@ -153,8 +153,8 @@ struct aiocblist {
 struct socket;
 struct sockbuf;
 
-void	aio_proc_rundown(struct proc *p);
-void	aio_swake(struct socket *, struct sockbuf *);
+void	aio_swake_cb(struct socket *, struct sockbuf *);
+extern void (*aio_swake)(struct socket *, struct sockbuf *);
 
 #endif
 
diff --git a/sys/sys/event.h b/sys/sys/event.h
index 806ab70..72035fb 100644
--- a/sys/sys/event.h
+++ b/sys/sys/event.h
@@ -177,6 +177,8 @@ extern void	knote_remove(struct thread *p, struct klist *list);
 extern void	knote_fdclose(struct thread *p, int fd);
 extern int 	kqueue_register(struct kqueue *kq,
 		    struct kevent *kev, struct thread *p);
+extern int	kqueue_add_filteropts(int filt, struct filterops *filtops);
+extern int	kqueue_del_filteropts(int filt);
 
 #else 	/* !_KERNEL */
 
diff --git a/sys/sys/sysent.h b/sys/sys/sysent.h
index c74ee41..e97ad64 100644
--- a/sys/sys/sysent.h
+++ b/sys/sys/sysent.h
@@ -112,10 +112,12 @@ static moduledata_t name##_mod = {                             \
 };                                                             \
 DECLARE_MODULE(name, name##_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE)
 
-#define SYSCALL_MODULE_HELPER(syscallname, argcount)    \
+#define SYSCALL_MODULE_HELPER(syscallname)              \
 static int syscallname##_syscall = SYS_##syscallname;   \
 static struct sysent syscallname##_sysent = {           \
-    argcount, (sy_call_t *)& syscallname                \
+    (sizeof(struct syscallname ## _args )               \
+     / sizeof(register_t)),                             \
+    (sy_call_t *)& syscallname                          \
 };                                                      \
 SYSCALL_MODULE(syscallname,                             \
     & syscallname##_syscall, & syscallname##_sysent,    \
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 70e99bd..37ace10 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -266,6 +266,12 @@ typedef void (*forklist_fn) __P((struct proc *parent, struct proc *child,
 int	at_fork __P((forklist_fn function));
 int	rm_at_fork __P((forklist_fn function));
 
+/* Exec callout list declarations. */
+typedef void (*execlist_fn) __P((struct proc *procp));
+
+int	at_exec __P((execlist_fn function));
+int	rm_at_exec __P((execlist_fn function));
+
 /*
  * Not exactly a callout LIST, but a callout entry.
  * Allow an external module to define a hardware watchdog tickler.