Refactor the AIO subsystem to permit file-type-specific handling and

improve cancellation robustness.

Introduce a new file operation, fo_aio_queue, which is responsible for
queueing and completing an asynchronous I/O request for a given file.
The AIO subystem now exports library of routines to manipulate AIO
requests as well as the ability to run a handler function in the
"default" pool of AIO daemons to service a request.

A default implementation for file types which do not include an
fo_aio_queue method queues requests to the "default" pool invoking the
fo_read or fo_write methods as before.

The AIO subsystem permits file types to install a private "cancel"
routine when a request is queued to permit safe dequeueing and cleanup
of cancelled requests.

Sockets now use their own pool of AIO daemons and service per-socket
requests in FIFO order.  Socket requests will not block indefinitely
permitting timely cancellation of all requests.

Due to the now-tight coupling of the AIO subsystem with file types,
the AIO subsystem is now a standard part of all kernels.  The VFS_AIO
kernel option and aio.ko module are gone.

Many file types may block indefinitely in their fo_read or fo_write
callbacks resulting in a hung AIO daemon.  This can result in hung
user processes (when processes attempt to cancel all outstanding
requests during exit) or a hung system.  To protect against this, AIO
requests are only permitted for known "safe" files by default.  AIO
requests for all file types can be enabled by setting the new
vfs.aio.enable_usafe sysctl to a non-zero value.  The AIO tests have
been updated to skip operations on unsafe file types if the sysctl is
zero.

Currently, AIO requests on sockets and raw disks are considered safe
and are enabled by default.  aio_mlock() is also enabled by default.

Reviewed by:	cem, jilles
Discussed with:	kib (earlier version)
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D5289

1 files changed, 110 insertions, 11 deletions

diff --git a/sys/sys/aio.h b/sys/sys/aio.h
index d6ef6aa..25ffb28 100644
--- a/sys/sys/aio.h
+++ b/sys/sys/aio.h
@@ -21,6 +21,11 @@
 
 #include <sys/types.h>
 #include <sys/signal.h>
+#ifdef _KERNEL
+#include <sys/queue.h>
+#include <sys/event.h>
+#include <sys/signalvar.h>
+#endif
 
 /*
  * Returned by aio_cancel:
@@ -51,6 +56,24 @@
  */
 #define	AIO_LISTIO_MAX		16
 
+#ifdef _KERNEL
+
+/* Default values of tunables for the AIO worker pool. */
+
+#ifndef MAX_AIO_PROCS
+#define MAX_AIO_PROCS		32
+#endif
+
+#ifndef TARGET_AIO_PROCS
+#define TARGET_AIO_PROCS	4
+#endif
+
+#ifndef AIOD_LIFETIME_DEFAULT
+#define AIOD_LIFETIME_DEFAULT	(30 * hz)
+#endif
+
+#endif
+
 /*
  * Private members for aiocb -- don't access
  * directly.
@@ -77,7 +100,91 @@ typedef struct aiocb {
 	struct	sigevent aio_sigevent;	/* Signal to deliver */
 } aiocb_t;
 
-#ifndef _KERNEL
+#ifdef _KERNEL
+
+typedef void aio_cancel_fn_t(struct kaiocb *);
+typedef void aio_handle_fn_t(struct kaiocb *);
+
+/*
+ * Kernel version of an I/O control block.
+ *
+ * Locking key:
+ * * - need not protected
+ * a - locked by kaioinfo lock
+ * b - locked by backend lock
+ * c - locked by aio_job_mtx
+ */
+struct kaiocb {
+	TAILQ_ENTRY(kaiocb) list;	/* (b) backend-specific list of jobs */
+	TAILQ_ENTRY(kaiocb) plist;	/* (a) lists of pending / done jobs */
+	TAILQ_ENTRY(kaiocb) allist;	/* (a) list of all jobs in proc */
+	int	jobflags;		/* (a) job flags */
+	int	inputcharge;		/* (*) input blocks */
+	int	outputcharge;		/* (*) output blocks */
+	struct	bio *bp;		/* (*) BIO backend BIO pointer */
+	struct	buf *pbuf;		/* (*) BIO backend buffer pointer */
+	struct	vm_page *pages[btoc(MAXPHYS)+1]; /* BIO backend pages */
+	int	npages;			/* BIO backend number of pages */
+	struct	proc *userproc;		/* (*) user process */
+	struct	ucred *cred;		/* (*) active credential when created */
+	struct	file *fd_file;		/* (*) pointer to file structure */
+	struct	aioliojob *lio;		/* (*) optional lio job */
+	struct	aiocb *ujob;		/* (*) pointer in userspace of aiocb */
+	struct	knlist klist;		/* (a) list of knotes */
+	struct	aiocb uaiocb;		/* (*) copy of user I/O control block */
+	ksiginfo_t ksi;			/* (a) realtime signal info */
+	uint64_t seqno;			/* (*) job number */
+	int	pending;		/* (a) number of pending I/O, aio_fsync only */
+	aio_cancel_fn_t *cancel_fn;	/* (a) backend cancel function */
+	aio_handle_fn_t *handle_fn;	/* (c) backend handle function */
+};
+
+struct socket;
+struct sockbuf;
+
+/*
+ * AIO backends should permit cancellation of queued requests waiting to
+ * be serviced by installing a cancel routine while the request is
+ * queued.  The cancellation routine should dequeue the request if
+ * necessary and cancel it.  Care must be used to handle races between
+ * queueing and dequeueing requests and cancellation.
+ *
+ * When queueing a request somewhere such that it can be cancelled, the
+ * caller should:
+ *
+ *  1) Acquire lock that protects the associated queue.
+ *  2) Call aio_set_cancel_function() to install the cancel routine.
+ *  3) If that fails, the request has a pending cancel and should be
+ *     cancelled via aio_cancel().
+ *  4) Queue the request.
+ *
+ * When dequeueing a request to service it or hand it off to somewhere else,
+ * the caller should:
+ *
+ *  1) Acquire the lock that protects the associated queue.
+ *  2) Dequeue the request.
+ *  3) Call aio_clear_cancel_function() to clear the cancel routine.
+ *  4) If that fails, the cancel routine is about to be called.  The
+ *     caller should ignore the request.
+ *
+ * The cancel routine should:
+ *
+ *  1) Acquire the lock that protects the associated queue.
+ *  2) Call aio_cancel_cleared() to determine if the request is already
+ *     dequeued due to a race with dequeueing thread.
+ *  3) If that fails, dequeue the request.
+ *  4) Cancel the request via aio_cancel().
+ */
+
+bool	aio_cancel_cleared(struct kaiocb *job);
+void	aio_cancel(struct kaiocb *job);
+bool	aio_clear_cancel_function(struct kaiocb *job);
+void	aio_complete(struct kaiocb *job, long status, int error);
+void	aio_schedule(struct kaiocb *job, aio_handle_fn_t *func);
+bool	aio_set_cancel_function(struct kaiocb *job, aio_cancel_fn_t *func);
+void	aio_switch_vmspace(struct kaiocb *job);
+
+#else /* !_KERNEL */
 
 struct timespec;
 
@@ -137,14 +244,6 @@ int	aio_waitcomplete(struct aiocb **, struct timespec *);
 int	aio_fsync(int op, struct aiocb *aiocbp);
 __END_DECLS
 
-#else
-
-/* Forward declarations for prototypes below. */
-struct socket;
-struct sockbuf;
-
-extern void (*aio_swake)(struct socket *, struct sockbuf *);
+#endif /* !_KERNEL */
 
-#endif
-
-#endif
+#endif /* !_SYS_AIO_H_ */