Bring in geom_sched, support for scheduling disk I/O requests

in a device independent manner. Also include an example anticipatory
scheduler, gsched_rr, which gives very nice performance improvements
in presence of competing random access patterns.

This is joint work with Fabio Checconi, developed last year
and presented at BSDCan 2009. You can find details in the
README file or at

http://info.iet.unipi.it/~luigi/geom_sched/

4 files changed, 304 insertions, 0 deletions

diff --git a/sbin/geom/class/Makefile b/sbin/geom/class/Makefile
index 591f79f..0611cdd 100644
--- a/sbin/geom/class/Makefile
+++ b/sbin/geom/class/Makefile
@@ -15,6 +15,7 @@ SUBDIR+=multipath
 SUBDIR+=nop
 SUBDIR+=part
 SUBDIR+=raid3
+SUBDIR+=sched
 SUBDIR+=shsec
 SUBDIR+=stripe
 SUBDIR+=virstor
diff --git a/sbin/geom/class/sched/Makefile b/sbin/geom/class/sched/Makefile
new file mode 100644
index 0000000..e58e221
--- /dev/null
+++ b/sbin/geom/class/sched/Makefile
@@ -0,0 +1,19 @@
+# GEOM_LIBRARY_PATH
+# $FreeBSD$
+
+.PATH: /usr/src/sbin/geom/misc
+
+CFLAGS += -I/usr/src/sbin/geom
+
+CLASS=sched
+
+WARNS?= 6
+CLASS_DIR?=/lib/geom
+
+SHLIBDIR?=${CLASS_DIR}
+SHLIB_NAME?=geom_${CLASS}.so
+LINKS=  ${BINDIR}/geom ${BINDIR}/g${CLASS}
+MAN=    g${CLASS}.8
+SRCS+=  geom_${CLASS}.c subr.c
+
+.include <bsd.lib.mk>
diff --git a/sbin/geom/class/sched/geom_sched.c b/sbin/geom/class/sched/geom_sched.c
new file mode 100644
index 0000000..4a38347
--- /dev/null
+++ b/sbin/geom/class/sched/geom_sched.c
@@ -0,0 +1,123 @@
+/*-
+ * Copyright (c) 2009 Fabio Checconi, Luigi Rizzo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $Id$
+ * $FreeBSD$
+ *
+ * This file implements the userspace library used by the 'geom'
+ * command to load and manipulate disk schedulers.
+ */
+  
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/linker.h>
+#include <sys/module.h>
+
+#include <stdio.h>
+#include <stdint.h>
+#include <libgeom.h>
+
+#include "core/geom.h"
+#include "misc/subr.h"
+
+#define	G_SCHED_VERSION	0
+
+uint32_t lib_version = G_LIB_VERSION;
+uint32_t version = G_SCHED_VERSION;
+
+/*
+ * storage for parameters used by this geom class.
+ * Right now only the scheduler name is used.
+ */
+static char algo[] = "rr";	/* default scheduler */
+
+/*
+ * Adapt to differences in geom library.
+ * in V1 struct g_command misses gc_argname, eld, and G_BOOL is undefined
+ */
+#if G_LIB_VERSION == 1
+#define G_ARGNAME
+#define G_TYPE_BOOL	G_TYPE_NUMBER
+#else
+#define G_ARGNAME	NULL,
+#endif
+
+static void
+gcmd_createinsert(struct gctl_req *req, unsigned flags __unused)
+{
+	const char *reqalgo;
+	char name[64];
+
+	if (gctl_has_param(req, "algo"))
+		reqalgo = gctl_get_ascii(req, "algo");
+	else
+		reqalgo = algo;
+
+	snprintf(name, sizeof(name), "gsched_%s", reqalgo);
+	/*
+	 * Do not complain about errors here, gctl_issue()
+	 * will fail anyway.
+	 */
+	if (modfind(name) < 0)
+		kldload(name);
+	gctl_issue(req);
+}
+
+struct g_command class_commands[] = {
+	{ "create", G_FLAG_VERBOSE | G_FLAG_LOADKLD, gcmd_createinsert,
+	    {
+		{ 'a', "algo", algo, G_TYPE_STRING },
+		G_OPT_SENTINEL
+	    },
+	    G_ARGNAME "[-v] [-a algorithm_name] dev ..."
+	},
+	{ "insert", G_FLAG_VERBOSE | G_FLAG_LOADKLD, gcmd_createinsert,
+	    {
+		{ 'a', "algo", algo, G_TYPE_STRING },
+		G_OPT_SENTINEL
+	    },
+	    G_ARGNAME "[-v] [-a algorithm_name] dev ..."
+	},
+	{ "configure", G_FLAG_VERBOSE, NULL,
+	    {
+		{ 'a', "algo", algo, G_TYPE_STRING },
+		G_OPT_SENTINEL
+	    },
+	    G_ARGNAME "[-v] [-a algorithm_name] prov ..."
+	},
+	{ "destroy", G_FLAG_VERBOSE, NULL,
+	    {
+		{ 'f', "force", NULL, G_TYPE_BOOL },
+		G_OPT_SENTINEL
+	    },
+	    G_ARGNAME "[-fv] prov ..."
+	},
+	{ "reset", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+	    G_ARGNAME "[-v] prov ..."
+	},
+	G_CMD_SENTINEL
+};
diff --git a/sbin/geom/class/sched/gsched.8 b/sbin/geom/class/sched/gsched.8
new file mode 100644
index 0000000..bde84d0
--- /dev/null
+++ b/sbin/geom/class/sched/gsched.8
@@ -0,0 +1,161 @@
+.\" Copyright (c) 2009-2010 Fabio Checconi, Luigi Rizzo
+.\" All rights reserved.
+.\" $FreeBSD$
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd April 12, 2010
+.Dt GSCHED 8
+.Os
+.Sh NAME
+.Nm gsched
+.Nd "control utility for disk scheduler GEOM class"
+.Sh SYNOPSIS
+.Nm
+.Cm create
+.Op Fl v
+.Op Fl a Ar algorithm
+.Ar provider ...
+.Nm
+.Cm insert
+.Op Fl v
+.Op Fl a Ar algorithm
+.Ar provider ...
+.Nm
+.Cm configure
+.Op Fl v
+.Op Fl a Ar algorithm
+.Ar node ...
+.Nm
+.Cm destroy
+.Op Fl fv
+.Ar node ...
+.Nm
+.Cm reset
+.Op Fl v
+.Ar node ...
+.Nm
+.Cm { list | status | load | unload }
+.Sh DESCRIPTION
+The
+.Nm
+utility (also callable as
+.Nm geom sched ... )
+changes the scheduling policy of the requests going to a provider.
+.Pp
+The first argument to
+.Nm
+indicates an action to be performed:
+.Bl -tag -width ".Cm configure"
+.It Cm create
+Create a new provider and geom node using the specified scheduling algorithm.
+.Ar algorithm
+is the name of the scheduling algorithm used for the provider.
+Available algorithms include:
+.Ar rr ,
+which implements anticipatory scheduling with round robin service
+among clients;
+.Ar as ,
+which implements a simple form of anticipatory scheduling with
+no per-client queue.
+.Pp
+If the operation succeeds, the new provider should appear with name
+.Pa /dev/ Ns Ao Ar dev Ac Ns Pa .sched. .
+The kernel module
+.Pa geom_sched.ko
+will be loaded if it is not loaded already.
+.It Cm insert
+Operates as "create", but the insertion is "transparent",
+i.e. the existing provider is rerouted to the newly created geom,
+which in turn forwards requests to the existing geom.
+This operation allows one to start/stop a scheduling service
+on an already existing provider.
+.Pp
+A subsequent 'destroy' will remove the newly created geom and
+hook the provider back to the original geom.
+.Ar algorithm
+.It Cm configure
+Configure existing scheduling provider.  It supports the same options
+as the 
+.Nm create
+command.
+.It Cm destroy
+Destroy the geom specified in the parameter.
+.It Cm reset
+Do nothing.
+.It Cm list | status | load | unload
+See
+.Xr geom 8 .
+.El
+.Pp
+Additional options:
+.Bl -tag -width ".Fl f"
+.It Fl f
+Force the removal of the specified provider.
+.It Fl v
+Be more verbose.
+.El
+.Sh SYSCTL VARIABLES
+The following
+.Xr sysctl 8
+variables can be used to control the behavior of the
+.Nm SCHED
+GEOM class.
+The default value is shown next to each variable.
+.Bl -tag -width indent
+.It Va kern.geom.sched.debug : No 0
+Debug level of the
+.Nm SCHED
+GEOM class.
+This can be set to a number between 0 and 2 inclusive.
+If set to 0 minimal debug information is printed, and if set to 2 the
+maximum amount of debug information is printed.
+.El
+.Sh EXIT STATUS
+Exit status is 0 on success, and 1 if the command fails.
+.Sh EXAMPLES
+The following example shows how to create a scheduling provider for disk
+.Pa /dev/da0
+, and how to destroy it.
+.Bd -literal -offset indent
+# Load the geom_sched module:
+kldload geom_sched
+# Load some scheduler classes used by geom_sched:
+kldload gsched_rr gsched_as
+# Configure device ad0 to use scheduler 'rr':
+geom sched insert -s rr ad0
+# Now provider ad0 uses the 'rr' algorithm;
+# the new geom is ad0.sched.
+# Remove the scheduler on the device:
+geom sched destroy -v ad0.sched.
+.Ed
+.Pp
+.Sh SEE ALSO
+.Xr geom 4 ,
+.Xr geom 8
+.Sh HISTORY
+The
+.Nm
+utility appeared in April 2010.
+.Sh AUTHORS
+.An Fabio Checconi Aq fabio@FreeBSD.org
+.An Luigi Rizzo Aq luigi@FreeBSD.org