Add support for methods to the OSD subsystem. Each object type has a

predefined set of methods, which are set in osd_register() and called
via osd_call().  Currently, no methods are defined, though prison
objects will have some in the future.

Expand the locking from a single per-type mutex to three different kinds
of locks (four if you include the requirement that the container
(e.g. prison) be locked when getting/setting data).  This clears up one
existing issue, as well as others added by the method support.

Approved by:	bz (mentor)

2 files changed, 180 insertions, 71 deletions

diff --git a/sys/kern/kern_osd.c b/sys/kern/kern_osd.c
index d9563e6..8fc2f05 100644
--- a/sys/kern/kern_osd.c
+++ b/sys/kern/kern_osd.c
@@ -35,6 +35,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/rmlock.h>
+#include <sys/sx.h>
 #include <sys/queue.h>
 #include <sys/proc.h>
 #include <sys/osd.h>
@@ -55,13 +57,26 @@ SYSCTL_INT(_debug, OID_AUTO, osd, CTLFLAG_RW, &osd_debug, 0, "OSD debug level");
 	}								\
 } while (0)
 
+static void do_osd_del(u_int type, struct osd *osd, u_int slot,
+    int list_locked);
+
 /*
  * Lists of objects with OSD.
+ *
+ * Lock key:
+ *  (m) osd_module_lock
+ *  (o) osd_object_lock
+ *  (l) osd_list_lock
  */
-static LIST_HEAD(, osd)	osd_list[OSD_LAST + 1];
-static osd_destructor_t *osd_destructors[OSD_LAST + 1];
-static u_int osd_nslots[OSD_LAST + 1];
-static struct mtx osd_lock[OSD_LAST + 1];
+static LIST_HEAD(, osd)	osd_list[OSD_LAST + 1];		/* (m) */
+static osd_method_t *osd_methods[OSD_LAST + 1];		/* (m) */
+static u_int osd_nslots[OSD_LAST + 1];			/* (m) */
+static osd_destructor_t *osd_destructors[OSD_LAST + 1];	/* (o) */
+static const u_int osd_nmethods[OSD_LAST + 1];
+
+static struct sx osd_module_lock[OSD_LAST + 1];
+static struct rmlock osd_object_lock[OSD_LAST + 1];
+static struct mtx osd_list_lock[OSD_LAST + 1];
 
 static void
 osd_default_destructor(void *value __unused)
@@ -70,10 +85,10 @@ osd_default_destructor(void *value __unused)
 }
 
 int
-osd_register(u_int type, osd_destructor_t destructor)
+osd_register(u_int type, osd_destructor_t destructor, osd_method_t *methods)
 {
 	void *newptr;
-	u_int i;
+	u_int i, m;
 
 	KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type."));
 
@@ -84,7 +99,7 @@ osd_register(u_int type, osd_destructor_t destructor)
 	if (destructor == NULL)
 		destructor = osd_default_destructor;
 
-	mtx_lock(&osd_lock[type]);
+	sx_xlock(&osd_module_lock[type]);
 	/*
 	 * First, we try to find unused slot.
 	 */
@@ -100,19 +115,29 @@ osd_register(u_int type, osd_destructor_t destructor)
 	 */
 	if (i == osd_nslots[type]) {
 		osd_nslots[type]++;
-		newptr = realloc(osd_destructors[type],
-		    sizeof(osd_destructor_t) * osd_nslots[type], M_OSD,
-		    M_NOWAIT | M_ZERO);
-		if (newptr == NULL) {
-			mtx_unlock(&osd_lock[type]);
-			return (0);
-		}
+		if (osd_nmethods[type] != 0)
+			osd_methods[type] = realloc(osd_methods[type],
+			    sizeof(osd_method_t) * osd_nslots[type] *
+			    osd_nmethods[type], M_OSD, M_WAITOK);
+		newptr = malloc(sizeof(osd_destructor_t) * osd_nslots[type],
+		    M_OSD, M_WAITOK);
+		rm_wlock(&osd_object_lock[type]);
+		bcopy(osd_destructors[type], newptr,
+		    sizeof(osd_destructor_t) * i);
+		free(osd_destructors[type], M_OSD);
 		osd_destructors[type] = newptr;
+		rm_wunlock(&osd_object_lock[type]);
 		OSD_DEBUG("New slot allocated (type=%u, slot=%u).",
 		    type, i + 1);
 	}
+
 	osd_destructors[type][i] = destructor;
-	mtx_unlock(&osd_lock[type]);
+	if (osd_nmethods[type] != 0) {
+		for (m = 0; m < osd_nmethods[type]; m++)
+			osd_methods[type][i * osd_nmethods[type] + m] =
+			    methods != NULL ? methods[m] : NULL;
+	}
+	sx_xunlock(&osd_module_lock[type]);
 	return (i + 1);
 }
 
@@ -125,13 +150,15 @@ osd_deregister(u_int type, u_int slot)
 	KASSERT(slot > 0, ("Invalid slot."));
 	KASSERT(osd_destructors[type][slot - 1] != NULL, ("Unused slot."));
 
-	mtx_lock(&osd_lock[type]);
+	sx_xlock(&osd_module_lock[type]);
+	rm_wlock(&osd_object_lock[type]);
 	/*
 	 * Free all OSD for the given slot.
 	 */
-	LIST_FOREACH_SAFE(osd, &osd_list[type], osd_next, tosd) {
-		osd_del(type, osd, slot);
-	}
+	mtx_lock(&osd_list_lock[type]);
+	LIST_FOREACH_SAFE(osd, &osd_list[type], osd_next, tosd)
+		do_osd_del(type, osd, slot, 1);
+	mtx_unlock(&osd_list_lock[type]);
 	/*
 	 * Set destructor to NULL to free the slot.
 	 */
@@ -141,83 +168,121 @@ osd_deregister(u_int type, u_int slot)
 		osd_destructors[type] = realloc(osd_destructors[type],
 		    sizeof(osd_destructor_t) * osd_nslots[type], M_OSD,
 		    M_NOWAIT | M_ZERO);
+		if (osd_nmethods[type] != 0)
+			osd_methods[type] = realloc(osd_methods[type],
+			    sizeof(osd_method_t) * osd_nslots[type] *
+			    osd_nmethods[type], M_OSD, M_NOWAIT | M_ZERO);
 		/*
 		 * We always reallocate to smaller size, so we assume it will
 		 * always succeed.
 		 */
-		KASSERT(osd_destructors[type] != NULL, ("realloc() failed"));
+		KASSERT(osd_destructors[type] != NULL &&
+		    (osd_nmethods[type] == 0 || osd_methods[type] != NULL),
+		    ("realloc() failed"));
 		OSD_DEBUG("Deregistration of the last slot (type=%u, slot=%u).",
 		    type, slot);
 	} else {
 		OSD_DEBUG("Slot deregistration (type=%u, slot=%u).",
 		    type, slot);
 	}
-	mtx_unlock(&osd_lock[type]);
+	rm_wunlock(&osd_object_lock[type]);
+	sx_xunlock(&osd_module_lock[type]);
 }
 
 int
 osd_set(u_int type, struct osd *osd, u_int slot, void *value)
 {
+	struct rm_priotracker tracker;
 
 	KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type."));
 	KASSERT(slot > 0, ("Invalid slot."));
 	KASSERT(osd_destructors[type][slot - 1] != NULL, ("Unused slot."));
 
-	if (osd->osd_nslots == 0) {
-		/*
-		 * First OSD for this object, so we need to allocate space and
-		 * put it onto the list.
-		 */
-		osd->osd_slots = malloc(sizeof(void *) * slot, M_OSD,
-		    M_NOWAIT | M_ZERO);
-		if (osd->osd_slots == NULL)
-			return (ENOMEM);
-		osd->osd_nslots = slot;
-		mtx_lock(&osd_lock[type]);
-		LIST_INSERT_HEAD(&osd_list[type], osd, osd_next);
-		mtx_unlock(&osd_lock[type]);
-		OSD_DEBUG("Setting first slot (type=%u).", type);
-	} else if (slot > osd->osd_nslots) {
-		void *newptr;
-
-		/*
-		 * Too few slots allocated here, needs to extend the array.
-		 */
-		newptr = realloc(osd->osd_slots, sizeof(void *) * slot, M_OSD,
-		    M_NOWAIT | M_ZERO);
-		if (newptr == NULL)
-			return (ENOMEM);
-		osd->osd_slots = newptr;
-		osd->osd_nslots = slot;
-		OSD_DEBUG("Growing slots array (type=%u).", type);
+	rm_rlock(&osd_object_lock[type], &tracker);
+	if (slot > osd->osd_nslots) {
+		if (value == NULL) {
+			OSD_DEBUG(
+			    "Not allocating null slot (type=%u, slot=%u).",
+			    type, slot);
+			rm_runlock(&osd_object_lock[type], &tracker);
+			return (0);
+		} else if (osd->osd_nslots == 0) {
+			/*
+			 * First OSD for this object, so we need to allocate
+			 * space and put it onto the list.
+			 */
+			osd->osd_slots = malloc(sizeof(void *) * slot, M_OSD,
+			    M_NOWAIT | M_ZERO);
+			if (osd->osd_slots == NULL) {
+				rm_runlock(&osd_object_lock[type], &tracker);
+				return (ENOMEM);
+			}
+			osd->osd_nslots = slot;
+			mtx_lock(&osd_list_lock[type]);
+			LIST_INSERT_HEAD(&osd_list[type], osd, osd_next);
+			mtx_unlock(&osd_list_lock[type]);
+			OSD_DEBUG("Setting first slot (type=%u).", type);
+		} else {
+			void *newptr;
+
+			/*
+			 * Too few slots allocated here, needs to extend
+			 * the array.
+			 */
+			newptr = realloc(osd->osd_slots, sizeof(void *) * slot,
+			    M_OSD, M_NOWAIT | M_ZERO);
+			if (newptr == NULL) {
+				rm_runlock(&osd_object_lock[type], &tracker);
+				return (ENOMEM);
+			}
+			osd->osd_slots = newptr;
+			osd->osd_nslots = slot;
+			OSD_DEBUG("Growing slots array (type=%u).", type);
+		}
 	}
 	OSD_DEBUG("Setting slot value (type=%u, slot=%u, value=%p).", type,
 	    slot, value);
 	osd->osd_slots[slot - 1] = value;
+	rm_runlock(&osd_object_lock[type], &tracker);
 	return (0);
 }
 
 void *
 osd_get(u_int type, struct osd *osd, u_int slot)
 {
+	struct rm_priotracker tracker;
+	void *value;
 
 	KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type."));
 	KASSERT(slot > 0, ("Invalid slot."));
 	KASSERT(osd_destructors[type][slot - 1] != NULL, ("Unused slot."));
 
+	rm_rlock(&osd_object_lock[type], &tracker);
 	if (slot > osd->osd_nslots) {
+		value = NULL;
 		OSD_DEBUG("Slot doesn't exist (type=%u, slot=%u).", type, slot);
-		return (NULL);
+	} else {
+		value = osd->osd_slots[slot - 1];
+		OSD_DEBUG("Returning slot value (type=%u, slot=%u, value=%p).",
+		    type, slot, value);
 	}
-
-	OSD_DEBUG("Returning slot value (type=%u, slot=%u, value=%p).", type,
-	    slot, osd->osd_slots[slot - 1]);
-	return (osd->osd_slots[slot - 1]);
+	rm_runlock(&osd_object_lock[type], &tracker);
+	return (value);
 }
 
 void
 osd_del(u_int type, struct osd *osd, u_int slot)
 {
+	struct rm_priotracker tracker;
+
+	rm_rlock(&osd_object_lock[type], &tracker);
+	do_osd_del(type, osd, slot, 0);
+	rm_runlock(&osd_object_lock[type], &tracker);
+}
+
+static void
+do_osd_del(u_int type, struct osd *osd, u_int slot, int list_locked)
+{
 	int i;
 
 	KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type."));
@@ -234,20 +299,19 @@ osd_del(u_int type, struct osd *osd, u_int slot)
 	osd->osd_slots[slot - 1] = NULL;
 	for (i = osd->osd_nslots - 1; i >= 0; i--) {
 		if (osd->osd_slots[i] != NULL) {
-			OSD_DEBUG("Slot still has a value (type=%u, slot=%u).", type, i + 1);
+			OSD_DEBUG("Slot still has a value (type=%u, slot=%u).",
+			    type, i + 1);
 			break;
 		}
 	}
 	if (i == -1) {
-		int unlock;
-
 		/* No values left for this object. */
 		OSD_DEBUG("No more slots left (type=%u).", type);
-		if ((unlock = !mtx_owned(&osd_lock[type])))
-			mtx_lock(&osd_lock[type]);
+		if (!list_locked)
+			mtx_lock(&osd_list_lock[type]);
 		LIST_REMOVE(osd, osd_next);
-		if (unlock)
-			mtx_unlock(&osd_lock[type]);
+		if (!list_locked)
+			mtx_unlock(&osd_list_lock[type]);
 		free(osd->osd_slots, M_OSD);
 		osd->osd_slots = NULL;
 		osd->osd_nslots = 0;
@@ -266,9 +330,35 @@ osd_del(u_int type, struct osd *osd, u_int slot)
 	}
 }
 
+int
+osd_call(u_int type, u_int method, void *obj, void *data)
+{
+	osd_method_t methodfun;
+	int error, i;
+
+	KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type."));
+	KASSERT(method < osd_nmethods[type], ("Invalid method."));
+
+	/*
+	 * Call this method for every slot that defines it, stopping if an
+	 * error is encountered.
+	 */
+	error = 0;
+	sx_slock(&osd_module_lock[type]);
+	for (i = 1; i <= osd_nslots[type]; i++) {
+		methodfun =
+		    osd_methods[type][(i - 1) * osd_nmethods[type] + method];
+		if (methodfun != NULL && (error = methodfun(obj, data)) != 0)
+			break;
+	}
+	sx_sunlock(&osd_module_lock[type]);
+	return (error);
+}
+
 void
 osd_exit(u_int type, struct osd *osd)
 {
+	struct rm_priotracker tracker;
 	u_int i;
 
 	KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type."));
@@ -279,10 +369,14 @@ osd_exit(u_int type, struct osd *osd)
 		return;
 	}
 
-	mtx_lock(&osd_lock[type]);
-	for (i = 1; i <= osd->osd_nslots; i++)
-		osd_del(type, osd, i);
-	mtx_unlock(&osd_lock[type]);
+	rm_rlock(&osd_object_lock[type], &tracker);
+	for (i = 1; i <= osd->osd_nslots; i++) {
+		if (osd_destructors[type][i - 1] != NULL)
+			do_osd_del(type, osd, i, 0);
+		else
+			OSD_DEBUG("Unused slot (type=%u, slot=%u).", type, i);
+	}
+	rm_runlock(&osd_object_lock[type], &tracker);
 	OSD_DEBUG("Object exit (type=%u).", type);
 }
 
@@ -294,8 +388,11 @@ osd_init(void *arg __unused)
 	for (i = OSD_FIRST; i <= OSD_LAST; i++) {
 		osd_nslots[i] = 0;
 		LIST_INIT(&osd_list[i]);
-		mtx_init(&osd_lock[i], "osd", NULL, MTX_DEF);
+		sx_init(&osd_module_lock[i], "osd_module");
+		rm_init(&osd_object_lock[i], "osd_object", 0);
+		mtx_init(&osd_list_lock[i], "osd_list", NULL, MTX_DEF);
 		osd_destructors[i] = NULL;
+		osd_methods[i] = NULL;
 	}
 }
 SYSINIT(osd, SI_SUB_LOCK, SI_ORDER_ANY, osd_init, NULL);
diff --git a/sys/sys/osd.h b/sys/sys/osd.h
index 281f580..c0f17a5 100644
--- a/sys/sys/osd.h
+++ b/sys/sys/osd.h
@@ -31,10 +31,15 @@
 
 #include <sys/queue.h>
 
+/*
+ * Lock key:
+ *   (c) container lock (e.g. jail's pr_mtx) and/or osd_object_lock
+ *   (l) osd_list_lock
+ */
 struct osd {
-	u_int		  osd_nslots;
-	void		**osd_slots;
-	LIST_ENTRY(osd)	  osd_next;
+	u_int		  osd_nslots;	/* (c) */
+	void		**osd_slots;	/* (c) */
+	LIST_ENTRY(osd)	  osd_next;	/* (l) */
 };
 
 #ifdef _KERNEL
@@ -46,18 +51,21 @@ struct osd {
 #define	OSD_LAST	OSD_JAIL
 
 typedef void (*osd_destructor_t)(void *value);
+typedef int (*osd_method_t)(void *obj, void *data);
 
-int osd_register(u_int type, osd_destructor_t destructor);
+int osd_register(u_int type, osd_destructor_t destructor,
+    osd_method_t *methods);
 void osd_deregister(u_int type, u_int slot);
 
 int osd_set(u_int type, struct osd *osd, u_int slot, void *value);
 void *osd_get(u_int type, struct osd *osd, u_int slot);
 void osd_del(u_int type, struct osd *osd, u_int slot);
+int osd_call(u_int type, u_int method, void *obj, void *data);
 
 void osd_exit(u_int type, struct osd *osd);
 
 #define	osd_thread_register(destructor)					\
-	osd_register(OSD_THREAD, (destructor))
+	osd_register(OSD_THREAD, (destructor), NULL)
 #define	osd_thread_deregister(slot)					\
 	osd_deregister(OSD_THREAD, (slot))
 #define	osd_thread_set(td, slot, value)					\
@@ -68,11 +76,13 @@ void osd_exit(u_int type, struct osd *osd);
 	KASSERT((td) == curthread, ("Not curthread."));			\
 	osd_del(OSD_THREAD, &(td)->td_osd, (slot));			\
 } while (0)
+#define	osd_thread_call(td, method, data)				\
+	osd_call(OSD_THREAD, (method), (td), (data))
 #define	osd_thread_exit(td)						\
 	osd_exit(OSD_THREAD, &(td)->td_osd)
 
 #define	osd_jail_register(destructor)					\
-	osd_register(OSD_JAIL, (destructor))
+	osd_register(OSD_JAIL, (destructor), NULL)
 #define	osd_jail_deregister(slot)					\
 	osd_deregister(OSD_JAIL, (slot))
 #define	osd_jail_set(pr, slot, value)					\
@@ -81,6 +91,8 @@ void osd_exit(u_int type, struct osd *osd);
 	osd_get(OSD_JAIL, &(pr)->pr_osd, (slot))
 #define	osd_jail_del(pr, slot)						\
 	osd_del(OSD_JAIL, &(pr)->pr_osd, (slot))
+#define	osd_jail_call(pr, method, data)					\
+	osd_call(OSD_JAIL, (method), (pr), (data))
 #define	osd_jail_exit(pr)						\
 	osd_exit(OSD_JAIL, &(pr)->pr_osd)