Add a new per-thread private flag: TDP_GEOM.

This flag gets set whenever the thread posts an event on the GEOM
event queue, and if the flag is set when the thread is prepared
to return to userland from the kernel, g_waitidle() will be called
to make sure that the posted events have completed.

This can replace an insufficient number of g_waitidle() calls in
various other places, and has the advantage of being failsafe:  Any
system call which does a VOP_OPEN()/VOP_CLOSE will now correctly
wait for any geom events it posted as part of spoils or tastes.

Assert that topology and Giant is not held in g_waitidle().

3 files changed, 16 insertions, 1 deletions

diff --git a/sys/geom/geom_event.c b/sys/geom/geom_event.c
index d26bac0..9166c30 100644
--- a/sys/geom/geom_event.c
+++ b/sys/geom/geom_event.c
@@ -47,12 +47,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
-#include <machine/stdarg.h>
+#include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/time.h>
 #include <geom/geom.h>
 #include <geom/geom_int.h>
 
+#include <machine/stdarg.h>
+
 TAILQ_HEAD(event_tailq_head, g_event);
 
 static struct event_tailq_head g_events = TAILQ_HEAD_INITIALIZER(g_events);
@@ -79,8 +81,12 @@ void
 g_waitidle(void)
 {
 
+	g_topology_assert_not();
+	mtx_assert(&Giant, MA_NOTOWNED);
+
 	while (g_pending_events)
 		tsleep(&g_pending_events, PPAUSE, "g_waitidle", hz/5);
+	curthread->td_pflags &= ~TDP_GEOM;
 }
 
 void
@@ -279,6 +285,7 @@ g_post_event_x(g_event_t *func, void *arg, int flag, int wuflag, struct g_event
 	wakeup(&g_wait_event);
 	if (epp != NULL)
 		*epp = ep;
+	curthread->td_pflags |= TDP_GEOM;
 	return (0);
 }
 
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index 86026a4..89fab49 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -95,6 +95,13 @@ userret(td, frame, oticks)
 #endif
 
 	/*
+	 * If this thread tickled GEOM, we need to wait for the giggling to
+	 * stop before we return to userland
+	 */
+	if (td->td_pflags & TDP_GEOM)
+		g_waitidle();
+
+	/*
 	 * Let the scheduler adjust our priority etc.
 	 */
 	sched_userret(td);
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 25d33c9..e6abd9a 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -375,6 +375,7 @@ struct thread {
 #define	TDP_SCHED2	0x00002000 /* Reserved for scheduler private use */
 #define	TDP_SCHED3	0x00004000 /* Reserved for scheduler private use */
 #define	TDP_SCHED4	0x00008000 /* Reserved for scheduler private use */
+#define	TDP_GEOM	0x00010000 /* Settle GEOM before finishing syscall */
 
 /*
  * Reasons that the current thread can not be run yet.