fix aio+kq... I've been running ambrisko's test program for much longer

w/o problems than I was before...  This simply brings back the knote_delete
as knlist_delete which will also drop the knote's, instead of just clearing
the list and seeing _ONESHOT...

Fix a race where if a note was _INFLUX and _DETACHED, it could end up being
modified... whoopse..

MFC after:	1 week
Prodded by:	ambrisko and dwhite

3 files changed, 18 insertions, 10 deletions

diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c
index 997962a..ee0ee82 100644
--- a/sys/kern/kern_event.c
+++ b/sys/kern/kern_event.c
@@ -1630,7 +1630,7 @@ knlist_destroy(struct knlist *knl)
  * knotes time to "settle".
  */
 void
-knlist_clear(struct knlist *knl, int islocked)
+knlist_cleardel(struct knlist *knl, struct thread *td, int islocked, int killkn)
 {
 	struct knote *kn;
 	struct kqueue *kq;
@@ -1646,15 +1646,20 @@ again:		/* need to reaquire lock since we have dropped it */
 	SLIST_FOREACH(kn, &knl->kl_list, kn_selnext) {
 		kq = kn->kn_kq;
 		KQ_LOCK(kq);
-		if ((kn->kn_status & KN_INFLUX) &&
-		    (kn->kn_status & KN_DETACHED) != KN_DETACHED) {
+		if ((kn->kn_status & KN_INFLUX)) {
 			KQ_UNLOCK(kq);
 			continue;
 		}
-		/* Make sure cleared knotes disappear soon */
-		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
 		knlist_remove_kq(knl, kn, 1, 1);
-		KQ_UNLOCK(kq);
+		if (killkn) {
+			kn->kn_status |= KN_INFLUX | KN_DETACHED;
+			KQ_UNLOCK(kq);
+			knote_drop(kn, td);
+		} else {
+			/* Make sure cleared knotes disappear soon */
+			kn->kn_flags |= (EV_EOF | EV_ONESHOT);
+			KQ_UNLOCK(kq);
+		}
 		kq = NULL;
 	}
 
@@ -1672,8 +1677,6 @@ again:		/* need to reaquire lock since we have dropped it */
 		goto again;
 	}
 
-	SLIST_INIT(&knl->kl_list);
-
 	if (islocked)
 		mtx_assert(knl->kl_lock, MA_OWNED);
 	else {
diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index 52c40a0..528a8ce 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -488,7 +488,7 @@ aio_free_entry(struct aiocblist *aiocbe)
 	 * OWNING thread? (or maybe the running thread?)
 	 * There is a semantic problem here...
 	 */
-	knlist_clear(&aiocbe->klist, 0); /* XXXKSE */
+	knlist_delete(&aiocbe->klist, FIRST_THREAD_IN_PROC(p), 0); /* XXXKSE */
 
 	if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags & KAIO_RUNDOWN)
 	    && ((ki->kaio_buffer_count == 0) && (ki->kaio_queue_count == 0)))) {
diff --git a/sys/sys/event.h b/sys/sys/event.h
index 86caab2..5fdc88f 100644
--- a/sys/sys/event.h
+++ b/sys/sys/event.h
@@ -207,7 +207,12 @@ extern void	knlist_remove_inevent(struct knlist *knl, struct knote *kn);
 extern int	knlist_empty(struct knlist *knl);
 extern void	knlist_init(struct knlist *knl, struct mtx *mtx);
 extern void	knlist_destroy(struct knlist *knl);
-extern void	knlist_clear(struct knlist *knl, int islocked);
+extern void	knlist_cleardel(struct knlist *knl, struct thread *td,
+	int islocked, int killkn);
+#define knlist_clear(knl, islocked)				\
+		knlist_cleardel((knl), NULL, (islocked), 0)
+#define knlist_delete(knl, td, islocked)			\
+		knlist_cleardel((knl), (td), (islocked), 1)
 extern void	knote_fdclose(struct thread *p, int fd);
 extern int 	kqueue_register(struct kqueue *kq,
 		    struct kevent *kev, struct thread *p, int waitok);