Introduce in_multi_mtx, which will protect IPv4-layer multicast address

lists, as well as accessor macros.  For now, this is a recursive mutex
due code sequences where IPv4 multicast calls into IGMP calls into
ip_output(), which then tests for a multicast forwarding case.

For support macros in in_var.h to check multicast address lists, assert
that in_multi_mtx is held.

Acquire in_multi_mtx around iteration over the IPv4 multicast address
lists, such as in ip_input() and ip_output().

Acquire in_multi_mtx when manipulating the IPv4 layer multicast addresses,
as well as over the manipulation of ifnet multicast address lists in order
to keep the two layers in sync.

Lock down accesses to IPv4 multicast addresses in IGMP, or assert the
lock when performing IGMP join/leave events.

Eliminate spl's associated with IPv4 multicast addresses, portions of
IGMP that weren't previously expunged by IGMP locking.

Add in_multi_mtx, igmp_mtx, and if_addr_mtx lock order to hard-coded
lock order in WITNESS, in that order.

Problem reported by:	Ed Maste <emaste at phaedrus dot sandvine dot ca>
MFC after:		10 days

1 files changed, 12 insertions, 8 deletions

diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c
index 2c68e3c..8612a23 100644
--- a/sys/netinet/igmp.c
+++ b/sys/netinet/igmp.c
@@ -286,6 +286,7 @@ igmp_input(register struct mbuf *m, int off)
 		 * - Use the value specified in the query message as
 		 *   the maximum timeout.
 		 */
+		IN_MULTI_LOCK();
 		IN_FIRST_MULTI(step, inm);
 		while (inm != NULL) {
 			if (inm->inm_ifp == ifp &&
@@ -301,6 +302,7 @@ igmp_input(register struct mbuf *m, int off)
 			}
 			IN_NEXT_MULTI(step, inm);
 		}
+		IN_MULTI_UNLOCK();
 
 		break;
 
@@ -343,14 +345,15 @@ igmp_input(register struct mbuf *m, int off)
 		 * If we belong to the group being reported, stop
 		 * our timer for that group.
 		 */
+		IN_MULTI_LOCK();
 		IN_LOOKUP_MULTI(igmp->igmp_group, ifp, inm);
-
 		if (inm != NULL) {
 			inm->inm_timer = 0;
 			++igmpstat.igps_rcv_ourreports;
 
 			inm->inm_state = IGMP_OTHERMEMBER;
 		}
+		IN_MULTI_UNLOCK();
 
 		break;
 	}
@@ -365,7 +368,8 @@ igmp_input(register struct mbuf *m, int off)
 void
 igmp_joingroup(struct in_multi *inm)
 {
-	int s = splnet();
+
+	IN_MULTI_LOCK_ASSERT();
 
 	if (inm->inm_addr.s_addr == igmp_all_hosts_group
 	    || inm->inm_ifp->if_flags & IFF_LOOPBACK) {
@@ -384,13 +388,14 @@ igmp_joingroup(struct in_multi *inm)
 		}
 		/* XXX handling of failure case? */
 	}
-	splx(s);
 }
 
 void
 igmp_leavegroup(struct in_multi *inm)
 {
 
+	IN_MULTI_LOCK_ASSERT();
+
 	if (inm->inm_state == IGMP_IREPORTEDLAST &&
 	    inm->inm_addr.s_addr != igmp_all_hosts_group &&
 	    !(inm->inm_ifp->if_flags & IFF_LOOPBACK) &&
@@ -403,7 +408,6 @@ igmp_fasttimo(void)
 {
 	register struct in_multi *inm;
 	struct in_multistep step;
-	int s;
 
 	/*
 	 * Quick check to see if any work needs to be done, in order
@@ -413,7 +417,7 @@ igmp_fasttimo(void)
 	if (!igmp_timers_are_running)
 		return;
 
-	s = splnet();
+	IN_MULTI_LOCK();
 	igmp_timers_are_running = 0;
 	IN_FIRST_MULTI(step, inm);
 	while (inm != NULL) {
@@ -427,13 +431,12 @@ igmp_fasttimo(void)
 		}
 		IN_NEXT_MULTI(step, inm);
 	}
-	splx(s);
+	IN_MULTI_UNLOCK();
 }
 
 void
 igmp_slowtimo(void)
 {
-	int s = splnet();
 	struct router_info *rti;
 
 	IGMP_PRINTF("[igmp.c,_slowtimo] -- > entering \n");
@@ -447,7 +450,6 @@ igmp_slowtimo(void)
 	}
 	mtx_unlock(&igmp_mtx);
 	IGMP_PRINTF("[igmp.c,_slowtimo] -- > exiting \n");
-	splx(s);
 }
 
 static void
@@ -458,6 +460,8 @@ igmp_sendpkt(struct in_multi *inm, int type, unsigned long addr)
 	struct ip *ip;
 	struct ip_moptions imo;
 
+	IN_MULTI_LOCK_ASSERT();
+
 	MGETHDR(m, M_DONTWAIT, MT_HEADER);
 	if (m == NULL)
 		return;