Take a first cut at cleaning up ifnet removal and multicast socket

panics, which occur when stale ifnet pointers are left in struct
moptions hung off of inpcbs:

- Add in_ifdetach(), which matches in6_ifdetach(), and allows the
  protocol to perform early tear-down on the interface early in
  if_detach().

- Annotate that if_detach() needs careful consideration.

- Remove calls to in_pcbpurgeif0() in the handling of SIOCDIFADDR --
  this is not the place to detect interface removal!  This also
  removes what is basically a nasty (and now unnecessary) hack.

- Invoke in_pcbpurgeif0() from in_ifdetach(), in both raw and UDP
  IPv4 sockets.

It is now possible to run the msocket_ifnet_remove regression test
using HEAD without panicking.

MFC after:	3 days

3 files changed, 20 insertions, 8 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index f7dbf47..b5c8295 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -600,6 +600,9 @@ if_purgeaddrs(struct ifnet *ifp)
 /*
  * Detach an interface, removing it from the
  * list of "active" interfaces and freeing the struct ifnet.
+ *
+ * XXXRW: There are some significant questions about event ordering, and
+ * how to prevent things from starting to use the interface during detach.
  */
 void
 if_detach(struct ifnet *ifp)
@@ -637,6 +640,10 @@ if_detach(struct ifnet *ifp)
 
 	if_purgeaddrs(ifp);
 
+#ifdef INET
+	in_ifdetach(ifp);
+#endif
+
 #ifdef INET6
 	/*
 	 * Remove all IPv6 kernel structs related to ifp.  This should be done
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index c5d148a..2eb5e94 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -459,14 +459,6 @@ in_control(so, cmd, data, ifp, td)
 		 * a routing process they will come back.
 		 */
 		in_ifadown(&ia->ia_ifa, 1);
-		/*
-		 * XXX horrible hack to detect that we are being called
-		 * from if_detach()
-		 */
-		if (ifaddr_byindex(ifp->if_index) == NULL) {
-			in_pcbpurgeif0(&ripcbinfo, ifp);
-			in_pcbpurgeif0(&udbinfo, ifp);
-		}
 		EVENTHANDLER_INVOKE(ifaddr_event, ifp);
 		error = 0;
 		break;
@@ -1046,3 +1038,15 @@ in_delmulti(inm)
 		igmp_leavegroup(&my_inm);
 	IN_MULTI_UNLOCK();
 }
+
+/*
+ * On interface removal, clean up IPv4 data structures hung off of the ifnet.
+ */
+void
+in_ifdetach(ifp)
+	struct ifnet *ifp;
+{
+
+	in_pcbpurgeif0(&ripcbinfo, ifp);
+	in_pcbpurgeif0(&udbinfo, ifp);
+}
diff --git a/sys/netinet/in.h b/sys/netinet/in.h
index f006ae3..eede911 100644
--- a/sys/netinet/in.h
+++ b/sys/netinet/in.h
@@ -572,6 +572,7 @@ int	 in_localaddr(struct in_addr);
 int	 in_localip(struct in_addr);
 char	*inet_ntoa(struct in_addr); /* in libkern */
 char	*inet_ntoa_r(struct in_addr ina, char *buf); /* in libkern */
+void	 in_ifdetach(struct ifnet *);
 
 #define	in_hosteq(s, t)	((s).s_addr == (t).s_addr)
 #define	in_nullhost(x)	((x).s_addr == INADDR_ANY)