Add some initial locking to gif(4). It doesn't covers the whole driver,

however IPv4-in-IPv4 tunnels are now stable on SMP. Details:

- Add per-softc mutex.
- Hold the mutex on output.

The main problem was the rtentry, placed in softc. It could be
freed by ip_output(). Meanwhile, another thread being in
in_gif_output() can read and write this rtentry.

Reported by:	many
Tested by:	Alexander Shiryaev <aixp mail.ru>

4 files changed, 26 insertions, 25 deletions

diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c
index 940e00e..c67f62d 100644
--- a/sys/net/if_gif.c
+++ b/sys/net/if_gif.c
@@ -90,7 +90,6 @@
 
 /*
  * gif_mtx protects the global gif_softc_list.
- * XXX: Per-softc locking is still required.
  */
 static struct mtx gif_mtx;
 static MALLOC_DEFINE(M_GIF, "gif", "Generic Tunnel Interface");
@@ -154,22 +153,11 @@ gif_clone_create(ifc, unit)
 		return (ENOSPC);
 	}
 
+	GIF_LOCK_INIT(sc);
+
 	GIF2IFP(sc)->if_softc = sc;
 	if_initname(GIF2IFP(sc), ifc->ifc_name, unit);
 
-	gifattach0(sc);
-
-	mtx_lock(&gif_mtx);
-	LIST_INSERT_HEAD(&gif_softc_list, sc, gif_list);
-	mtx_unlock(&gif_mtx);
-	return (0);
-}
-
-void
-gifattach0(sc)
-	struct gif_softc *sc;
-{
-
 	sc->encap_cookie4 = sc->encap_cookie6 = NULL;
 
 	GIF2IFP(sc)->if_addrlen = 0;
@@ -187,6 +175,12 @@ gifattach0(sc)
 	bpfattach(GIF2IFP(sc), DLT_NULL, sizeof(u_int32_t));
 	if (ng_gif_attach_p != NULL)
 		(*ng_gif_attach_p)(GIF2IFP(sc));
+
+	mtx_lock(&gif_mtx);
+	LIST_INSERT_HEAD(&gif_softc_list, sc, gif_list);
+	mtx_unlock(&gif_mtx);
+
+	return (0);
 }
 
 static void
@@ -220,6 +214,8 @@ gif_clone_destroy(ifp)
 	if_detach(ifp);
 	if_free(ifp);
 
+	GIF_LOCK_DESTROY(sc);
+
 	free(sc, M_GIF);
 }
 
@@ -411,6 +407,9 @@ gif_output(ifp, m, dst, rt)
 	m_tag_prepend(m, mtag);
 
 	m->m_flags &= ~(M_BCAST|M_MCAST);
+
+	GIF_LOCK(sc);
+
 	if (!(ifp->if_flags & IFF_UP) ||
 	    sc->gif_psrc == NULL || sc->gif_pdst == NULL) {
 		m_freem(m);
@@ -460,7 +459,8 @@ gif_output(ifp, m, dst, rt)
   end:
 	if (error)
 		ifp->if_oerrors++;
-	return error;
+	GIF_UNLOCK(sc);
+	return (error);
 }
 
 void
@@ -827,11 +827,8 @@ gif_set_tunnel(ifp, src, dst)
 	struct gif_softc *sc = ifp->if_softc;
 	struct gif_softc *sc2;
 	struct sockaddr *osrc, *odst, *sa;
-	int s;
 	int error = 0; 
 
-	s = splnet();
-
 	mtx_lock(&gif_mtx);
 	LIST_FOREACH(sc2, &gif_softc_list, gif_list) {
 		if (sc2 == sc)
@@ -925,7 +922,6 @@ gif_set_tunnel(ifp, src, dst)
 		ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	else
 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
-	splx(s);
 
 	return 0;
 
@@ -934,7 +930,6 @@ gif_set_tunnel(ifp, src, dst)
 		ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	else
 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
-	splx(s);
 
 	return error;
 }
@@ -944,9 +939,6 @@ gif_delete_tunnel(ifp)
 	struct ifnet *ifp;
 {
 	struct gif_softc *sc = ifp->if_softc;
-	int s;
-
-	s = splnet();
 
 	if (sc->gif_psrc) {
 		free((caddr_t)sc->gif_psrc, M_IFADDR);
@@ -968,5 +960,4 @@ gif_delete_tunnel(ifp)
 		ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	else
 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
-	splx(s);
 }
diff --git a/sys/net/if_gif.h b/sys/net/if_gif.h
index 698c6703..8e9ceb1 100644
--- a/sys/net/if_gif.h
+++ b/sys/net/if_gif.h
@@ -57,6 +57,7 @@ extern	void (*ng_gif_detach_p)(struct ifnet *ifp);
 
 struct gif_softc {
 	struct ifnet	*gif_ifp;
+	struct mtx	gif_mtx;
 	struct sockaddr	*gif_psrc; /* Physical src addr */
 	struct sockaddr	*gif_pdst; /* Physical dst addr */
 	union {
@@ -72,6 +73,12 @@ struct gif_softc {
 	LIST_ENTRY(gif_softc) gif_list; /* all gif's are linked */
 };
 #define	GIF2IFP(sc)	((sc)->gif_ifp)
+#define	GIF_LOCK_INIT(sc)	mtx_init(&(sc)->gif_mtx, "gif softc",	\
+				     NULL, MTX_DEF)
+#define	GIF_LOCK_DESTROY(sc)	mtx_destroy(&(sc)->gif_mtx)
+#define	GIF_LOCK(sc)		mtx_lock(&(sc)->gif_mtx)
+#define	GIF_UNLOCK(sc)		mtx_unlock(&(sc)->gif_mtx)
+#define	GIF_LOCK_ASSERT(sc)	mtx_assert(&(sc)->gif_mtx, MA_OWNED)
 
 #define gif_ro gifsc_gifscr.gifscr_ro
 #ifdef INET6
@@ -94,7 +101,6 @@ struct etherip_header {
 #define ETHERIP_VERSION         0x03
 
 /* Prototypes */
-void gifattach0(struct gif_softc *);
 void gif_input(struct mbuf *, int, struct ifnet *);
 int gif_output(struct ifnet *, struct mbuf *, struct sockaddr *,
 	       struct rtentry *);
diff --git a/sys/netinet/in_gif.c b/sys/netinet/in_gif.c
index cc6b64d..b69346c 100644
--- a/sys/netinet/in_gif.c
+++ b/sys/netinet/in_gif.c
@@ -104,6 +104,8 @@ in_gif_output(ifp, family, m)
 	int proto, error;
 	u_int8_t tos;
 
+	GIF_LOCK_ASSERT(sc);
+
 	if (sin_src == NULL || sin_dst == NULL ||
 	    sin_src->sin_family != AF_INET ||
 	    sin_dst->sin_family != AF_INET) {
diff --git a/sys/netinet6/in6_gif.c b/sys/netinet6/in6_gif.c
index 41c34f2..00ca2f2 100644
--- a/sys/netinet6/in6_gif.c
+++ b/sys/netinet6/in6_gif.c
@@ -97,6 +97,8 @@ in6_gif_output(ifp, family, m)
 	int proto, error;
 	u_int8_t itos, otos;
 
+	GIF_LOCK_ASSERT(sc);
+
 	if (sin6_src == NULL || sin6_dst == NULL ||
 	    sin6_src->sin6_family != AF_INET6 ||
 	    sin6_dst->sin6_family != AF_INET6) {