Commit a first pass at in6pcb and pcbinfo locking for IPv6,

synchronizing IPv6 protocol control blocks and lists.  These changes
are modeled on the inpcb locking for IPv4, submitted by Jennifer Yang,
and committed by Jeffrey Hsu.  With these locking changes, IPv6 use of
inpcbs is now substantially more MPSAFE, and permits IPv4 inpcb locking
assertions to be run in the presence of IPv6 compiled into the kernel.

3 files changed, 160 insertions, 35 deletions

diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 8a66e4f..7627c11 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -130,6 +130,9 @@ in6_pcbbind(inp, nam, cred)
 	u_short	lport = 0;
 	int wild = 0, reuseport = (so->so_options & SO_REUSEPORT);
 
+	INP_INFO_WLOCK_ASSERT(pcbinfo);
+	INP_LOCK_ASSERT(inp);
+
 	if (!in6_ifaddr) /* XXX broken! */
 		return (EADDRNOTAVAIL);
 	if (inp->inp_lport || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))
@@ -304,6 +307,9 @@ in6_pcbladdr(inp, nam, plocal_addr6)
 	if (sin6->sin6_port == 0)
 		return (EADDRNOTAVAIL);
 
+	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
+	INP_LOCK_ASSERT(inp);
+
 	/* KAME hack: embed scopeid */
 	if (in6_embedscope(&sin6->sin6_addr, sin6, inp, &ifp) != 0)
 		return EINVAL;
@@ -356,6 +362,9 @@ in6_pcbconnect(inp, nam, cred)
 	register struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
 	int error;
 
+	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
+	INP_LOCK_ASSERT(inp);
+
 	/*
 	 * Call inner routine, to assign local interface address.
 	 * in6_pcbladdr() may automatically fill in sin6_scope_id.
@@ -402,6 +411,10 @@ void
 in6_pcbdisconnect(inp)
 	struct inpcb *inp;
 {
+
+	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
+	INP_LOCK_ASSERT(inp);
+
 	bzero((caddr_t)&inp->in6p_faddr, sizeof(inp->in6p_faddr));
 	inp->inp_fport = 0;
 	/* clear flowinfo - draft-itojun-ipv6-flowlabel-api-00 */
@@ -421,6 +434,9 @@ in6_pcbdetach(inp)
 	struct socket *so = inp->inp_socket;
 	struct inpcbinfo *ipi = inp->inp_pcbinfo;
 
+	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
+	INP_LOCK_ASSERT(inp);
+
 #if defined(IPSEC) || defined(FAST_IPSEC)
 	if (inp->in6p_sp != NULL)
 		ipsec6_delete_pcbpolicy(inp);
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index 23131de..994cf5f 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -149,24 +149,29 @@ rip6_input(mp, offp, proto)
 
 	init_sin6(&fromsa, m); /* general init */
 
+	INP_INFO_RLOCK(&ripcbinfo);
 	LIST_FOREACH(in6p, &ripcb, inp_list) {
-		if ((in6p->in6p_vflag & INP_IPV6) == 0)
+		INP_LOCK(in6p);
+		if ((in6p->in6p_vflag & INP_IPV6) == 0) {
+docontinue:
+			INP_UNLOCK(in6p);
 			continue;
+		}
 		if (in6p->in6p_ip6_nxt &&
 		    in6p->in6p_ip6_nxt != proto)
-			continue;
+			goto docontinue;
 		if (!IN6_IS_ADDR_UNSPECIFIED(&in6p->in6p_laddr) &&
 		    !IN6_ARE_ADDR_EQUAL(&in6p->in6p_laddr, &ip6->ip6_dst))
-			continue;
+			goto docontinue;
 		if (!IN6_IS_ADDR_UNSPECIFIED(&in6p->in6p_faddr) &&
 		    !IN6_ARE_ADDR_EQUAL(&in6p->in6p_faddr, &ip6->ip6_src))
-			continue;
+			goto docontinue;
 		if (in6p->in6p_cksum != -1) {
 			rip6stat.rip6s_isum++;
 			if (in6_cksum(m, ip6->ip6_nxt, *offp,
 			    m->m_pkthdr.len - *offp)) {
 				rip6stat.rip6s_badsum++;
-				continue;
+				goto docontinue;
 			}
 		}
 		if (last) {
@@ -201,6 +206,7 @@ rip6_input(mp, offp, proto)
 					sorwakeup(last->in6p_socket);
 				opts = NULL;
 			}
+			INP_UNLOCK(last);
 		}
 		last = in6p;
 	}
@@ -231,6 +237,7 @@ rip6_input(mp, offp, proto)
 			rip6stat.rip6s_fullsock++;
 		} else
 			sorwakeup(last->in6p_socket);
+		INP_UNLOCK(last);
 	} else {
 		rip6stat.rip6s_nosock++;
 		if (m->m_flags & M_MCAST)
@@ -245,6 +252,7 @@ rip6_input(mp, offp, proto)
 		}
 		ip6stat.ip6s_delivered--;
 	}
+	INP_INFO_RLOCK(&ripcbinfo);
 	return IPPROTO_DONE;
 }
 
@@ -329,6 +337,13 @@ rip6_output(m, va_alist)
 	va_end(ap);
 
 	in6p = sotoin6pcb(so);
+	/*
+	 * XXXRW: In IPv6, we don't start referencing the contents of the
+	 * inpcb until after all M_TRYWAIT allocations have finished.  We may
+	 * want to reorder this function to provide similar guarantees here,
+	 * so as to avoid holding a mutex over M_TRYWAIT.
+	 */
+	INP_LOCK(in6p);
 	stickyopt = in6p->in6p_outputopts;
 
 	priv = 0;
@@ -472,6 +487,7 @@ rip6_output(m, va_alist)
 		in6p->in6p_outputopts = stickyopt;
 		m_freem(control);
 	}
+	INP_UNLOCK(in6p);
 	return (error);
 }
 
@@ -547,21 +563,31 @@ rip6_attach(struct socket *so, int proto, struct thread *td)
 	struct inpcb *inp;
 	int error, s;
 
+	INP_INFO_WLOCK(&ripcbinfo);
 	inp = sotoinpcb(so);
-	if (inp)
+	if (inp) {
+		INP_INFO_WUNLOCK(&ripcbinfo);
 		panic("rip6_attach");
-	if (td && (error = suser(td)) != 0)
+	}
+	if (td && (error = suser(td)) != 0) {
+		INP_INFO_WUNLOCK(&ripcbinfo);
 		return error;
-
+	}
 	error = soreserve(so, rip_sendspace, rip_recvspace);
-	if (error)
+	if (error) {
+		INP_INFO_WUNLOCK(&ripcbinfo);
 		return error;
+	}
 	s = splnet();
 	error = in_pcballoc(so, &ripcbinfo, "raw6inp");
 	splx(s);
-	if (error)
+	if (error) {
+		INP_INFO_WUNLOCK(&ripcbinfo);
 		return error;
+	}
 	inp = (struct inpcb *)so->so_pcb;
+	INP_LOCK(inp);
+	INP_INFO_WUNLOCK(&ripcbinfo);
 	inp->inp_vflag |= INP_IPV6;
 	inp->in6p_ip6_nxt = (long)proto;
 	inp->in6p_hops = -1;	/* use kernel default */
@@ -569,6 +595,7 @@ rip6_attach(struct socket *so, int proto, struct thread *td)
 	MALLOC(inp->in6p_icmp6filt, struct icmp6_filter *,
 	       sizeof(struct icmp6_filter), M_PCB, M_NOWAIT);
 	ICMP6_FILTER_SETPASSALL(inp->in6p_icmp6filt);
+	INP_UNLOCK(inp);
 	return 0;
 }
 
@@ -577,9 +604,12 @@ rip6_detach(struct socket *so)
 {
 	struct inpcb *inp;
 
+	INP_INFO_WLOCK(&ripcbinfo);
 	inp = sotoinpcb(so);
-	if (inp == 0)
+	if (inp == 0) {
+		INP_INFO_WUNLOCK(&ripcbinfo);
 		panic("rip6_detach");
+	}
 	/* xxx: RSVP */
 	if (so == ip6_mrouter)
 		ip6_mrouter_done();
@@ -587,7 +617,9 @@ rip6_detach(struct socket *so)
 		FREE(inp->in6p_icmp6filt, M_PCB);
 		inp->in6p_icmp6filt = NULL;
 	}
+	INP_LOCK(inp);
 	in6_pcbdetach(inp);
+	INP_INFO_WUNLOCK(&ripcbinfo);
 	return 0;
 }
 
@@ -634,7 +666,11 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	     IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) {
 		return (EADDRNOTAVAIL);
 	}
+	INP_INFO_WLOCK(&ripcbinfo);
+	INP_LOCK(inp);
 	inp->in6p_laddr = addr->sin6_addr;
+	INP_UNLOCK(inp);
+	INP_INFO_WUNLOCK(&ripcbinfo);
 	return 0;
 }
 
@@ -663,22 +699,36 @@ rip6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		addr->sin6_scope_id = scope6_addr2default(&addr->sin6_addr);
 	}
 #endif
+	INP_INFO_WLOCK(&ripcbinfo);
+	INP_LOCK(inp);
 	/* Source address selection. XXX: need pcblookup? */
 	in6a = in6_selectsrc(addr, inp->in6p_outputopts,
 			     inp->in6p_moptions, NULL,
 			     &inp->in6p_laddr, &error);
-	if (in6a == NULL)
+	if (in6a == NULL) {
+		INP_UNLOCK(inp);
+		INP_INFO_WUNLOCK(&ripcbinfo);
 		return (error ? error : EADDRNOTAVAIL);
+	}
 	inp->in6p_laddr = *in6a;
 	inp->in6p_faddr = addr->sin6_addr;
 	soisconnected(so);
+	INP_UNLOCK(inp);
+	INP_INFO_WUNLOCK(&ripcbinfo);
 	return 0;
 }
 
 static int
 rip6_shutdown(struct socket *so)
 {
+	struct inpcb *inp;
+
+	INP_INFO_RLOCK(&ripcbinfo);
+	inp = sotoinpcb(so);
+	INP_LOCK(inp);
+	INP_INFO_RUNLOCK(&ripcbinfo);
 	socantsendmore(so);
+	INP_UNLOCK(inp);
 	return 0;
 }
 
@@ -689,10 +739,14 @@ rip6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 	struct inpcb *inp = sotoinpcb(so);
 	struct sockaddr_in6 tmp;
 	struct sockaddr_in6 *dst;
+	int ret;
 
+	INP_INFO_WLOCK(&ripcbinfo);
 	/* always copy sockaddr to avoid overwrites */
+	/* Unlocked read. */
 	if (so->so_state & SS_ISCONNECTED) {
 		if (nam) {
+			INP_INFO_WUNLOCK(&ripcbinfo);
 			m_freem(m);
 			return EISCONN;
 		}
@@ -705,6 +759,7 @@ rip6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 		dst = &tmp;
 	} else {
 		if (nam == NULL) {
+			INP_INFO_WUNLOCK(&ripcbinfo);
 			m_freem(m);
 			return ENOTCONN;
 		}
@@ -716,7 +771,9 @@ rip6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 		dst->sin6_scope_id = scope6_addr2default(&dst->sin6_addr);
 	}
 #endif
-	return rip6_output(m, so, dst, control);
+	ret = rip6_output(m, so, dst, control);
+	INP_INFO_WUNLOCK(&ripcbinfo);
+	return (ret);
 }
 
 struct pr_usrreqs rip6_usrreqs = {
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index 23842e5..c248ce2 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -514,21 +514,30 @@ udp6_attach(struct socket *so, int proto, struct thread *td)
 	struct inpcb *inp;
 	int s, error;
 
+	INP_INFO_WLOCK(&udbinfo);
 	inp = sotoinpcb(so);
-	if (inp != 0)
+	if (inp != 0) {
+		INP_INFO_WUNLOCK(&udbinfo);
 		return EINVAL;
+	}
 
 	if (so->so_snd.sb_hiwat == 0 || so->so_rcv.sb_hiwat == 0) {
 		error = soreserve(so, udp_sendspace, udp_recvspace);
-		if (error)
+		if (error) {
+			INP_INFO_WUNLOCK(&udbinfo);
 			return error;
+		}
 	}
 	s = splnet();
 	error = in_pcballoc(so, &udbinfo, "udp6inp");
 	splx(s);
-	if (error)
+	if (error) {
+		INP_INFO_WUNLOCK(&udbinfo);
 		return error;
+	}
 	inp = (struct inpcb *)so->so_pcb;
+	INP_LOCK(inp);
+	INP_INFO_WUNLOCK(&udbinfo);
 	inp->inp_vflag |= INP_IPV6;
 	if (!ip6_v6only)
 		inp->inp_vflag |= INP_IPV4;
@@ -541,6 +550,7 @@ udp6_attach(struct socket *so, int proto, struct thread *td)
 	 * which may match an IPv4-mapped IPv6 address.
 	 */
 	inp->inp_ip_ttl = ip_defttl;
+	INP_UNLOCK(inp);
 	return 0;
 }
 
@@ -550,9 +560,13 @@ udp6_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	struct inpcb *inp;
 	int s, error;
 
+	INP_INFO_WLOCK(&udbinfo);
 	inp = sotoinpcb(so);
-	if (inp == 0)
+	if (inp == 0) {
+		INP_INFO_WUNLOCK(&udbinfo);
 		return EINVAL;
+	}
+	INP_LOCK(inp);
 
 	inp->inp_vflag &= ~INP_IPV4;
 	inp->inp_vflag |= INP_IPV6;
@@ -572,13 +586,15 @@ udp6_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 			s = splnet();
 			error = in_pcbbind(inp, (struct sockaddr *)&sin,
 			    td->td_ucred);
-			splx(s);
-			return error;
+			goto out;
 		}
 	}
 
 	s = splnet();
 	error = in6_pcbbind(inp, nam, td->td_ucred);
+out:
+	INP_UNLOCK(inp);
+	INP_INFO_WUNLOCK(&udbinfo);
 	splx(s);
 	return error;
 }
@@ -589,9 +605,13 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	struct inpcb *inp;
 	int s, error;
 
+	INP_INFO_WLOCK(&udbinfo);
 	inp = sotoinpcb(so);
-	if (inp == 0)
+	if (inp == 0) {
+		INP_INFO_WUNLOCK(&udbinfo);
 		return EINVAL;
+	}
+	INP_LOCK(inp);
 
 	if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0) {
 		struct sockaddr_in6 *sin6_p;
@@ -612,11 +632,13 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 				inp->inp_vflag &= ~INP_IPV6;
 				soisconnected(so);
 			}
-			return error;
+			goto out;
 		}
 	}
-	if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
-		return EISCONN;
+	if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
+		error = EISCONN;
+		goto out;
+	}
 	s = splnet();
 	error = in6_pcbconnect(inp, nam, td->td_ucred);
 	splx(s);
@@ -627,6 +649,9 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		}
 		soisconnected(so);
 	}
+out:
+	INP_UNLOCK(inp);
+	INP_INFO_WUNLOCK(&udbinfo);
 	return error;
 }
 
@@ -636,12 +661,17 @@ udp6_detach(struct socket *so)
 	struct inpcb *inp;
 	int s;
 
+	INP_INFO_WLOCK(&udbinfo);
 	inp = sotoinpcb(so);
-	if (inp == 0)
+	if (inp == 0) {
+		INP_INFO_WUNLOCK(&udbinfo);
 		return EINVAL;
+	}
+	INP_LOCK(inp);
 	s = splnet();
 	in6_pcbdetach(inp);
 	splx(s);
+	INP_INFO_WUNLOCK(&udbinfo);
 	return 0;
 }
 
@@ -649,29 +679,40 @@ static int
 udp6_disconnect(struct socket *so)
 {
 	struct inpcb *inp;
-	int s;
+	int error, s;
 
+	INP_INFO_WLOCK(&udbinfo);
 	inp = sotoinpcb(so);
-	if (inp == 0)
+	if (inp == 0) {
+		INP_INFO_WUNLOCK(&udbinfo);
 		return EINVAL;
+	}
+	INP_LOCK(inp);
 
 #ifdef INET
 	if (inp->inp_vflag & INP_IPV4) {
 		struct pr_usrreqs *pru;
 
 		pru = inetsw[ip_protox[IPPROTO_UDP]].pr_usrreqs;
-		return ((*pru->pru_disconnect)(so));
+		error = (*pru->pru_disconnect)(so);
+		goto out;
 	}
 #endif
 
-	if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
-		return ENOTCONN;
+	if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
+		error = ENOTCONN;
+		goto out;
+	}
 
 	s = splnet();
 	in6_pcbdisconnect(inp);
 	inp->in6p_laddr = in6addr_any;
 	splx(s);
+	/* XXXRW: so_state locking? */
 	so->so_state &= ~SS_ISCONNECTED;		/* XXX */
+out:
+	INP_UNLOCK(inp);
+	INP_INFO_WUNLOCK(&udbinfo);
 	return 0;
 }
 
@@ -682,11 +723,14 @@ udp6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 	struct inpcb *inp;
 	int error = 0;
 
+	INP_INFO_WLOCK(&udbinfo);
 	inp = sotoinpcb(so);
 	if (inp == 0) {
-		error = EINVAL;
-		goto bad;
+		INP_INFO_WUNLOCK(&udbinfo);
+		m_freem(m);
+		return EINVAL;
 	}
+	INP_LOCK(inp);
 
 	if (addr) {
 		if (addr->sa_len != sizeof(struct sockaddr_in6)) { 
@@ -720,7 +764,8 @@ udp6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 				 * IPV6_V6ONLY flag, we discard this
 				 * datagram destined to a v4 addr.
 				 */
-				return EINVAL;
+				error = EINVAL;
+				goto out;
 			}
 			if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) &&
 			    !IN6_IS_ADDR_V4MAPPED(&inp->in6p_laddr)) {
@@ -731,7 +776,8 @@ udp6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 				 * determine how to map IPv6 source
 				 * address to IPv4.
 				 */
-				return EINVAL;
+				error = EINVAL;
+				goto out;
 			}
 			if (sin6)
 				in6_sin6_2_sin_in_sock(addr);
@@ -739,14 +785,20 @@ udp6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 			error = ((*pru->pru_send)(so, flags, m, addr, control,
 						  td));
 			/* addr will just be freed in sendit(). */
-			return error;
+			goto out;
 		}
 	}
 #endif
 
-	return udp6_output(inp, m, addr, control, td);
+	error = udp6_output(inp, m, addr, control, td);
+out:
+	INP_UNLOCK(inp);
+	INP_INFO_WUNLOCK(&udbinfo);
+	return error;
 
   bad:
+	INP_UNLOCK(inp);
+	INP_INFO_WUNLOCK(&udbinfo);
 	m_freem(m);
 	return (error);
 }