Convert pcbinfo and inpcb mutexes to rwlocks, and modify macros to

explicitly select write locking for all use of the inpcb mutex.
Update some pcbinfo lock assertions to assert locked rather than
write-locked, although in practice almost all uses of the pcbinfo
rwlock main exclusive, and all instances of inpcb lock acquisition
are exclusive.

This change should introduce (ideally) little functional change.
However, it lays the groundwork for significantly increased
parallelism in the TCP/IP code.

MFC after:	3 months
Tested by:	kris (superset of committered patch)

5 files changed, 65 insertions, 65 deletions

diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index ea5a646..7039edd 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -1891,10 +1891,10 @@ icmp6_rip6_input(struct mbuf **mp, int off)
 
 	INP_INFO_RLOCK(&ripcbinfo);
 	LIST_FOREACH(in6p, &ripcb, inp_list) {
-		INP_LOCK(in6p);
+		INP_WLOCK(in6p);
 		if ((in6p->inp_vflag & INP_IPV6) == 0) {
 	docontinue:
-			INP_UNLOCK(in6p);
+			INP_WUNLOCK(in6p);
 			continue;
 		}
 		if (in6p->in6p_ip6_nxt != IPPROTO_ICMPV6)
@@ -1965,7 +1965,7 @@ icmp6_rip6_input(struct mbuf **mp, int off)
 					sorwakeup_locked(last->in6p_socket);
 				opts = NULL;
 			}
-			INP_UNLOCK(last);
+			INP_WUNLOCK(last);
 		}
 		last = in6p;
 	}
@@ -2003,7 +2003,7 @@ icmp6_rip6_input(struct mbuf **mp, int off)
 			SOCKBUF_UNLOCK(&last->in6p_socket->so_rcv);
 		} else
 			sorwakeup_locked(last->in6p_socket);
-		INP_UNLOCK(last);
+		INP_WUNLOCK(last);
 	} else {
 		m_freem(m);
 		ip6stat.ip6s_delivered--;
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index a93b45a..9986e69 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -122,7 +122,7 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 	int wild = 0, reuseport = (so->so_options & SO_REUSEPORT);
 
 	INP_INFO_WLOCK_ASSERT(pcbinfo);
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 	if (!in6_ifaddr) /* XXX broken! */
 		return (EADDRNOTAVAIL);
@@ -288,7 +288,7 @@ in6_pcbladdr(register struct inpcb *inp, struct sockaddr *nam,
 	int scope_ambiguous = 0;
 
 	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 	if (nam->sa_len != sizeof (*sin6))
 		return (EINVAL);
@@ -354,7 +354,7 @@ in6_pcbconnect(register struct inpcb *inp, struct sockaddr *nam,
 	int error;
 
 	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 	/*
 	 * Call inner routine, to assign local interface address.
@@ -396,7 +396,7 @@ in6_pcbdisconnect(struct inpcb *inp)
 {
 
 	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 	bzero((caddr_t)&inp->in6p_faddr, sizeof(inp->in6p_faddr));
 	inp->inp_fport = 0;
@@ -421,7 +421,7 @@ in6_pcbfree(struct inpcb *inp)
 
 	KASSERT(inp->inp_socket == NULL, ("in6_pcbfree: inp_socket != NULL"));
 	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 #ifdef IPSEC
 	if (inp->in6p_sp != NULL)
@@ -440,7 +440,7 @@ in6_pcbfree(struct inpcb *inp)
 #ifdef MAC
 	mac_inpcb_destroy(inp);
 #endif
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	uma_zfree(ipi->ipi_zone, inp);
 }
 
@@ -489,10 +489,10 @@ in6_getsockaddr(struct socket *so, struct sockaddr **nam)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("in6_getsockaddr: inp == NULL"));
 
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	port = inp->inp_lport;
 	addr = inp->in6p_laddr;
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 
 	*nam = in6_sockaddr(port, &addr);
 	return 0;
@@ -508,10 +508,10 @@ in6_getpeeraddr(struct socket *so, struct sockaddr **nam)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("in6_getpeeraddr: inp == NULL"));
 
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	port = inp->inp_fport;
 	addr = inp->in6p_faddr;
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 
 	*nam = in6_sockaddr(port, &addr);
 	return 0;
@@ -611,9 +611,9 @@ in6_pcbnotify(struct inpcbinfo *pcbinfo, struct sockaddr *dst,
 	errno = inet6ctlerrmap[cmd];
 	INP_INFO_WLOCK(pcbinfo);
 	LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
-		INP_LOCK(inp);
+		INP_WLOCK(inp);
 		if ((inp->inp_vflag & INP_IPV6) == 0) {
-			INP_UNLOCK(inp);
+			INP_WUNLOCK(inp);
 			continue;
 		}
 
@@ -654,16 +654,16 @@ in6_pcbnotify(struct inpcbinfo *pcbinfo, struct sockaddr *dst,
 			  !IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr,
 					      &sa6_src.sin6_addr)) ||
 			 (fport && inp->inp_fport != fport)) {
-			INP_UNLOCK(inp);
+			INP_WUNLOCK(inp);
 			continue;
 		}
 
 	  do_notify:
 		if (notify) {
 			if ((*notify)(inp, errno))
-				INP_UNLOCK(inp);
+				INP_WUNLOCK(inp);
 		} else
-			INP_UNLOCK(inp);
+			INP_WUNLOCK(inp);
 	}
 	INP_INFO_WUNLOCK(pcbinfo);
 }
@@ -765,7 +765,7 @@ in6_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
 
 	INP_INFO_RLOCK(pcbinfo);
 	LIST_FOREACH(in6p, pcbinfo->ipi_listhead, inp_list) {
-		INP_LOCK(in6p);
+		INP_WLOCK(in6p);
 		im6o = in6p->in6p_moptions;
 		if ((in6p->inp_vflag & INP_IPV6) &&
 		    im6o) {
@@ -792,7 +792,7 @@ in6_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
 				}
 			}
 		}
-		INP_UNLOCK(in6p);
+		INP_WUNLOCK(in6p);
 	}
 	INP_INFO_RUNLOCK(pcbinfo);
 }
@@ -839,7 +839,7 @@ in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
 	u_short fport = fport_arg, lport = lport_arg;
 	int faith;
 
-	INP_INFO_RLOCK_ASSERT(pcbinfo);
+	INP_INFO_LOCK_ASSERT(pcbinfo);
 
 	if (faithprefix_p != NULL)
 		faith = (*faithprefix_p)(laddr);
diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c
index eeba82b..b6e2856 100644
--- a/sys/netinet6/in6_src.c
+++ b/sys/netinet6/in6_src.c
@@ -763,7 +763,7 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct ucred *cred)
 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 
 	INP_INFO_WLOCK_ASSERT(pcbinfo);
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 	/* XXX: this is redundant when called from in6_pcbbind */
 	if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) == 0)
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index 30fcbea..c12bde1 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -154,10 +154,10 @@ rip6_input(struct mbuf **mp, int *offp, int proto)
 
 	INP_INFO_RLOCK(&ripcbinfo);
 	LIST_FOREACH(in6p, &ripcb, inp_list) {
-		INP_LOCK(in6p);
+		INP_WLOCK(in6p);
 		if ((in6p->in6p_vflag & INP_IPV6) == 0) {
 docontinue:
-			INP_UNLOCK(in6p);
+			INP_WUNLOCK(in6p);
 			continue;
 		}
 		if (in6p->in6p_ip6_nxt &&
@@ -207,7 +207,7 @@ docontinue:
 					sorwakeup(last->in6p_socket);
 				opts = NULL;
 			}
-			INP_UNLOCK(last);
+			INP_WUNLOCK(last);
 		}
 		last = in6p;
 	}
@@ -220,7 +220,7 @@ docontinue:
 		ipsec6stat.in_polvio++;
 		ip6stat.ip6s_delivered--;
 		/* do not inject data into pcb */
-		INP_UNLOCK(last);
+		INP_WUNLOCK(last);
 	} else
 #endif /* IPSEC */
 	if (last) {
@@ -237,7 +237,7 @@ docontinue:
 			rip6stat.rip6s_fullsock++;
 		} else
 			sorwakeup(last->in6p_socket);
-		INP_UNLOCK(last);
+		INP_WUNLOCK(last);
 	} else {
 		rip6stat.rip6s_nosock++;
 		if (m->m_flags & M_MCAST)
@@ -335,7 +335,7 @@ rip6_output(m, va_alist)
 	va_end(ap);
 
 	in6p = sotoin6pcb(so);
-	INP_LOCK(in6p);
+	INP_WLOCK(in6p);
 
 	dst = &dstsock->sin6_addr;
 	if (control) {
@@ -465,7 +465,7 @@ rip6_output(m, va_alist)
 		ip6_clearpktopts(&opt, -1);
 		m_freem(control);
 	}
-	INP_UNLOCK(in6p);
+	INP_WUNLOCK(in6p);
 	return (error);
 }
 
@@ -569,7 +569,7 @@ rip6_attach(struct socket *so, int proto, struct thread *td)
 	inp->in6p_cksum = -1;
 	inp->in6p_icmp6filt = filter;
 	ICMP6_FILTER_SETPASSALL(inp->in6p_icmp6filt);
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	return 0;
 }
 
@@ -585,7 +585,7 @@ rip6_detach(struct socket *so)
 		ip6_mrouter_done();
 	/* xxx: RSVP */
 	INP_INFO_WLOCK(&ripcbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	if (inp->in6p_icmp6filt) {
 		FREE(inp->in6p_icmp6filt, M_PCB);
 		inp->in6p_icmp6filt = NULL;
@@ -656,9 +656,9 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 		return (EADDRNOTAVAIL);
 	}
 	INP_INFO_WLOCK(&ripcbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	inp->in6p_laddr = addr->sin6_addr;
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&ripcbinfo);
 	return 0;
 }
@@ -694,13 +694,13 @@ rip6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		return(error);
 
 	INP_INFO_WLOCK(&ripcbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	/* Source address selection. XXX: need pcblookup? */
 	in6a = in6_selectsrc(addr, inp->in6p_outputopts,
 			     inp->in6p_moptions, NULL,
 			     &inp->in6p_laddr, &ifp, &error);
 	if (in6a == NULL) {
-		INP_UNLOCK(inp);
+		INP_WUNLOCK(inp);
 		INP_INFO_WUNLOCK(&ripcbinfo);
 		return (error ? error : EADDRNOTAVAIL);
 	}
@@ -708,14 +708,14 @@ rip6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	/* XXX: see above */
 	if (ifp && scope_ambiguous &&
 	    (error = in6_setscope(&addr->sin6_addr, ifp, NULL)) != 0) {
-		INP_UNLOCK(inp);
+		INP_WUNLOCK(inp);
 		INP_INFO_WUNLOCK(&ripcbinfo);
 		return(error);
 	}
 	inp->in6p_faddr = addr->sin6_addr;
 	inp->in6p_laddr = *in6a;
 	soisconnected(so);
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&ripcbinfo);
 	return 0;
 }
@@ -727,9 +727,9 @@ rip6_shutdown(struct socket *so)
 
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("rip6_shutdown: inp == NULL"));
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	socantsendmore(so);
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	return 0;
 }
 
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index 803e32d..b019ca7 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -129,7 +129,7 @@ udp6_append(struct inpcb *inp, struct mbuf *n, int off,
 	struct socket *so;
 	struct mbuf *opts;
 
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 #ifdef IPSEC
 	/* Check AH/ESP integrity. */
@@ -277,9 +277,9 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 				struct mbuf *n;
 
 				if ((n = m_copy(m, 0, M_COPYALL)) != NULL) {
-					INP_LOCK(last);
+					INP_WLOCK(last);
 					udp6_append(last, n, off, &fromsa);
-					INP_UNLOCK(last);
+					INP_WUNLOCK(last);
 				}
 			}
 			last = inp;
@@ -306,9 +306,9 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 			udpstat.udps_noportmcast++;
 			goto badheadlocked;
 		}
-		INP_LOCK(last);
+		INP_WLOCK(last);
 		udp6_append(last, m, off, &fromsa);
-		INP_UNLOCK(last);
+		INP_WUNLOCK(last);
 		INP_INFO_RUNLOCK(&udbinfo);
 		return (IPPROTO_DONE);
 	}
@@ -343,9 +343,9 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 		icmp6_error(m, ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_NOPORT, 0);
 		return (IPPROTO_DONE);
 	}
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	udp6_append(inp, m, off, &fromsa);
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&udbinfo);
 	return (IPPROTO_DONE);
 
@@ -453,7 +453,7 @@ udp6_getcred(SYSCTL_HANDLER_ARGS)
 		INP_INFO_RUNLOCK(&udbinfo);
 		return (ENOENT);
 	}
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	if (inp->inp_socket == NULL) {
 		error = ENOENT;
 		goto out;
@@ -463,7 +463,7 @@ udp6_getcred(SYSCTL_HANDLER_ARGS)
 		goto out;
 	cru2x(inp->inp_socket->so_cred, &xuc);
 out:
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&udbinfo);
 	if (error == 0)
 		error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
@@ -492,7 +492,7 @@ udp6_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr6,
 	int flags;
 	struct sockaddr_in6 tmp;
 
-	INP_LOCK_ASSERT(inp);
+	INP_WLOCK_ASSERT(inp);
 
 	if (addr6) {
 		/* addr6 has been validated in udp6_send(). */
@@ -708,13 +708,13 @@ udp6_abort(struct socket *so)
 #endif
 
 	INP_INFO_WLOCK(&udbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
 		in6_pcbdisconnect(inp);
 		inp->in6p_laddr = in6addr_any;
 		soisdisconnected(so);
 	}
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
 }
 
@@ -752,7 +752,7 @@ udp6_attach(struct socket *so, int proto, struct thread *td)
 	 * which may match an IPv4-mapped IPv6 address.
 	 */
 	inp->inp_ip_ttl = ip_defttl;
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	return (0);
 }
 
@@ -766,7 +766,7 @@ udp6_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	KASSERT(inp != NULL, ("udp6_bind: inp == NULL"));
 
 	INP_INFO_WLOCK(&udbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	inp->inp_vflag &= ~INP_IPV4;
 	inp->inp_vflag |= INP_IPV6;
 	if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0) {
@@ -790,7 +790,7 @@ udp6_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 
 	error = in6_pcbbind(inp, nam, td->td_ucred);
 out:
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
 	return (error);
 }
@@ -813,13 +813,13 @@ udp6_close(struct socket *so)
 	}
 #endif
 	INP_INFO_WLOCK(&udbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
 		in6_pcbdisconnect(inp);
 		inp->in6p_laddr = in6addr_any;
 		soisdisconnected(so);
 	}
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
 }
 
@@ -833,7 +833,7 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	KASSERT(inp != NULL, ("udp6_connect: inp == NULL"));
 
 	INP_INFO_WLOCK(&udbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0) {
 		struct sockaddr_in6 *sin6_p;
 
@@ -870,7 +870,7 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		soisconnected(so);
 	}
 out:
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
 	return (error);
 }
@@ -884,7 +884,7 @@ udp6_detach(struct socket *so)
 	KASSERT(inp != NULL, ("udp6_detach: inp == NULL"));
 
 	INP_INFO_WLOCK(&udbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	in6_pcbdetach(inp);
 	in6_pcbfree(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
@@ -900,7 +900,7 @@ udp6_disconnect(struct socket *so)
 	KASSERT(inp != NULL, ("udp6_disconnect: inp == NULL"));
 
 	INP_INFO_WLOCK(&udbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 
 #ifdef INET
 	if (inp->inp_vflag & INP_IPV4) {
@@ -922,7 +922,7 @@ udp6_disconnect(struct socket *so)
 	/* XXXRW: so_state locking? */
 	so->so_state &= ~SS_ISCONNECTED;		/* XXX */
 out:
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
 	return (0);
 }
@@ -938,7 +938,7 @@ udp6_send(struct socket *so, int flags, struct mbuf *m,
 	KASSERT(inp != NULL, ("udp6_send: inp == NULL"));
 
 	INP_INFO_WLOCK(&udbinfo);
-	INP_LOCK(inp);
+	INP_WLOCK(inp);
 	if (addr) {
 		if (addr->sa_len != sizeof(struct sockaddr_in6)) {
 			error = EINVAL;
@@ -991,12 +991,12 @@ udp6_send(struct socket *so, int flags, struct mbuf *m,
 #endif
 	error = udp6_output(inp, m, addr, control, td);
 out:
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
 	return (error);
 
 bad:
-	INP_UNLOCK(inp);
+	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&udbinfo);
 	m_freem(m);
 	return (error);