When we receive an ICMP unreach need fragmentation datagram, we take

proposed MTU value from it and update the TCP host cache. Then
tcp_mss_update() is called on the corresponding tcpcb. It finds the
just allocated entry in the TCP host cache and updates MSS on the
tcpcb. And then we do a fast retransmit of what we have in the tcp
send buffer.

This sequence gets broken if the TCP host cache is exausted. In this
case allocation fails, and later called tcp_mss_update() finds nothing
in cache. The fast retransmit is done with not reduced MSS and is
immidiately replied by remote host with new ICMP datagrams and the
cycle repeats. This ping-pong can go up to wirespeed.

To fix this:
- tcp_mss_update() gets new parameter - mtuoffer, that is like
  offer, but needs to have min_protoh subtracted.
- tcp_mtudisc() as notification method renamed to tcp_mtudisc_notify().
- tcp_mtudisc() now accepts not a useless error argument, but proposed
  MTU value, that is passed to tcp_mss_update() as mtuoffer.

Reported by:	az
Reported by:	Andrey Zonov <andrey zonov.org>
Reviewed by:	andre (previous version of patch)

4 files changed, 32 insertions, 19 deletions

diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 1a31c5e..7c8310f 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -3288,22 +3288,19 @@ tcp_xmit_timer(struct tcpcb *tp, int rtt)
  * are present.  Store the upper limit of the length of options plus
  * data in maxopd.
  *
- * In case of T/TCP, we call this routine during implicit connection
- * setup as well (offer = -1), to initialize maxseg from the cached
- * MSS of our peer.
- *
  * NOTE that this routine is only called when we process an incoming
- * segment. Outgoing SYN/ACK MSS settings are handled in tcp_mssopt().
+ * segment, or an ICMP need fragmentation datagram. Outgoing SYN/ACK MSS
+ * settings are handled in tcp_mssopt().
  */
 void
-tcp_mss_update(struct tcpcb *tp, int offer,
+tcp_mss_update(struct tcpcb *tp, int offer, int mtuoffer,
     struct hc_metrics_lite *metricptr, int *mtuflags)
 {
 	int mss = 0;
 	u_long maxmtu = 0;
 	struct inpcb *inp = tp->t_inpcb;
 	struct hc_metrics_lite metrics;
-	int origoffer = offer;
+	int origoffer;
 #ifdef INET6
 	int isipv6 = ((inp->inp_vflag & INP_IPV6) != 0) ? 1 : 0;
 	size_t min_protoh = isipv6 ?
@@ -3315,6 +3312,12 @@ tcp_mss_update(struct tcpcb *tp, int offer,
 
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
+	if (mtuoffer != -1) {
+		KASSERT(offer == -1, ("%s: conflict", __func__));
+		offer = mtuoffer - min_protoh;
+	}
+	origoffer = offer;
+
 	/* Initialize. */
 #ifdef INET6
 	if (isipv6) {
@@ -3473,7 +3476,7 @@ tcp_mss(struct tcpcb *tp, int offer)
 
 	KASSERT(tp != NULL, ("%s: tp == NULL", __func__));
 	
-	tcp_mss_update(tp, offer, &metrics, &mtuflags);
+	tcp_mss_update(tp, offer, -1, &metrics, &mtuflags);
 
 	mss = tp->t_maxseg;
 	inp = tp->t_inpcb;
diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c
index 1965db1..645fb37 100644
--- a/sys/netinet/tcp_output.c
+++ b/sys/netinet/tcp_output.c
@@ -1293,7 +1293,7 @@ out:
 			 */
 			if (tso)
 				tp->t_flags &= ~TF_TSO;
-			tcp_mtudisc(tp->t_inpcb, 0);
+			tcp_mtudisc(tp->t_inpcb, -1);
 			return (0);
 		case EHOSTDOWN:
 		case EHOSTUNREACH:
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index a86d8f7..561749a 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -222,6 +222,7 @@ VNET_DEFINE(uma_zone_t, sack_hole_zone);
 VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]);
 
 static struct inpcb *tcp_notify(struct inpcb *, int);
+static struct inpcb *tcp_mtudisc_notify(struct inpcb *, int);
 static char *	tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th,
 		    void *ip4hdr, const void *ip6hdr);
 
@@ -1337,7 +1338,7 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
 		return;
 
 	if (cmd == PRC_MSGSIZE)
-		notify = tcp_mtudisc;
+		notify = tcp_mtudisc_notify;
 	else if (V_icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB ||
 		cmd == PRC_UNREACH_PORT || cmd == PRC_TIMXCEED_INTRANS) && ip)
 		notify = tcp_drop_syn_sent;
@@ -1410,9 +1411,10 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
 					     */
 					    if (mtu <= tcp_maxmtu(&inc, NULL))
 						tcp_hc_updatemtu(&inc, mtu);
-					}
-
-					inp = (*notify)(inp, inetctlerrmap[cmd]);
+					    tcp_mtudisc(inp, mtu);
+					} else
+						inp = (*notify)(inp,
+						    inetctlerrmap[cmd]);
 				}
 			}
 			if (inp != NULL)
@@ -1452,7 +1454,7 @@ tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d)
 		return;
 
 	if (cmd == PRC_MSGSIZE)
-		notify = tcp_mtudisc;
+		notify = tcp_mtudisc_notify;
 	else if (!PRC_IS_REDIRECT(cmd) &&
 		 ((unsigned)cmd >= PRC_NCMDS || inet6ctlerrmap[cmd] == 0))
 		return;
@@ -1653,12 +1655,19 @@ tcp_drop_syn_sent(struct inpcb *inp, int errno)
 
 /*
  * When `need fragmentation' ICMP is received, update our idea of the MSS
- * based on the new value in the route.  Also nudge TCP to send something,
- * since we know the packet we just sent was dropped.
+ * based on the new value. Also nudge TCP to send something, since we
+ * know the packet we just sent was dropped.
  * This duplicates some code in the tcp_mss() function in tcp_input.c.
  */
+static struct inpcb *
+tcp_mtudisc_notify(struct inpcb *inp, int error)
+{
+
+	return (tcp_mtudisc(inp, -1));
+}
+
 struct inpcb *
-tcp_mtudisc(struct inpcb *inp, int errno)
+tcp_mtudisc(struct inpcb *inp, int mtuoffer)
 {
 	struct tcpcb *tp;
 	struct socket *so;
@@ -1671,7 +1680,7 @@ tcp_mtudisc(struct inpcb *inp, int errno)
 	tp = intotcpcb(inp);
 	KASSERT(tp != NULL, ("tcp_mtudisc: tp == NULL"));
 
-	tcp_mss_update(tp, -1, NULL, NULL);
+	tcp_mss_update(tp, -1, mtuoffer, NULL, NULL);
   
 	so = inp->inp_socket;
 	SOCKBUF_LOCK(&so->so_snd);
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index b602c27..3b8bdf7 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -675,7 +675,8 @@ void	 tcp_reass_destroy(void);
 void	 tcp_input(struct mbuf *, int);
 u_long	 tcp_maxmtu(struct in_conninfo *, int *);
 u_long	 tcp_maxmtu6(struct in_conninfo *, int *);
-void	 tcp_mss_update(struct tcpcb *, int, struct hc_metrics_lite *, int *);
+void	 tcp_mss_update(struct tcpcb *, int, int, struct hc_metrics_lite *,
+	    int *);
 void	 tcp_mss(struct tcpcb *, int);
 int	 tcp_mssopt(struct in_conninfo *);
 struct inpcb *