Unify the "send high" and "recover" variables as specified in the

lastest rev of the spec.  Use an explicit flag for Fast Recovery. [1]

Fix bug with exiting Fast Recovery on a retransmit timeout
diagnosed by Lu Guohan. [2]

Reviewed by:		Thomas Henderson <thomas.r.henderson@boeing.com>
Reported and tested by:	Lu Guohan <lguohan00@mails.tsinghua.edu.cn> [2]
Approved by:		Thomas Henderson <thomas.r.henderson@boeing.com>,
			Sally Floyd <floyd@acm.org> [1]

5 files changed, 83 insertions, 63 deletions

diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index f112022..a58e830 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -992,8 +992,7 @@ after_listen:
 			    tp->snd_cwnd >= tp->snd_wnd &&
 			    ((!tcp_do_newreno &&
 			      tp->t_dupacks < tcprexmtthresh) ||
-			     (tcp_do_newreno &&
-			      !SEQ_LT(tp->snd_una, tp->snd_recover)))) {
+			     (tcp_do_newreno && !IN_FASTRECOVERY(tp)))) {
 				KASSERT(headlocked, ("headlocked"));
 				INP_INFO_WUNLOCK(&tcbinfo);
 				/*
@@ -1009,7 +1008,9 @@ after_listen:
 					tp->snd_cwnd = tp->snd_cwnd_prev;
 					tp->snd_ssthresh =
 					    tp->snd_ssthresh_prev;
-					tp->snd_high = tp->snd_high_prev;
+					tp->snd_recover = tp->snd_recover_prev;
+					if (tp->t_flags & TF_WASFRECOVERY)
+					    ENTER_FASTRECOVERY(tp);
 					tp->snd_nxt = tp->snd_max;
 					tp->t_badrxtwin = 0;
 				}
@@ -1036,10 +1037,10 @@ after_listen:
 				tcpstat.tcps_rcvackpack++;
 				tcpstat.tcps_rcvackbyte += acked;
 				sbdrop(&so->so_snd, acked);
-				if (SEQ_GT(tp->snd_una, tp->snd_high) &&
-				    SEQ_LEQ(th->th_ack, tp->snd_high))
-					tp->snd_high = th->th_ack - 1;
-				tp->snd_una = tp->snd_recover = th->th_ack;
+				if (SEQ_GT(tp->snd_una, tp->snd_recover) &&
+				    SEQ_LEQ(th->th_ack, tp->snd_recover))
+					tp->snd_recover = th->th_ack - 1;
+				tp->snd_una = th->th_ack;
 				/*
 				 * pull snd_wl2 up to prevent seq wrap relative
 				 * to th_ack.
@@ -1739,8 +1740,7 @@ trimthenstep6:
 					tp->t_dupacks = 0;
 				else if (++tp->t_dupacks > tcprexmtthresh ||
 					 (tcp_do_newreno &&
-					  SEQ_LT(tp->snd_una,
-					  	 tp->snd_recover))) {
+					  IN_FASTRECOVERY(tp))) {
 					tp->snd_cwnd += tp->t_maxseg;
 					(void) tcp_output(tp);
 					goto drop;
@@ -1748,7 +1748,8 @@ trimthenstep6:
 					tcp_seq onxt = tp->snd_nxt;
 					u_int win;
 					if (tcp_do_newreno &&
-					    SEQ_LEQ(th->th_ack, tp->snd_high)) {
+					    SEQ_LEQ(th->th_ack,
+					            tp->snd_recover)) {
 						tp->t_dupacks = 0;
 						break;
 					}
@@ -1757,6 +1758,7 @@ trimthenstep6:
 					if (win < 2)
 						win = 2;
 					tp->snd_ssthresh = win * tp->t_maxseg;
+					ENTER_FASTRECOVERY(tp);
 					tp->snd_recover = tp->snd_max;
 					callout_stop(tp->tt_rexmt);
 					tp->t_rtttime = 0;
@@ -1809,7 +1811,7 @@ trimthenstep6:
 		 * for the other side's cached packets, retract it.
 		 */
 		if (tcp_do_newreno) {
-			if (SEQ_LT(tp->snd_una, tp->snd_recover)) {
+			if (IN_FASTRECOVERY(tp)) {
 				if (SEQ_LT(th->th_ack, tp->snd_recover)) {
 					tcp_newreno_partial_ack(tp, th);
 				} else {
@@ -1879,7 +1881,9 @@ process_ACK:
 			++tcpstat.tcps_sndrexmitbad;
 			tp->snd_cwnd = tp->snd_cwnd_prev;
 			tp->snd_ssthresh = tp->snd_ssthresh_prev;
-			tp->snd_high = tp->snd_high_prev;
+			tp->snd_recover = tp->snd_recover_prev;
+			if (tp->t_flags & TF_WASFRECOVERY)
+				ENTER_FASTRECOVERY(tp);
 			tp->snd_nxt = tp->snd_max;
 			tp->t_badrxtwin = 0;	/* XXX probably not required */ 
 		}
@@ -1933,7 +1937,7 @@ process_ACK:
 		 * Otherwise open linearly: maxseg per window
 		 * (maxseg^2 / cwnd per packet).
 		 */
-		if (!tcp_do_newreno || SEQ_GEQ(tp->snd_una, tp->snd_recover)) {
+		if (!tcp_do_newreno || !IN_FASTRECOVERY(tp)) {
 			register u_int cw = tp->snd_cwnd;
 			register u_int incr = tp->t_maxseg;
 			if (cw > tp->snd_ssthresh)
@@ -1951,12 +1955,13 @@ process_ACK:
 		}
 		sowwakeup(so);
 		/* detect una wraparound */
-		if (SEQ_GEQ(tp->snd_una, tp->snd_recover) &&
-		    SEQ_LT(th->th_ack, tp->snd_recover))
-			tp->snd_recover = th->th_ack;
-		if (SEQ_GT(tp->snd_una, tp->snd_high) &&
-		    SEQ_LEQ(th->th_ack, tp->snd_high))
-			tp->snd_high = th->th_ack - 1;
+		if (tcp_do_newreno && !IN_FASTRECOVERY(tp) &&
+		    SEQ_GT(tp->snd_una, tp->snd_recover) &&
+		    SEQ_LEQ(th->th_ack, tp->snd_recover))
+			tp->snd_recover = th->th_ack - 1;
+		if (tcp_do_newreno && IN_FASTRECOVERY(tp) &&
+		    SEQ_GEQ(th->th_ack, tp->snd_recover))
+			EXIT_FASTRECOVERY(tp);
 		tp->snd_una = th->th_ack;
 		if (SEQ_LT(tp->snd_nxt, tp->snd_una))
 			tp->snd_nxt = tp->snd_una;
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index f112022..a58e830 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -992,8 +992,7 @@ after_listen:
 			    tp->snd_cwnd >= tp->snd_wnd &&
 			    ((!tcp_do_newreno &&
 			      tp->t_dupacks < tcprexmtthresh) ||
-			     (tcp_do_newreno &&
-			      !SEQ_LT(tp->snd_una, tp->snd_recover)))) {
+			     (tcp_do_newreno && !IN_FASTRECOVERY(tp)))) {
 				KASSERT(headlocked, ("headlocked"));
 				INP_INFO_WUNLOCK(&tcbinfo);
 				/*
@@ -1009,7 +1008,9 @@ after_listen:
 					tp->snd_cwnd = tp->snd_cwnd_prev;
 					tp->snd_ssthresh =
 					    tp->snd_ssthresh_prev;
-					tp->snd_high = tp->snd_high_prev;
+					tp->snd_recover = tp->snd_recover_prev;
+					if (tp->t_flags & TF_WASFRECOVERY)
+					    ENTER_FASTRECOVERY(tp);
 					tp->snd_nxt = tp->snd_max;
 					tp->t_badrxtwin = 0;
 				}
@@ -1036,10 +1037,10 @@ after_listen:
 				tcpstat.tcps_rcvackpack++;
 				tcpstat.tcps_rcvackbyte += acked;
 				sbdrop(&so->so_snd, acked);
-				if (SEQ_GT(tp->snd_una, tp->snd_high) &&
-				    SEQ_LEQ(th->th_ack, tp->snd_high))
-					tp->snd_high = th->th_ack - 1;
-				tp->snd_una = tp->snd_recover = th->th_ack;
+				if (SEQ_GT(tp->snd_una, tp->snd_recover) &&
+				    SEQ_LEQ(th->th_ack, tp->snd_recover))
+					tp->snd_recover = th->th_ack - 1;
+				tp->snd_una = th->th_ack;
 				/*
 				 * pull snd_wl2 up to prevent seq wrap relative
 				 * to th_ack.
@@ -1739,8 +1740,7 @@ trimthenstep6:
 					tp->t_dupacks = 0;
 				else if (++tp->t_dupacks > tcprexmtthresh ||
 					 (tcp_do_newreno &&
-					  SEQ_LT(tp->snd_una,
-					  	 tp->snd_recover))) {
+					  IN_FASTRECOVERY(tp))) {
 					tp->snd_cwnd += tp->t_maxseg;
 					(void) tcp_output(tp);
 					goto drop;
@@ -1748,7 +1748,8 @@ trimthenstep6:
 					tcp_seq onxt = tp->snd_nxt;
 					u_int win;
 					if (tcp_do_newreno &&
-					    SEQ_LEQ(th->th_ack, tp->snd_high)) {
+					    SEQ_LEQ(th->th_ack,
+					            tp->snd_recover)) {
 						tp->t_dupacks = 0;
 						break;
 					}
@@ -1757,6 +1758,7 @@ trimthenstep6:
 					if (win < 2)
 						win = 2;
 					tp->snd_ssthresh = win * tp->t_maxseg;
+					ENTER_FASTRECOVERY(tp);
 					tp->snd_recover = tp->snd_max;
 					callout_stop(tp->tt_rexmt);
 					tp->t_rtttime = 0;
@@ -1809,7 +1811,7 @@ trimthenstep6:
 		 * for the other side's cached packets, retract it.
 		 */
 		if (tcp_do_newreno) {
-			if (SEQ_LT(tp->snd_una, tp->snd_recover)) {
+			if (IN_FASTRECOVERY(tp)) {
 				if (SEQ_LT(th->th_ack, tp->snd_recover)) {
 					tcp_newreno_partial_ack(tp, th);
 				} else {
@@ -1879,7 +1881,9 @@ process_ACK:
 			++tcpstat.tcps_sndrexmitbad;
 			tp->snd_cwnd = tp->snd_cwnd_prev;
 			tp->snd_ssthresh = tp->snd_ssthresh_prev;
-			tp->snd_high = tp->snd_high_prev;
+			tp->snd_recover = tp->snd_recover_prev;
+			if (tp->t_flags & TF_WASFRECOVERY)
+				ENTER_FASTRECOVERY(tp);
 			tp->snd_nxt = tp->snd_max;
 			tp->t_badrxtwin = 0;	/* XXX probably not required */ 
 		}
@@ -1933,7 +1937,7 @@ process_ACK:
 		 * Otherwise open linearly: maxseg per window
 		 * (maxseg^2 / cwnd per packet).
 		 */
-		if (!tcp_do_newreno || SEQ_GEQ(tp->snd_una, tp->snd_recover)) {
+		if (!tcp_do_newreno || !IN_FASTRECOVERY(tp)) {
 			register u_int cw = tp->snd_cwnd;
 			register u_int incr = tp->t_maxseg;
 			if (cw > tp->snd_ssthresh)
@@ -1951,12 +1955,13 @@ process_ACK:
 		}
 		sowwakeup(so);
 		/* detect una wraparound */
-		if (SEQ_GEQ(tp->snd_una, tp->snd_recover) &&
-		    SEQ_LT(th->th_ack, tp->snd_recover))
-			tp->snd_recover = th->th_ack;
-		if (SEQ_GT(tp->snd_una, tp->snd_high) &&
-		    SEQ_LEQ(th->th_ack, tp->snd_high))
-			tp->snd_high = th->th_ack - 1;
+		if (tcp_do_newreno && !IN_FASTRECOVERY(tp) &&
+		    SEQ_GT(tp->snd_una, tp->snd_recover) &&
+		    SEQ_LEQ(th->th_ack, tp->snd_recover))
+			tp->snd_recover = th->th_ack - 1;
+		if (tcp_do_newreno && IN_FASTRECOVERY(tp) &&
+		    SEQ_GEQ(th->th_ack, tp->snd_recover))
+			EXIT_FASTRECOVERY(tp);
 		tp->snd_una = th->th_ack;
 		if (SEQ_LT(tp->snd_nxt, tp->snd_una))
 			tp->snd_nxt = tp->snd_una;
diff --git a/sys/netinet/tcp_seq.h b/sys/netinet/tcp_seq.h
index 90dd97d..5850ccc 100644
--- a/sys/netinet/tcp_seq.h
+++ b/sys/netinet/tcp_seq.h
@@ -73,7 +73,7 @@
 
 #define	tcp_sendseqinit(tp) \
 	(tp)->snd_una = (tp)->snd_nxt = (tp)->snd_max = (tp)->snd_up = \
-	    (tp)->snd_recover = (tp)->snd_high = (tp)->iss
+	    (tp)->snd_recover = (tp)->iss
 
 #define TCP_PAWS_IDLE	(24 * 24 * 60 * 60 * hz)
 					/* timestamp wrap-around time */
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 4dfdba4..12b63a5 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -528,7 +528,11 @@ tcp_timer_rexmt(xtp)
 		 */
 		tp->snd_cwnd_prev = tp->snd_cwnd;
 		tp->snd_ssthresh_prev = tp->snd_ssthresh;
-		tp->snd_high_prev = tp->snd_high;
+		tp->snd_recover_prev = tp->snd_recover;
+		if (IN_FASTRECOVERY(tp))
+		  tp->t_flags |= TF_WASFRECOVERY;
+		else
+		  tp->t_flags &= ~TF_WASFRECOVERY;
 		tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
 	}
 	tcpstat.tcps_rexmttimeo++;
@@ -566,7 +570,7 @@ tcp_timer_rexmt(xtp)
 		tp->t_srtt = 0;
 	}
 	tp->snd_nxt = tp->snd_una;
-	tp->snd_high = tp->snd_max;
+	tp->snd_recover = tp->snd_max;
 	/*
 	 * Force a segment to be sent.
 	 */
@@ -607,6 +611,7 @@ tcp_timer_rexmt(xtp)
 		tp->snd_ssthresh = win * tp->t_maxseg;
 		tp->t_dupacks = 0;
 	}
+	EXIT_FASTRECOVERY(tp);
 	(void) tcp_output(tp);
 
 out:
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index 11474b0..23898b9 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -83,26 +83,28 @@ struct tcpcb {
 	struct	inpcb *t_inpcb;		/* back pointer to internet pcb */
 	int	t_state;		/* state of this connection */
 	u_int	t_flags;
-#define	TF_ACKNOW	0x00001		/* ack peer immediately */
-#define	TF_DELACK	0x00002		/* ack, but try to delay it */
-#define	TF_NODELAY	0x00004		/* don't delay packets to coalesce */
-#define	TF_NOOPT	0x00008		/* don't use tcp options */
-#define	TF_SENTFIN	0x00010		/* have sent FIN */
-#define	TF_REQ_SCALE	0x00020		/* have/will request window scaling */
-#define	TF_RCVD_SCALE	0x00040		/* other side has requested scaling */
-#define	TF_REQ_TSTMP	0x00080		/* have/will request timestamps */
-#define	TF_RCVD_TSTMP	0x00100		/* a timestamp was received in SYN */
-#define	TF_SACK_PERMIT	0x00200		/* other side said I could SACK */
-#define	TF_NEEDSYN	0x00400		/* send SYN (implicit state) */
-#define	TF_NEEDFIN	0x00800		/* send FIN (implicit state) */
-#define	TF_NOPUSH	0x01000		/* don't push */
-#define	TF_REQ_CC	0x02000		/* have/will request CC */
-#define	TF_RCVD_CC	0x04000		/* a CC was received in SYN */
-#define	TF_SENDCCNEW	0x08000		/* send CCnew instead of CC in SYN */
-#define	TF_MORETOCOME	0x10000		/* More data to be appended to sock */
-#define	TF_LQ_OVERFLOW	0x20000		/* listen queue overflow */
-#define	TF_LASTIDLE	0x40000		/* connection was previously idle */
-#define TF_RXWIN0SENT	0x80000		/* sent a receiver win 0 in response */
+#define	TF_ACKNOW	0x000001	/* ack peer immediately */
+#define	TF_DELACK	0x000002	/* ack, but try to delay it */
+#define	TF_NODELAY	0x000004	/* don't delay packets to coalesce */
+#define	TF_NOOPT	0x000008	/* don't use tcp options */
+#define	TF_SENTFIN	0x000010	/* have sent FIN */
+#define	TF_REQ_SCALE	0x000020	/* have/will request window scaling */
+#define	TF_RCVD_SCALE	0x000040	/* other side has requested scaling */
+#define	TF_REQ_TSTMP	0x000080	/* have/will request timestamps */
+#define	TF_RCVD_TSTMP	0x000100	/* a timestamp was received in SYN */
+#define	TF_SACK_PERMIT	0x000200	/* other side said I could SACK */
+#define	TF_NEEDSYN	0x000400	/* send SYN (implicit state) */
+#define	TF_NEEDFIN	0x000800	/* send FIN (implicit state) */
+#define	TF_NOPUSH	0x001000	/* don't push */
+#define	TF_REQ_CC	0x002000	/* have/will request CC */
+#define	TF_RCVD_CC	0x004000	/* a CC was received in SYN */
+#define	TF_SENDCCNEW	0x008000	/* send CCnew instead of CC in SYN */
+#define	TF_MORETOCOME	0x010000	/* More data to be appended to sock */
+#define	TF_LQ_OVERFLOW	0x020000	/* listen queue overflow */
+#define	TF_LASTIDLE	0x040000	/* connection was previously idle */
+#define	TF_RXWIN0SENT	0x080000	/* sent a receiver win 0 in response */
+#define	TF_FASTRECOVERY	0x100000	/* in NewReno Fast Recovery */
+#define	TF_WASFRECOVERY	0x200000	/* was in NewReno Fast Recovery */
 	int	t_force;		/* 1 if forcing out a byte */
 
 	tcp_seq	snd_una;		/* send unacknowledged */
@@ -131,7 +133,6 @@ struct tcpcb {
 					 */
 	u_long	snd_bandwidth;		/* calculated bandwidth or 0 */
 	tcp_seq	snd_recover;		/* for use in NewReno Fast Recovery */
-	tcp_seq snd_high;		/* for use in NewReno Fast Recovery */
 
 	u_int	t_maxopd;		/* mss plus options */
 
@@ -175,11 +176,15 @@ struct tcpcb {
 /* experimental */
 	u_long	snd_cwnd_prev;		/* cwnd prior to retransmit */
 	u_long	snd_ssthresh_prev;	/* ssthresh prior to retransmit */
-	tcp_seq	snd_high_prev;		/* snd_high prior to retransmit */
+	tcp_seq	snd_recover_prev;	/* snd_recover prior to retransmit */
 	u_long	t_badrxtwin;		/* window for retransmit recovery */
 	u_char	snd_limited;		/* segments limited transmitted */
 };
 
+#define IN_FASTRECOVERY(tp)	(tp->t_flags & TF_FASTRECOVERY)
+#define ENTER_FASTRECOVERY(tp)	tp->t_flags |= TF_FASTRECOVERY
+#define EXIT_FASTRECOVERY(tp)	tp->t_flags &= ~TF_FASTRECOVERY
+
 /*
  * Structure to hold TCP options that are only used during segment
  * processing (in tcp_input), but not held in the tcpcb.