- fix sctp_ifn initial refcount issue (prevents deletion)

- fix a bug during cookie collision that prevented an
  association from coming up in a specific restart case.
- Fix it so the shutdown-pending flag gets removed (this is
  more for correctness then needed) when we enter shutdown-sent
  or shutdown-ack-sent states.
- Fix a bug that caused the receiver to sometimes NOT send
  a SACK when a duplicate TSN arrived. Without this fix
  it was possible for the association to fall down if the
- Deleted primary destination is also stored when SCTP_MOBILITY_BASE.
  (Previously, it is stored when only SCTP_MOBILITY_FASTHANDOFF)
- Fix a locking issue where we might call send_initiate_ack() and
  incorrectly state the lock held/not held. Also fix it so that
  when we release the lock the inp cannot be deleted on us.
- Add the debug option that can cause the stack to panic instead
  of aborting an assoc. This does not and should never show up
  in options but is useful for debugging unexpected aborts.
- Add cumack_log sent to track sending cumack information for
  the debug case where we are running a special log per assoc.
- Added extra () aroudn sctp_sbspace macro to avoid compile warnings.
MFC after:	1 week

10 files changed, 64 insertions, 54 deletions

diff --git a/sys/netinet/sctp_constants.h b/sys/netinet/sctp_constants.h
index fab56a3..f2174df 100644
--- a/sys/netinet/sctp_constants.h
+++ b/sys/netinet/sctp_constants.h
@@ -506,6 +506,8 @@ __FBSDID("$FreeBSD$");
 
 #define SCTP_GET_STATE(asoc)	((asoc)->state & SCTP_STATE_MASK)
 #define SCTP_SET_STATE(asoc, newstate)  ((asoc)->state = ((asoc)->state & ~SCTP_STATE_MASK) |  newstate)
+#define SCTP_CLEAR_SUBSTATE(asoc, substate) ((asoc)->state &= ~substate)
+#define SCTP_ADD_SUBSTATE(asoc, substate) ((asoc)->state |= substate)
 
 /* SCTP reachability state for each address */
 #define SCTP_ADDR_REACHABLE		0x001
diff --git a/sys/netinet/sctp_indata.c b/sys/netinet/sctp_indata.c
index e001a0c..575d671 100644
--- a/sys/netinet/sctp_indata.c
+++ b/sys/netinet/sctp_indata.c
@@ -1477,6 +1477,7 @@ sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc,
 			asoc->dup_tsns[asoc->numduptsns] = tsn;
 			asoc->numduptsns++;
 		}
+		asoc->send_sack = 1;
 		return (0);
 	}
 	/* Calculate the number of TSN's between the base and this TSN */
@@ -1571,9 +1572,6 @@ sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc,
 		    asoc->highest_tsn_inside_map, MAX_TSN)) {
 
 			/* Nope not in the valid range dump it */
-			SCTPDBG(SCTP_DEBUG_INDATA1, "My rwnd overrun1:tsn:%x rwnd %x sbspace:%x\n",
-			    tsn, asoc->my_rwnd,
-			    sctp_sbspace(&stcb->asoc, &stcb->sctp_socket->so_rcv));
 			sctp_set_rwnd(stcb, asoc);
 			if ((asoc->cnt_on_all_streams +
 			    asoc->cnt_on_reasm_queue +
@@ -2774,6 +2772,7 @@ sctp_process_data(struct mbuf **mm, int iphlen, int *offset, int length,
 	    (stcb->asoc.mapping_array[0] != 0xff)) {
 		if ((stcb->asoc.data_pkts_seen >= stcb->asoc.sack_freq) ||
 		    (stcb->asoc.delayed_ack == 0) ||
+		    (stcb->asoc.numduptsns) ||
 		    (stcb->asoc.send_sack == 1)) {
 			if (SCTP_OS_TIMER_PENDING(&stcb->asoc.dack_timer.timer)) {
 				(void)SCTP_OS_TIMER_STOP(&stcb->asoc.dack_timer.timer);
@@ -4279,6 +4278,7 @@ again:
 					SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 				}
 				SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT);
+				SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 				sctp_stop_timers_for_shutdown(stcb);
 				sctp_send_shutdown(stcb,
 				    stcb->asoc.primary_destination);
@@ -4294,6 +4294,7 @@ again:
 			}
 			SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 			SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_ACK_SENT);
+			SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 			sctp_send_shutdown_ack(stcb,
 			    stcb->asoc.primary_destination);
 
@@ -4970,6 +4971,7 @@ done_with_it:
 					SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 				}
 				SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT);
+				SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 				sctp_stop_timers_for_shutdown(stcb);
 				sctp_send_shutdown(stcb,
 				    stcb->asoc.primary_destination);
@@ -4986,6 +4988,7 @@ done_with_it:
 			}
 			SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 			SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_ACK_SENT);
+			SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 			sctp_send_shutdown_ack(stcb,
 			    stcb->asoc.primary_destination);
 
diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c
index 10f305c..ea7456f 100644
--- a/sys/netinet/sctp_input.c
+++ b/sys/netinet/sctp_input.c
@@ -168,7 +168,7 @@ sctp_handle_init(struct mbuf *m, int iphlen, int offset, struct sctphdr *sh,
 	/* send an INIT-ACK w/cookie */
 	SCTPDBG(SCTP_DEBUG_INPUT3, "sctp_handle_init: sending INIT-ACK\n");
 	sctp_send_initiate_ack(inp, stcb, m, iphlen, offset, sh, cp, vrf_id,
-	    SCTP_HOLDS_LOCK);
+	    ((stcb == NULL) ? SCTP_HOLDS_LOCK : SCTP_NOT_LOCKED));
 outnow:
 	if (stcb == NULL) {
 		SCTP_INP_RUNLOCK(inp);
@@ -739,6 +739,7 @@ sctp_handle_shutdown(struct sctp_shutdown_chunk *cp,
 		    (SCTP_GET_STATE(asoc) != SCTP_STATE_SHUTDOWN_ACK_SENT) &&
 		    (SCTP_GET_STATE(asoc) != SCTP_STATE_SHUTDOWN_SENT)) {
 			SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_RECEIVED);
+			SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 			/*
 			 * notify upper layer that peer has initiated a
 			 * shutdown
@@ -774,7 +775,7 @@ sctp_handle_shutdown(struct sctp_shutdown_chunk *cp,
 			SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 		}
 		SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_ACK_SENT);
-
+		SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 		sctp_timer_stop(SCTP_TIMER_TYPE_RECV, stcb->sctp_ep, stcb, net,
 		    SCTP_FROM_SCTP_INPUT + SCTP_LOC_7);
 		/* start SHUTDOWN timer */
@@ -4387,51 +4388,31 @@ process_control_chunks:
 			 * closed. We opened and bound.. and are now no
 			 * longer listening.
 			 */
-			if (inp->sctp_socket->so_qlimit == 0) {
-				if ((stcb) && (inp->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL)) {
-					/*
-					 * special case, is this a retran'd
-					 * COOKIE-ECHO or a restarting assoc
-					 * that is a peeled off or
-					 * one-to-one style socket.
-					 */
-					goto process_cookie_anyway;
-				}
-				sctp_abort_association(inp, stcb, m, iphlen,
-				    sh, NULL, vrf_id);
-				*offset = length;
-				return (NULL);
-			} else if (inp->sctp_socket->so_qlimit) {
-				/* we are accepting so check limits like TCP */
-				if (inp->sctp_socket->so_qlen >=
-				    inp->sctp_socket->so_qlimit) {
-					/* no space */
+
+			if ((stcb == NULL) && (inp->sctp_socket->so_qlen >= inp->sctp_socket->so_qlimit)) {
+				if ((inp->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) &&
+				    (sctp_abort_if_one_2_one_hits_limit)) {
 					struct mbuf *oper;
 					struct sctp_paramhdr *phdr;
 
-					if (sctp_abort_if_one_2_one_hits_limit) {
-						oper = NULL;
-						oper = sctp_get_mbuf_for_msg(sizeof(struct sctp_paramhdr),
-						    0, M_DONTWAIT, 1, MT_DATA);
-						if (oper) {
-							SCTP_BUF_LEN(oper) =
-							    sizeof(struct sctp_paramhdr);
-							phdr = mtod(oper,
-							    struct sctp_paramhdr *);
-							phdr->param_type =
-							    htons(SCTP_CAUSE_OUT_OF_RESC);
-							phdr->param_length =
-							    htons(sizeof(struct sctp_paramhdr));
-						}
-						sctp_abort_association(inp, stcb, m,
-						    iphlen, sh, oper, vrf_id);
+					oper = sctp_get_mbuf_for_msg(sizeof(struct sctp_paramhdr),
+					    0, M_DONTWAIT, 1, MT_DATA);
+					if (oper) {
+						SCTP_BUF_LEN(oper) =
+						    sizeof(struct sctp_paramhdr);
+						phdr = mtod(oper,
+						    struct sctp_paramhdr *);
+						phdr->param_type =
+						    htons(SCTP_CAUSE_OUT_OF_RESC);
+						phdr->param_length =
+						    htons(sizeof(struct sctp_paramhdr));
 					}
-					*offset = length;
-					return (NULL);
+					sctp_abort_association(inp, stcb, m,
+					    iphlen, sh, oper, vrf_id);
 				}
-			}
-	process_cookie_anyway:
-			{
+				*offset = length;
+				return (NULL);
+			} else {
 				struct mbuf *ret_buf;
 				struct sctp_inpcb *linp;
 
diff --git a/sys/netinet/sctp_output.c b/sys/netinet/sctp_output.c
index 48bf4b0..d384a65 100644
--- a/sys/netinet/sctp_output.c
+++ b/sys/netinet/sctp_output.c
@@ -4949,6 +4949,7 @@ sctp_send_initiate_ack(struct sctp_inpcb *inp, struct sctp_tcb *stcb,
 		uint32_t vtag, itsn;
 
 		if (hold_inp_lock) {
+			SCTP_INP_INCR_REF(inp);
 			SCTP_INP_RUNLOCK(inp);
 		}
 		if (asoc) {
@@ -4969,6 +4970,7 @@ sctp_send_initiate_ack(struct sctp_inpcb *inp, struct sctp_tcb *stcb,
 		}
 		if (hold_inp_lock) {
 			SCTP_INP_RLOCK(inp);
+			SCTP_INP_DECR_REF(inp);
 		}
 	}
 	/* save away my tag to */
@@ -5824,6 +5826,7 @@ sctp_sendall_iterator(struct sctp_inpcb *inp, struct sctp_tcb *stcb, void *ptr,
 						SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 					}
 					SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT);
+					SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 					sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN, stcb->sctp_ep, stcb,
 					    asoc->primary_destination);
 					sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWNGUARD, stcb->sctp_ep, stcb,
@@ -9321,6 +9324,13 @@ sctp_send_sack(struct sctp_tcb *stcb)
 		sack->ch.chunk_flags |= (asoc->cmt_dac_pkts_rcvd << 6);
 		asoc->cmt_dac_pkts_rcvd = 0;
 	}
+#ifdef SCTP_ASOCLOG_OF_TSNS
+	stcb->asoc.cumack_logsnt[stcb->asoc.cumack_log_atsnt] = asoc->cumulative_tsn;
+	stcb->asoc.cumack_log_atsnt++;
+	if (stcb->asoc.cumack_log_atsnt >= SCTP_TSN_LOG_SIZE) {
+		stcb->asoc.cumack_log_atsnt = 0;
+	}
+#endif
 	sack->sack.cum_tsn_ack = htonl(asoc->cumulative_tsn);
 	sack->sack.a_rwnd = htonl(asoc->my_rwnd);
 	asoc->my_last_reported_rwnd = asoc->my_rwnd;
@@ -12050,6 +12060,7 @@ dataless_eof:
 					SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 				}
 				SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT);
+				SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 				sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN, stcb->sctp_ep, stcb,
 				    asoc->primary_destination);
 				sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWNGUARD, stcb->sctp_ep, stcb,
diff --git a/sys/netinet/sctp_pcb.c b/sys/netinet/sctp_pcb.c
index d7c2475..c5ee0fe 100644
--- a/sys/netinet/sctp_pcb.c
+++ b/sys/netinet/sctp_pcb.c
@@ -523,9 +523,8 @@ sctp_add_addr_to_vrf(uint32_t vrf_id, void *ifn, uint32_t ifn_index,
 		sctp_ifnp->ifn_index = ifn_index;
 		sctp_ifnp->ifn_p = ifn;
 		sctp_ifnp->ifn_type = ifn_type;
-		sctp_ifnp->refcount = 1;
+		sctp_ifnp->refcount = 0;
 		sctp_ifnp->vrf = vrf;
-
 		atomic_add_int(&vrf->refcount, 1);
 		sctp_ifnp->ifn_mtu = SCTP_GATHER_MTU_FROM_IFN_INFO(ifn, ifn_index, addr->sa_family);
 		if (if_name != NULL) {
@@ -2941,6 +2940,7 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate, int from)
 						SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 					}
 					SCTP_SET_STATE(&asoc->asoc, SCTP_STATE_SHUTDOWN_SENT);
+					SCTP_CLEAR_SUBSTATE(&asoc->asoc, SCTP_STATE_SHUTDOWN_PENDING);
 					sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN, asoc->sctp_ep, asoc,
 					    asoc->asoc.primary_destination);
 					sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWNGUARD, asoc->sctp_ep, asoc,
@@ -3075,6 +3075,9 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate, int from)
 
 			}
 			asoc->sctp_ep->last_abort_code = SCTP_FROM_SCTP_PCB + SCTP_LOC_7;
+#if defined(SCTP_PANIC_ON_ABORT)
+			panic("inpcb_free does an abort");
+#endif
 			sctp_send_abort_tcb(asoc, op_err, SCTP_SO_LOCKED);
 			SCTP_STAT_INCR_COUNTER32(sctps_aborted);
 		} else if (asoc->asoc.state & SCTP_STATE_ABOUT_TO_BE_FREED) {
@@ -3821,6 +3824,8 @@ sctp_remove_net(struct sctp_tcb *stcb, struct sctp_nets *net)
 		 * the subsequent SET PRIMARY. (by micchie)
 		 */
 		if (sctp_is_mobility_feature_on(stcb->sctp_ep,
+		    SCTP_MOBILITY_BASE) ||
+		    sctp_is_mobility_feature_on(stcb->sctp_ep,
 		    SCTP_MOBILITY_FASTHANDOFF)) {
 			SCTPDBG(SCTP_DEBUG_ASCONF1, "remove_net: primary dst is deleting\n");
 			if (asoc->deleted_primary != NULL) {
diff --git a/sys/netinet/sctp_structs.h b/sys/netinet/sctp_structs.h
index 09e08f0..1d145e2 100644
--- a/sys/netinet/sctp_structs.h
+++ b/sys/netinet/sctp_structs.h
@@ -775,12 +775,13 @@ struct sctp_association {
 	struct sctp_tsn_log in_tsnlog[SCTP_TSN_LOG_SIZE];
 	struct sctp_tsn_log out_tsnlog[SCTP_TSN_LOG_SIZE];
 	uint32_t cumack_log[SCTP_TSN_LOG_SIZE];
+	uint32_t cumack_logsnt[SCTP_TSN_LOG_SIZE];
 	uint16_t tsn_in_at;
 	uint16_t tsn_out_at;
 	uint16_t tsn_in_wrapped;
 	uint16_t tsn_out_wrapped;
 	uint16_t cumack_log_at;
-
+	uint16_t cumack_log_atsnt;
 #endif				/* SCTP_ASOCLOG_OF_TSNS */
 #ifdef SCTP_FS_SPEC_LOG
 	struct sctp_fs_spec_log fslog[SCTP_FS_SPEC_LOG_SIZE];
diff --git a/sys/netinet/sctp_timer.c b/sys/netinet/sctp_timer.c
index e9382a5..6f6d188 100644
--- a/sys/netinet/sctp_timer.c
+++ b/sys/netinet/sctp_timer.c
@@ -1762,6 +1762,7 @@ sctp_autoclose_timer(struct sctp_inpcb *inp,
 						SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 					}
 					SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT);
+					SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 					sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN,
 					    stcb->sctp_ep, stcb,
 					    asoc->primary_destination);
diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c
index 20de9c3..24b916e 100644
--- a/sys/netinet/sctp_usrreq.c
+++ b/sys/netinet/sctp_usrreq.c
@@ -826,6 +826,7 @@ sctp_disconnect(struct socket *so)
 						SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 					}
 					SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT);
+					SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 					sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN,
 					    stcb->sctp_ep, stcb,
 					    asoc->primary_destination);
@@ -977,6 +978,7 @@ sctp_shutdown(struct socket *so)
 					SCTP_STAT_DECR_GAUGE32(sctps_currestab);
 				}
 				SCTP_SET_STATE(asoc, SCTP_STATE_SHUTDOWN_SENT);
+				SCTP_CLEAR_SUBSTATE(asoc, SCTP_STATE_SHUTDOWN_PENDING);
 				sctp_timer_start(SCTP_TIMER_TYPE_SHUTDOWN,
 				    stcb->sctp_ep, stcb,
 				    asoc->primary_destination);
diff --git a/sys/netinet/sctp_var.h b/sys/netinet/sctp_var.h
index c4af433..e2fe3f8 100644
--- a/sys/netinet/sctp_var.h
+++ b/sys/netinet/sctp_var.h
@@ -56,7 +56,7 @@ extern struct pr_usrreqs sctp_usrreqs;
 
 #define sctp_maxspace(sb) (max((sb)->sb_hiwat,SCTP_MINIMAL_RWND))
 
-#define	sctp_sbspace(asoc, sb) ((long) (sctp_maxspace(sb) > (asoc)->sb_cc) ? (sctp_maxspace(sb) - (asoc)->sb_cc) : 0)
+#define	sctp_sbspace(asoc, sb) ((long) ((sctp_maxspace(sb) > (asoc)->sb_cc) ? (sctp_maxspace(sb) - (asoc)->sb_cc) : 0))
 
 #define	sctp_sbspace_failedmsgs(sb) ((long) ((sctp_maxspace(sb) > (sb)->sb_cc) ? (sctp_maxspace(sb) - (sb)->sb_cc) : 0))
 
diff --git a/sys/netinet/sctputil.c b/sys/netinet/sctputil.c
index 76a7902..c1ffe81 100644
--- a/sys/netinet/sctputil.c
+++ b/sys/netinet/sctputil.c
@@ -934,6 +934,7 @@ sctp_init_asoc(struct sctp_inpcb *m, struct sctp_tcb *stcb,
 	asoc->tsn_in_wrapped = 0;
 	asoc->tsn_out_wrapped = 0;
 	asoc->cumack_log_at = 0;
+	asoc->cumack_log_atsnt = 0;
 #endif
 #ifdef SCTP_FS_SPEC_LOG
 	asoc->fs_index = 0;
@@ -3911,6 +3912,9 @@ sctp_abort_an_association(struct sctp_inpcb *inp, struct sctp_tcb *stcb,
 	if ((inp->sctp_flags & SCTP_PCB_FLAGS_SOCKET_GONE) == 0)
 		sctp_abort_notification(stcb, error, so_locked);
 	/* notify the peer */
+#if defined(SCTP_PANIC_ON_ABORT)
+	panic("aborting an association");
+#endif
 	sctp_send_abort_tcb(stcb, op_err, so_locked);
 	SCTP_STAT_INCR_COUNTER32(sctps_aborted);
 	if ((SCTP_GET_STATE(&stcb->asoc) == SCTP_STATE_OPEN) ||
@@ -5575,11 +5579,11 @@ get_more_data:
 				if (TAILQ_NEXT(control, next) == NULL) {
 					/*
 					 * If we don't have a next we need a
-					 * lock, if there is a next interrupt
-					 * is filling ahead of us and we
-					 * don't need a lock to remove this
-					 * guy (which is the head of the
-					 * queue).
+					 * lock, if there is a next
+					 * interrupt is filling ahead of us
+					 * and we don't need a lock to
+					 * remove this guy (which is the
+					 * head of the queue).
 					 */
 					if (hold_rlock == 0) {
 						SCTP_INP_READ_LOCK(inp);