MFC 303867,303901

303867
    hyperv/hn: Move gpa array out of netvsc_packet.

    Prepare to deprecate the netvsc_packet.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D7436

303901
    hyperv/hn: Reorganize send done callback.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D7450

6 files changed, 249 insertions, 187 deletions

diff --git a/sys/dev/hyperv/netvsc/hv_net_vsc.c b/sys/dev/hyperv/netvsc/hv_net_vsc.c
index 5b9281f..637ba63 100644
--- a/sys/dev/hyperv/netvsc/hv_net_vsc.c
+++ b/sys/dev/hyperv/netvsc/hv_net_vsc.c
@@ -67,6 +67,12 @@ static void hv_nv_on_receive_completion(struct vmbus_channel *chan,
 static void hv_nv_on_receive(netvsc_dev *net_dev,
     struct hn_rx_ring *rxr, struct vmbus_channel *chan,
     const struct vmbus_chanpkt_hdr *pkt);
+static void hn_nvs_sent_none(struct hn_send_ctx *sndc,
+    struct netvsc_dev_ *net_dev, struct vmbus_channel *chan,
+    const struct nvsp_msg_ *msg);
+
+static struct hn_send_ctx	hn_send_ctx_none =
+    HN_SEND_CTX_INITIALIZER(hn_nvs_sent_none, NULL);
 
 /*
  *
@@ -140,6 +146,7 @@ hv_nv_get_next_send_section(netvsc_dev *net_dev)
 static int 
 hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc)
 {
+	struct hn_send_ctx sndc;
 	netvsc_dev *net_dev;
 	nvsp_msg *init_pkt;
 	int ret = 0;
@@ -188,9 +195,10 @@ hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc)
 
 	/* Send the gpadl notification request */
 
+	hn_send_ctx_init_simple(&sndc, hn_nvs_sent_wakeup, NULL);
 	ret = vmbus_chan_send(sc->hn_prichan,
 	    VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
-	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
+	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)&sndc);
 	if (ret != 0) {
 		goto cleanup;
 	}
@@ -239,6 +247,7 @@ exit:
 static int 
 hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc)
 {
+	struct hn_send_ctx sndc;
 	netvsc_dev *net_dev;
 	nvsp_msg *init_pkt;
 	int ret = 0;
@@ -286,9 +295,10 @@ hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc)
 
 	/* Send the gpadl notification request */
 
+	hn_send_ctx_init_simple(&sndc, hn_nvs_sent_wakeup, NULL);
 	ret = vmbus_chan_send(sc->hn_prichan,
 	    VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
-  	    init_pkt, sizeof(nvsp_msg), (uint64_t)init_pkt);
+  	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)&sndc);
 	if (ret != 0) {
 		goto cleanup;
 	}
@@ -347,8 +357,7 @@ hv_nv_destroy_rx_buffer(netvsc_dev *net_dev)
 
 		ret = vmbus_chan_send(net_dev->sc->hn_prichan,
 		    VMBUS_CHANPKT_TYPE_INBAND, 0, revoke_pkt, sizeof(nvsp_msg),
-		    (uint64_t)(uintptr_t)revoke_pkt);
-
+		    (uint64_t)(uintptr_t)&hn_send_ctx_none);
 		/*
 		 * If we failed here, we might as well return and have a leak 
 		 * rather than continue and a bugchk
@@ -413,9 +422,8 @@ hv_nv_destroy_send_buffer(netvsc_dev *net_dev)
 		    NETVSC_SEND_BUFFER_ID;
 
 		ret = vmbus_chan_send(net_dev->sc->hn_prichan,
-		    VMBUS_CHANPKT_TYPE_INBAND, 0,
-		    revoke_pkt, sizeof(nvsp_msg),
-		    (uint64_t)(uintptr_t)revoke_pkt);
+		    VMBUS_CHANPKT_TYPE_INBAND, 0, revoke_pkt, sizeof(nvsp_msg),
+		    (uint64_t)(uintptr_t)&hn_send_ctx_none);
 		/*
 		 * If we failed here, we might as well return and have a leak 
 		 * rather than continue and a bugchk
@@ -465,6 +473,7 @@ static int
 hv_nv_negotiate_nvsp_protocol(struct hn_softc *sc, netvsc_dev *net_dev,
     uint32_t nvsp_ver)
 {
+	struct hn_send_ctx sndc;
 	nvsp_msg *init_pkt;
 	int ret;
 
@@ -479,9 +488,10 @@ hv_nv_negotiate_nvsp_protocol(struct hn_softc *sc, netvsc_dev *net_dev,
 	init_pkt->msgs.init_msgs.init.protocol_version_2 = nvsp_ver;
 
 	/* Send the init request */
+	hn_send_ctx_init_simple(&sndc, hn_nvs_sent_wakeup, NULL);
 	ret = vmbus_chan_send(sc->hn_prichan,
 	    VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
-	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
+	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)&sndc);
 	if (ret != 0)
 		return (-1);
 
@@ -523,7 +533,7 @@ hv_nv_send_ndis_config(struct hn_softc *sc, uint32_t mtu)
 
 	/* Send the configuration packet */
 	ret = vmbus_chan_send(sc->hn_prichan, VMBUS_CHANPKT_TYPE_INBAND, 0,
-	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
+	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)&hn_send_ctx_none);
 	if (ret != 0)
 		return (-EINVAL);
 
@@ -601,7 +611,7 @@ hv_nv_connect_to_vsp(struct hn_softc *sc)
 	/* Send the init request */
 
 	ret = vmbus_chan_send(sc->hn_prichan, VMBUS_CHANPKT_TYPE_INBAND, 0,
-	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
+	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)&hn_send_ctx_none);
 	if (ret != 0) {
 		goto cleanup;
 	}
@@ -730,6 +740,43 @@ hv_nv_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel)
 	return (0);
 }
 
+void
+hn_nvs_sent_wakeup(struct hn_send_ctx *sndc __unused,
+    struct netvsc_dev_ *net_dev, struct vmbus_channel *chan __unused,
+    const struct nvsp_msg_ *msg)
+{
+	/* Copy the response back */
+	memcpy(&net_dev->channel_init_packet, msg, sizeof(nvsp_msg));
+	sema_post(&net_dev->channel_init_sema);
+}
+
+static void
+hn_nvs_sent_none(struct hn_send_ctx *sndc __unused,
+    struct netvsc_dev_ *net_dev __unused, struct vmbus_channel *chan __unused,
+    const struct nvsp_msg_ *msg __unused)
+{
+	/* EMPTY */
+}
+
+void
+hn_chim_free(struct netvsc_dev_ *net_dev, uint32_t chim_idx)
+{
+	u_long mask;
+	uint32_t idx;
+
+	idx = chim_idx / BITS_PER_LONG;
+	KASSERT(idx < net_dev->bitsmap_words,
+	    ("invalid chimney index 0x%x", chim_idx));
+
+	mask = 1UL << (chim_idx % BITS_PER_LONG);
+	KASSERT(net_dev->send_section_bitsmap[idx] & mask,
+	    ("index bitmap 0x%lx, chimney index %u, "
+	     "bitmap idx %d, bitmask 0x%lx",
+	     net_dev->send_section_bitsmap[idx], chim_idx, idx, mask));
+
+	atomic_clear_long(&net_dev->send_section_bitsmap[idx], mask);
+}
+
 /*
  * Net VSC on send completion
  */
@@ -737,59 +784,15 @@ static void
 hv_nv_on_send_completion(netvsc_dev *net_dev, struct vmbus_channel *chan,
     const struct vmbus_chanpkt_hdr *pkt)
 {
-	const nvsp_msg *nvsp_msg_pkt;
-	netvsc_packet *net_vsc_pkt;
-
-	nvsp_msg_pkt = VMBUS_CHANPKT_CONST_DATA(pkt);
+	struct hn_send_ctx *sndc;
 
-	if (nvsp_msg_pkt->hdr.msg_type == nvsp_msg_type_init_complete
-		|| nvsp_msg_pkt->hdr.msg_type
-			== nvsp_msg_1_type_send_rx_buf_complete
-		|| nvsp_msg_pkt->hdr.msg_type
-			== nvsp_msg_1_type_send_send_buf_complete
-		|| nvsp_msg_pkt->hdr.msg_type
-			== nvsp_msg5_type_subchannel) {
-		/* Copy the response back */
-		memcpy(&net_dev->channel_init_packet, nvsp_msg_pkt,
-		    sizeof(nvsp_msg));
-		sema_post(&net_dev->channel_init_sema);
-	} else if (nvsp_msg_pkt->hdr.msg_type ==
-		    nvsp_msg_1_type_send_rndis_pkt_complete) {
-		/* Get the send context */
-		net_vsc_pkt =
-		    (netvsc_packet *)(unsigned long)pkt->cph_xactid;
-		if (NULL != net_vsc_pkt) {
-			if (net_vsc_pkt->send_buf_section_idx !=
-			    NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) {
-				u_long mask;
-				int idx;
-
-				idx = net_vsc_pkt->send_buf_section_idx /
-				    BITS_PER_LONG;
-				KASSERT(idx < net_dev->bitsmap_words,
-				    ("invalid section index %u",
-				     net_vsc_pkt->send_buf_section_idx));
-				mask = 1UL <<
-				    (net_vsc_pkt->send_buf_section_idx %
-				     BITS_PER_LONG);
-
-				KASSERT(net_dev->send_section_bitsmap[idx] &
-				    mask,
-				    ("index bitmap 0x%lx, section index %u, "
-				     "bitmap idx %d, bitmask 0x%lx",
-				     net_dev->send_section_bitsmap[idx],
-				     net_vsc_pkt->send_buf_section_idx,
-				     idx, mask));
-				atomic_clear_long(
-				    &net_dev->send_section_bitsmap[idx], mask);
-			}
-			
-			/* Notify the layer above us */
-			net_vsc_pkt->compl.send.on_send_completion(chan,
-			    net_vsc_pkt->compl.send.send_completion_context);
-
-		}
-	}
+	sndc = (struct hn_send_ctx *)(uintptr_t)pkt->cph_xactid;
+	sndc->hn_cb(sndc, net_dev, chan, VMBUS_CHANPKT_CONST_DATA(pkt));
+	/*
+	 * NOTE:
+	 * 'sndc' CAN NOT be accessed anymore, since it can be freed by
+	 * its callback.
+	 */
 }
 
 /*
@@ -798,13 +801,14 @@ hv_nv_on_send_completion(netvsc_dev *net_dev, struct vmbus_channel *chan,
  * Returns 0 on success, non-zero on failure.
  */
 int
-hv_nv_on_send(struct vmbus_channel *chan, netvsc_packet *pkt)
+hv_nv_on_send(struct vmbus_channel *chan, bool is_data_pkt,
+    struct hn_send_ctx *sndc, struct vmbus_gpa *gpa, int gpa_cnt)
 {
 	nvsp_msg send_msg;
 	int ret;
 
 	send_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt;
-	if (pkt->is_data_pkt) {
+	if (is_data_pkt) {
 		/* 0 is RMC_DATA */
 		send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 0;
 	} else {
@@ -813,17 +817,17 @@ hv_nv_on_send(struct vmbus_channel *chan, netvsc_packet *pkt)
 	}
 
 	send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_idx =
-	    pkt->send_buf_section_idx;
+	    sndc->hn_chim_idx;
 	send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_size =
-	    pkt->send_buf_section_size;
+	    sndc->hn_chim_sz;
 
-	if (pkt->gpa_cnt) {
-		ret = vmbus_chan_send_sglist(chan, pkt->gpa, pkt->gpa_cnt,
-		    &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
+	if (gpa_cnt) {
+		ret = vmbus_chan_send_sglist(chan, gpa, gpa_cnt,
+		    &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)sndc);
 	} else {
 		ret = vmbus_chan_send(chan,
 		    VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
-		    &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt);
+		    &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)sndc);
 	}
 
 	return (ret);
diff --git a/sys/dev/hyperv/netvsc/hv_net_vsc.h b/sys/dev/hyperv/netvsc/hv_net_vsc.h
index cdae2fb..abd45f4 100644
--- a/sys/dev/hyperv/netvsc/hv_net_vsc.h
+++ b/sys/dev/hyperv/netvsc/hv_net_vsc.h
@@ -1112,33 +1112,10 @@ typedef void (*pfn_on_send_rx_completion)(struct vmbus_channel *, void *);
 #endif
 
 typedef struct netvsc_packet_ {
-	uint8_t                    is_data_pkt;      /* One byte */
-	uint16_t		   vlan_tci;
-	uint32_t status;
-
-	/* Completion */
-	union {
-		struct {
-			uint64_t   rx_completion_tid;
-			void	   *rx_completion_context;
-			/* This is no longer used */
-			pfn_on_send_rx_completion   on_rx_completion;
-		} rx;
-		struct {
-			uint64_t    send_completion_tid;
-			void	    *send_completion_context;
-			/* Still used in netvsc and filter code */
-			pfn_on_send_rx_completion   on_send_completion;
-		} send;
-	} compl;
-	uint32_t	send_buf_section_idx;
-	uint32_t	send_buf_section_size;
-
-	void		*rndis_mesg;
+	uint16_t	vlan_tci;
+	uint32_t	status;
 	uint32_t	tot_data_buf_len;
 	void		*data;
-	uint32_t	gpa_cnt;
-	struct vmbus_gpa gpa[NETVSC_PACKET_MAXPAGE];
 } netvsc_packet;
 
 typedef struct {
@@ -1216,6 +1193,9 @@ struct hn_tx_ring {
 	bus_dma_tag_t	hn_tx_data_dtag;
 	uint64_t	hn_csum_assist;
 
+	int		hn_gpa_cnt;
+	struct vmbus_gpa hn_gpa[NETVSC_PACKET_MAXPAGE];
+
 	u_long		hn_no_txdescs;
 	u_long		hn_send_failed;
 	u_long		hn_txdma_failed;
@@ -1270,13 +1250,15 @@ typedef struct hn_softc {
  * Externs
  */
 extern int hv_promisc_mode;
+struct hn_send_ctx;
 
 void netvsc_linkstatus_callback(struct hn_softc *sc, uint32_t status);
 netvsc_dev *hv_nv_on_device_add(struct hn_softc *sc,
     void *additional_info, struct hn_rx_ring *rxr);
 int hv_nv_on_device_remove(struct hn_softc *sc,
     boolean_t destroy_channel);
-int hv_nv_on_send(struct vmbus_channel *chan, netvsc_packet *pkt);
+int hv_nv_on_send(struct vmbus_channel *chan, bool is_data_pkt,
+	struct hn_send_ctx *sndc, struct vmbus_gpa *gpa, int gpa_cnt);
 int hv_nv_get_next_send_section(netvsc_dev *net_dev);
 void hv_nv_subchan_attach(struct vmbus_channel *chan,
     struct hn_rx_ring *rxr);
diff --git a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
index 54301af..a278c5b 100644
--- a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
+++ b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
@@ -168,7 +168,7 @@ struct hn_txdesc {
 	struct hn_tx_ring *txr;
 	int		refs;
 	uint32_t	flags;		/* HN_TXD_FLAG_ */
-	netvsc_packet	netvsc_pkt;	/* XXX to be removed */
+	struct hn_send_ctx send_ctx;
 
 	bus_dmamap_t	data_dmap;
 
@@ -793,14 +793,14 @@ hn_txeof(struct hn_tx_ring *txr)
 }
 
 static void
-hn_tx_done(struct vmbus_channel *chan, void *xpkt)
+hn_tx_done(struct hn_send_ctx *sndc, struct netvsc_dev_ *net_dev,
+    struct vmbus_channel *chan, const struct nvsp_msg_ *msg __unused)
 {
-	netvsc_packet *packet = xpkt;
-	struct hn_txdesc *txd;
+	struct hn_txdesc *txd = sndc->hn_cbarg;
 	struct hn_tx_ring *txr;
 
-	txd = (struct hn_txdesc *)(uintptr_t)
-	    packet->compl.send.send_completion_tid;
+	if (sndc->hn_chim_idx != NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX)
+		hn_chim_free(net_dev, sndc->hn_chim_idx);
 
 	txr = txd->txr;
 	KASSERT(txr->hn_chan == chan,
@@ -853,16 +853,14 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0)
 	bus_dma_segment_t segs[HN_TX_DATA_SEGCNT_MAX];
 	int error, nsegs, i;
 	struct mbuf *m_head = *m_head0;
-	netvsc_packet *packet;
 	rndis_msg *rndis_mesg;
 	rndis_packet *rndis_pkt;
 	rndis_per_packet_info *rppi;
 	struct rndis_hash_value *hash_value;
-	uint32_t rndis_msg_size;
+	uint32_t rndis_msg_size, tot_data_buf_len, send_buf_section_idx;
+	int send_buf_section_size;
 
-	packet = &txd->netvsc_pkt;
-	packet->is_data_pkt = TRUE;
-	packet->tot_data_buf_len = m_head->m_pkthdr.len;
+	tot_data_buf_len = m_head->m_pkthdr.len;
 
 	/*
 	 * extension points to the area reserved for the
@@ -877,7 +875,7 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0)
 
 	rndis_pkt = &rndis_mesg->msg.packet;
 	rndis_pkt->data_offset = sizeof(rndis_packet);
-	rndis_pkt->data_length = packet->tot_data_buf_len;
+	rndis_pkt->data_length = tot_data_buf_len;
 	rndis_pkt->per_pkt_info_offset = sizeof(rndis_packet);
 
 	rndis_msg_size = RNDIS_MESSAGE_SIZE(rndis_packet);
@@ -985,15 +983,14 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0)
 		}
 	}
 
-	rndis_mesg->msg_len = packet->tot_data_buf_len + rndis_msg_size;
-	packet->tot_data_buf_len = rndis_mesg->msg_len;
+	rndis_mesg->msg_len = tot_data_buf_len + rndis_msg_size;
+	tot_data_buf_len = rndis_mesg->msg_len;
 
 	/*
 	 * Chimney send, if the packet could fit into one chimney buffer.
 	 */
-	if (packet->tot_data_buf_len < txr->hn_tx_chimney_size) {
+	if (tot_data_buf_len < txr->hn_tx_chimney_size) {
 		netvsc_dev *net_dev = txr->hn_sc->net_dev;
-		uint32_t send_buf_section_idx;
 
 		txr->hn_tx_chimney_tried++;
 		send_buf_section_idx =
@@ -1008,10 +1005,8 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0)
 			dest += rndis_msg_size;
 			m_copydata(m_head, 0, m_head->m_pkthdr.len, dest);
 
-			packet->send_buf_section_idx = send_buf_section_idx;
-			packet->send_buf_section_size =
-			    packet->tot_data_buf_len;
-			packet->gpa_cnt = 0;
+			send_buf_section_size = tot_data_buf_len;
+			txr->hn_gpa_cnt = 0;
 			txr->hn_tx_chimney++;
 			goto done;
 		}
@@ -1037,19 +1032,19 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0)
 	}
 	*m_head0 = m_head;
 
-	packet->gpa_cnt = nsegs + HV_RF_NUM_TX_RESERVED_PAGE_BUFS;
+	txr->hn_gpa_cnt = nsegs + HV_RF_NUM_TX_RESERVED_PAGE_BUFS;
 
 	/* send packet with page buffer */
-	packet->gpa[0].gpa_page = atop(txd->rndis_msg_paddr);
-	packet->gpa[0].gpa_ofs = txd->rndis_msg_paddr & PAGE_MASK;
-	packet->gpa[0].gpa_len = rndis_msg_size;
+	txr->hn_gpa[0].gpa_page = atop(txd->rndis_msg_paddr);
+	txr->hn_gpa[0].gpa_ofs = txd->rndis_msg_paddr & PAGE_MASK;
+	txr->hn_gpa[0].gpa_len = rndis_msg_size;
 
 	/*
 	 * Fill the page buffers with mbuf info starting at index
 	 * HV_RF_NUM_TX_RESERVED_PAGE_BUFS.
 	 */
 	for (i = 0; i < nsegs; ++i) {
-		struct vmbus_gpa *gpa = &packet->gpa[
+		struct vmbus_gpa *gpa = &txr->hn_gpa[
 		    i + HV_RF_NUM_TX_RESERVED_PAGE_BUFS];
 
 		gpa->gpa_page = atop(segs[i].ds_addr);
@@ -1057,16 +1052,14 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0)
 		gpa->gpa_len = segs[i].ds_len;
 	}
 
-	packet->send_buf_section_idx =
-	    NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
-	packet->send_buf_section_size = 0;
+	send_buf_section_idx = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
+	send_buf_section_size = 0;
 done:
 	txd->m = m_head;
 
 	/* Set the completion routine */
-	packet->compl.send.on_send_completion = hn_tx_done;
-	packet->compl.send.send_completion_context = packet;
-	packet->compl.send.send_completion_tid = (uint64_t)(uintptr_t)txd;
+	hn_send_ctx_init(&txd->send_ctx, hn_tx_done, txd,
+	    send_buf_section_idx, send_buf_section_size);
 
 	return 0;
 }
@@ -1086,7 +1079,8 @@ again:
 	 * Make sure that txd is not freed before ETHER_BPF_MTAP.
 	 */
 	hn_txdesc_hold(txd);
-	error = hv_nv_on_send(txr->hn_chan, &txd->netvsc_pkt);
+	error = hv_nv_on_send(txr->hn_chan, true, &txd->send_ctx,
+	    txr->hn_gpa, txr->hn_gpa_cnt);
 	if (!error) {
 		ETHER_BPF_MTAP(ifp, txd->m);
 		if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
diff --git a/sys/dev/hyperv/netvsc/hv_rndis_filter.c b/sys/dev/hyperv/netvsc/hv_rndis_filter.c
index 120ad1a..b4adf90 100644
--- a/sys/dev/hyperv/netvsc/hv_rndis_filter.c
+++ b/sys/dev/hyperv/netvsc/hv_rndis_filter.c
@@ -86,11 +86,17 @@ static int  hv_rf_set_packet_filter(rndis_device *device, uint32_t new_filter);
 static int  hv_rf_init_device(rndis_device *device);
 static int  hv_rf_open_device(rndis_device *device);
 static int  hv_rf_close_device(rndis_device *device);
-static void hv_rf_on_send_request_completion(struct vmbus_channel *, void *context);
-static void hv_rf_on_send_request_halt_completion(struct vmbus_channel *, void *context);
 int
 hv_rf_send_offload_request(struct hn_softc *sc,
     rndis_offload_params *offloads);
+
+static void hn_rndis_sent_halt(struct hn_send_ctx *sndc,
+    struct netvsc_dev_ *net_dev, struct vmbus_channel *chan,
+    const struct nvsp_msg_ *msg);
+static void hn_rndis_sent_cb(struct hn_send_ctx *sndc,
+    struct netvsc_dev_ *net_dev, struct vmbus_channel *chan,
+    const struct nvsp_msg_ *msg);
+
 /*
  * Set the Per-Packet-Info with the specified type
  */
@@ -239,45 +245,36 @@ static int
 hv_rf_send_request(rndis_device *device, rndis_request *request,
     uint32_t message_type)
 {
-	int ret;
-	netvsc_packet *packet;
 	netvsc_dev      *net_dev = device->net_dev;
-	int send_buf_section_idx;
+	uint32_t send_buf_section_idx, tot_data_buf_len;
+	struct vmbus_gpa gpa[2];
+	int gpa_cnt, send_buf_section_size;
+	hn_sent_callback_t cb;
 
 	/* Set up the packet to send it */
-	packet = &request->pkt;
-	
-	packet->is_data_pkt = FALSE;
-	packet->tot_data_buf_len = request->request_msg.msg_len;
-	packet->gpa_cnt = 1;
-
-	packet->gpa[0].gpa_page =
-	    hv_get_phys_addr(&request->request_msg) >> PAGE_SHIFT;
-	packet->gpa[0].gpa_len = request->request_msg.msg_len;
-	packet->gpa[0].gpa_ofs =
-	    (unsigned long)&request->request_msg & (PAGE_SIZE - 1);
-
-	if (packet->gpa[0].gpa_ofs + packet->gpa[0].gpa_len > PAGE_SIZE) {
-		packet->gpa_cnt = 2;
-		packet->gpa[0].gpa_len = PAGE_SIZE - packet->gpa[0].gpa_ofs;
-		packet->gpa[1].gpa_page =
-		        hv_get_phys_addr((char*)&request->request_msg +
-                		packet->gpa[0].gpa_len) >> PAGE_SHIFT;
-		packet->gpa[1].gpa_ofs = 0;
-		packet->gpa[1].gpa_len = request->request_msg.msg_len -
-		    packet->gpa[0].gpa_len;
+	tot_data_buf_len = request->request_msg.msg_len;
+
+	gpa_cnt = 1;
+	gpa[0].gpa_page = hv_get_phys_addr(&request->request_msg) >> PAGE_SHIFT;
+	gpa[0].gpa_len = request->request_msg.msg_len;
+	gpa[0].gpa_ofs = (unsigned long)&request->request_msg & (PAGE_SIZE - 1);
+
+	if (gpa[0].gpa_ofs + gpa[0].gpa_len > PAGE_SIZE) {
+		gpa_cnt = 2;
+		gpa[0].gpa_len = PAGE_SIZE - gpa[0].gpa_ofs;
+		gpa[1].gpa_page =
+		    hv_get_phys_addr((char*)&request->request_msg +
+		    gpa[0].gpa_len) >> PAGE_SHIFT;
+		gpa[1].gpa_ofs = 0;
+		gpa[1].gpa_len = request->request_msg.msg_len - gpa[0].gpa_len;
 	}
 
-	packet->compl.send.send_completion_context = request; /* packet */
-	if (message_type != REMOTE_NDIS_HALT_MSG) {
-		packet->compl.send.on_send_completion =
-		    hv_rf_on_send_request_completion;
-	} else {
-		packet->compl.send.on_send_completion =
-		    hv_rf_on_send_request_halt_completion;
-	}
-	packet->compl.send.send_completion_tid = (unsigned long)device;
-	if (packet->tot_data_buf_len < net_dev->send_section_size) {
+	if (message_type != REMOTE_NDIS_HALT_MSG)
+		cb = hn_rndis_sent_cb;
+	else
+		cb = hn_rndis_sent_halt;
+
+	if (tot_data_buf_len < net_dev->send_section_size) {
 		send_buf_section_idx = hv_nv_get_next_send_section(net_dev);
 		if (send_buf_section_idx !=
 			NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) {
@@ -285,20 +282,20 @@ hv_rf_send_request(rndis_device *device, rndis_request *request,
 				send_buf_section_idx * net_dev->send_section_size);
 
 			memcpy(dest, &request->request_msg, request->request_msg.msg_len);
-			packet->send_buf_section_idx = send_buf_section_idx;
-			packet->send_buf_section_size = packet->tot_data_buf_len;
-			packet->gpa_cnt = 0;
+			send_buf_section_size = tot_data_buf_len;
+			gpa_cnt = 0;
 			goto sendit;
 		}
 		/* Failed to allocate chimney send buffer; move on */
 	}
-	packet->send_buf_section_idx = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
-	packet->send_buf_section_size = 0;
+	send_buf_section_idx = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX;
+	send_buf_section_size = 0;
 
 sendit:
-	ret = hv_nv_on_send(device->net_dev->sc->hn_prichan, packet);
-
-	return (ret);
+	hn_send_ctx_init(&request->send_ctx, cb, request,
+	    send_buf_section_idx, send_buf_section_size);
+	return hv_nv_on_send(device->net_dev->sc->hn_prichan, false,
+	    &request->send_ctx, gpa, gpa_cnt);
 }
 
 /*
@@ -1060,6 +1057,7 @@ int
 hv_rf_on_device_add(struct hn_softc *sc, void *additl_info,
     int nchan, struct hn_rx_ring *rxr)
 {
+	struct hn_send_ctx sndc;
 	int ret;
 	netvsc_dev *net_dev;
 	rndis_device *rndis_dev;
@@ -1166,9 +1164,10 @@ hv_rf_on_device_add(struct hn_softc *sc, void *additl_info,
 	init_pkt->msgs.vers_5_msgs.subchannel_request.num_subchannels =
 	    net_dev->num_channel - 1;
 
+	hn_send_ctx_init_simple(&sndc, hn_nvs_sent_wakeup, NULL);
 	ret = vmbus_chan_send(sc->hn_prichan,
 	    VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
-	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt);
+	    init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)&sndc);
 	if (ret != 0) {
 		device_printf(dev, "Fail to allocate subchannel\n");
 		goto out;
@@ -1239,23 +1238,22 @@ hv_rf_on_close(struct hn_softc *sc)
 	return (hv_rf_close_device((rndis_device *)net_dev->extension));
 }
 
-/*
- * RNDIS filter on send request completion callback
- */
-static void 
-hv_rf_on_send_request_completion(struct vmbus_channel *chan __unused,
-    void *context __unused)
+static void
+hn_rndis_sent_cb(struct hn_send_ctx *sndc, struct netvsc_dev_ *net_dev,
+    struct vmbus_channel *chan __unused, const struct nvsp_msg_ *msg __unused)
 {
+	if (sndc->hn_chim_idx != NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX)
+		hn_chim_free(net_dev, sndc->hn_chim_idx);
 }
 
-/*
- * RNDIS filter on send request (halt only) completion callback
- */
-static void 
-hv_rf_on_send_request_halt_completion(struct vmbus_channel *chan __unused,
-    void *context)
+static void
+hn_rndis_sent_halt(struct hn_send_ctx *sndc, struct netvsc_dev_ *net_dev,
+    struct vmbus_channel *chan __unused, const struct nvsp_msg_ *msg __unused)
 {
-	rndis_request *request = context;
+	rndis_request *request = sndc->hn_cbarg;
+
+	if (sndc->hn_chim_idx != NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX)
+		hn_chim_free(net_dev, sndc->hn_chim_idx);
 
 	/*
 	 * Notify hv_rf_halt_device() about halt completion.
diff --git a/sys/dev/hyperv/netvsc/hv_rndis_filter.h b/sys/dev/hyperv/netvsc/hv_rndis_filter.h
index 2f940db..ebfda20 100644
--- a/sys/dev/hyperv/netvsc/hv_rndis_filter.h
+++ b/sys/dev/hyperv/netvsc/hv_rndis_filter.h
@@ -33,6 +33,7 @@
 
 #include <sys/param.h>
 #include <net/ethernet.h>
+#include <dev/hyperv/netvsc/if_hnvar.h>
 
 /*
  * Defines
@@ -75,7 +76,7 @@ typedef struct rndis_request_ {
 	uint8_t				buf_resp[PAGE_SIZE];
 
 	/* Simplify allocation by having a netvsc packet inline */
-	netvsc_packet			pkt;
+	struct hn_send_ctx		send_ctx;
 
 	/*
 	 * The max request size is sizeof(rndis_msg) + PAGE_SIZE.
diff --git a/sys/dev/hyperv/netvsc/if_hnvar.h b/sys/dev/hyperv/netvsc/if_hnvar.h
new file mode 100644
index 0000000..6e5fb50
--- /dev/null
+++ b/sys/dev/hyperv/netvsc/if_hnvar.h
@@ -0,0 +1,83 @@
+/*-
+ * Copyright (c) 2016 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _IF_HNVAR_H_
+#define _IF_HNVAR_H_
+
+#include <sys/param.h>
+#include <dev/hyperv/netvsc/hv_net_vsc.h>
+
+struct netvsc_dev_;
+struct nvsp_msg_;
+
+struct vmbus_channel;
+struct hn_send_ctx;
+
+typedef void		(*hn_sent_callback_t)
+			(struct hn_send_ctx *, struct netvsc_dev_ *,
+			 struct vmbus_channel *, const struct nvsp_msg_ *);
+
+struct hn_send_ctx {
+	hn_sent_callback_t	hn_cb;
+	void			*hn_cbarg;
+	uint32_t		hn_chim_idx;
+	int			hn_chim_sz;
+};
+
+#define HN_SEND_CTX_INITIALIZER(cb, cbarg)				\
+{									\
+	.hn_cb		= cb,						\
+	.hn_cbarg	= cbarg,					\
+	.hn_chim_idx	= NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX,	\
+	.hn_chim_sz	= 0						\
+}
+
+static __inline void
+hn_send_ctx_init(struct hn_send_ctx *sndc, hn_sent_callback_t cb,
+    void *cbarg, uint32_t chim_idx, int chim_sz)
+{
+	sndc->hn_cb = cb;
+	sndc->hn_cbarg = cbarg;
+	sndc->hn_chim_idx = chim_idx;
+	sndc->hn_chim_sz = chim_sz;
+}
+
+static __inline void
+hn_send_ctx_init_simple(struct hn_send_ctx *sndc, hn_sent_callback_t cb,
+    void *cbarg)
+{
+	hn_send_ctx_init(sndc, cb, cbarg,
+	    NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX, 0);
+}
+
+void		hn_nvs_sent_wakeup(struct hn_send_ctx *sndc,
+		    struct netvsc_dev_ *net_dev, struct vmbus_channel *chan,
+		    const struct nvsp_msg_ *msg);
+void		hn_chim_free(struct netvsc_dev_ *net_dev, uint32_t chim_idx);
+
+#endif	/* !_IF_HNVAR_H_ */