MFC r282212:

Microsoft vmbus, storage and other related driver enhancements for HyperV.
    - Vmbus multi channel support.
    - Vector interrupt support.
    - Signal optimization.
    - Storvsc driver performance improvement.
    - Scatter and gather support for storvsc driver.
    - Minor bug fix for KVP driver.
Thanks royger, jhb and delphij from FreeBSD community for the reviews
and comments. Also thanks Hovy Xu from NetApp for the contributions to
the storvsc driver.

PR:     195238
Submitted by:   whu
Reviewed by:    royger
Approved by:    royger
Relnotes:       yes
Sponsored by:   Microsoft OSTC
Differential Revision:  https://reviews.freebsd.org/D2575

1 files changed, 210 insertions, 79 deletions

diff --git a/sys/dev/hyperv/vmbus/hv_connection.c b/sys/dev/hyperv/vmbus/hv_connection.c
index c8e0b48..cc83037 100644
--- a/sys/dev/hyperv/vmbus/hv_connection.c
+++ b/sys/dev/hyperv/vmbus/hv_connection.c
@@ -26,6 +26,9 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include <sys/param.h>
 #include <sys/malloc.h>
 #include <sys/systm.h>
@@ -45,14 +48,113 @@ hv_vmbus_connection hv_vmbus_g_connection =
 	{ .connect_state = HV_DISCONNECTED,
 	  .next_gpadl_handle = 0xE1E10, };
 
+uint32_t hv_vmbus_protocal_version = HV_VMBUS_VERSION_WS2008;
+
+static uint32_t
+hv_vmbus_get_next_version(uint32_t current_ver)
+{
+	switch (current_ver) {
+	case (HV_VMBUS_VERSION_WIN7):
+		return(HV_VMBUS_VERSION_WS2008);
+
+	case (HV_VMBUS_VERSION_WIN8):
+		return(HV_VMBUS_VERSION_WIN7);
+
+	case (HV_VMBUS_VERSION_WIN8_1):
+		return(HV_VMBUS_VERSION_WIN8);
+
+	case (HV_VMBUS_VERSION_WS2008):
+	default:
+		return(HV_VMBUS_VERSION_INVALID);
+	}
+}
+
+/**
+ * Negotiate the highest supported hypervisor version.
+ */
+static int
+hv_vmbus_negotiate_version(hv_vmbus_channel_msg_info *msg_info,
+	uint32_t version)
+{
+	int					ret = 0;
+	hv_vmbus_channel_initiate_contact	*msg;
+
+	sema_init(&msg_info->wait_sema, 0, "Msg Info Sema");
+	msg = (hv_vmbus_channel_initiate_contact*) msg_info->msg;
+
+	msg->header.message_type = HV_CHANNEL_MESSAGE_INITIATED_CONTACT;
+	msg->vmbus_version_requested = version;
+
+	msg->interrupt_page = hv_get_phys_addr(
+		hv_vmbus_g_connection.interrupt_page);
+
+	msg->monitor_page_1 = hv_get_phys_addr(
+		hv_vmbus_g_connection.monitor_pages);
+
+	msg->monitor_page_2 =
+		hv_get_phys_addr(
+			((uint8_t *) hv_vmbus_g_connection.monitor_pages
+			+ PAGE_SIZE));
+
+	/**
+	 * Add to list before we send the request since we may receive the
+	 * response before returning from this routine
+	 */
+	mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
+
+	TAILQ_INSERT_TAIL(
+		&hv_vmbus_g_connection.channel_msg_anchor,
+		msg_info,
+		msg_list_entry);
+
+	mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
+
+	ret = hv_vmbus_post_message(
+		msg,
+		sizeof(hv_vmbus_channel_initiate_contact));
+
+	if (ret != 0) {
+		mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
+		TAILQ_REMOVE(
+			&hv_vmbus_g_connection.channel_msg_anchor,
+			msg_info,
+			msg_list_entry);
+		mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
+		return (ret);
+	}
+
+	/**
+	 * Wait for the connection response
+	 */
+	ret = sema_timedwait(&msg_info->wait_sema, 500); /* KYS 5 seconds */
+
+	mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
+	TAILQ_REMOVE(
+		&hv_vmbus_g_connection.channel_msg_anchor,
+		msg_info,
+		msg_list_entry);
+	mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
+
+	/**
+	 * Check if successful
+	 */
+	if (msg_info->response.version_response.version_supported) {
+		hv_vmbus_g_connection.connect_state = HV_CONNECTED;
+	} else {
+		ret = ECONNREFUSED;
+	}
+
+	return (ret);
+}
+
 /**
  * Send a connect request on the partition service connection
  */
 int
 hv_vmbus_connect(void) {
 	int					ret = 0;
+	uint32_t				version;
 	hv_vmbus_channel_msg_info*		msg_info = NULL;
-	hv_vmbus_channel_initiate_contact*	msg;
 
 	/**
 	 * Make sure we are not connecting or connected
@@ -74,7 +176,7 @@ hv_vmbus_connect(void) {
 
 	TAILQ_INIT(&hv_vmbus_g_connection.channel_anchor);
 	mtx_init(&hv_vmbus_g_connection.channel_lock, "vmbus channel",
-		NULL, MTX_SPIN);
+		NULL, MTX_DEF);
 
 	/**
 	 * Setup the vmbus event connection for channel interrupt abstraction
@@ -130,71 +232,30 @@ hv_vmbus_connect(void) {
 	    goto cleanup;
 	}
 
-	sema_init(&msg_info->wait_sema, 0, "Msg Info Sema");
-	msg = (hv_vmbus_channel_initiate_contact*) msg_info->msg;
-
-	msg->header.message_type = HV_CHANNEL_MESSAGE_INITIATED_CONTACT;
-	msg->vmbus_version_requested = HV_VMBUS_REVISION_NUMBER;
-
-	msg->interrupt_page = hv_get_phys_addr(
-		hv_vmbus_g_connection.interrupt_page);
-
-	msg->monitor_page_1 = hv_get_phys_addr(
-		hv_vmbus_g_connection.monitor_pages);
-
-	msg->monitor_page_2 =
-		hv_get_phys_addr(
-			((uint8_t *) hv_vmbus_g_connection.monitor_pages
-			+ PAGE_SIZE));
-
-	/**
-	 * Add to list before we send the request since we may receive the
-	 * response before returning from this routine
+	/*
+	 * Find the highest vmbus version number we can support.
 	 */
-	mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
-
-	TAILQ_INSERT_TAIL(
-		&hv_vmbus_g_connection.channel_msg_anchor,
-		msg_info,
-		msg_list_entry);
-
-	mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
-
-	ret = hv_vmbus_post_message(
-		msg,
-		sizeof(hv_vmbus_channel_initiate_contact));
-
-	if (ret != 0) {
-		mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
-		TAILQ_REMOVE(
-			&hv_vmbus_g_connection.channel_msg_anchor,
-			msg_info,
-			msg_list_entry);
-		mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
-		goto cleanup;
-	}
+	version = HV_VMBUS_VERSION_CURRENT;
+
+	do {
+		ret = hv_vmbus_negotiate_version(msg_info, version);
+		if (ret == EWOULDBLOCK) {
+			/*
+			 * We timed out.
+			 */
+			goto cleanup;
+		}
 
-	/**
-	 * Wait for the connection response
-	 */
-	ret = sema_timedwait(&msg_info->wait_sema, 500); /* KYS 5 seconds */
+		if (hv_vmbus_g_connection.connect_state == HV_CONNECTED)
+			break;
 
-	mtx_lock_spin(&hv_vmbus_g_connection.channel_msg_lock);
-	TAILQ_REMOVE(
-		&hv_vmbus_g_connection.channel_msg_anchor,
-		msg_info,
-		msg_list_entry);
-	mtx_unlock_spin(&hv_vmbus_g_connection.channel_msg_lock);
+		version = hv_vmbus_get_next_version(version);
+	} while (version != HV_VMBUS_VERSION_INVALID);
 
-	/**
-	 * Check if successful
-	 */
-	if (msg_info->response.version_response.version_supported) {
-		hv_vmbus_g_connection.connect_state = HV_CONNECTED;
-	} else {
-		ret = ECONNREFUSED;
-		goto cleanup;
-	}
+	hv_vmbus_protocal_version = version;
+	if (bootverbose)
+		printf("VMBUS: Portocal Version: %d.%d\n",
+		    version >> 16, version & 0xFFFF);
 
 	sema_destroy(&msg_info->wait_sema);
 	free(msg_info, M_DEVBUF);
@@ -286,7 +347,7 @@ hv_vmbus_get_channel_from_rel_id(uint32_t rel_id) {
 	 *  and channels are accessed without the need to take this lock or search
 	 *  the list.
 	 */
-	mtx_lock_spin(&hv_vmbus_g_connection.channel_lock);
+	mtx_lock(&hv_vmbus_g_connection.channel_lock);
 	TAILQ_FOREACH(channel,
 		&hv_vmbus_g_connection.channel_anchor, list_entry) {
 
@@ -295,7 +356,7 @@ hv_vmbus_get_channel_from_rel_id(uint32_t rel_id) {
 		break;
 	    }
 	}
-	mtx_unlock_spin(&hv_vmbus_g_connection.channel_lock);
+	mtx_unlock(&hv_vmbus_g_connection.channel_lock);
 
 	return (foundChannel);
 }
@@ -306,7 +367,10 @@ hv_vmbus_get_channel_from_rel_id(uint32_t rel_id) {
 static void
 VmbusProcessChannelEvent(uint32_t relid) 
 {
+	void* arg;
+	uint32_t bytes_to_read;
 	hv_vmbus_channel* channel;
+	boolean_t is_batched_reading;
 
 	/**
 	 * Find the channel based on this relid and invokes
@@ -327,31 +391,98 @@ VmbusProcessChannelEvent(uint32_t relid)
 	 * callback to NULL. This closes the window.
 	 */
 
-	mtx_lock(&channel->inbound_lock);
+	/*
+	 * Disable the lock due to newly added WITNESS check in r277723.
+	 * Will seek other way to avoid race condition.
+	 * -- whu
+	 */
+	// mtx_lock(&channel->inbound_lock);
 	if (channel->on_channel_callback != NULL) {
-		channel->on_channel_callback(channel->channel_callback_context);
+		arg = channel->channel_callback_context;
+		is_batched_reading = channel->batched_reading;
+		/*
+		 * Optimize host to guest signaling by ensuring:
+		 * 1. While reading the channel, we disable interrupts from
+		 *    host.
+		 * 2. Ensure that we process all posted messages from the host
+		 *    before returning from this callback.
+		 * 3. Once we return, enable signaling from the host. Once this
+		 *    state is set we check to see if additional packets are
+		 *    available to read. In this case we repeat the process.
+		 */
+		do {
+			if (is_batched_reading)
+				hv_ring_buffer_read_begin(&channel->inbound);
+
+			channel->on_channel_callback(arg);
+
+			if (is_batched_reading)
+				bytes_to_read =
+				    hv_ring_buffer_read_end(&channel->inbound);
+			else
+				bytes_to_read = 0;
+		} while (is_batched_reading && (bytes_to_read != 0));
 	}
-	mtx_unlock(&channel->inbound_lock);
+	// mtx_unlock(&channel->inbound_lock);
 }
 
+#ifdef HV_DEBUG_INTR
+extern uint32_t hv_intr_count;
+extern uint32_t hv_vmbus_swintr_event_cpu[MAXCPU];
+extern uint32_t hv_vmbus_intr_cpu[MAXCPU];
+#endif
+
 /**
  * Handler for events
  */
 void
 hv_vmbus_on_events(void *arg) 
 {
-	int dword;
 	int bit;
+	int cpu;
+	int dword;
+	void *page_addr;
+	uint32_t* recv_interrupt_page = NULL;
 	int rel_id;
-	int maxdword = HV_MAX_NUM_CHANNELS_SUPPORTED >> 5;
+	int maxdword;
+	hv_vmbus_synic_event_flags *event;
 	/* int maxdword = PAGE_SIZE >> 3; */
 
-	/*
-	 * receive size is 1/2 page and divide that by 4 bytes
-	 */
-
-	uint32_t* recv_interrupt_page =
-	    hv_vmbus_g_connection.recv_interrupt_page;
+	cpu = (int)(long)arg;
+	KASSERT(cpu <= mp_maxid, ("VMBUS: hv_vmbus_on_events: "
+	    "cpu out of range!"));
+
+#ifdef HV_DEBUG_INTR
+	int i;
+	hv_vmbus_swintr_event_cpu[cpu]++;
+	if (hv_intr_count % 10000 == 0) {
+                printf("VMBUS: Total interrupt %d\n", hv_intr_count);
+                for (i = 0; i < mp_ncpus; i++)
+                        printf("VMBUS: hw cpu[%d]: %d, event sw intr cpu[%d]: %d\n",
+			    i, hv_vmbus_intr_cpu[i], i, hv_vmbus_swintr_event_cpu[i]);
+        }
+#endif
+
+	if ((hv_vmbus_protocal_version == HV_VMBUS_VERSION_WS2008) ||
+	    (hv_vmbus_protocal_version == HV_VMBUS_VERSION_WIN7)) {
+		maxdword = HV_MAX_NUM_CHANNELS_SUPPORTED >> 5;
+		/*
+		 * receive size is 1/2 page and divide that by 4 bytes
+		 */
+		recv_interrupt_page =
+		    hv_vmbus_g_connection.recv_interrupt_page;
+	} else {
+		/*
+		 * On Host with Win8 or above, the event page can be
+		 * checked directly to get the id of the channel
+		 * that has the pending interrupt.
+		 */
+		maxdword = HV_EVENT_FLAGS_DWORD_COUNT;
+		page_addr = hv_vmbus_g_context.syn_ic_event_page[cpu];
+		event = (hv_vmbus_synic_event_flags *)
+		    page_addr + HV_VMBUS_MESSAGE_SINT;
+		recv_interrupt_page = event->flags32;
+	}
 
 	/*
 	 * Check events
@@ -416,16 +547,16 @@ int hv_vmbus_post_message(void *buffer, size_t bufferLen) {
  * Send an event notification to the parent
  */
 int
-hv_vmbus_set_event(uint32_t child_rel_id) {
+hv_vmbus_set_event(hv_vmbus_channel *channel) {
 	int ret = 0;
+	uint32_t child_rel_id = channel->offer_msg.child_rel_id;
 
 	/* Each uint32_t represents 32 channels */
 
 	synch_set_bit(child_rel_id & 31,
 		(((uint32_t *)hv_vmbus_g_connection.send_interrupt_page
 			+ (child_rel_id >> 5))));
-	ret = hv_vmbus_signal_event();
+	ret = hv_vmbus_signal_event(channel->signal_event_param);
 
 	return (ret);
 }
-