MFC 309240,309242,309244,309245,309670

309240
    hyperv/vmbus: Add result polling support for xact API.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D8633

309242
    hyperv/vmbus: Add result polling support for message Hypercall API.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D8634

309244
    hyperv/vmbus: Add exec cancel support for message Hypercall API.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D8635

309245
    hyperv/vmbus: Use poll/cancel APIs to wait for the CHOPEN response.

    Since hypervisor does not respond CHOPEN to a revoked channel.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D8636

309670
    hyperv/vmbus: Use pause if possible.

    This makes booting on Hyper-V w/ small # of vCPUs work properly.

    Reported by:    Hongxiong Xian <v-hoxian microsoft com>, Hongjiang Zhang <honzhan microsoft com>
    Sponsored by:   Microsoft

5 files changed, 127 insertions, 27 deletions

diff --git a/sys/dev/hyperv/include/vmbus_xact.h b/sys/dev/hyperv/include/vmbus_xact.h
index 280c5fe..90711a0 100644
--- a/sys/dev/hyperv/include/vmbus_xact.h
+++ b/sys/dev/hyperv/include/vmbus_xact.h
@@ -55,6 +55,8 @@ const void		*vmbus_xact_wait(struct vmbus_xact *xact,
 			    size_t *resp_len);
 const void		*vmbus_xact_busywait(struct vmbus_xact *xact,
 			    size_t *resp_len);
+const void		*vmbus_xact_poll(struct vmbus_xact *xact,
+			    size_t *resp_len);
 void			vmbus_xact_wakeup(struct vmbus_xact *xact,
 			    const void *data, size_t dlen);
 void			vmbus_xact_ctx_wakeup(struct vmbus_xact_ctx *ctx,
diff --git a/sys/dev/hyperv/vmbus/vmbus.c b/sys/dev/hyperv/vmbus/vmbus.c
index d1e0616..026f3e8 100644
--- a/sys/dev/hyperv/vmbus/vmbus.c
+++ b/sys/dev/hyperv/vmbus/vmbus.c
@@ -310,6 +310,13 @@ vmbus_msghc_exec(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh)
 	return error;
 }
 
+void
+vmbus_msghc_exec_cancel(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh)
+{
+
+	vmbus_xact_deactivate(mh->mh_xact);
+}
+
 const struct vmbus_message *
 vmbus_msghc_wait_result(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh)
 {
@@ -318,6 +325,14 @@ vmbus_msghc_wait_result(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh)
 	return (vmbus_xact_wait(mh->mh_xact, &resp_len));
 }
 
+const struct vmbus_message *
+vmbus_msghc_poll_result(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh)
+{
+	size_t resp_len;
+
+	return (vmbus_xact_poll(mh->mh_xact, &resp_len));
+}
+
 void
 vmbus_msghc_wakeup(struct vmbus_softc *sc, const struct vmbus_message *msg)
 {
diff --git a/sys/dev/hyperv/vmbus/vmbus_chan.c b/sys/dev/hyperv/vmbus/vmbus_chan.c
index 47333ee..5a58320 100644
--- a/sys/dev/hyperv/vmbus/vmbus_chan.c
+++ b/sys/dev/hyperv/vmbus/vmbus_chan.c
@@ -67,7 +67,7 @@ static void			vmbus_chan_set_chmap(struct vmbus_channel *);
 static void			vmbus_chan_clear_chmap(struct vmbus_channel *);
 static void			vmbus_chan_detach(struct vmbus_channel *);
 static bool			vmbus_chan_wait_revoke(
-				    const struct vmbus_channel *);
+				    const struct vmbus_channel *, bool);
 
 static void			vmbus_chan_ins_prilist(struct vmbus_softc *,
 				    struct vmbus_channel *);
@@ -349,7 +349,6 @@ vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr,
     const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg)
 {
 	struct vmbus_softc *sc = chan->ch_vmbus;
-	const struct vmbus_chanmsg_chopen_resp *resp;
 	const struct vmbus_message *msg;
 	struct vmbus_chanmsg_chopen *req;
 	struct vmbus_msghc *mh;
@@ -453,9 +452,50 @@ vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr,
 		goto failed;
 	}
 
-	msg = vmbus_msghc_wait_result(sc, mh);
-	resp = (const struct vmbus_chanmsg_chopen_resp *)msg->msg_data;
-	status = resp->chm_status;
+	for (;;) {
+		msg = vmbus_msghc_poll_result(sc, mh);
+		if (msg != NULL)
+			break;
+		if (vmbus_chan_is_revoked(chan)) {
+			int i;
+
+			/*
+			 * NOTE:
+			 * Hypervisor does _not_ send response CHOPEN to
+			 * a revoked channel.
+			 */
+			vmbus_chan_printf(chan,
+			    "chan%u is revoked, when it is being opened\n",
+			    chan->ch_id);
+
+			/*
+			 * XXX
+			 * Add extra delay before cancel the hypercall
+			 * execution; mainly to close any possible
+			 * CHRESCIND and CHOPEN_RESP races on the
+			 * hypervisor side.
+			 */
+#define REVOKE_LINGER	100
+			for (i = 0; i < REVOKE_LINGER; ++i) {
+				msg = vmbus_msghc_poll_result(sc, mh);
+				if (msg != NULL)
+					break;
+				pause("rchopen", 1);
+			}
+#undef REVOKE_LINGER
+			if (msg == NULL)
+				vmbus_msghc_exec_cancel(sc, mh);
+			break;
+		}
+		pause("chopen", 1);
+	}
+	if (msg != NULL) {
+		status = ((const struct vmbus_chanmsg_chopen_resp *)
+		    msg->msg_data)->chm_status;
+	} else {
+		/* XXX any non-0 value is ok here. */
+		status = 0xff;
+	}
 
 	vmbus_msghc_put(sc, mh);
 
@@ -620,7 +660,7 @@ vmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr,
 }
 
 static bool
-vmbus_chan_wait_revoke(const struct vmbus_channel *chan)
+vmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep)
 {
 #define WAIT_COUNT	200	/* 200ms */
 
@@ -629,8 +669,10 @@ vmbus_chan_wait_revoke(const struct vmbus_channel *chan)
 	for (i = 0; i < WAIT_COUNT; ++i) {
 		if (vmbus_chan_is_revoked(chan))
 			return (true);
-		/* Not sure about the context; use busy-wait. */
-		DELAY(1000);
+		if (can_sleep)
+			pause("wchrev", 1);
+		else
+			DELAY(1000);
 	}
 	return (false);
 
@@ -667,7 +709,7 @@ vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl)
 	if (error) {
 		vmbus_msghc_put(sc, mh);
 
-		if (vmbus_chan_wait_revoke(chan)) {
+		if (vmbus_chan_wait_revoke(chan, true)) {
 			/*
 			 * Error is benign; this channel is revoked,
 			 * so this GPADL will not be touched anymore.
diff --git a/sys/dev/hyperv/vmbus/vmbus_var.h b/sys/dev/hyperv/vmbus/vmbus_var.h
index b16548a..2f39c91 100644
--- a/sys/dev/hyperv/vmbus/vmbus_var.h
+++ b/sys/dev/hyperv/vmbus/vmbus_var.h
@@ -160,9 +160,14 @@ void		vmbus_msghc_put(struct vmbus_softc *, struct vmbus_msghc *);
 void		*vmbus_msghc_dataptr(struct vmbus_msghc *);
 int		vmbus_msghc_exec_noresult(struct vmbus_msghc *);
 int		vmbus_msghc_exec(struct vmbus_softc *, struct vmbus_msghc *);
+void		vmbus_msghc_exec_cancel(struct vmbus_softc *,
+		    struct vmbus_msghc *);
 const struct vmbus_message *
 		vmbus_msghc_wait_result(struct vmbus_softc *,
 		    struct vmbus_msghc *);
+const struct vmbus_message *
+		vmbus_msghc_poll_result(struct vmbus_softc *,
+		    struct vmbus_msghc *);
 void		vmbus_msghc_wakeup(struct vmbus_softc *,
 		    const struct vmbus_message *);
 void		vmbus_msghc_reset(struct vmbus_msghc *, size_t);
diff --git a/sys/dev/hyperv/vmbus/vmbus_xact.c b/sys/dev/hyperv/vmbus/vmbus_xact.c
index c0219e8..90bdba7 100644
--- a/sys/dev/hyperv/vmbus/vmbus_xact.c
+++ b/sys/dev/hyperv/vmbus/vmbus_xact.c
@@ -71,8 +71,10 @@ static struct vmbus_xact	*vmbus_xact_alloc(struct vmbus_xact_ctx *,
 static void			vmbus_xact_free(struct vmbus_xact *);
 static struct vmbus_xact	*vmbus_xact_get1(struct vmbus_xact_ctx *,
 				    uint32_t);
-const void			*vmbus_xact_wait1(struct vmbus_xact *, size_t *,
+static const void		*vmbus_xact_wait1(struct vmbus_xact *, size_t *,
 				    bool);
+static const void		*vmbus_xact_return(struct vmbus_xact *,
+				    size_t *);
 static void			vmbus_xact_save_resp(struct vmbus_xact *,
 				    const void *, size_t);
 static void			vmbus_xact_ctx_free(struct vmbus_xact_ctx *);
@@ -277,27 +279,13 @@ vmbus_xact_deactivate(struct vmbus_xact *xact)
 	mtx_unlock(&ctx->xc_lock);
 }
 
-const void *
-vmbus_xact_wait1(struct vmbus_xact *xact, size_t *resp_len,
-    bool can_sleep)
+static const void *
+vmbus_xact_return(struct vmbus_xact *xact, size_t *resp_len)
 {
 	struct vmbus_xact_ctx *ctx = xact->x_ctx;
 	const void *resp;
 
-	mtx_lock(&ctx->xc_lock);
-
-	KASSERT(ctx->xc_active == xact, ("xact mismatch"));
-	while (xact->x_resp == NULL &&
-	    (ctx->xc_flags & VMBUS_XACT_CTXF_DESTROY) == 0) {
-		if (can_sleep) {
-			mtx_sleep(&ctx->xc_active, &ctx->xc_lock, 0,
-			    "wxact", 0);
-		} else {
-			mtx_unlock(&ctx->xc_lock);
-			DELAY(1000);
-			mtx_lock(&ctx->xc_lock);
-		}
-	}
+	mtx_assert(&ctx->xc_lock, MA_OWNED);
 	KASSERT(ctx->xc_active == xact, ("xact trashed"));
 
 	if ((ctx->xc_flags & VMBUS_XACT_CTXF_DESTROY) && xact->x_resp == NULL) {
@@ -317,6 +305,32 @@ vmbus_xact_wait1(struct vmbus_xact *xact, size_t *resp_len,
 	resp = xact->x_resp;
 	*resp_len = xact->x_resp_len;
 
+	return (resp);
+}
+
+static const void *
+vmbus_xact_wait1(struct vmbus_xact *xact, size_t *resp_len,
+    bool can_sleep)
+{
+	struct vmbus_xact_ctx *ctx = xact->x_ctx;
+	const void *resp;
+
+	mtx_lock(&ctx->xc_lock);
+
+	KASSERT(ctx->xc_active == xact, ("xact mismatch"));
+	while (xact->x_resp == NULL &&
+	    (ctx->xc_flags & VMBUS_XACT_CTXF_DESTROY) == 0) {
+		if (can_sleep) {
+			mtx_sleep(&ctx->xc_active, &ctx->xc_lock, 0,
+			    "wxact", 0);
+		} else {
+			mtx_unlock(&ctx->xc_lock);
+			DELAY(1000);
+			mtx_lock(&ctx->xc_lock);
+		}
+	}
+	resp = vmbus_xact_return(xact, resp_len);
+
 	mtx_unlock(&ctx->xc_lock);
 
 	return (resp);
@@ -336,6 +350,28 @@ vmbus_xact_busywait(struct vmbus_xact *xact, size_t *resp_len)
 	return (vmbus_xact_wait1(xact, resp_len, false /* can't sleep */));
 }
 
+const void *
+vmbus_xact_poll(struct vmbus_xact *xact, size_t *resp_len)
+{
+	struct vmbus_xact_ctx *ctx = xact->x_ctx;
+	const void *resp;
+
+	mtx_lock(&ctx->xc_lock);
+
+	KASSERT(ctx->xc_active == xact, ("xact mismatch"));
+	if (xact->x_resp == NULL &&
+	    (ctx->xc_flags & VMBUS_XACT_CTXF_DESTROY) == 0) {
+		mtx_unlock(&ctx->xc_lock);
+		*resp_len = 0;
+		return (NULL);
+	}
+	resp = vmbus_xact_return(xact, resp_len);
+
+	mtx_unlock(&ctx->xc_lock);
+
+	return (resp);
+}
+
 static void
 vmbus_xact_save_resp(struct vmbus_xact *xact, const void *data, size_t dlen)
 {