MFC 258860,260167,260238,260397:

- Restructure the VMX code to enter and exit the guest. In large part this
  change hides the setjmp/longjmp semantics of VM enter/exit.
  vmx_enter_guest() is used to enter guest context and vmx_exit_guest() is
  used to transition back into host context.

  Fix a longstanding race where a vcpu interrupt notification might be
  ignored if it happens after vmx_inject_interrupts() but before host
  interrupts are disabled in vmx_resume/vmx_launch. We now call
  vmx_inject_interrupts() with host interrupts disabled to prevent this.
- The 'protection' field in the VM exit collateral for the PAGING exit is
  not used - get rid of it.

Reviewed by:	grehan

7 files changed, 232 insertions, 423 deletions

diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
index d6f1a5a..92b767f 100644
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -286,7 +286,6 @@ struct vm_exit {
 		struct {
 			uint64_t	gpa;
 			int		fault_type;
-			int		protection;
 		} paging;
 		struct {
 			uint64_t	gpa;
@@ -299,9 +298,19 @@ struct vm_exit {
 		 * exitcode to represent the VM-exit.
 		 */
 		struct {
-			int		error;		/* vmx inst error */
+			int		status;		/* vmx inst status */
+			/*
+			 * 'exit_reason' and 'exit_qualification' are valid
+			 * only if 'status' is zero.
+			 */
 			uint32_t	exit_reason;
 			uint64_t	exit_qualification;
+			/*
+			 * 'inst_error' and 'inst_type' are valid
+			 * only if 'status' is non-zero.
+			 */
+			int		inst_type;
+			int		inst_error;
 		} vmx;
 		struct {
 			uint32_t	code;		/* ecx value */
diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
index 859da35..bcaed4e 100644
--- a/sys/amd64/vmm/intel/vmx.c
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -286,82 +286,6 @@ exit_reason_to_str(int reason)
 		return (reasonbuf);
 	}
 }
-
-#ifdef SETJMP_TRACE
-static const char *
-vmx_setjmp_rc2str(int rc)
-{
-	switch (rc) {
-	case VMX_RETURN_DIRECT:
-		return "direct";
-	case VMX_RETURN_LONGJMP:
-		return "longjmp";
-	case VMX_RETURN_VMRESUME:
-		return "vmresume";
-	case VMX_RETURN_VMLAUNCH:
-		return "vmlaunch";
-	case VMX_RETURN_AST:
-		return "ast";
-	default:
-		return "unknown";
-	}
-}
-
-#define	SETJMP_TRACE(vmx, vcpu, vmxctx, regname)			    \
-	VCPU_CTR1((vmx)->vm, (vcpu), "setjmp trace " #regname " 0x%016lx",  \
-		 (vmxctx)->regname)
-
-static void
-vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
-{
-	uint64_t host_rip, host_rsp;
-
-	if (vmxctx != &vmx->ctx[vcpu])
-		panic("vmx_setjmp_trace: invalid vmxctx %p; should be %p",
-			vmxctx, &vmx->ctx[vcpu]);
-
-	VCPU_CTR1((vmx)->vm, (vcpu), "vmxctx = %p", vmxctx);
-	VCPU_CTR2((vmx)->vm, (vcpu), "setjmp return code %s(%d)",
-		 vmx_setjmp_rc2str(rc), rc);
-
-	host_rip = vmcs_read(VMCS_HOST_RIP);
-	host_rsp = vmcs_read(VMCS_HOST_RSP);
-	VCPU_CTR2((vmx)->vm, (vcpu), "vmcs host_rip 0x%016lx, host_rsp %#lx",
-		 host_rip, host_rsp);
-
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r15);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r14);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r13);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r12);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbp);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rsp);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbx);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rip);
-
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdi);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rsi);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdx);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rcx);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r8);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r9);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rax);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbx);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbp);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r10);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r11);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r12);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r13);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r14);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r15);
-	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_cr2);
-}
-#endif
-#else
-static void __inline
-vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
-{
-	return;
-}
 #endif	/* KTR */
 
 u_long
@@ -825,7 +749,7 @@ vmx_vminit(struct vm *vm, pmap_t pmap)
 		}
 
 		error = vmcs_set_defaults(&vmx->vmcs[i],
-					  (u_long)vmx_longjmp,
+					  (u_long)vmx_exit_guest,
 					  (u_long)&vmx->ctx[i],
 					  vmx->eptp,
 					  pinbased_ctls,
@@ -1222,21 +1146,6 @@ ept_fault_type(uint64_t ept_qual)
 	return (fault_type);
 }
 
-static int
-ept_protection(uint64_t ept_qual)
-{
-	int prot = 0;
-
-	if (ept_qual & EPT_VIOLATION_GPA_READABLE)
-		prot |= VM_PROT_READ;
-	if (ept_qual & EPT_VIOLATION_GPA_WRITEABLE)
-		prot |= VM_PROT_WRITE;
-	if (ept_qual & EPT_VIOLATION_GPA_EXECUTABLE)
-		prot |= VM_PROT_EXECUTE;
-
-	return (prot);
-}
-
 static boolean_t
 ept_emulation_fault(uint64_t ept_qual)
 {
@@ -1269,15 +1178,14 @@ static int
 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 {
 	int error, handled;
-	struct vmcs *vmcs;
 	struct vmxctx *vmxctx;
 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, reason;
 	uint64_t qual, gpa;
 	bool retu;
 
 	handled = 0;
-	vmcs = &vmx->vmcs[vcpu];
 	vmxctx = &vmx->ctx[vcpu];
+
 	qual = vmexit->u.vmx.exit_qualification;
 	reason = vmexit->u.vmx.exit_reason;
 	vmexit->exitcode = VM_EXITCODE_BOGUS;
@@ -1421,7 +1329,6 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 			vmexit->exitcode = VM_EXITCODE_PAGING;
 			vmexit->u.paging.gpa = gpa;
 			vmexit->u.paging.fault_type = ept_fault_type(qual);
-			vmexit->u.paging.protection = ept_protection(qual);
 		} else if (ept_emulation_fault(qual)) {
 			vmexit->exitcode = VM_EXITCODE_INST_EMUL;
 			vmexit->u.inst_emul.gpa = gpa;
@@ -1455,7 +1362,7 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 			 * treat it as a generic VMX exit.
 			 */
 			vmexit->exitcode = VM_EXITCODE_VMX;
-			vmexit->u.vmx.error = 0;
+			vmexit->u.vmx.status = VM_SUCCESS;
 		} else {
 			/*
 			 * The exitcode and collateral have been populated.
@@ -1466,32 +1373,69 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 	return (handled);
 }
 
+static __inline int
+vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
+{
+
+	vmexit->rip = vmcs_guest_rip();
+	vmexit->inst_length = 0;
+	vmexit->exitcode = VM_EXITCODE_BOGUS;
+	vmx_astpending_trace(vmx, vcpu, vmexit->rip);
+	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
+
+	return (HANDLED);
+}
+
+static __inline int
+vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
+{
+
+	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
+	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
+	    vmxctx->inst_fail_status));
+
+	vmexit->inst_length = 0;
+	vmexit->exitcode = VM_EXITCODE_VMX;
+	vmexit->u.vmx.status = vmxctx->inst_fail_status;
+	vmexit->u.vmx.inst_error = vmcs_instruction_error();
+	vmexit->u.vmx.exit_reason = ~0;
+	vmexit->u.vmx.exit_qualification = ~0;
+
+	switch (rc) {
+	case VMX_VMRESUME_ERROR:
+	case VMX_VMLAUNCH_ERROR:
+	case VMX_INVEPT_ERROR:
+		vmexit->u.vmx.inst_type = rc;
+		break;
+	default:
+		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
+	}
+
+	return (UNHANDLED);
+}
+
 static int
-vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap)
+vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap)
 {
-	int vie, rc, handled, astpending;
-	uint32_t exit_reason;
+	int rc, handled, launched;
 	struct vmx *vmx;
 	struct vmxctx *vmxctx;
 	struct vmcs *vmcs;
 	struct vm_exit *vmexit;
+	uint64_t rip;
+	uint32_t exit_reason;
 
 	vmx = arg;
 	vmcs = &vmx->vmcs[vcpu];
 	vmxctx = &vmx->ctx[vcpu];
-	vmxctx->launched = 0;
-
-	astpending = 0;
 	vmexit = vm_exitinfo(vmx->vm, vcpu);
+	launched = 0;
 
 	KASSERT(vmxctx->pmap == pmap,
 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
 	KASSERT(vmxctx->eptp == vmx->eptp,
 	    ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp));
 
-	/*
-	 * XXX Can we avoid doing this every time we do a vm run?
-	 */
 	VMPTRLD(vmcs);
 
 	/*
@@ -1503,76 +1447,55 @@ vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap)
 	 * of a single process we could do this once in vmcs_set_defaults().
 	 */
 	vmcs_write(VMCS_HOST_CR3, rcr3());
-	vmcs_write(VMCS_GUEST_RIP, rip);
-	vmx_set_pcpu_defaults(vmx, vcpu);
 
+	vmcs_write(VMCS_GUEST_RIP, startrip);
+	vmx_set_pcpu_defaults(vmx, vcpu);
 	do {
-		vmx_inject_interrupts(vmx, vcpu);
-		vmx_run_trace(vmx, vcpu);
-		rc = vmx_setjmp(vmxctx);
-#ifdef SETJMP_TRACE
-		vmx_setjmp_trace(vmx, vcpu, vmxctx, rc);
-#endif
-		switch (rc) {
-		case VMX_RETURN_DIRECT:
-			if (vmxctx->launched == 0) {
-				vmxctx->launched = 1;
-				vmx_launch(vmxctx);
-			} else
-				vmx_resume(vmxctx);
-			panic("vmx_launch/resume should not return");
-			break;
-		case VMX_RETURN_LONGJMP:
-			break;			/* vm exit */
-		case VMX_RETURN_AST:
-			astpending = 1;
-			break;
-		case VMX_RETURN_VMRESUME:
-			vie = vmcs_instruction_error();
-			if (vmxctx->launch_error == VM_FAIL_INVALID ||
-			    vie != VMRESUME_WITH_NON_LAUNCHED_VMCS) {
-				printf("vmresume error %d vmcs inst error %d\n",
-					vmxctx->launch_error, vie);
-				goto err_exit;
-			}
-			vmx_launch(vmxctx);	/* try to launch the guest */
-			panic("vmx_launch should not return");
+		/*
+		 * Interrupts are disabled from this point on until the
+		 * guest starts executing. This is done for the following
+		 * reasons:
+		 *
+		 * If an AST is asserted on this thread after the check below,
+		 * then the IPI_AST notification will not be lost, because it
+		 * will cause a VM exit due to external interrupt as soon as
+		 * the guest state is loaded.
+		 *
+		 * A posted interrupt after 'vmx_inject_interrupts()' will
+		 * not be "lost" because it will be held pending in the host
+		 * APIC because interrupts are disabled. The pending interrupt
+		 * will be recognized as soon as the guest state is loaded.
+		 *
+		 * The same reasoning applies to the IPI generated by
+		 * pmap_invalidate_ept().
+		 */
+		disable_intr();
+		if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
+			enable_intr();
+			handled = vmx_exit_astpending(vmx, vcpu, vmexit);
 			break;
-		case VMX_RETURN_VMLAUNCH:
-			vie = vmcs_instruction_error();
-#if 1
-			printf("vmlaunch error %d vmcs inst error %d\n",
-				vmxctx->launch_error, vie);
-#endif
-			goto err_exit;
-		case VMX_RETURN_INVEPT:
-			panic("vm %s:%d invept error %d",
-			      vm_name(vmx->vm), vcpu, vmxctx->launch_error);
-		default:
-			panic("vmx_setjmp returned %d", rc);
 		}
-		
-		/* enable interrupts */
+
+		vmx_inject_interrupts(vmx, vcpu);
+		vmx_run_trace(vmx, vcpu);
+		rc = vmx_enter_guest(vmxctx, launched);
+
 		enable_intr();
 
-		/* collect some basic information for VM exit processing */
+		/* Collect some information for VM exit processing */
 		vmexit->rip = rip = vmcs_guest_rip();
 		vmexit->inst_length = vmexit_instruction_length();
 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
 
-		if (astpending) {
-			handled = 1;
-			vmexit->inst_length = 0;
-			vmexit->exitcode = VM_EXITCODE_BOGUS;
-			vmx_astpending_trace(vmx, vcpu, rip);
-			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
-			break;
+		if (rc == VMX_GUEST_VMEXIT) {
+			launched = 1;
+			handled = vmx_exit_process(vmx, vcpu, vmexit);
+		} else {
+			handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
 		}
 
-		handled = vmx_exit_process(vmx, vcpu, vmexit);
 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
-
 	} while (handled);
 
 	/*
@@ -1588,26 +1511,11 @@ vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap)
 	if (!handled)
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_USERSPACE, 1);
 
-	VCPU_CTR1(vmx->vm, vcpu, "goto userland: exitcode %d",vmexit->exitcode);
+	VCPU_CTR1(vmx->vm, vcpu, "returning from vmx_run: exitcode %d",
+	    vmexit->exitcode);
 
-	/*
-	 * XXX
-	 * We need to do this to ensure that any VMCS state cached by the
-	 * processor is flushed to memory. We need to do this in case the
-	 * VM moves to a different cpu the next time it runs.
-	 *
-	 * Can we avoid doing this?
-	 */
 	VMCLEAR(vmcs);
 	return (0);
-
-err_exit:
-	vmexit->exitcode = VM_EXITCODE_VMX;
-	vmexit->u.vmx.exit_reason = (uint32_t)-1;
-	vmexit->u.vmx.exit_qualification = (uint32_t)-1;
-	vmexit->u.vmx.error = vie;
-	VMCLEAR(vmcs);
-	return (ENOEXEC);
 }
 
 static void
diff --git a/sys/amd64/vmm/intel/vmx.h b/sys/amd64/vmm/intel/vmx.h
index 450bb66..67ef631 100644
--- a/sys/amd64/vmm/intel/vmx.h
+++ b/sys/amd64/vmm/intel/vmx.h
@@ -36,9 +36,6 @@ struct pmap;
 #define	GUEST_MSR_MAX_ENTRIES	64		/* arbitrary */
 
 struct vmxctx {
-	register_t	tmpstk[32];		/* vmx_return() stack */
-	register_t	tmpstktop;
-
 	register_t	guest_rdi;		/* Guest state */
 	register_t	guest_rsi;
 	register_t	guest_rdx;
@@ -68,8 +65,7 @@ struct vmxctx {
 	 * XXX todo debug registers and fpu state
 	 */
 	
-	int		launched;		/* vmcs launch state */
-	int		launch_error;
+	int		inst_fail_status;
 
 	long		eptgen[MAXCPU];		/* cached pmap->pm_eptgen */
 
@@ -107,25 +103,12 @@ CTASSERT((offsetof(struct vmx, vmcs) & PAGE_MASK) == 0);
 CTASSERT((offsetof(struct vmx, msr_bitmap) & PAGE_MASK) == 0);
 CTASSERT((offsetof(struct vmx, guest_msrs) & 15) == 0);
 
-#define	VMX_RETURN_DIRECT	0
-#define	VMX_RETURN_LONGJMP	1
-#define	VMX_RETURN_VMRESUME	2
-#define	VMX_RETURN_VMLAUNCH	3
-#define	VMX_RETURN_AST		4
-#define	VMX_RETURN_INVEPT	5
-/*
- * vmx_setjmp() returns:
- * - 0 when it returns directly
- * - 1 when it returns from vmx_longjmp
- * - 2 when it returns from vmx_resume (which would only be in the error case)
- * - 3 when it returns from vmx_launch (which would only be in the error case)
- * - 4 when it returns from vmx_resume or vmx_launch because of AST pending
- * - 5 when it returns from vmx_launch/vmx_resume because of invept error
- */
-int	vmx_setjmp(struct vmxctx *ctx);
-void	vmx_longjmp(void);			/* returns via vmx_setjmp */
-void	vmx_launch(struct vmxctx *ctx) __dead2;	/* may return via vmx_setjmp */
-void	vmx_resume(struct vmxctx *ctx) __dead2;	/* may return via vmx_setjmp */
+#define	VMX_GUEST_VMEXIT	0
+#define	VMX_VMRESUME_ERROR	1
+#define	VMX_VMLAUNCH_ERROR	2
+#define	VMX_INVEPT_ERROR	3
+int	vmx_enter_guest(struct vmxctx *ctx, int launched);
+void	vmx_exit_guest(void);
 
 u_long	vmx_fix_cr0(u_long cr0);
 u_long	vmx_fix_cr4(u_long cr4);
diff --git a/sys/amd64/vmm/intel/vmx_genassym.c b/sys/amd64/vmm/intel/vmx_genassym.c
index 4f39ef9..bf463dc 100644
--- a/sys/amd64/vmm/intel/vmx_genassym.c
+++ b/sys/amd64/vmm/intel/vmx_genassym.c
@@ -31,7 +31,6 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
-#include <sys/malloc.h>
 #include <sys/proc.h>
 #include <sys/assym.h>
 
@@ -42,7 +41,6 @@ __FBSDID("$FreeBSD$");
 #include "vmx_cpufunc.h"
 #include "vmx.h"
 
-ASSYM(VMXCTX_TMPSTKTOP, offsetof(struct vmxctx, tmpstktop));
 ASSYM(VMXCTX_GUEST_RDI, offsetof(struct vmxctx, guest_rdi));
 ASSYM(VMXCTX_GUEST_RSI, offsetof(struct vmxctx, guest_rsi));
 ASSYM(VMXCTX_GUEST_RDX, offsetof(struct vmxctx, guest_rdx));
@@ -69,27 +67,19 @@ ASSYM(VMXCTX_HOST_RSP, offsetof(struct vmxctx, host_rsp));
 ASSYM(VMXCTX_HOST_RBX, offsetof(struct vmxctx, host_rbx));
 ASSYM(VMXCTX_HOST_RIP, offsetof(struct vmxctx, host_rip));
 
-ASSYM(VMXCTX_LAUNCH_ERROR, offsetof(struct vmxctx, launch_error));
+ASSYM(VMXCTX_INST_FAIL_STATUS, offsetof(struct vmxctx, inst_fail_status));
 ASSYM(VMXCTX_EPTGEN, offsetof(struct vmxctx, eptgen));
 
 ASSYM(VMXCTX_PMAP, offsetof(struct vmxctx, pmap));
 ASSYM(VMXCTX_EPTP, offsetof(struct vmxctx, eptp));
 
-ASSYM(VM_SUCCESS,	VM_SUCCESS);
 ASSYM(VM_FAIL_INVALID,	VM_FAIL_INVALID);
 ASSYM(VM_FAIL_VALID,	VM_FAIL_VALID);
+ASSYM(VMX_GUEST_VMEXIT, VMX_GUEST_VMEXIT);
+ASSYM(VMX_VMRESUME_ERROR, VMX_VMRESUME_ERROR);
+ASSYM(VMX_VMLAUNCH_ERROR, VMX_VMLAUNCH_ERROR);
+ASSYM(VMX_INVEPT_ERROR, VMX_INVEPT_ERROR);
 
-ASSYM(VMX_RETURN_DIRECT,	VMX_RETURN_DIRECT);
-ASSYM(VMX_RETURN_LONGJMP,	VMX_RETURN_LONGJMP);
-ASSYM(VMX_RETURN_VMRESUME,	VMX_RETURN_VMRESUME);
-ASSYM(VMX_RETURN_VMLAUNCH,	VMX_RETURN_VMLAUNCH);
-ASSYM(VMX_RETURN_AST,		VMX_RETURN_AST);
-ASSYM(VMX_RETURN_INVEPT,	VMX_RETURN_INVEPT);
-
-ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
-ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
-ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
-ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
 ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
 
 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
diff --git a/sys/amd64/vmm/intel/vmx_support.S b/sys/amd64/vmm/intel/vmx_support.S
index fd7b4aa..d616984 100644
--- a/sys/amd64/vmm/intel/vmx_support.S
+++ b/sys/amd64/vmm/intel/vmx_support.S
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2011 NetApp, Inc.
+ * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,32 +38,6 @@
 #endif
 
 /*
- * Disable interrupts before updating %rsp in VMX_CHECK_AST or
- * VMX_GUEST_RESTORE.
- *
- * The location that %rsp points to is a 'vmxctx' and not a
- * real stack so we don't want an interrupt handler to trash it
- */
-#define	VMX_DISABLE_INTERRUPTS		cli
-
-/*
- * If the thread hosting the vcpu has an ast pending then take care of it
- * by returning from vmx_setjmp() with a return value of VMX_RETURN_AST.
- *
- * Assumes that %rdi holds a pointer to the 'vmxctx' and that interrupts
- * are disabled.
- */
-#define	VMX_CHECK_AST							\
-	movq	PCPU(CURTHREAD),%rax;					\
-	testl	$TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax);	\
-	je	9f;							\
-	movq	$VMX_RETURN_AST,%rsi;					\
-	movq	%rdi,%rsp;						\
-	addq	$VMXCTX_TMPSTKTOP,%rsp;					\
-	callq	vmx_return;						\
-9:
-
-/*
  * Assumes that %rdi holds a pointer to the 'vmxctx'.
  *
  * On "return" all registers are updated to reflect guest state. The two
@@ -93,132 +68,132 @@
 	movq	VMXCTX_GUEST_RDI(%rdi),%rdi; /* restore rdi the last */
 
 /*
- * Check for an error after executing a VMX instruction.
- * 'errreg' will be zero on success and non-zero otherwise.
- * 'ctxreg' points to the 'struct vmxctx' associated with the vcpu.
+ * Save and restore the host context.
+ *
+ * Assumes that %rdi holds a pointer to the 'vmxctx'.
  */
-#define	VM_INSTRUCTION_ERROR(errreg, ctxreg)				\
-	jnc 	1f;							\
-	movl 	$VM_FAIL_INVALID,errreg;		/* CF is set */	\
-	jmp 	3f;							\
-1:	jnz 	2f;							\
-	movl 	$VM_FAIL_VALID,errreg;		/* ZF is set */		\
-	jmp 	3f;							\
-2:	movl 	$VM_SUCCESS,errreg;					\
-3:	movl	errreg,VMXCTX_LAUNCH_ERROR(ctxreg)
+#define	VMX_HOST_SAVE(tmpreg)						\
+	movq	(%rsp), tmpreg;			/* return address */	\
+	movq    %r15, VMXCTX_HOST_R15(%rdi);				\
+	movq    %r14, VMXCTX_HOST_R14(%rdi);				\
+	movq    %r13, VMXCTX_HOST_R13(%rdi);				\
+	movq    %r12, VMXCTX_HOST_R12(%rdi);				\
+	movq    %rbp, VMXCTX_HOST_RBP(%rdi);				\
+	movq    %rsp, VMXCTX_HOST_RSP(%rdi);				\
+	movq    %rbx, VMXCTX_HOST_RBX(%rdi);				\
+	movq    tmpreg, VMXCTX_HOST_RIP(%rdi)
+
+#define	VMX_HOST_RESTORE(tmpreg)					\
+	movq	VMXCTX_HOST_R15(%rdi), %r15;				\
+	movq	VMXCTX_HOST_R14(%rdi), %r14;				\
+	movq	VMXCTX_HOST_R13(%rdi), %r13;				\
+	movq	VMXCTX_HOST_R12(%rdi), %r12;				\
+	movq	VMXCTX_HOST_RBP(%rdi), %rbp;				\
+	movq	VMXCTX_HOST_RSP(%rdi), %rsp;				\
+	movq	VMXCTX_HOST_RBX(%rdi), %rbx;				\
+	movq	VMXCTX_HOST_RIP(%rdi), tmpreg;				\
+	movq	tmpreg, (%rsp)			/* return address */
 
 /*
- * set or clear the appropriate bit in 'pm_active'
- * %rdi = vmxctx
- * %rax, %r11 = scratch registers
+ * vmx_enter_guest(struct vmxctx *vmxctx, int launched)
+ * %rdi: pointer to the 'vmxctx'
+ * %esi: launch state of the VMCS
+ * Interrupts must be disabled on entry.
  */
-#define	VMX_SET_PM_ACTIVE						\
-	movq	VMXCTX_PMAP(%rdi), %r11;				\
-	movl	PCPU(CPUID), %eax;					\
-	LK btsl	%eax, PM_ACTIVE(%r11)
+ENTRY(vmx_enter_guest)
+	/*
+	 * Save host state before doing anything else.
+	 */
+	VMX_HOST_SAVE(%r10)
 
-#define	VMX_CLEAR_PM_ACTIVE						\
-	movq	VMXCTX_PMAP(%rdi), %r11;				\
-	movl	PCPU(CPUID), %eax;					\
-	LK btrl	%eax, PM_ACTIVE(%r11)
+	/*
+	 * Activate guest pmap on this cpu.
+	 */
+	movq	VMXCTX_PMAP(%rdi), %r11
+	movl	PCPU(CPUID), %eax
+	LK btsl	%eax, PM_ACTIVE(%r11)
 
-/*
- * If 'vmxctx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen'
- * then we must invalidate all mappings associated with this eptp.
- *
- * %rdi = vmxctx
- * %rax, %rbx, %r11 = scratch registers
- */
-#define	VMX_CHECK_EPTGEN						\
-	movl	PCPU(CPUID), %ebx;					\
-	movq	VMXCTX_PMAP(%rdi), %r11;				\
-	movq	PM_EPTGEN(%r11), %rax;					\
-	cmpq	%rax, VMXCTX_EPTGEN(%rdi, %rbx, 8);			\
-	je	9f;							\
-									\
-	/* Refresh 'vmxctx->eptgen[curcpu]' */				\
-	movq	%rax, VMXCTX_EPTGEN(%rdi, %rbx, 8);			\
-									\
-	/* Setup the invept descriptor at the top of tmpstk */		\
-	mov	%rdi, %r11;						\
-	addq	$VMXCTX_TMPSTKTOP, %r11;				\
-	movq	VMXCTX_EPTP(%rdi), %rax;				\
-	movq	%rax, -16(%r11);					\
-	movq	$0x0, -8(%r11);						\
-	mov	$0x1, %eax;	/* Single context invalidate */		\
-	invept	-16(%r11), %rax;					\
-									\
-	/* Check for invept error */					\
-	VM_INSTRUCTION_ERROR(%eax, %rdi);				\
-	testl	%eax, %eax;						\
-	jz	9f;							\
-									\
-	/* Return via vmx_setjmp with retval of VMX_RETURN_INVEPT */	\
-	movq	$VMX_RETURN_INVEPT, %rsi;				\
-	movq	%rdi,%rsp;						\
-	addq	$VMXCTX_TMPSTKTOP, %rsp;				\
-	callq	vmx_return;						\
-9:	;
+	/*
+	 * If 'vmxctx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen'
+	 * then we must invalidate all mappings associated with this EPTP.
+	 */
+	movq	PM_EPTGEN(%r11), %r10
+	cmpq	%r10, VMXCTX_EPTGEN(%rdi, %rax, 8)
+	je	guest_restore
+
+	/* Refresh 'vmxctx->eptgen[curcpu]' */
+	movq	%r10, VMXCTX_EPTGEN(%rdi, %rax, 8)
+
+	/* Setup the invept descriptor on the host stack */
+	mov	%rsp, %r11
+	movq	VMXCTX_EPTP(%rdi), %rax
+	movq	%rax, -16(%r11)
+	movq	$0x0, -8(%r11)
+	mov	$0x1, %eax		/* Single context invalidate */
+	invept	-16(%r11), %rax
+	jbe	invept_error		/* Check invept instruction error */
+
+guest_restore:
+	cmpl	$0, %esi
+	je	do_launch
 
-	.text
-/*
- * int vmx_setjmp(ctxp)
- * %rdi = ctxp
- *
- * Return value is '0' when it returns directly from here.
- * Return value is '1' when it returns after a vm exit through vmx_longjmp.
- */
-ENTRY(vmx_setjmp)
-	movq	(%rsp),%rax			/* return address */
-	movq    %r15,VMXCTX_HOST_R15(%rdi)
-	movq    %r14,VMXCTX_HOST_R14(%rdi)
-	movq    %r13,VMXCTX_HOST_R13(%rdi)
-	movq    %r12,VMXCTX_HOST_R12(%rdi)
-	movq    %rbp,VMXCTX_HOST_RBP(%rdi)
-	movq    %rsp,VMXCTX_HOST_RSP(%rdi)
-	movq    %rbx,VMXCTX_HOST_RBX(%rdi)
-	movq    %rax,VMXCTX_HOST_RIP(%rdi)
+	VMX_GUEST_RESTORE
+	vmresume
+	/*
+	 * In the common case 'vmresume' returns back to the host through
+	 * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'.
+	 *
+	 * If there is an error we return VMX_VMRESUME_ERROR to the caller.
+	 */
+	movq	%rsp, %rdi		/* point %rdi back to 'vmxctx' */
+	movl	$VMX_VMRESUME_ERROR, %eax
+	jmp	decode_inst_error
 
+do_launch:
+	VMX_GUEST_RESTORE
+	vmlaunch
 	/*
-	 * XXX save host debug registers
+	 * In the common case 'vmlaunch' returns back to the host through
+	 * 'vmx_exit_guest' with %rsp pointing to 'vmxctx'.
+	 *
+	 * If there is an error we return VMX_VMLAUNCH_ERROR to the caller.
 	 */
-	movl	$VMX_RETURN_DIRECT,%eax
-	ret
-END(vmx_setjmp)
+	movq	%rsp, %rdi		/* point %rdi back to 'vmxctx' */
+	movl	$VMX_VMLAUNCH_ERROR, %eax
+	jmp	decode_inst_error
 
-/*
- * void vmx_return(struct vmxctx *ctxp, int retval)
- * %rdi = ctxp
- * %rsi = retval
- * Return to vmm context through vmx_setjmp() with a value of 'retval'.
- */
-ENTRY(vmx_return)
-	/* The pmap is no longer active on the host cpu */
-	VMX_CLEAR_PM_ACTIVE
+invept_error:
+	movl	$VMX_INVEPT_ERROR, %eax
+	jmp	decode_inst_error
+
+decode_inst_error:
+	movl	$VM_FAIL_VALID, %r11d
+	jz	inst_error
+	movl	$VM_FAIL_INVALID, %r11d
+inst_error:
+	movl	%r11d, VMXCTX_INST_FAIL_STATUS(%rdi)
 
-	/* Restore host context. */
-	movq	VMXCTX_HOST_R15(%rdi),%r15
-	movq	VMXCTX_HOST_R14(%rdi),%r14
-	movq	VMXCTX_HOST_R13(%rdi),%r13
-	movq	VMXCTX_HOST_R12(%rdi),%r12
-	movq	VMXCTX_HOST_RBP(%rdi),%rbp
-	movq	VMXCTX_HOST_RSP(%rdi),%rsp
-	movq	VMXCTX_HOST_RBX(%rdi),%rbx
-	movq	VMXCTX_HOST_RIP(%rdi),%rax
-	movq	%rax,(%rsp)			/* return address */
+	/*
+	 * The return value is already populated in %eax so we cannot use
+	 * it as a scratch register beyond this point.
+	 */
 
 	/*
-	 * XXX restore host debug registers
+	 * Deactivate guest pmap from this cpu.
 	 */
-	movl	%esi,%eax
+	movq	VMXCTX_PMAP(%rdi), %r11
+	movl	PCPU(CPUID), %r10d
+	LK btrl	%r10d, PM_ACTIVE(%r11)
+
+	VMX_HOST_RESTORE(%r10)
 	ret
-END(vmx_return)
+END(vmx_enter_guest)
 
 /*
- * void vmx_longjmp(void)
+ * void vmx_exit_guest(void)
  * %rsp points to the struct vmxctx
  */
-ENTRY(vmx_longjmp)
+ENTRY(vmx_exit_guest)
 	/*
 	 * Save guest state that is not automatically saved in the vmcs.
 	 */
@@ -242,80 +217,20 @@ ENTRY(vmx_longjmp)
 	movq	%rdi,VMXCTX_GUEST_CR2(%rsp)
 
 	movq	%rsp,%rdi
-	movq	$VMX_RETURN_LONGJMP,%rsi
-
-	addq	$VMXCTX_TMPSTKTOP,%rsp
-	callq	vmx_return
-END(vmx_longjmp)
-
-/*
- * void vmx_resume(struct vmxctx *ctxp)
- * %rdi = ctxp
- *
- * Although the return type is a 'void' this function may return indirectly
- * through vmx_setjmp() with a return value of 2.
- */
-ENTRY(vmx_resume)
-	VMX_DISABLE_INTERRUPTS
-
-	VMX_CHECK_AST
-
-	VMX_SET_PM_ACTIVE	/* This vcpu is now active on the host cpu */
-
-	VMX_CHECK_EPTGEN	/* Check if we have to invalidate TLB */
-
-	/*
-	 * Restore guest state that is not automatically loaded from the vmcs.
-	 */
-	VMX_GUEST_RESTORE
-
-	vmresume
-
-	/*
-	 * Capture the reason why vmresume failed.
-	 */
-	VM_INSTRUCTION_ERROR(%eax, %rsp)
-
-	/* Return via vmx_setjmp with return value of VMX_RETURN_VMRESUME */
-	movq	%rsp,%rdi
-	movq	$VMX_RETURN_VMRESUME,%rsi
-
-	addq	$VMXCTX_TMPSTKTOP,%rsp
-	callq	vmx_return
-END(vmx_resume)
-
-/*
- * void vmx_launch(struct vmxctx *ctxp)
- * %rdi = ctxp
- *
- * Although the return type is a 'void' this function may return indirectly
- * through vmx_setjmp() with a return value of 3.
- */
-ENTRY(vmx_launch)
-	VMX_DISABLE_INTERRUPTS
-
-	VMX_CHECK_AST
-
-	VMX_SET_PM_ACTIVE	/* This vcpu is now active on the host cpu */
-
-	VMX_CHECK_EPTGEN	/* Check if we have to invalidate TLB */
 
 	/*
-	 * Restore guest state that is not automatically loaded from the vmcs.
+	 * Deactivate guest pmap from this cpu.
 	 */
-	VMX_GUEST_RESTORE
+	movq	VMXCTX_PMAP(%rdi), %r11
+	movl	PCPU(CPUID), %r10d
+	LK btrl	%r10d, PM_ACTIVE(%r11)
 
-	vmlaunch
+	VMX_HOST_RESTORE(%r10)
 
 	/*
-	 * Capture the reason why vmlaunch failed.
+	 * This will return to the caller of 'vmx_enter_guest()' with a return
+	 * value of VMX_GUEST_VMEXIT.
 	 */
-	VM_INSTRUCTION_ERROR(%eax, %rsp)
-
-	/* Return via vmx_setjmp with return value of VMX_RETURN_VMLAUNCH */
-	movq	%rsp,%rdi
-	movq	$VMX_RETURN_VMLAUNCH,%rsi
-
-	addq	$VMXCTX_TMPSTKTOP,%rsp
-	callq	vmx_return
-END(vmx_launch)
+	movl	$VMX_GUEST_VMEXIT, %eax
+	ret
+END(vmx_exit_guest)
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index aef3314..c1bc0d2 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -388,10 +388,12 @@ vmexit_vmx(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 	fprintf(stderr, "\treason\t\tVMX\n");
 	fprintf(stderr, "\trip\t\t0x%016lx\n", vmexit->rip);
 	fprintf(stderr, "\tinst_length\t%d\n", vmexit->inst_length);
-	fprintf(stderr, "\terror\t\t%d\n", vmexit->u.vmx.error);
+	fprintf(stderr, "\tstatus\t\t%d\n", vmexit->u.vmx.status);
 	fprintf(stderr, "\texit_reason\t%u\n", vmexit->u.vmx.exit_reason);
 	fprintf(stderr, "\tqualification\t0x%016lx\n",
 	    vmexit->u.vmx.exit_qualification);
+	fprintf(stderr, "\tinst_type\t\t%d\n", vmexit->u.vmx.inst_type);
+	fprintf(stderr, "\tinst_error\t\t%d\n", vmexit->u.vmx.inst_error);
 
 	return (VMEXIT_ABORT);
 }
diff --git a/usr.sbin/bhyvectl/bhyvectl.c b/usr.sbin/bhyvectl/bhyvectl.c
index c697492..efb111f 100644
--- a/usr.sbin/bhyvectl/bhyvectl.c
+++ b/usr.sbin/bhyvectl/bhyvectl.c
@@ -273,11 +273,13 @@ dump_vm_run_exitcode(struct vm_exit *vmexit, int vcpu)
 		break;
 	case VM_EXITCODE_VMX:
 		printf("\treason\t\tVMX\n");
-		printf("\terror\t\t%d\n", vmexit->u.vmx.error);
+		printf("\tstatus\t\t%d\n", vmexit->u.vmx.status);
 		printf("\texit_reason\t0x%08x (%u)\n",
 		    vmexit->u.vmx.exit_reason, vmexit->u.vmx.exit_reason);
 		printf("\tqualification\t0x%016lx\n",
 			vmexit->u.vmx.exit_qualification);
+		printf("\tinst_type\t\t%d\n", vmexit->u.vmx.inst_type);
+		printf("\tinst_error\t\t%d\n", vmexit->u.vmx.inst_error);
 		break;
 	default:
 		printf("*** unknown vm run exitcode %d\n", vmexit->exitcode);