Add an explicit exit code 'SPINUP_AP' to tell the controlling process that an

AP needs to be activated by spinning up an execution context for it.

The local apic emulation is now completely done in the hypervisor and it will
detect writes to the ICR_LO register that try to bring up the AP. In response
to such writes it will return to userspace with an exit code of SPINUP_AP.

Reviewed by: grehan

3 files changed, 67 insertions, 5 deletions

diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
index 61faf56..e841963 100644
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -228,6 +228,7 @@ enum vm_exitcode {
 	VM_EXITCODE_MTRAP,
 	VM_EXITCODE_PAUSE,
 	VM_EXITCODE_PAGING,
+	VM_EXITCODE_SPINUP_AP,
 	VM_EXITCODE_MAX
 };
 
@@ -260,6 +261,10 @@ struct vm_exit {
 			uint32_t	code;		/* ecx value */
 			uint64_t	wval;
 		} msr;
+		struct {
+			int		vcpu;
+			uint64_t	rip;
+		} spinup_ap;
 	} u;
 };
 
diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
index 88f870c..6689013 100644
--- a/sys/amd64/vmm/intel/vmx.c
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -1253,6 +1253,14 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 		vm_exit_update_rip(vmexit);
 		vmexit->rip += vmexit->inst_length;
 		vmexit->inst_length = 0;
+
+		/*
+		 * Special case for spinning up an AP - exit to userspace to
+		 * give the controlling process a chance to intercept and
+		 * spin up a thread for the AP.
+		 */
+		if (vmexit->exitcode == VM_EXITCODE_SPINUP_AP)
+			handled = 0;
 	} else {
 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
 			/*
diff --git a/sys/amd64/vmm/io/vlapic.c b/sys/amd64/vmm/io/vlapic.c
index f1d363f..9b7d3cb 100644
--- a/sys/amd64/vmm/io/vlapic.c
+++ b/sys/amd64/vmm/io/vlapic.c
@@ -89,6 +89,12 @@ static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
 
 #define	x2apic(vlapic)		((vlapic)->msr_apicbase & APICBASE_X2APIC)
 
+enum boot_state {
+	BS_INIT,
+	BS_SIPI,
+	BS_RUNNING
+};
+
 struct vlapic {
 	struct vm		*vm;
 	int			vcpuid;
@@ -112,6 +118,7 @@ struct vlapic {
 	int			 isrvec_stk_top;
 
 	uint64_t		msr_apicbase;
+	enum boot_state		boot_state;
 };
 
 static void
@@ -168,6 +175,11 @@ vlapic_op_reset(void* dev)
 	memset(lapic, 0, sizeof(*lapic));
 	lapic->apr = vlapic->vcpuid;
 	vlapic_init_ipi(vlapic);
+
+	if (vlapic->vcpuid == 0)
+		vlapic->boot_state = BS_RUNNING;	/* BSP */
+	else
+		vlapic->boot_state = BS_INIT;		/* AP */
 	
 	return 0;
 
@@ -418,6 +430,8 @@ lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
 	int i;
 	cpuset_t dmask;
 	uint32_t dest, vec, mode;
+	struct vlapic *vlapic2;
+	struct vm_exit *vmexit;
 	
 	dest = icrval >> 32;
 	vec = icrval & APIC_VECTOR_MASK;
@@ -452,11 +466,46 @@ lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
 		return (0);	/* handled completely in the kernel */
 	}
 
-	/*
-	 * XXX this assumes that the startup IPI always succeeds
-	 */
-	if (mode == APIC_DELMODE_STARTUP)
-		vm_activate_cpu(vlapic->vm, dest);
+	if (mode == APIC_DELMODE_INIT) {
+		if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
+			return (0);
+
+		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
+			vlapic2 = vm_lapic(vlapic->vm, dest);
+
+			/* move from INIT to waiting-for-SIPI state */
+			if (vlapic2->boot_state == BS_INIT) {
+				vlapic2->boot_state = BS_SIPI;
+			}
+
+			return (0);
+		}
+	}
+
+	if (mode == APIC_DELMODE_STARTUP) {
+		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
+			vlapic2 = vm_lapic(vlapic->vm, dest);
+
+			/*
+			 * Ignore SIPIs in any state other than wait-for-SIPI
+			 */
+			if (vlapic2->boot_state != BS_SIPI)
+				return (0);
+
+			vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
+			vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
+			vmexit->u.spinup_ap.vcpu = dest;
+			vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
+
+			/*
+			 * XXX this assumes that the startup IPI always succeeds
+			 */
+			vlapic2->boot_state = BS_RUNNING;
+			vm_activate_cpu(vlapic2->vm, dest);
+
+			return (0);
+		}
+	}
 
 	/*
 	 * This will cause a return to userland.