The "unrestricted guest" capability is a feature of Intel VT-x that allows

the guest to execute real or unpaged protected mode code - bhyve relies on
this feature to execute the AP bootstrap code.

Get rid of the hack that allowed bhyve to support SMP guests on processors
that do not have the "unrestricted guest" capability. This hack was entirely
FreeBSD-specific and would not work with any other guest OS.

Instead, limit the number of vcpus to 1 when executing on processors without
"unrestricted guest" capability.

Suggested by:	grehan
Obtained from:	NetApp

2 files changed, 31 insertions, 80 deletions

diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index fd3d8a2..999040f 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -573,11 +573,28 @@ vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
 	fprintf(stderr, "vm_run error %d, errno %d\n", error, errno);
 }
 
+static int
+num_vcpus_allowed(struct vmctx *ctx)
+{
+	int tmp, error;
+
+	error = vm_get_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, &tmp);
+
+	/*
+	 * The guest is allowed to spinup more than one processor only if the
+	 * UNRESTRICTED_GUEST capability is available.
+	 */
+	if (error == 0)
+		return (VM_MAXCPU);
+	else
+		return (1);
+}
 
 int
 main(int argc, char *argv[])
 {
 	int c, error, gdb_port, inject_bkpt, tmp, err, ioapic, bvmcons;
+	int max_vcpus;
 	struct vmctx *ctx;
 	uint64_t rip;
 
@@ -660,12 +677,6 @@ main(int argc, char *argv[])
 	if (guest_ncpus <= 1)
 		guest_vcpu_mux = 0;
 
-	if (guest_ncpus > VM_MAXCPU) {
-		fprintf(stderr, "%d vCPUs requested, max %d\n",
-		    guest_ncpus, VM_MAXCPU);
-		exit(1);
-	}
-
 	/* vmexit on hlt if guest is muxed */
 	if (guest_vcpu_mux) {
 		guest_vmexit_on_hlt = 1;
@@ -680,6 +691,13 @@ main(int argc, char *argv[])
 		exit(1);
 	}
 
+	max_vcpus = num_vcpus_allowed(ctx);
+	if (guest_ncpus > max_vcpus) {
+		fprintf(stderr, "%d vCPUs requested but only %d available\n",
+			guest_ncpus, max_vcpus);
+		exit(1);
+	}
+
 	if (fbsdrun_vmexit_on_hlt()) {
 		err = vm_get_capability(ctx, BSP, VM_CAP_HALT_EXIT, &tmp);
 		if (err < 0) {
diff --git a/usr.sbin/bhyve/spinup_ap.c b/usr.sbin/bhyve/spinup_ap.c
index b5e553f..2632aed 100644
--- a/usr.sbin/bhyve/spinup_ap.c
+++ b/usr.sbin/bhyve/spinup_ap.c
@@ -42,26 +42,6 @@ __FBSDID("$FreeBSD$");
 #include "bhyverun.h"
 #include "spinup_ap.h"
 
-/*
- * Trampoline for hypervisor direct 64-bit jump.
- *
- *   0  - signature for guest->host verification
- *   8  - kernel virtual address of trampoline
- *  16  - instruction virtual address
- *  24  - stack pointer virtual address
- *  32  - CR3, physical address of kernel page table
- *  40  - 24-byte area for null/code/data GDT entries
- */
-#define MP_V64T_SIG	0xcafebabecafebabeULL
-struct mp_v64tramp {
-	uint64_t	mt_sig;
-	uint64_t	mt_virt;
-	uint64_t	mt_eip;
-	uint64_t	mt_rsp;
-	uint64_t	mt_cr3;
-	uint64_t	mt_gdtr[3];
-};
-
 static void
 spinup_ap_realmode(struct vmctx *ctx, int newcpu, uint64_t *rip)
 {
@@ -94,46 +74,6 @@ spinup_ap_realmode(struct vmctx *ctx, int newcpu, uint64_t *rip)
 	assert(error == 0);
 }
 
-static void
-spinup_ap_direct64(struct vmctx *ctx, int newcpu, uint64_t *rip)
-{
-	struct mp_v64tramp *mvt;
-	char *errstr;
-	int error;
-	uint64_t gdtbase;
-
-	mvt = paddr_guest2host(*rip);
-
-	assert(mvt->mt_sig == MP_V64T_SIG);
-
-	/*
-	 * Set up the 3-entry GDT using memory supplied in the
-	 * guest's trampoline structure.
-	 */
-	vm_setup_freebsd_gdt(mvt->mt_gdtr);
-
-#define  CHECK_ERROR(msg) \
-	if (error != 0) { \
-		errstr = msg; \
-		goto err_exit; \
-	}
-
-        /* entry point */
-	*rip = mvt->mt_eip;
-
-	/* Get the guest virtual address of the GDT */
-        gdtbase = mvt->mt_virt + __offsetof(struct mp_v64tramp, mt_gdtr);
-
-	error = vm_setup_freebsd_registers(ctx, newcpu, mvt->mt_eip,
-					   mvt->mt_cr3, gdtbase, mvt->mt_rsp);
-	CHECK_ERROR("vm_setup_freebsd_registers");
-
-	return;
-err_exit:
-	printf("spinup_ap_direct64: machine state error: %s", errstr);
-	exit(1);
-}
-
 int
 spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip)
 {
@@ -163,22 +103,15 @@ spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip)
 	assert(error == 0);
 
 	/*
-	 * There are 2 startup modes possible here:
-	 *  - if the CPU supports 'unrestricted guest' mode, the spinup can
-	 *    set up the processor state in power-on 16-bit mode, with the CS:IP
-	 *    init'd to the specified low-mem 4K page.
-	 *  - if the guest has requested a 64-bit trampoline in the low-mem 4K
-	 *    page by placing in the specified signature, set up the register
-	 *    state using register state in the signature. Note that this
-	 *    requires accessing guest physical memory to read the signature
-	 *    while 'unrestricted mode' does not.
+	 * Enable the 'unrestricted guest' mode for 'newcpu'.
+	 *
+	 * Set up the processor state in power-on 16-bit mode, with the CS:IP
+	 * init'd to the specified low-mem 4K page.
 	 */
 	error = vm_set_capability(ctx, newcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
-	if (error) {
-		spinup_ap_direct64(ctx, newcpu, &rip);
-	} else {
-		spinup_ap_realmode(ctx, newcpu, &rip);
-	}
+	assert(error == 0);
+
+	spinup_ap_realmode(ctx, newcpu, &rip);
 
 	fbsdrun_addcpu(ctx, newcpu, rip);