Add a new capability, VM_CAP_ENABLE_INVPCID, that can be enabled to expose

'invpcid' instruction to the guest. Currently bhyve will try to enable this
capability unconditionally if it is available.

Consolidate code in bhyve to set the capabilities so it is no longer
duplicated in BSP and AP bringup.

Add a sysctl 'vm.pmap.invpcid_works' to display whether the 'invpcid'
instruction is available.

Reviewed by:	grehan
MFC after:	3 days

3 files changed, 48 insertions, 50 deletions

diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index 5359442..f679ad5 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -484,10 +484,54 @@ num_vcpus_allowed(struct vmctx *ctx)
 		return (1);
 }
 
+void
+fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
+{
+	int err, tmp;
+
+	if (fbsdrun_vmexit_on_hlt()) {
+		err = vm_get_capability(ctx, cpu, VM_CAP_HALT_EXIT, &tmp);
+		if (err < 0) {
+			fprintf(stderr, "VM exit on HLT not supported\n");
+			exit(1);
+		}
+		vm_set_capability(ctx, cpu, VM_CAP_HALT_EXIT, 1);
+		if (cpu == BSP)
+			handler[VM_EXITCODE_HLT] = vmexit_hlt;
+	}
+
+        if (fbsdrun_vmexit_on_pause()) {
+		/*
+		 * pause exit support required for this mode
+		 */
+		err = vm_get_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, &tmp);
+		if (err < 0) {
+			fprintf(stderr,
+			    "SMP mux requested, no pause support\n");
+			exit(1);
+		}
+		vm_set_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, 1);
+		if (cpu == BSP)
+			handler[VM_EXITCODE_PAUSE] = vmexit_pause;
+        }
+
+	if (fbsdrun_disable_x2apic())
+		err = vm_set_x2apic_state(ctx, cpu, X2APIC_DISABLED);
+	else
+		err = vm_set_x2apic_state(ctx, cpu, X2APIC_ENABLED);
+
+	if (err) {
+		fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
+		exit(1);
+	}
+
+	vm_set_capability(ctx, cpu, VM_CAP_ENABLE_INVPCID, 1);
+}
+
 int
 main(int argc, char *argv[])
 {
-	int c, error, gdb_port, tmp, err, ioapic, bvmcons;
+	int c, error, gdb_port, err, ioapic, bvmcons;
 	int max_vcpus;
 	struct vmctx *ctx;
 	uint64_t rip;
@@ -574,39 +618,7 @@ main(int argc, char *argv[])
 		exit(1);
 	}
 
-	if (fbsdrun_vmexit_on_hlt()) {
-		err = vm_get_capability(ctx, BSP, VM_CAP_HALT_EXIT, &tmp);
-		if (err < 0) {
-			fprintf(stderr, "VM exit on HLT not supported\n");
-			exit(1);
-		}
-		vm_set_capability(ctx, BSP, VM_CAP_HALT_EXIT, 1);
-		handler[VM_EXITCODE_HLT] = vmexit_hlt;
-	}
-
-        if (fbsdrun_vmexit_on_pause()) {
-		/*
-		 * pause exit support required for this mode
-		 */
-		err = vm_get_capability(ctx, BSP, VM_CAP_PAUSE_EXIT, &tmp);
-		if (err < 0) {
-			fprintf(stderr,
-			    "SMP mux requested, no pause support\n");
-			exit(1);
-		}
-		vm_set_capability(ctx, BSP, VM_CAP_PAUSE_EXIT, 1);
-		handler[VM_EXITCODE_PAUSE] = vmexit_pause;
-        }
-
-	if (fbsdrun_disable_x2apic())
-		err = vm_set_x2apic_state(ctx, BSP, X2APIC_DISABLED);
-	else
-		err = vm_set_x2apic_state(ctx, BSP, X2APIC_ENABLED);
-
-	if (err) {
-		fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
-		exit(1);
-	}
+	fbsdrun_set_capabilities(ctx, BSP);
 
 	err = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
 	if (err) {
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index 28ffb3c..d074d65 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -41,6 +41,7 @@ extern char *vmname;
 
 void *paddr_guest2host(struct vmctx *ctx, uintptr_t addr, size_t len);
 
+void fbsdrun_set_capabilities(struct vmctx *ctx, int cpu);
 void fbsdrun_addcpu(struct vmctx *ctx, int cpu, uint64_t rip);
 int  fbsdrun_muxed(void);
 int  fbsdrun_vmexit_on_hlt(void);
diff --git a/usr.sbin/bhyve/spinup_ap.c b/usr.sbin/bhyve/spinup_ap.c
index 2632aed..729108a 100644
--- a/usr.sbin/bhyve/spinup_ap.c
+++ b/usr.sbin/bhyve/spinup_ap.c
@@ -85,22 +85,7 @@ spinup_ap(struct vmctx *ctx, int vcpu, int newcpu, uint64_t rip)
 	error = vcpu_reset(ctx, newcpu);
 	assert(error == 0);
 
-	/* Set up capabilities */
-	if (fbsdrun_vmexit_on_hlt()) {
-		error = vm_set_capability(ctx, newcpu, VM_CAP_HALT_EXIT, 1);
-		assert(error == 0);
-	}
-
-	if (fbsdrun_vmexit_on_pause()) {
-		error = vm_set_capability(ctx, newcpu, VM_CAP_PAUSE_EXIT, 1);
-		assert(error == 0);
-	}
-
-	if (fbsdrun_disable_x2apic())
-		error = vm_set_x2apic_state(ctx, newcpu, X2APIC_DISABLED);
-	else
-		error = vm_set_x2apic_state(ctx, newcpu, X2APIC_ENABLED);
-	assert(error == 0);
+	fbsdrun_set_capabilities(ctx, newcpu);
 
 	/*
 	 * Enable the 'unrestricted guest' mode for 'newcpu'.