MFC 261504:

Add support for FreeBSD/i386 guests under bhyve.

3 files changed, 112 insertions, 27 deletions

diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
index 16d750e..28d6504 100644
--- a/sys/amd64/vmm/intel/vmx.c
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -1337,6 +1337,30 @@ vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
 	return (HANDLED);
 }
 
+static enum vie_cpu_mode
+vmx_cpu_mode(void)
+{
+
+	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA)
+		return (CPU_MODE_64BIT);
+	else
+		return (CPU_MODE_COMPATIBILITY);
+}
+
+static enum vie_paging_mode
+vmx_paging_mode(void)
+{
+
+	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
+		return (PAGING_MODE_FLAT);
+	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
+		return (PAGING_MODE_32);
+	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
+		return (PAGING_MODE_64);
+	else
+		return (PAGING_MODE_PAE);
+}
+
 static int
 ept_fault_type(uint64_t ept_qual)
 {
@@ -1496,6 +1520,8 @@ vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
 		vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
 		vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
 		vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
+		vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
+		vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
 	}
 
 	/*
@@ -1714,6 +1740,8 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 			vmexit->u.inst_emul.gpa = gpa;
 			vmexit->u.inst_emul.gla = vmcs_gla();
 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
+			vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
+			vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
 		}
 		/*
diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c
index 2bb6f1c..8fc3df3 100644
--- a/sys/amd64/vmm/vmm.c
+++ b/sys/amd64/vmm/vmm.c
@@ -1082,6 +1082,8 @@ vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
 	struct vm_exit *vme;
 	int error, inst_length;
 	uint64_t rip, gla, gpa, cr3;
+	enum vie_cpu_mode cpu_mode;
+	enum vie_paging_mode paging_mode;
 	mem_region_read_t mread;
 	mem_region_write_t mwrite;
 
@@ -1094,15 +1096,18 @@ vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
 	gla = vme->u.inst_emul.gla;
 	gpa = vme->u.inst_emul.gpa;
 	cr3 = vme->u.inst_emul.cr3;
+	cpu_mode = vme->u.inst_emul.cpu_mode;
+	paging_mode = vme->u.inst_emul.paging_mode;
 	vie = &vme->u.inst_emul.vie;
 
 	vie_init(vie);
 
 	/* Fetch, decode and emulate the faulting instruction */
-	if (vmm_fetch_instruction(vm, vcpuid, rip, inst_length, cr3, vie) != 0)
+	if (vmm_fetch_instruction(vm, vcpuid, rip, inst_length, cr3,
+	    paging_mode, vie) != 0)
 		return (EFAULT);
 
-	if (vmm_decode_instruction(vm, vcpuid, gla, vie) != 0)
+	if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, vie) != 0)
 		return (EFAULT);
 
 	/* return to userland unless this is an in-kernel emulated device */
diff --git a/sys/amd64/vmm/vmm_instruction_emul.c b/sys/amd64/vmm/vmm_instruction_emul.c
index 9277274..f7d109c 100644
--- a/sys/amd64/vmm/vmm_instruction_emul.c
+++ b/sys/amd64/vmm/vmm_instruction_emul.c
@@ -49,11 +49,6 @@ __FBSDID("$FreeBSD$");
 #include <vmmapi.h>
 #endif	/* _KERNEL */
 
-enum cpu_mode {
-	CPU_MODE_COMPATIBILITY,		/* IA-32E mode (CS.L = 0) */
-	CPU_MODE_64BIT,			/* IA-32E mode (CS.L = 1) */
-};
-
 /* struct vie_op.op_type */
 enum {
 	VIE_OP_TYPE_NONE = 0,
@@ -578,16 +573,76 @@ vie_init(struct vie *vie)
 
 static int
 gla2gpa(struct vm *vm, uint64_t gla, uint64_t ptpphys,
-	uint64_t *gpa, uint64_t *gpaend)
+	uint64_t *gpa, enum vie_paging_mode paging_mode)
 {
 	int nlevels, ptpshift, ptpindex;
 	uint64_t *ptpbase, pte, pgsize;
+	uint32_t *ptpbase32, pte32;
 	void *cookie;
 
-	/*
-	 * XXX assumes 64-bit guest with 4 page walk levels
-	 */
-	nlevels = 4;
+	if (paging_mode == PAGING_MODE_FLAT) {
+		*gpa = gla;
+		return (0);
+	}
+
+	if (paging_mode == PAGING_MODE_32) {
+		nlevels = 2;
+		while (--nlevels >= 0) {
+			/* Zero out the lower 12 bits. */
+			ptpphys &= ~0xfff;
+
+			ptpbase32 = vm_gpa_hold(vm, ptpphys, PAGE_SIZE,
+						VM_PROT_READ, &cookie);
+			
+			if (ptpbase32 == NULL)
+				goto error;
+
+			ptpshift = PAGE_SHIFT + nlevels * 10;
+			ptpindex = (gla >> ptpshift) & 0x3FF;
+			pgsize = 1UL << ptpshift;
+
+			pte32 = ptpbase32[ptpindex];
+
+			vm_gpa_release(cookie);
+
+			if ((pte32 & PG_V) == 0)
+				goto error;
+
+			if (pte32 & PG_PS)
+				break;
+
+			ptpphys = pte32;
+		}
+
+		/* Zero out the lower 'ptpshift' bits */
+		pte32 >>= ptpshift; pte32 <<= ptpshift;
+		*gpa = pte32 | (gla & (pgsize - 1));
+		return (0);
+	}
+
+	if (paging_mode == PAGING_MODE_PAE) {
+		/* Zero out the lower 5 bits and the upper 12 bits */
+		ptpphys >>= 5; ptpphys <<= 17; ptpphys >>= 12;
+
+		ptpbase = vm_gpa_hold(vm, ptpphys, sizeof(*ptpbase) * 4,
+				      VM_PROT_READ, &cookie);
+		if (ptpbase == NULL)
+			goto error;
+
+		ptpindex = (gla >> 30) & 0x3;
+
+		pte = ptpbase[ptpindex];
+
+		vm_gpa_release(cookie);
+
+		if ((pte & PG_V) == 0)
+			goto error;
+
+		ptpphys = pte;
+
+		nlevels = 2;
+	} else
+		nlevels = 4;
 	while (--nlevels >= 0) {
 		/* Zero out the lower 12 bits and the upper 12 bits */
 		ptpphys >>= 12; ptpphys <<= 24; ptpphys >>= 12;
@@ -621,7 +676,6 @@ gla2gpa(struct vm *vm, uint64_t gla, uint64_t ptpphys,
 	/* Zero out the lower 'ptpshift' bits and the upper 12 bits */
 	pte >>= ptpshift; pte <<= (ptpshift + 12); pte >>= 12;
 	*gpa = pte | (gla & (pgsize - 1));
-	*gpaend = pte + pgsize;
 	return (0);
 
 error:
@@ -630,10 +684,11 @@ error:
 
 int
 vmm_fetch_instruction(struct vm *vm, int cpuid, uint64_t rip, int inst_length,
-		      uint64_t cr3, struct vie *vie)
+		      uint64_t cr3, enum vie_paging_mode paging_mode,
+		      struct vie *vie)
 {
 	int n, err, prot;
-	uint64_t gpa, gpaend, off;
+	uint64_t gpa, off;
 	void *hpa, *cookie;
 
 	/*
@@ -646,7 +701,7 @@ vmm_fetch_instruction(struct vm *vm, int cpuid, uint64_t rip, int inst_length,
 
 	/* Copy the instruction into 'vie' */
 	while (vie->num_valid < inst_length) {
-		err = gla2gpa(vm, rip, cr3, &gpa, &gpaend);
+		err = gla2gpa(vm, rip, cr3, &gpa, paging_mode);
 		if (err)
 			break;
 
@@ -749,15 +804,9 @@ decode_opcode(struct vie *vie)
 }
 
 static int
-decode_modrm(struct vie *vie)
+decode_modrm(struct vie *vie, enum vie_cpu_mode cpu_mode)
 {
 	uint8_t x;
-	enum cpu_mode cpu_mode;
-
-	/*
-	 * XXX assuming that guest is in IA-32E 64-bit mode
-	 */
-	cpu_mode = CPU_MODE_64BIT;
 
 	if (vie_peek(vie, &x))
 		return (-1);
@@ -1034,16 +1083,19 @@ verify_gla(struct vm *vm, int cpuid, uint64_t gla, struct vie *vie)
 }
 
 int
-vmm_decode_instruction(struct vm *vm, int cpuid, uint64_t gla, struct vie *vie)
+vmm_decode_instruction(struct vm *vm, int cpuid, uint64_t gla,
+		       enum vie_cpu_mode cpu_mode, struct vie *vie)
 {
 
-	if (decode_rex(vie))
-		return (-1);
+	if (cpu_mode == CPU_MODE_64BIT) {
+		if (decode_rex(vie))
+			return (-1);
+	}
 
 	if (decode_opcode(vie))
 		return (-1);
 
-	if (decode_modrm(vie))
+	if (decode_modrm(vie, cpu_mode))
 		return (-1);
 
 	if (decode_sib(vie))