MFC 261504:

Add support for FreeBSD/i386 guests under bhyve.

16 files changed, 413 insertions, 101 deletions

diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index 8428987..4a3db5d 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -111,5 +111,8 @@ int	vcpu_reset(struct vmctx *ctx, int vcpu);
 int	vm_setup_freebsd_registers(struct vmctx *ctx, int vcpu,
 				uint64_t rip, uint64_t cr3, uint64_t gdtbase,
 				uint64_t rsp);
+int	vm_setup_freebsd_registers_i386(struct vmctx *vmctx, int vcpu,
+					uint32_t eip, uint32_t gdtbase,
+					uint32_t esp);
 void	vm_setup_freebsd_gdt(uint64_t *gdtr);
 #endif	/* _VMMAPI_H_ */
diff --git a/lib/libvmmapi/vmmapi_freebsd.c b/lib/libvmmapi/vmmapi_freebsd.c
index 9bd2988..d801184 100644
--- a/lib/libvmmapi/vmmapi_freebsd.c
+++ b/lib/libvmmapi/vmmapi_freebsd.c
@@ -35,14 +35,176 @@ __FBSDID("$FreeBSD$");
 #include <machine/segments.h>
 #include <machine/vmm.h>
 
+#include <errno.h>
+#include <string.h>
+
 #include "vmmapi.h"
 
+#define	I386_TSS_SIZE		104
+
+#define	DESC_PRESENT		0x00000080
+#define	DESC_LONGMODE		0x00002000
+#define	DESC_DEF32		0x00004000
+#define	DESC_GRAN		0x00008000
 #define	DESC_UNUSABLE		0x00010000
 
 #define	GUEST_NULL_SEL		0
 #define	GUEST_CODE_SEL		1
 #define	GUEST_DATA_SEL		2
-#define	GUEST_GDTR_LIMIT	(3 * 8 - 1)
+#define	GUEST_TSS_SEL		3
+#define	GUEST_GDTR_LIMIT64	(3 * 8 - 1)
+
+static struct segment_descriptor i386_gdt[] = {
+	{},						/* NULL */
+	{ .sd_lolimit = 0xffff, .sd_type = SDT_MEMER,	/* CODE */
+	  .sd_p = 1, .sd_hilimit = 0xf, .sd_def32 = 1, .sd_gran = 1 }, 
+	{ .sd_lolimit = 0xffff, .sd_type = SDT_MEMRW,	/* DATA */
+	  .sd_p = 1, .sd_hilimit = 0xf, .sd_def32 = 1, .sd_gran = 1 },
+	{ .sd_lolimit = I386_TSS_SIZE - 1,		/* TSS */
+	  .sd_type = SDT_SYS386TSS, .sd_p = 1 }
+};
+
+/*
+ * Setup the 'vcpu' register set such that it will begin execution at
+ * 'eip' in flat mode.
+ */
+int
+vm_setup_freebsd_registers_i386(struct vmctx *vmctx, int vcpu, uint32_t eip,
+				uint32_t gdtbase, uint32_t esp)
+{
+	uint64_t cr0, rflags, desc_base;
+	uint32_t desc_access, desc_limit, tssbase;
+	uint16_t gsel;
+	struct segment_descriptor *gdt;
+	int error, tmp;
+
+	/* A 32-bit guest requires unrestricted mode. */	
+	error = vm_get_capability(vmctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp);
+	if (error)
+		goto done;
+	error = vm_set_capability(vmctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
+	if (error)
+		goto done;
+
+	cr0 = CR0_PE | CR0_NE;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR0, cr0)) != 0)
+		goto done;
+
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR4, 0)) != 0)
+		goto done;
+
+	/*
+	 * Forcing EFER to 0 causes bhyve to clear the "IA-32e guest
+	 * mode" entry control.
+	 */
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_EFER, 0)))
+		goto done;
+
+	gdt = vm_map_gpa(vmctx, gdtbase, 0x1000);
+	if (gdt == NULL)
+		return (EFAULT);
+	memcpy(gdt, i386_gdt, sizeof(i386_gdt));
+	desc_base = gdtbase;
+	desc_limit = sizeof(i386_gdt) - 1;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GDTR,
+			    desc_base, desc_limit, 0);
+	if (error != 0)
+		goto done;
+
+	/* Place the TSS one page above the GDT. */
+	tssbase = gdtbase + 0x1000;
+	gdt[3].sd_lobase = tssbase;	
+
+	rflags = 0x2;
+	error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RFLAGS, rflags);
+	if (error)
+		goto done;
+
+	desc_base = 0;
+	desc_limit = 0xffffffff;
+	desc_access = DESC_GRAN | DESC_DEF32 | DESC_PRESENT | SDT_MEMERA;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_CS,
+			    desc_base, desc_limit, desc_access);
+
+	desc_access = DESC_GRAN | DESC_DEF32 | DESC_PRESENT | SDT_MEMRWA;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_DS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_ES,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_FS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_SS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	desc_base = tssbase;
+	desc_limit = I386_TSS_SIZE - 1;
+	desc_access = DESC_PRESENT | SDT_SYS386BSY;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_TR,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_LDTR, 0, 0,
+			    DESC_UNUSABLE);
+	if (error)
+		goto done;
+
+	gsel = GSEL(GUEST_CODE_SEL, SEL_KPL);
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CS, gsel)) != 0)
+		goto done;
+	
+	gsel = GSEL(GUEST_DATA_SEL, SEL_KPL);
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_DS, gsel)) != 0)
+		goto done;
+	
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_ES, gsel)) != 0)
+		goto done;
+
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_FS, gsel)) != 0)
+		goto done;
+	
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_GS, gsel)) != 0)
+		goto done;
+	
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_SS, gsel)) != 0)
+		goto done;
+
+	gsel = GSEL(GUEST_TSS_SEL, SEL_KPL);
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_TR, gsel)) != 0)
+		goto done;
+
+	/* LDTR is pointing to the null selector */
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_LDTR, 0)) != 0)
+		goto done;
+
+	/* entry point */
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RIP, eip)) != 0)
+		goto done;
+
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSP, esp)) != 0)
+		goto done;
+
+	error = 0;
+done:
+	return (error);
+}
 
 void     
 vm_setup_freebsd_gdt(uint64_t *gdtr)
@@ -168,7 +330,7 @@ vm_setup_freebsd_registers(struct vmctx *vmctx, int vcpu,
 		goto done;
 
 	desc_base = gdtbase;
-	desc_limit = GUEST_GDTR_LIMIT;
+	desc_limit = GUEST_GDTR_LIMIT64;
 	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GDTR,
 			    desc_base, desc_limit, 0);
 	if (error != 0)
diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
index cc7c7ad..8b6933a 100644
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -323,6 +323,8 @@ struct vm_exit {
 			uint64_t	gpa;
 			uint64_t	gla;
 			uint64_t	cr3;
+			enum vie_cpu_mode cpu_mode;
+			enum vie_paging_mode paging_mode;
 			struct vie	vie;
 		} inst_emul;
 		/*
diff --git a/sys/amd64/include/vmm_instruction_emul.h b/sys/amd64/include/vmm_instruction_emul.h
index a7480e7..0901aa2 100644
--- a/sys/amd64/include/vmm_instruction_emul.h
+++ b/sys/amd64/include/vmm_instruction_emul.h
@@ -29,6 +29,18 @@
 #ifndef	_VMM_INSTRUCTION_EMUL_H_
 #define	_VMM_INSTRUCTION_EMUL_H_
 
+enum vie_cpu_mode {
+	CPU_MODE_COMPATIBILITY,		/* IA-32E mode (CS.L = 0) */
+	CPU_MODE_64BIT,			/* IA-32E mode (CS.L = 1) */
+};
+
+enum vie_paging_mode {
+	PAGING_MODE_FLAT,
+	PAGING_MODE_32,
+	PAGING_MODE_PAE,
+	PAGING_MODE_64,
+};
+
 /*
  * The data structures 'vie' and 'vie_op' are meant to be opaque to the
  * consumers of instruction decoding. The only reason why their contents
@@ -107,7 +119,7 @@ int vmm_emulate_instruction(void *vm, int cpuid, uint64_t gpa, struct vie *vie,
  */
 int vmm_fetch_instruction(struct vm *vm, int cpuid,
 			  uint64_t rip, int inst_length, uint64_t cr3,
-			  struct vie *vie);
+			  enum vie_paging_mode paging_mode, struct vie *vie);
 
 void vie_init(struct vie *vie);
 
@@ -123,8 +135,8 @@ void vie_init(struct vie *vie);
  * in VIE_INVALID_GLA instead.
  */
 #define	VIE_INVALID_GLA		(1UL << 63)	/* a non-canonical address */
-int vmm_decode_instruction(struct vm *vm, int cpuid,
-			   uint64_t gla, struct vie *vie);
+int vmm_decode_instruction(struct vm *vm, int cpuid, uint64_t gla,
+			   enum vie_cpu_mode cpu_mode, struct vie *vie);
 #endif	/* _KERNEL */
 
 #endif	/* _VMM_INSTRUCTION_EMUL_H_ */
diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
index 16d750e..28d6504 100644
--- a/sys/amd64/vmm/intel/vmx.c
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -1337,6 +1337,30 @@ vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
 	return (HANDLED);
 }
 
+static enum vie_cpu_mode
+vmx_cpu_mode(void)
+{
+
+	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA)
+		return (CPU_MODE_64BIT);
+	else
+		return (CPU_MODE_COMPATIBILITY);
+}
+
+static enum vie_paging_mode
+vmx_paging_mode(void)
+{
+
+	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
+		return (PAGING_MODE_FLAT);
+	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
+		return (PAGING_MODE_32);
+	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
+		return (PAGING_MODE_64);
+	else
+		return (PAGING_MODE_PAE);
+}
+
 static int
 ept_fault_type(uint64_t ept_qual)
 {
@@ -1496,6 +1520,8 @@ vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
 		vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
 		vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
 		vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
+		vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
+		vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
 	}
 
 	/*
@@ -1714,6 +1740,8 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 			vmexit->u.inst_emul.gpa = gpa;
 			vmexit->u.inst_emul.gla = vmcs_gla();
 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
+			vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
+			vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
 		}
 		/*
diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c
index 2bb6f1c..8fc3df3 100644
--- a/sys/amd64/vmm/vmm.c
+++ b/sys/amd64/vmm/vmm.c
@@ -1082,6 +1082,8 @@ vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
 	struct vm_exit *vme;
 	int error, inst_length;
 	uint64_t rip, gla, gpa, cr3;
+	enum vie_cpu_mode cpu_mode;
+	enum vie_paging_mode paging_mode;
 	mem_region_read_t mread;
 	mem_region_write_t mwrite;
 
@@ -1094,15 +1096,18 @@ vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
 	gla = vme->u.inst_emul.gla;
 	gpa = vme->u.inst_emul.gpa;
 	cr3 = vme->u.inst_emul.cr3;
+	cpu_mode = vme->u.inst_emul.cpu_mode;
+	paging_mode = vme->u.inst_emul.paging_mode;
 	vie = &vme->u.inst_emul.vie;
 
 	vie_init(vie);
 
 	/* Fetch, decode and emulate the faulting instruction */
-	if (vmm_fetch_instruction(vm, vcpuid, rip, inst_length, cr3, vie) != 0)
+	if (vmm_fetch_instruction(vm, vcpuid, rip, inst_length, cr3,
+	    paging_mode, vie) != 0)
 		return (EFAULT);
 
-	if (vmm_decode_instruction(vm, vcpuid, gla, vie) != 0)
+	if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, vie) != 0)
 		return (EFAULT);
 
 	/* return to userland unless this is an in-kernel emulated device */
diff --git a/sys/amd64/vmm/vmm_instruction_emul.c b/sys/amd64/vmm/vmm_instruction_emul.c
index 9277274..f7d109c 100644
--- a/sys/amd64/vmm/vmm_instruction_emul.c
+++ b/sys/amd64/vmm/vmm_instruction_emul.c
@@ -49,11 +49,6 @@ __FBSDID("$FreeBSD$");
 #include <vmmapi.h>
 #endif	/* _KERNEL */
 
-enum cpu_mode {
-	CPU_MODE_COMPATIBILITY,		/* IA-32E mode (CS.L = 0) */
-	CPU_MODE_64BIT,			/* IA-32E mode (CS.L = 1) */
-};
-
 /* struct vie_op.op_type */
 enum {
 	VIE_OP_TYPE_NONE = 0,
@@ -578,16 +573,76 @@ vie_init(struct vie *vie)
 
 static int
 gla2gpa(struct vm *vm, uint64_t gla, uint64_t ptpphys,
-	uint64_t *gpa, uint64_t *gpaend)
+	uint64_t *gpa, enum vie_paging_mode paging_mode)
 {
 	int nlevels, ptpshift, ptpindex;
 	uint64_t *ptpbase, pte, pgsize;
+	uint32_t *ptpbase32, pte32;
 	void *cookie;
 
-	/*
-	 * XXX assumes 64-bit guest with 4 page walk levels
-	 */
-	nlevels = 4;
+	if (paging_mode == PAGING_MODE_FLAT) {
+		*gpa = gla;
+		return (0);
+	}
+
+	if (paging_mode == PAGING_MODE_32) {
+		nlevels = 2;
+		while (--nlevels >= 0) {
+			/* Zero out the lower 12 bits. */
+			ptpphys &= ~0xfff;
+
+			ptpbase32 = vm_gpa_hold(vm, ptpphys, PAGE_SIZE,
+						VM_PROT_READ, &cookie);
+			
+			if (ptpbase32 == NULL)
+				goto error;
+
+			ptpshift = PAGE_SHIFT + nlevels * 10;
+			ptpindex = (gla >> ptpshift) & 0x3FF;
+			pgsize = 1UL << ptpshift;
+
+			pte32 = ptpbase32[ptpindex];
+
+			vm_gpa_release(cookie);
+
+			if ((pte32 & PG_V) == 0)
+				goto error;
+
+			if (pte32 & PG_PS)
+				break;
+
+			ptpphys = pte32;
+		}
+
+		/* Zero out the lower 'ptpshift' bits */
+		pte32 >>= ptpshift; pte32 <<= ptpshift;
+		*gpa = pte32 | (gla & (pgsize - 1));
+		return (0);
+	}
+
+	if (paging_mode == PAGING_MODE_PAE) {
+		/* Zero out the lower 5 bits and the upper 12 bits */
+		ptpphys >>= 5; ptpphys <<= 17; ptpphys >>= 12;
+
+		ptpbase = vm_gpa_hold(vm, ptpphys, sizeof(*ptpbase) * 4,
+				      VM_PROT_READ, &cookie);
+		if (ptpbase == NULL)
+			goto error;
+
+		ptpindex = (gla >> 30) & 0x3;
+
+		pte = ptpbase[ptpindex];
+
+		vm_gpa_release(cookie);
+
+		if ((pte & PG_V) == 0)
+			goto error;
+
+		ptpphys = pte;
+
+		nlevels = 2;
+	} else
+		nlevels = 4;
 	while (--nlevels >= 0) {
 		/* Zero out the lower 12 bits and the upper 12 bits */
 		ptpphys >>= 12; ptpphys <<= 24; ptpphys >>= 12;
@@ -621,7 +676,6 @@ gla2gpa(struct vm *vm, uint64_t gla, uint64_t ptpphys,
 	/* Zero out the lower 'ptpshift' bits and the upper 12 bits */
 	pte >>= ptpshift; pte <<= (ptpshift + 12); pte >>= 12;
 	*gpa = pte | (gla & (pgsize - 1));
-	*gpaend = pte + pgsize;
 	return (0);
 
 error:
@@ -630,10 +684,11 @@ error:
 
 int
 vmm_fetch_instruction(struct vm *vm, int cpuid, uint64_t rip, int inst_length,
-		      uint64_t cr3, struct vie *vie)
+		      uint64_t cr3, enum vie_paging_mode paging_mode,
+		      struct vie *vie)
 {
 	int n, err, prot;
-	uint64_t gpa, gpaend, off;
+	uint64_t gpa, off;
 	void *hpa, *cookie;
 
 	/*
@@ -646,7 +701,7 @@ vmm_fetch_instruction(struct vm *vm, int cpuid, uint64_t rip, int inst_length,
 
 	/* Copy the instruction into 'vie' */
 	while (vie->num_valid < inst_length) {
-		err = gla2gpa(vm, rip, cr3, &gpa, &gpaend);
+		err = gla2gpa(vm, rip, cr3, &gpa, paging_mode);
 		if (err)
 			break;
 
@@ -749,15 +804,9 @@ decode_opcode(struct vie *vie)
 }
 
 static int
-decode_modrm(struct vie *vie)
+decode_modrm(struct vie *vie, enum vie_cpu_mode cpu_mode)
 {
 	uint8_t x;
-	enum cpu_mode cpu_mode;
-
-	/*
-	 * XXX assuming that guest is in IA-32E 64-bit mode
-	 */
-	cpu_mode = CPU_MODE_64BIT;
 
 	if (vie_peek(vie, &x))
 		return (-1);
@@ -1034,16 +1083,19 @@ verify_gla(struct vm *vm, int cpuid, uint64_t gla, struct vie *vie)
 }
 
 int
-vmm_decode_instruction(struct vm *vm, int cpuid, uint64_t gla, struct vie *vie)
+vmm_decode_instruction(struct vm *vm, int cpuid, uint64_t gla,
+		       enum vie_cpu_mode cpu_mode, struct vie *vie)
 {
 
-	if (decode_rex(vie))
-		return (-1);
+	if (cpu_mode == CPU_MODE_64BIT) {
+		if (decode_rex(vie))
+			return (-1);
+	}
 
 	if (decode_opcode(vie))
 		return (-1);
 
-	if (decode_modrm(vie))
+	if (decode_modrm(vie, cpu_mode))
 		return (-1);
 
 	if (decode_sib(vie))
diff --git a/sys/boot/common/load_elf32.c b/sys/boot/common/load_elf32.c
index 1de5dc1..0c9f460 100644
--- a/sys/boot/common/load_elf32.c
+++ b/sys/boot/common/load_elf32.c
@@ -2,5 +2,6 @@
 __FBSDID("$FreeBSD$");
 
 #define __ELF_WORD_SIZE 32
+#define	_MACHINE_ELF_WANT_32BIT
 
 #include "load_elf.c"
diff --git a/sys/boot/common/load_elf32_obj.c b/sys/boot/common/load_elf32_obj.c
index fed25ef..94b0896 100644
--- a/sys/boot/common/load_elf32_obj.c
+++ b/sys/boot/common/load_elf32_obj.c
@@ -2,5 +2,6 @@
 __FBSDID("$FreeBSD$");
 
 #define __ELF_WORD_SIZE 32
+#define	_MACHINE_ELF_WANT_32BIT
 
 #include "load_elf_obj.c"
diff --git a/sys/boot/userboot/userboot/Makefile b/sys/boot/userboot/userboot/Makefile
index 1df1f30..076b4b2 100644
--- a/sys/boot/userboot/userboot/Makefile
+++ b/sys/boot/userboot/userboot/Makefile
@@ -11,6 +11,7 @@ STRIP=
 LIBDIR=		/boot
 
 SRCS=		autoload.c
+SRCS+=		biossmap.c
 SRCS+=		bootinfo.c
 SRCS+=		bootinfo32.c
 SRCS+=		bootinfo64.c
diff --git a/sys/boot/userboot/userboot/biossmap.c b/sys/boot/userboot/userboot/biossmap.c
new file mode 100644
index 0000000..9e556be
--- /dev/null
+++ b/sys/boot/userboot/userboot/biossmap.c
@@ -0,0 +1,74 @@
+/*-
+ * Copyright (c) 1998 Michael Smith <msmith@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <stand.h>
+#include <sys/param.h>
+#include <sys/reboot.h>
+#include <sys/linker.h>
+#include <machine/pc/bios.h>
+#include <machine/metadata.h>
+
+#include "bootstrap.h"
+#include "libuserboot.h"
+
+#define GB (1024UL * 1024 * 1024)
+
+void
+bios_addsmapdata(struct preloaded_file *kfp)
+{
+	uint64_t lowmem, highmem;
+	int smapnum, len;
+	struct bios_smap smap[3], *sm;
+
+	CALLBACK(getmem, &lowmem, &highmem);
+
+	sm = &smap[0];
+
+	sm->base = 0;				/* base memory */
+	sm->length = 640 * 1024;
+	sm->type = SMAP_TYPE_MEMORY;
+	sm++;
+
+	sm->base = 0x100000;			/* extended memory */
+	sm->length = lowmem - 0x100000;
+	sm->type = SMAP_TYPE_MEMORY;
+	sm++;
+
+	smapnum = 2;
+
+        if (highmem != 0) {
+                sm->base = 4 * GB;
+                sm->length = highmem;
+                sm->type = SMAP_TYPE_MEMORY;
+		smapnum++;
+        }
+
+        len = smapnum * sizeof(struct bios_smap);
+        file_addmetadata(kfp, MODINFOMD_SMAP, len, &smap[0]);
+}
diff --git a/sys/boot/userboot/userboot/bootinfo32.c b/sys/boot/userboot/userboot/bootinfo32.c
index 4e6447a..6498461 100644
--- a/sys/boot/userboot/userboot/bootinfo32.c
+++ b/sys/boot/userboot/userboot/bootinfo32.c
@@ -66,7 +66,7 @@ static struct bootinfo  bi;
     COPY32(strlen(s) + 1, a, c);		\
     if (c)					\
         CALLBACK(copyin, s, a, strlen(s) + 1);  \
-    a += roundup(strlen(s) + 1, sizeof(u_long));\
+    a += roundup(strlen(s) + 1, sizeof(uint32_t));\
 }
 
 #define MOD_NAME(a, s, c)	MOD_STR(MODINFO_NAME, a, s, c)
@@ -78,7 +78,7 @@ static struct bootinfo  bi;
     COPY32(sizeof(s), a, c);			\
     if (c)					\
         CALLBACK(copyin, &s, a, sizeof(s));	\
-    a += roundup(sizeof(s), sizeof(u_long));	\
+    a += roundup(sizeof(s), sizeof(uint32_t));	\
 }
 
 #define MOD_ADDR(a, s, c)	MOD_VAR(MODINFO_ADDR, a, s, c)
@@ -89,7 +89,7 @@ static struct bootinfo  bi;
     COPY32(mm->md_size, a, c);			\
     if (c)					\
         CALLBACK(copyin, mm->md_data, a, mm->md_size);    \
-    a += roundup(mm->md_size, sizeof(u_long));\
+    a += roundup(mm->md_size, sizeof(uint32_t));\
 }
 
 #define MOD_END(a, c) {				\
@@ -146,6 +146,7 @@ bi_load32(char *args, int *howtop, int *bootdevp, vm_offset_t *bip, vm_offset_t
     int				bootdevnr, howto;
     char			*kernelname;
     const char			*kernelpath;
+    uint64_t			lowmem, highmem;
 
     howto = bi_getboothowto(args);
 
@@ -198,9 +199,7 @@ bi_load32(char *args, int *howtop, int *bootdevp, vm_offset_t *bip, vm_offset_t
     file_addmetadata(kfp, MODINFOMD_HOWTO, sizeof howto, &howto);
     file_addmetadata(kfp, MODINFOMD_ENVP, sizeof envp, &envp);
     file_addmetadata(kfp, MODINFOMD_KERNEND, sizeof kernend, &kernend);
-#if 0
     bios_addsmapdata(kfp);
-#endif
 
     /* Figure out the size and location of the metadata */
     *modulep = addr;
@@ -237,11 +236,10 @@ bi_load32(char *args, int *howtop, int *bootdevp, vm_offset_t *bip, vm_offset_t
         bi.bi_bios_geom[i] = bd_getbigeom(i);
 #endif
     bi.bi_size = sizeof(bi);
+    CALLBACK(getmem, &lowmem, &highmem);
     bi.bi_memsizes_valid = 1;
-#if 0
-    bi.bi_basemem = bios_basemem / 1024;
-    bi.bi_extmem = bios_extmem / 1024;
-#endif
+    bi.bi_basemem = 640;
+    bi.bi_extmem = (lowmem - 0x100000) / 1024;
     bi.bi_envp = envp;
     bi.bi_modulep = *modulep;
     bi.bi_kernend = kernend;
@@ -251,7 +249,7 @@ bi_load32(char *args, int *howtop, int *bootdevp, vm_offset_t *bip, vm_offset_t
     /*
      * Copy the legacy bootinfo and kernel name to the guest at 0x2000
      */
-    bi.bi_kernelname = (char *) (0x2000 + sizeof(bi));
+    bi.bi_kernelname = 0x2000 + sizeof(bi);
     CALLBACK(copyin, &bi, 0x2000, sizeof(bi));
     CALLBACK(copyin, kernelname, 0x2000 + sizeof(bi), strlen(kernelname) + 1);
 
diff --git a/sys/boot/userboot/userboot/bootinfo64.c b/sys/boot/userboot/userboot/bootinfo64.c
index fc7c14d..10ff133 100644
--- a/sys/boot/userboot/userboot/bootinfo64.c
+++ b/sys/boot/userboot/userboot/bootinfo64.c
@@ -169,53 +169,6 @@ bi_checkcpu(void)
 #endif
 }
 
-struct smap {
-        uint64_t       base;
-        uint64_t       length;
-        uint32_t       type;
-} __packed;
-
-/* From FreeBSD <machine/pc/bios.h> */
-#define SMAP_TYPE_MEMORY	1
-
-#define GB (1024UL * 1024 * 1024)
-
-#define MODINFOMD_SMAP          0x1001
-
-static void
-bios_addsmapdata(struct preloaded_file *kfp)
-{
-	uint64_t lowmem, highmem;
-	int smapnum, len;
-	struct smap smap[3], *sm;
-
-	CALLBACK(getmem, &lowmem, &highmem);
-
-	sm = &smap[0];
-
-	sm->base = 0;				/* base memory */
-	sm->length = 640 * 1024;
-	sm->type = SMAP_TYPE_MEMORY;
-	sm++;
-
-	sm->base = 0x100000;			/* extended memory */
-	sm->length = lowmem - 0x100000;
-	sm->type = SMAP_TYPE_MEMORY;
-	sm++;
-
-	smapnum = 2;
-
-        if (highmem != 0) {
-                sm->base = 4 * GB;
-                sm->length = highmem;
-                sm->type = SMAP_TYPE_MEMORY;
-		smapnum++;
-        }
-
-        len = smapnum * sizeof (struct smap);
-        file_addmetadata(kfp, MODINFOMD_SMAP, len, &smap[0]);
-}
-
 /*
  * Load the information expected by an amd64 kernel.
  *
diff --git a/sys/boot/userboot/userboot/elf32_freebsd.c b/sys/boot/userboot/userboot/elf32_freebsd.c
index bb5b693..d8ccc33 100644
--- a/sys/boot/userboot/userboot/elf32_freebsd.c
+++ b/sys/boot/userboot/userboot/elf32_freebsd.c
@@ -45,6 +45,9 @@ static int	elf32_obj_exec(struct preloaded_file *amp);
 struct file_format i386_elf = { elf32_loadfile, elf32_exec };
 struct file_format i386_elf_obj = { elf32_obj_loadfile, elf32_obj_exec };
 
+#define	GUEST_STACK	0x1000		/* Initial stack base */
+#define	GUEST_GDT	0x3000		/* Address of initial GDT */
+
 /*
  * There is an ELF kernel and one or more ELF modules loaded.  
  * We wish to start executing the kernel image, so make such 
@@ -57,7 +60,7 @@ elf32_exec(struct preloaded_file *fp)
 	Elf_Ehdr 		*ehdr;
 	vm_offset_t		entry, bootinfop, modulep, kernend;
 	int			boothowto, err, bootdev;
-	uint32_t		stack[1024];
+	uint32_t		stack[1024], *sp;
 
 
 	if ((md = file_findmetadata(fp, MODINFOMD_ELFHDR)) == NULL)
@@ -78,16 +81,27 @@ elf32_exec(struct preloaded_file *fp)
 	/*
 	 * Build a scratch stack at physical 0x1000
 	 */
-	stack[0] = boothowto;
-	stack[1] = bootdev;
-	stack[2] = 0;
-	stack[3] = 0;
-	stack[4] = 0;
-	stack[5] = bootinfop;
-	stack[6] = modulep;
-	stack[7] = kernend;
-	CALLBACK(copyin, stack, 0x1000, sizeof(stack));
-	CALLBACK(setreg, 4, 0x1000);
+	memset(stack, 0, sizeof(stack));
+	sp = (uint32_t *)((char *)stack + sizeof(stack));
+	*--sp = kernend;
+	*--sp = modulep;
+	*--sp = bootinfop;
+	*--sp = 0;
+	*--sp = 0;
+	*--sp = 0;
+	*--sp = bootdev;
+	*--sp = boothowto;
+
+	/*
+	 * Fake return address to mimic "new" boot blocks.  For more
+	 * details see recover_bootinfo in locore.S.
+	 */
+	*--sp = 0xbeefface;
+	CALLBACK(copyin, stack, GUEST_STACK, sizeof(stack));
+	CALLBACK(setreg, 4, (char *)sp - (char *)stack + GUEST_STACK);
+
+	CALLBACK(setgdt, GUEST_GDT, 8 * 4 - 1);
+
         CALLBACK(exec, entry);
 
 	panic("exec returned");
diff --git a/sys/boot/userboot/userboot/libuserboot.h b/sys/boot/userboot/userboot/libuserboot.h
index d795188..e2048d5 100644
--- a/sys/boot/userboot/userboot/libuserboot.h
+++ b/sys/boot/userboot/userboot/libuserboot.h
@@ -65,3 +65,4 @@ vm_offset_t	bi_copyenv(vm_offset_t addr);
 int	bi_load32(char *args, int *howtop, int *bootdevp, vm_offset_t *bip,
     vm_offset_t *modulep, vm_offset_t *kernend);
 int	bi_load64(char *args, vm_offset_t *modulep, vm_offset_t *kernend);
+void	bios_addsmapdata(struct preloaded_file *kfp);
diff --git a/usr.sbin/bhyveload/bhyveload.c b/usr.sbin/bhyveload/bhyveload.c
index 94cb1b1..701e9c3 100644
--- a/usr.sbin/bhyveload/bhyveload.c
+++ b/usr.sbin/bhyveload/bhyveload.c
@@ -465,7 +465,12 @@ cb_exec(void *arg, uint64_t rip)
 {
 	int error;
 
-	error = vm_setup_freebsd_registers(ctx, BSP, rip, cr3, gdtbase, rsp);
+	if (cr3 == 0)
+		error = vm_setup_freebsd_registers_i386(ctx, BSP, rip, gdtbase,
+		    rsp);
+	else
+		error = vm_setup_freebsd_registers(ctx, BSP, rip, cr3, gdtbase,
+		    rsp);
 	if (error) {
 		perror("vm_setup_freebsd_registers");
 		cb_exit(NULL, USERBOOT_EXIT_QUIT);