Get rid of assumptions in the hypervisor that the host physical memory

associated with guest physical memory is contiguous.

Rewrite vm_gpa2hpa() to get the GPA to HPA mapping by querying the nested
page tables.

6 files changed, 134 insertions, 45 deletions

diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
index bb2f778..be22eec 100644
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -47,9 +47,11 @@ typedef int	(*vmm_cleanup_func_t)(void);
 typedef void *	(*vmi_init_func_t)(struct vm *vm); /* instance specific apis */
 typedef int	(*vmi_run_func_t)(void *vmi, int vcpu, register_t rip);
 typedef void	(*vmi_cleanup_func_t)(void *vmi);
-typedef int	(*vmi_mmap_func_t)(void *vmi, vm_paddr_t gpa, vm_paddr_t hpa,
-				   size_t length, vm_memattr_t attr,
-				   int prot, boolean_t superpages_ok);
+typedef int	(*vmi_mmap_set_func_t)(void *vmi, vm_paddr_t gpa,
+				       vm_paddr_t hpa, size_t length,
+				       vm_memattr_t attr, int prot,
+				       boolean_t superpages_ok);
+typedef vm_paddr_t (*vmi_mmap_get_func_t)(void *vmi, vm_paddr_t gpa);
 typedef int	(*vmi_get_register_t)(void *vmi, int vcpu, int num,
 				      uint64_t *retval);
 typedef int	(*vmi_set_register_t)(void *vmi, int vcpu, int num,
@@ -72,7 +74,8 @@ struct vmm_ops {
 	vmi_init_func_t		vminit;		/* vm-specific initialization */
 	vmi_run_func_t		vmrun;
 	vmi_cleanup_func_t	vmcleanup;
-	vmi_mmap_func_t		vmmmap;
+	vmi_mmap_set_func_t	vmmmap_set;
+	vmi_mmap_get_func_t	vmmmap_get;
 	vmi_get_register_t	vmgetreg;
 	vmi_set_register_t	vmsetreg;
 	vmi_get_desc_t		vmgetdesc;
diff --git a/sys/amd64/vmm/amd/amdv.c b/sys/amd64/vmm/amd/amdv.c
index 674337d..b50f972 100644
--- a/sys/amd64/vmm/amd/amdv.c
+++ b/sys/amd64/vmm/amd/amdv.c
@@ -78,11 +78,19 @@ amdv_vmcleanup(void *arg)
 }
 
 static int
-amdv_vmmmap(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
+amdv_vmmmap_set(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
 	    vm_memattr_t attr, int prot, boolean_t spok)
 {
 
-	printf("amdv_vmmmap: not implemented\n");
+	printf("amdv_vmmmap_set: not implemented\n");
+	return (EINVAL);
+}
+
+static vm_paddr_t
+amdv_vmmmap_get(void *arg, vm_paddr_t gpa)
+{
+
+	printf("amdv_vmmmap_get: not implemented\n");
 	return (EINVAL);
 }
 
@@ -157,7 +165,8 @@ struct vmm_ops vmm_ops_amd = {
 	amdv_vminit,
 	amdv_vmrun,
 	amdv_vmcleanup,
-	amdv_vmmmap,
+	amdv_vmmmap_set,
+	amdv_vmmmap_get,
 	amdv_getreg,
 	amdv_setreg,
 	amdv_getdesc,
diff --git a/sys/amd64/vmm/intel/ept.c b/sys/amd64/vmm/intel/ept.c
index c9fca9d..4f91601 100644
--- a/sys/amd64/vmm/intel/ept.c
+++ b/sys/amd64/vmm/intel/ept.c
@@ -115,6 +115,40 @@ ept_init(void)
 	return (0);
 }
 
+#if 0
+static void
+ept_dump(uint64_t *ptp, int nlevels)
+{
+	int i, t, tabs;
+	uint64_t *ptpnext, ptpval;
+
+	if (--nlevels < 0)
+		return;
+
+	tabs = 3 - nlevels;
+	for (t = 0; t < tabs; t++)
+		printf("\t");
+	printf("PTP = %p\n", ptp);
+
+	for (i = 0; i < 512; i++) {
+		ptpval = ptp[i];
+
+		if (ptpval == 0)
+			continue;
+		
+		for (t = 0; t < tabs; t++)
+			printf("\t");
+		printf("%3d 0x%016lx\n", i, ptpval);
+
+		if (nlevels != 0 && (ptpval & EPT_PG_SUPERPAGE) == 0) {
+			ptpnext = (uint64_t *)
+				  PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK);
+			ept_dump(ptpnext, nlevels);
+		}
+	}
+}
+#endif
+
 static size_t
 ept_create_mapping(uint64_t *ptp, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
 		   vm_memattr_t attr, vm_prot_t prot, boolean_t spok)
@@ -179,29 +213,64 @@ ept_create_mapping(uint64_t *ptp, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
 		      "mismatch\n", gpa, ptpshift);
 	}
 
-	/* Do the mapping */
-	ptp[ptpindex] = hpa;
+	if (prot != VM_PROT_NONE) {
+		/* Do the mapping */
+		ptp[ptpindex] = hpa;
 
-	/* Apply the access controls */
-	if (prot & VM_PROT_READ)
-		ptp[ptpindex] |= EPT_PG_RD;
-	if (prot & VM_PROT_WRITE)
-		ptp[ptpindex] |= EPT_PG_WR;
-	if (prot & VM_PROT_EXECUTE)
-		ptp[ptpindex] |= EPT_PG_EX;
+		/* Apply the access controls */
+		if (prot & VM_PROT_READ)
+			ptp[ptpindex] |= EPT_PG_RD;
+		if (prot & VM_PROT_WRITE)
+			ptp[ptpindex] |= EPT_PG_WR;
+		if (prot & VM_PROT_EXECUTE)
+			ptp[ptpindex] |= EPT_PG_EX;
 
-	/*
-	 * XXX should we enforce this memory type by setting the ignore PAT
-	 * bit to 1.
-	 */
-	ptp[ptpindex] |= EPT_PG_MEMORY_TYPE(attr);
+		/*
+		 * XXX should we enforce this memory type by setting the
+		 * ignore PAT bit to 1.
+		 */
+		ptp[ptpindex] |= EPT_PG_MEMORY_TYPE(attr);
 
-	if (nlevels > 0)
-		ptp[ptpindex] |= EPT_PG_SUPERPAGE;
+		if (nlevels > 0)
+			ptp[ptpindex] |= EPT_PG_SUPERPAGE;
+	} else {
+		/* Remove the mapping */
+		ptp[ptpindex] = 0;
+	}
 
 	return (1UL << ptpshift);
 }
 
+static vm_paddr_t
+ept_lookup_mapping(uint64_t *ptp, vm_paddr_t gpa)
+{
+	int nlevels, ptpshift, ptpindex;
+	uint64_t ptpval, hpabase, pgmask;
+
+	nlevels = EPT_PWLEVELS;
+	while (--nlevels >= 0) {
+		ptpshift = PAGE_SHIFT + nlevels * 9;
+		ptpindex = (gpa >> ptpshift) & 0x1FF;
+
+		ptpval = ptp[ptpindex];
+
+		/* Cannot make progress beyond this point */
+		if ((ptpval & (EPT_PG_RD | EPT_PG_WR | EPT_PG_EX)) == 0)
+			break;
+
+		if (nlevels == 0 || (ptpval & EPT_PG_SUPERPAGE)) {
+			pgmask = (1UL << ptpshift) - 1;
+			hpabase = ptpval & ~pgmask;
+			return (hpabase | (gpa & pgmask));
+		}
+
+		/* Work our way down to the next level page table page */
+		ptp = (uint64_t *)PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK);
+	}
+
+	return ((vm_paddr_t)-1);
+}
+
 static void
 ept_free_pt_entry(pt_entry_t pte)
 {
@@ -276,8 +345,8 @@ ept_vmcleanup(struct vmx *vmx)
 }
 
 int
-ept_vmmmap(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t len,
-	   vm_memattr_t attr, int prot, boolean_t spok)
+ept_vmmmap_set(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t len,
+		vm_memattr_t attr, int prot, boolean_t spok)
 {
 	size_t n;
 	struct vmx *vmx = arg;
@@ -293,6 +362,17 @@ ept_vmmmap(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t len,
 	return (0);
 }
 
+vm_paddr_t
+ept_vmmmap_get(void *arg, vm_paddr_t gpa)
+{
+	vm_paddr_t hpa;
+	struct vmx *vmx;
+
+	vmx = arg;
+	hpa = ept_lookup_mapping(vmx->pml4ept, gpa);
+	return (hpa);
+}
+
 static void
 invept_single_context(void *arg)
 {
diff --git a/sys/amd64/vmm/intel/ept.h b/sys/amd64/vmm/intel/ept.h
index 013c330..2d7258d 100644
--- a/sys/amd64/vmm/intel/ept.h
+++ b/sys/amd64/vmm/intel/ept.h
@@ -35,8 +35,9 @@ struct vmx;
 #define	EPTP(pml4)	((pml4) | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK)
 
 int	ept_init(void);
-int	ept_vmmmap(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
+int	ept_vmmmap_set(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
 	    vm_memattr_t attr, int prot, boolean_t allow_superpage_mappings);
+vm_paddr_t ept_vmmmap_get(void *arg, vm_paddr_t gpa);
 void	ept_invalidate_mappings(u_long ept_pml4);
 void	ept_vmcleanup(struct vmx *vmx);
 #endif
diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
index a2c8e76..3fbe5a1 100644
--- a/sys/amd64/vmm/intel/vmx.c
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -1813,7 +1813,8 @@ struct vmm_ops vmm_ops_intel = {
 	vmx_vminit,
 	vmx_run,
 	vmx_vmcleanup,
-	ept_vmmmap,
+	ept_vmmmap_set,
+	ept_vmmmap_get,
 	vmx_getreg,
 	vmx_setreg,
 	vmx_getdesc,
diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c
index 06109b1..62bb753 100644
--- a/sys/amd64/vmm/vmm.c
+++ b/sys/amd64/vmm/vmm.c
@@ -115,8 +115,12 @@ static struct vmm_ops *ops;
 #define	VMRUN(vmi, vcpu, rip) \
 	(ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip) : ENXIO)
 #define	VMCLEANUP(vmi)	(ops != NULL ? (*ops->vmcleanup)(vmi) : NULL)
-#define	VMMMAP(vmi, gpa, hpa, len, attr, prot, spm)	\
-    (ops != NULL ? (*ops->vmmmap)(vmi, gpa, hpa, len, attr, prot, spm) : ENXIO)
+#define	VMMMAP_SET(vmi, gpa, hpa, len, attr, prot, spm)			\
+    	(ops != NULL ? 							\
+    	(*ops->vmmmap_set)(vmi, gpa, hpa, len, attr, prot, spm) :	\
+	ENXIO)
+#define	VMMMAP_GET(vmi, gpa) \
+	(ops != NULL ? (*ops->vmmmap_get)(vmi, gpa) : ENXIO)
 #define	VMGETREG(vmi, vcpu, num, retval)		\
 	(ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO)
 #define	VMSETREG(vmi, vcpu, num, val)		\
@@ -302,8 +306,8 @@ vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
 {
 	const boolean_t spok = TRUE;	/* superpage mappings are ok */
 
-	return (VMMMAP(vm->cookie, gpa, hpa, len, VM_MEMATTR_UNCACHEABLE,
-		       VM_PROT_RW, spok));
+	return (VMMMAP_SET(vm->cookie, gpa, hpa, len, VM_MEMATTR_UNCACHEABLE,
+			   VM_PROT_RW, spok));
 }
 
 int
@@ -311,8 +315,8 @@ vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
 {
 	const boolean_t spok = TRUE;	/* superpage mappings are ok */
 
-	return (VMMMAP(vm->cookie, gpa, 0, len, VM_MEMATTR_UNCACHEABLE,
-		       VM_PROT_NONE, spok));
+	return (VMMMAP_SET(vm->cookie, gpa, 0, len, 0,
+			   VM_PROT_NONE, spok));
 }
 
 /*
@@ -380,8 +384,8 @@ vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len)
 	if (hpa == 0)
 		return (ENOMEM);
 
-	error = VMMMAP(vm->cookie, gpa, hpa, len, VM_MEMATTR_WRITE_BACK,
-		       VM_PROT_ALL, spok);
+	error = VMMMAP_SET(vm->cookie, gpa, hpa, len, VM_MEMATTR_WRITE_BACK,
+			   VM_PROT_ALL, spok);
 	if (error) {
 		vmm_mem_free(hpa, len);
 		return (error);
@@ -400,17 +404,8 @@ vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len)
 vm_paddr_t
 vm_gpa2hpa(struct vm *vm, vm_paddr_t gpa, size_t len)
 {
-	int i;
-	vm_paddr_t gpabase, gpalimit, hpabase;
 
-	for (i = 0; i < vm->num_mem_segs; i++) {
-		hpabase = vm->mem_segs[i].hpa;
-		gpabase = vm->mem_segs[i].gpa;
-		gpalimit = gpabase + vm->mem_segs[i].len;
-		if (gpa >= gpabase && gpa + len <= gpalimit)
-			return ((gpa - gpabase) + hpabase);
-	}
-	return ((vm_paddr_t)-1);
+	return (VMMMAP_GET(vm->cookie, gpa));
 }
 
 int