xen: unify gnttab initialization for PVHVM and PVH

Switch the initialization of gnttab to use an unused physical memory
range for both PVHVM and PVH.

In the past PVHVM was using the xenpci BAR, but there's no reason to
do that, and in fact FreeBSD was probably doing it because it was the
way it was done in Windows, were drivers cannot probably request for
unused physical memory ranges, but it was never enforced in the
hypervisor.

Sponsored by: Citrix Systems R&D
Approved by: gibbs

xen/gnttab.c:
 - Allocate contiguous physical memory for grant table frames for both
   PVHVM and PVH.
 - Since gnttab is not a device, use the xenpv device in order to
   request for this allocation.

dev/xen/xenpci/xenpcivar.h:
dev/xen/xenpci/xenpci.c:
 - Remove the now unused xenpci_alloc_space and xenpci_alloc_space_int
   functions.

xen/gnttab.h:
 - Change the prototype of gnttab_init and gnttab_resume, that now
   takes a device_t parameter.

dev/xen/control/control.c:
x86/xen/xenpv.c:
 - Changes to accomodate the new prototype of gnttab_init and
   gnttab_resume.

5 files changed, 31 insertions, 74 deletions

diff --git a/sys/dev/xen/control/control.c b/sys/dev/xen/control/control.c
index 78894ba..665a5ac 100644
--- a/sys/dev/xen/control/control.c
+++ b/sys/dev/xen/control/control.c
@@ -287,7 +287,7 @@ xctrl_suspend()
 	}
 	HYPERVISOR_shared_info->arch.max_pfn = max_pfn;
 
-	gnttab_resume();
+	gnttab_resume(NULL);
 	intr_resume(suspend_cancelled != 0);
 	local_irq_enable();
 	xencons_resume();
@@ -385,7 +385,7 @@ xctrl_suspend()
 	/*
 	 * Reset grant table info.
 	 */
-	gnttab_resume();
+	gnttab_resume(NULL);
 
 #ifdef SMP
 	if (smp_started && !CPU_EMPTY(&cpu_suspend_map)) {
diff --git a/sys/dev/xen/xenpci/xenpci.c b/sys/dev/xen/xenpci/xenpci.c
index e334051..3fdd78c 100644
--- a/sys/dev/xen/xenpci/xenpci.c
+++ b/sys/dev/xen/xenpci/xenpci.c
@@ -108,13 +108,6 @@ xenpci_deallocate_resources(device_t dev)
 			scp->rid_irq, scp->res_irq);
 		scp->res_irq = 0;
 	}
-	if (scp->res_memory != 0) {
-		bus_deactivate_resource(dev, SYS_RES_MEMORY,
-			scp->rid_memory, scp->res_memory);
-		bus_release_resource(dev, SYS_RES_MEMORY,
-			scp->rid_memory, scp->res_memory);
-		scp->res_memory = 0;
-	}
 
 	return (0);
 }
@@ -134,16 +127,6 @@ xenpci_allocate_resources(device_t dev)
 		goto errexit;
 	}
 
-	scp->rid_memory = PCIR_BAR(1);
-	scp->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
-			&scp->rid_memory, RF_ACTIVE);
-	if (scp->res_memory == NULL) {
-		printf("xenpci Could not allocate memory bar.\n");
-		goto errexit;
-	}
-
-	scp->phys_next = rman_get_start(scp->res_memory);
-
 	return (0);
 
 errexit:
@@ -153,40 +136,6 @@ errexit:
 }
 
 /*
- * Allocate a physical address range from our mmio region.
- */
-static int
-xenpci_alloc_space_int(struct xenpci_softc *scp, size_t sz,
-    vm_paddr_t *pa)
-{
-
-	if (scp->phys_next + sz > rman_get_end(scp->res_memory)) {
-		return (ENOMEM);
-	}
-
-	*pa = scp->phys_next;
-	scp->phys_next += sz;
-
-	return (0);
-}
-
-/*
- * Allocate a physical address range from our mmio region.
- */
-int
-xenpci_alloc_space(size_t sz, vm_paddr_t *pa)
-{
-	device_t dev = devclass_get_device(xenpci_devclass, 0);
-
-	if (dev) {
-		return (xenpci_alloc_space_int(device_get_softc(dev),
-			sz, pa));
-	} else {
-		return (ENOMEM);
-	}
-}
-
-/*
  * Probe - just check device ID.
  */
 static int
diff --git a/sys/dev/xen/xenpci/xenpcivar.h b/sys/dev/xen/xenpci/xenpcivar.h
index 527a291..45d1215 100644
--- a/sys/dev/xen/xenpci/xenpcivar.h
+++ b/sys/dev/xen/xenpci/xenpcivar.h
@@ -31,13 +31,7 @@
  */
 struct xenpci_softc {
 	int rid_ioport;
-	int rid_memory;
 	int rid_irq;
-	struct resource* res_memory;	/* Resource for mem range. */
 	struct resource* res_irq;	/* Resource for irq range. */
 	void	*intr_cookie;
-
-	vm_paddr_t phys_next;		/* next page from mem range */
 };
-
-extern int xenpci_alloc_space(size_t sz, vm_paddr_t *pa);
diff --git a/sys/xen/gnttab.c b/sys/xen/gnttab.c
index 03c32b7..ca6fa3b 100644
--- a/sys/xen/gnttab.c
+++ b/sys/xen/gnttab.c
@@ -25,6 +25,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mman.h>
+#include <sys/limits.h>
+#include <sys/rman.h>
+#include <machine/resource.h>
 
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
@@ -51,6 +54,17 @@ static int gnttab_free_count;
 static grant_ref_t gnttab_free_head;
 static struct mtx gnttab_list_lock;
 
+#ifdef XENHVM
+/*
+ * Resource representing allocated physical address space
+ * for the grant table metainfo
+ */
+static struct resource *gnttab_pseudo_phys_res;
+
+/* Resource id for allocated physical address space. */
+static int gnttab_pseudo_phys_res_id;
+#endif
+
 static grant_entry_t *shared;
 
 static struct gnttab_free_callback *gnttab_free_callback_list = NULL;
@@ -542,7 +556,7 @@ gnttab_map(unsigned int start_idx, unsigned int end_idx)
 }
 
 int
-gnttab_resume(void)
+gnttab_resume(device_t dev)
 {
 
 	if (max_nr_grant_frames() < nr_grant_frames)
@@ -563,8 +577,6 @@ gnttab_suspend(void)
 
 #else /* XENHVM */
 
-#include <dev/xen/xenpci/xenpcivar.h>
-
 static vm_paddr_t resume_frames;
 
 static int
@@ -603,9 +615,8 @@ gnttab_map(unsigned int start_idx, unsigned int end_idx)
 }
 
 int
-gnttab_resume(void)
+gnttab_resume(device_t dev)
 {
-	int error;
 	unsigned int max_nr_gframes, nr_gframes;
 
 	nr_gframes = nr_grant_frames;
@@ -614,12 +625,15 @@ gnttab_resume(void)
 		return (ENOSYS);
 
 	if (!resume_frames) {
-		error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes,
-		    &resume_frames);
-		if (error) {
-			printf("error mapping gnttab share frames\n");
-			return (error);
-		}
+		KASSERT(dev != NULL,
+		    ("No resume frames and no device provided"));
+
+		gnttab_pseudo_phys_res = bus_alloc_resource(dev,
+		    SYS_RES_MEMORY, &gnttab_pseudo_phys_res_id, 0, ~0,
+		    PAGE_SIZE * max_nr_gframes, RF_ACTIVE);
+		if (gnttab_pseudo_phys_res == NULL)
+			panic("Unable to reserve physical memory for gnttab");
+		resume_frames = rman_get_start(gnttab_pseudo_phys_res);
 	}
 
 	return (gnttab_map(0, nr_gframes - 1));
@@ -647,7 +661,7 @@ gnttab_expand(unsigned int req_entries)
 }
 
 int 
-gnttab_init()
+gnttab_init(device_t dev)
 {
 	int i;
 	unsigned int max_nr_glist_frames;
@@ -679,7 +693,7 @@ gnttab_init()
 			goto ini_nomem;
 	}
 
-	if (gnttab_resume())
+	if (gnttab_resume(dev))
 		return (ENODEV);
 
 	nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
diff --git a/sys/xen/gnttab.h b/sys/xen/gnttab.h
index d81e965..afbe600 100644
--- a/sys/xen/gnttab.h
+++ b/sys/xen/gnttab.h
@@ -51,7 +51,7 @@ struct gnttab_free_callback {
 	uint16_t count;
 };
 
-int gnttab_init(void);
+int gnttab_init(device_t);
 
 /*
  * Allocate a grant table reference and return it in *result. Returns
@@ -116,7 +116,7 @@ void gnttab_grant_foreign_transfer_ref(grant_ref_t, domid_t domid,
 				       unsigned long pfn);
 
 int gnttab_suspend(void);
-int gnttab_resume(void);
+int gnttab_resume(device_t);
 
 #if 0