Simplify the assignment of memory to virtual machines by requiring a single

command line option "-m <memsize in MB>" to specify the memory size.

Prior to this change the user needed to explicitly specify the amount of
memory allocated below 4G (-m <lowmem>) and the amount above 4G (-M <highmem>).

The "-M" option is no longer supported by 'bhyveload' and 'bhyve'.

The start of the PCI hole is fixed at 3GB and cannot be directly changed
using command line options. However it is still possible to change this in
special circumstances via the 'vm_set_lowmem_limit()' API provided by
libvmmapi.

Submitted by:	Dinakar Medavaram (initial version)
Reviewed by:	grehan
Obtained from:	NetApp

7 files changed, 48 insertions, 70 deletions

diff --git a/usr.sbin/bhyve/acpi.c b/usr.sbin/bhyve/acpi.c
index f9504f8..ed0c545 100644
--- a/usr.sbin/bhyve/acpi.c
+++ b/usr.sbin/bhyve/acpi.c
@@ -680,29 +680,26 @@ basl_end(struct basl_fio *in, struct basl_fio *out)
 }
 
 static int
-basl_load(int fd, uint64_t off)
+basl_load(struct vmctx *ctx, int fd, uint64_t off)
 {
-        struct stat sb;
+	struct stat sb;
 	void *gaddr;
-	int err;
 
-	err = 0;
-        if (fstat(fd, &sb) < 0) {
-		err = errno;
-        } else {
-		gaddr = paddr_guest2host(basl_acpi_base + off, sb.st_size);
-		if (gaddr != NULL) {
-			if (read(fd, gaddr, sb.st_size) < 0)
-				err = errno;
-		} else
-			err = EFAULT;
-        }
+	if (fstat(fd, &sb) < 0)
+		return (errno);
+		
+	gaddr = paddr_guest2host(ctx, basl_acpi_base + off, sb.st_size);
+	if (gaddr == NULL)
+		return (EFAULT);
 
-	return (err);
+	if (read(fd, gaddr, sb.st_size) < 0)
+		return (errno);
+
+	return (0);
 }
 
 static int
-basl_compile(int (*fwrite_section)(FILE *fp), uint64_t offset)
+basl_compile(struct vmctx *ctx, int (*fwrite_section)(FILE *), uint64_t offset)
 {
 	struct basl_fio io[2];
 	static char iaslbuf[3*MAXPATHLEN + 10];
@@ -736,7 +733,7 @@ basl_compile(int (*fwrite_section)(FILE *fp), uint64_t offset)
 				 * Copy the aml output file into guest
 				 * memory at the specified location
 				 */
-				err = basl_load(io[1].fd, offset);
+				err = basl_load(ctx, io[1].fd, offset);
 			}
 		}
 		basl_end(&io[0], &io[1]);
@@ -842,7 +839,7 @@ acpi_build(struct vmctx *ctx, int ncpu, int ioapic)
 	 * copying them into guest memory
 	 */
 	while (!err && basl_ftables[i].wsect != NULL) {
-		err = basl_compile(basl_ftables[i].wsect,
+		err = basl_compile(ctx, basl_ftables[i].wsect,
 				   basl_ftables[i].offset);
 		i++;
 	}
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index 17d60d6..5794b39 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -80,9 +80,6 @@ int guest_tslice = DEFAULT_GUEST_TSLICE;
 int guest_hz = DEFAULT_GUEST_HZ;
 char *vmname;
 
-u_long lomem_sz;
-u_long himem_sz;
-
 int guest_ncpus;
 
 static int pincpu = -1;
@@ -95,9 +92,6 @@ static int strictio;
 
 static int acpi;
 
-static char *lomem_addr;
-static char *himem_addr;
-
 static char *progname;
 static const int BSP = 0;
 
@@ -147,8 +141,7 @@ usage(int code)
 		"       -z: guest hz (default is %d)\n"
 		"       -s: <slot,driver,configinfo> PCI slot config\n"
 		"       -S: <slot,driver,configinfo> legacy PCI slot config\n"
-		"       -m: lowmem in MB\n"
-		"       -M: highmem in MB\n"
+		"       -m: memory size in MB\n"
 		"       -x: mux vcpus to 1 hcpu\n"
 		"       -t: mux vcpu timeslice hz (default %d)\n",
 		progname, DEFAULT_GDB_PORT, DEFAULT_GUEST_HZ,
@@ -157,19 +150,10 @@ usage(int code)
 }
 
 void *
-paddr_guest2host(uintptr_t gaddr, size_t len)
+paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
 {
 
-	if (gaddr < lomem_sz && gaddr + len <= lomem_sz)
-		return ((void *)(lomem_addr + gaddr));
-
-	if (gaddr >= 4*GB) {
-		gaddr -= 4*GB;
-		if (gaddr < himem_sz && gaddr + len <= himem_sz)
-			return ((void *)(himem_addr + gaddr));
-	}
-
-	return (NULL);
+	return (vm_map_gpa(ctx, gaddr, len));
 }
 
 int
@@ -604,6 +588,7 @@ main(int argc, char *argv[])
 	int max_vcpus;
 	struct vmctx *ctx;
 	uint64_t rip;
+	size_t memsize;
 
 	bvmcons = 0;
 	inject_bkpt = 0;
@@ -611,8 +596,9 @@ main(int argc, char *argv[])
 	gdb_port = DEFAULT_GDB_PORT;
 	guest_ncpus = 1;
 	ioapic = 0;
+	memsize = 256 * MB;
 
-	while ((c = getopt(argc, argv, "abehABHIPxp:g:c:z:s:S:n:m:M:")) != -1) {
+	while ((c = getopt(argc, argv, "abehABHIPxp:g:c:z:s:S:n:m:")) != -1) {
 		switch (c) {
 		case 'a':
 			disable_x2apic = 1;
@@ -651,10 +637,7 @@ main(int argc, char *argv[])
 			pci_parse_slot(optarg, 1);
 			break;
                 case 'm':
-			lomem_sz = strtoul(optarg, NULL, 0) * MB;
-			break;
-                case 'M':
-			himem_sz = strtoul(optarg, NULL, 0) * MB;
+			memsize = strtoul(optarg, NULL, 0) * MB;
 			break;
 		case 'H':
 			guest_vmexit_on_hlt = 1;
@@ -739,17 +722,10 @@ main(int argc, char *argv[])
 		exit(1);
 	}
 
-	if (lomem_sz != 0) {
-		lomem_addr = vm_map_memory(ctx, 0, lomem_sz);
-		if (lomem_addr == (char *) MAP_FAILED) {
-			lomem_sz = 0;
-		} else if (himem_sz != 0) {
-			himem_addr = vm_map_memory(ctx, 4*GB, himem_sz);
-			if (himem_addr == (char *) MAP_FAILED) {
-				lomem_sz = 0;
-				himem_sz = 0;
-			}
-		}
+	err = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
+	if (err) {
+		fprintf(stderr, "Unable to setup memory (%d)\n", err);
+		exit(1);
 	}
 
 	init_inout();
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index 70455bf..ffe3018 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -41,9 +41,7 @@ extern int guest_tslice;
 extern int guest_ncpus;
 extern char *vmname;
 
-extern u_long lomem_sz, himem_sz;
-
-void *paddr_guest2host(uintptr_t addr, size_t len);
+void *paddr_guest2host(struct vmctx *ctx, uintptr_t addr, size_t len);
 
 void fbsdrun_addcpu(struct vmctx *ctx, int cpu, uint64_t rip);
 int  fbsdrun_muxed(void);
diff --git a/usr.sbin/bhyve/mptbl.c b/usr.sbin/bhyve/mptbl.c
index 9c68b3d..ece71cd 100644
--- a/usr.sbin/bhyve/mptbl.c
+++ b/usr.sbin/bhyve/mptbl.c
@@ -349,7 +349,7 @@ mptable_build(struct vmctx *ctx, int ncpu, int ioapic)
 	char 			*curraddr;
 	char 			*startaddr;
 
-	startaddr = paddr_guest2host(MPTABLE_BASE, MPTABLE_MAX_LENGTH);
+	startaddr = paddr_guest2host(ctx, MPTABLE_BASE, MPTABLE_MAX_LENGTH);
 	if (startaddr == NULL) {
 		printf("mptable requires mapped mem\n");
 		return (ENOMEM);
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index cb09b7c..b5e923c 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -86,6 +86,8 @@ static struct lirqinfo {
 
 SET_DECLARE(pci_devemu_set, struct pci_devemu);
 
+static uint32_t	pci_hole_startaddr;
+
 static uint64_t pci_emul_iobase;
 static uint64_t pci_emul_membase32;
 static uint64_t pci_emul_membase64;
@@ -93,7 +95,6 @@ static uint64_t pci_emul_membase64;
 #define	PCI_EMUL_IOBASE		0x2000
 #define	PCI_EMUL_IOLIMIT	0x10000
 
-#define	PCI_EMUL_MEMBASE32	(lomem_sz)
 #define	PCI_EMUL_MEMLIMIT32	0xE0000000		/* 3.5GB */
 
 #define	PCI_EMUL_MEMBASE64	0xD000000000UL
@@ -870,8 +871,10 @@ init_pci(struct vmctx *ctx)
 	int slot, func;
 	int error;
 
+	pci_hole_startaddr = vm_get_lowmem_limit(ctx);
+
 	pci_emul_iobase = PCI_EMUL_IOBASE;
-	pci_emul_membase32 = PCI_EMUL_MEMBASE32;
+	pci_emul_membase32 = pci_hole_startaddr;
 	pci_emul_membase64 = PCI_EMUL_MEMBASE64;
 
 	for (slot = 0; slot < MAXSLOTS; slot++) {
@@ -904,8 +907,8 @@ init_pci(struct vmctx *ctx)
 	memset(&memp, 0, sizeof(struct mem_range));
 	memp.name = "PCI hole";
 	memp.flags = MEM_F_RW;
-	memp.base = lomem_sz;
-	memp.size = (4ULL * 1024 * 1024 * 1024) - lomem_sz;
+	memp.base = pci_hole_startaddr;
+	memp.size = (4ULL * 1024 * 1024 * 1024) - pci_hole_startaddr;
 	memp.handler = pci_emul_fallback_handler;
 
 	error = register_mem_fallback(&memp);
diff --git a/usr.sbin/bhyve/pci_virtio_block.c b/usr.sbin/bhyve/pci_virtio_block.c
index 0c34666..a325e97 100644
--- a/usr.sbin/bhyve/pci_virtio_block.c
+++ b/usr.sbin/bhyve/pci_virtio_block.c
@@ -141,6 +141,7 @@ struct pci_vtblk_softc {
 	uint16_t	msix_table_idx_req;
 	uint16_t	msix_table_idx_cfg;
 };
+#define	vtblk_ctx(sc)	((sc)->vbsc_pi->pi_vmctx)
 
 /* 
  * Return the size of IO BAR that maps virtio header and device specific
@@ -227,13 +228,14 @@ pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vring_hqueue *hq)
 	assert(nsegs >= 3);
 	assert(nsegs < VTBLK_MAXSEGS + 2);
 
-	vid = paddr_guest2host(vd->vd_addr, vd->vd_len);
+	vid = paddr_guest2host(vtblk_ctx(sc), vd->vd_addr, vd->vd_len);
 	assert((vid->vd_flags & VRING_DESC_F_INDIRECT) == 0);
 
 	/*
 	 * The first descriptor will be the read-only fixed header
 	 */
-	vbh = paddr_guest2host(vid[0].vd_addr, sizeof(struct virtio_blk_hdr));
+	vbh = paddr_guest2host(vtblk_ctx(sc), vid[0].vd_addr,
+			    sizeof(struct virtio_blk_hdr));
 	assert(vid[0].vd_len == sizeof(struct virtio_blk_hdr));
 	assert(vid[0].vd_flags & VRING_DESC_F_NEXT);
 	assert((vid[0].vd_flags & VRING_DESC_F_WRITE) == 0);
@@ -252,8 +254,8 @@ pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vring_hqueue *hq)
 	 * Build up the iovec based on the guest's data descriptors
 	 */
 	for (i = 1, iolen = 0; i < nsegs - 1; i++) {
-		iov[i-1].iov_base = paddr_guest2host(vid[i].vd_addr,
-						     vid[i].vd_len);
+		iov[i-1].iov_base = paddr_guest2host(vtblk_ctx(sc),
+						vid[i].vd_addr, vid[i].vd_len);
 		iov[i-1].iov_len = vid[i].vd_len;
 		iolen += vid[i].vd_len;
 
@@ -271,7 +273,7 @@ pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vring_hqueue *hq)
 	}
 
 	/* Lastly, get the address of the status byte */
-	status = paddr_guest2host(vid[nsegs - 1].vd_addr, 1);
+	status = paddr_guest2host(vtblk_ctx(sc), vid[nsegs - 1].vd_addr, 1);
 	assert(vid[nsegs - 1].vd_len == 1);
 	assert((vid[nsegs - 1].vd_flags & VRING_DESC_F_NEXT) == 0);
 	assert(vid[nsegs - 1].vd_flags & VRING_DESC_F_WRITE);
@@ -347,7 +349,7 @@ pci_vtblk_ring_init(struct pci_vtblk_softc *sc, uint64_t pfn)
 	hq = &sc->vbsc_q;
 	hq->hq_size = VTBLK_RINGSZ;
 
-	hq->hq_dtable = paddr_guest2host(pfn << VRING_PFN,
+	hq->hq_dtable = paddr_guest2host(vtblk_ctx(sc), pfn << VRING_PFN,
 					 vring_size(VTBLK_RINGSZ));
 	hq->hq_avail_flags =  (uint16_t *)(hq->hq_dtable + hq->hq_size);
 	hq->hq_avail_idx = hq->hq_avail_flags + 1;
diff --git a/usr.sbin/bhyve/pci_virtio_net.c b/usr.sbin/bhyve/pci_virtio_net.c
index 7129f6b..783c45e 100644
--- a/usr.sbin/bhyve/pci_virtio_net.c
+++ b/usr.sbin/bhyve/pci_virtio_net.c
@@ -148,6 +148,7 @@ struct pci_vtnet_softc {
 	struct	vring_hqueue vsc_hq[VTNET_MAXQ];
 	uint16_t	vsc_msix_table_idx[VTNET_MAXQ];
 };
+#define	vtnet_ctx(sc)	((sc)->vsc_pi->pi_vmctx)
 
 /*
  * Return the size of IO BAR that maps virtio header and device specific
@@ -331,7 +332,7 @@ pci_vtnet_tap_rx(struct pci_vtnet_softc *sc)
 		 * Get a pointer to the rx header, and use the
 		 * data immediately following it for the packet buffer.
 		 */
-		vrx = paddr_guest2host(vd->vd_addr, vd->vd_len);
+		vrx = paddr_guest2host(vtnet_ctx(sc), vd->vd_addr, vd->vd_len);
 		buf = (uint8_t *)(vrx + 1);
 
 		len = read(sc->vsc_tapfd, buf,
@@ -439,7 +440,8 @@ pci_vtnet_proctx(struct pci_vtnet_softc *sc, struct vring_hqueue *hq)
 	for (i = 0, plen = 0;
 	     i < VTNET_MAXSEGS;
 	     i++, vd = &hq->hq_dtable[vd->vd_next]) {
-		iov[i].iov_base = paddr_guest2host(vd->vd_addr, vd->vd_len);
+		iov[i].iov_base = paddr_guest2host(vtnet_ctx(sc),
+						   vd->vd_addr, vd->vd_len);
 		iov[i].iov_len = vd->vd_len;
 		plen += vd->vd_len;
 		tlen += vd->vd_len;
@@ -522,7 +524,7 @@ pci_vtnet_ring_init(struct pci_vtnet_softc *sc, uint64_t pfn)
 	hq = &sc->vsc_hq[qnum];
 	hq->hq_size = pci_vtnet_qsize(qnum);
 
-	hq->hq_dtable = paddr_guest2host(pfn << VRING_PFN,
+	hq->hq_dtable = paddr_guest2host(vtnet_ctx(sc), pfn << VRING_PFN,
 					 vring_size(hq->hq_size));
 	hq->hq_avail_flags =  (uint16_t *)(hq->hq_dtable + hq->hq_size);
 	hq->hq_avail_idx = hq->hq_avail_flags + 1;