NTB: Expose 32-bit BAR limits to consumers

32-bit BARs can only address memory mapped in the low 32 bits of
physical RAM.  Expose this as a 'plimit' out parameter from
ntb_mw_get_range().

Fix if_ntb to allocate memory within this limit.

Sponsored by:	EMC / Isilon Storage Division

3 files changed, 24 insertions, 8 deletions

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 08846cc..7838f01 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -207,6 +207,7 @@ struct ntb_transport_mw {
 	size_t		phys_size;
 	size_t		xlat_align;
 	size_t		xlat_align_size;
+	bus_addr_t	addr_limit;
 	/* Tx buff is off vbase / phys_addr */
 	caddr_t		vbase;
 	size_t		xlat_size;
@@ -576,7 +577,8 @@ ntb_transport_probe(struct ntb_softc *ntb)
 		mw = &nt->mw_vec[i];
 
 		rc = ntb_mw_get_range(ntb, i, &mw->phys_addr, &mw->vbase,
-		    &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size);
+		    &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size,
+		    &mw->addr_limit);
 		if (rc != 0)
 			goto err;
 
@@ -1306,7 +1308,7 @@ ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, size_t size)
 	mw->buff_size = buff_size;
 
 	mw->virt_addr = contigmalloc(mw->buff_size, M_NTB_IF, M_ZERO, 0,
-	    BUS_SPACE_MAXADDR, mw->xlat_align, 0);
+	    mw->addr_limit, mw->xlat_align, 0);
 	if (mw->virt_addr == NULL) {
 		mw->xlat_size = 0;
 		mw->buff_size = 0;
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 36f0bec..220d635 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -2485,15 +2485,19 @@ ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val)
  */
 int
 ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
-    caddr_t *vbase, size_t *size, size_t *align, size_t *align_size)
+    caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
+    bus_addr_t *plimit)
 {
 	struct ntb_pci_bar_info *bar;
+	bus_addr_t limit;
 	size_t bar_b2b_off;
+	enum ntb_bar bar_num;
 
 	if (mw_idx >= ntb_mw_count(ntb))
 		return (EINVAL);
 
-	bar = &ntb->bar_info[ntb_mw_to_bar(ntb, mw_idx)];
+	bar_num = ntb_mw_to_bar(ntb, mw_idx);
+	bar = &ntb->bar_info[bar_num];
 	bar_b2b_off = 0;
 	if (mw_idx == ntb->b2b_mw_idx) {
 		KASSERT(ntb->b2b_off != 0,
@@ -2501,6 +2505,11 @@ ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
 		bar_b2b_off = ntb->b2b_off;
 	}
 
+	if (bar_is_64bit(ntb, bar_num))
+		limit = BUS_SPACE_MAXADDR;
+	else
+		limit = BUS_SPACE_MAXADDR_32BIT;
+
 	if (base != NULL)
 		*base = bar->pbase + bar_b2b_off;
 	if (vbase != NULL)
@@ -2511,6 +2520,8 @@ ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
 		*align = bar->size;
 	if (align_size != NULL)
 		*align_size = 1;
+	if (plimit != NULL)
+		*plimit = limit;
 	return (0);
 }
 
@@ -2524,7 +2535,9 @@ ntb_mw_get_range(struct ntb_softc *ntb, unsigned mw_idx, vm_paddr_t *base,
  * Set the translation of a memory window.  The peer may access local memory
  * through the window starting at the address, up to the size.  The address
  * must be aligned to the alignment specified by ntb_mw_get_range().  The size
- * must be aligned to the size alignment specified by ntb_mw_get_range().
+ * must be aligned to the size alignment specified by ntb_mw_get_range().  The
+ * address must be below the plimit specified by ntb_mw_get_range() (i.e. for
+ * 32-bit BARs).
  *
  * Return: Zero on success, otherwise an error number.
  */
@@ -2586,9 +2599,9 @@ ntb_mw_set_trans(struct ntb_softc *ntb, unsigned idx, bus_addr_t addr,
 		/* Configure 32-bit (split) BAR MW */
 
 		if ((addr & UINT32_MAX) != addr)
-			return (EINVAL);
+			return (ERANGE);
 		if (((addr + size) & UINT32_MAX) != (addr + size))
-			return (EINVAL);
+			return (ERANGE);
 
 		base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK;
 
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.h b/sys/dev/ntb/ntb_hw/ntb_hw.h
index ac92a80..d4f4930 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.h
@@ -77,7 +77,8 @@ void ntb_clear_ctx(struct ntb_softc *);
 
 uint8_t ntb_mw_count(struct ntb_softc *);
 int ntb_mw_get_range(struct ntb_softc *, unsigned mw_idx, vm_paddr_t *base,
-    caddr_t *vbase, size_t *size, size_t *align, size_t *align_size);
+    caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
+    bus_addr_t *plimit);
 int ntb_mw_set_trans(struct ntb_softc *, unsigned mw_idx, bus_addr_t, size_t);
 int ntb_mw_clear_trans(struct ntb_softc *, unsigned mw_idx);