MFC r273856,273863,273963-273965

- Add two new functions to the AGP driver KPI to bind/unbind arbitrary sets
  of pages into the GTT.
- Avoid possible overflow in agp_generic_alloc_memory.
- In agp(4) avoid the need to flush all cpu caches with wbinvd between
  updating the GTT and flushing the AGP TLB by storing the GTT in
  write-combining memory.
- In agp_amd_bind_page don't flush the AGP TLB.  It's done by the calling
  function.
- agp_generic_unbind_memory: flush AGP TLB before unwiring pages
  agp_bind_pages: assert that pages have been wired down

8 files changed, 130 insertions, 69 deletions

diff --git a/sys/dev/agp/agp.c b/sys/dev/agp/agp.c
index 5970943..497271c 100644
--- a/sys/dev/agp/agp.c
+++ b/sys/dev/agp/agp.c
@@ -50,6 +50,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pcireg.h>
 
 #include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
 #include <vm/vm_param.h>
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
@@ -84,14 +86,6 @@ static devclass_t agp_devclass;
 
 /* Helper functions for implementing chipset mini drivers. */
 
-void
-agp_flush_cache()
-{
-#if defined(__i386__) || defined(__amd64__)
-	wbinvd();
-#endif
-}
-
 u_int8_t
 agp_find_caps(device_t dev)
 {
@@ -158,17 +152,16 @@ agp_alloc_gatt(device_t dev)
 		return 0;
 
 	gatt->ag_entries = entries;
-	gatt->ag_virtual = contigmalloc(entries * sizeof(u_int32_t), M_AGP, 0,
-					0, ~0, PAGE_SIZE, 0);
+	gatt->ag_virtual = (void *)kmem_alloc_contig(kernel_arena,
+	    entries * sizeof(u_int32_t), M_NOWAIT | M_ZERO, 0, ~0, PAGE_SIZE,
+	    0, VM_MEMATTR_WRITE_COMBINING);
 	if (!gatt->ag_virtual) {
 		if (bootverbose)
 			device_printf(dev, "contiguous allocation failed\n");
 		free(gatt, M_AGP);
 		return 0;
 	}
-	bzero(gatt->ag_virtual, entries * sizeof(u_int32_t));
 	gatt->ag_physical = vtophys((vm_offset_t) gatt->ag_virtual);
-	agp_flush_cache();
 
 	return gatt;
 }
@@ -176,8 +169,8 @@ agp_alloc_gatt(device_t dev)
 void
 agp_free_gatt(struct agp_gatt *gatt)
 {
-	contigfree(gatt->ag_virtual,
-		   gatt->ag_entries * sizeof(u_int32_t), M_AGP);
+	kmem_free(kernel_arena, (vm_offset_t)gatt->ag_virtual,
+	    gatt->ag_entries * sizeof(u_int32_t));
 	free(gatt, M_AGP);
 }
 
@@ -280,7 +273,6 @@ agp_free_res(device_t dev)
 		bus_release_resource(dev, SYS_RES_MEMORY, sc->as_aperture_rid,
 		    sc->as_aperture);
 	mtx_destroy(&sc->as_lock);
-	agp_flush_cache();
 }
 
 int
@@ -485,7 +477,7 @@ agp_generic_alloc_memory(device_t dev, int type, vm_size_t size)
 	if ((size & (AGP_PAGE_SIZE - 1)) != 0)
 		return 0;
 
-	if (sc->as_allocated + size > sc->as_maxmem)
+	if (size > sc->as_maxmem - sc->as_allocated)
 		return 0;
 
 	if (type != 0) {
@@ -605,12 +597,6 @@ agp_generic_bind_memory(device_t dev, struct agp_memory *mem,
 	VM_OBJECT_WUNLOCK(mem->am_obj);
 
 	/*
-	 * Flush the cpu cache since we are providing a new mapping
-	 * for these pages.
-	 */
-	agp_flush_cache();
-
-	/*
 	 * Make sure the chipset gets the new mappings.
 	 */
 	AGP_FLUSH_TLB(dev);
@@ -659,6 +645,9 @@ agp_generic_unbind_memory(device_t dev, struct agp_memory *mem)
 	 */
 	for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE)
 		AGP_UNBIND_PAGE(dev, mem->am_offset + i);
+
+	AGP_FLUSH_TLB(dev);
+
 	VM_OBJECT_WLOCK(mem->am_obj);
 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
 		m = vm_page_lookup(mem->am_obj, atop(i));
@@ -667,9 +656,6 @@ agp_generic_unbind_memory(device_t dev, struct agp_memory *mem)
 		vm_page_unlock(m);
 	}
 	VM_OBJECT_WUNLOCK(mem->am_obj);
-		
-	agp_flush_cache();
-	AGP_FLUSH_TLB(dev);
 
 	mem->am_offset = 0;
 	mem->am_is_bound = 0;
@@ -996,3 +982,76 @@ void agp_memory_info(device_t dev, void *handle, struct
 	mi->ami_offset = mem->am_offset;
 	mi->ami_is_bound = mem->am_is_bound;
 }
+
+int
+agp_bind_pages(device_t dev, vm_page_t *pages, vm_size_t size,
+    vm_offset_t offset)
+{
+	struct agp_softc *sc;
+	vm_offset_t i, j, k, pa;
+	vm_page_t m;
+	int error;
+
+	if ((size & (AGP_PAGE_SIZE - 1)) != 0 ||
+	    (offset & (AGP_PAGE_SIZE - 1)) != 0)
+		return (EINVAL);
+
+	sc = device_get_softc(dev);
+
+	mtx_lock(&sc->as_lock);
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		m = pages[OFF_TO_IDX(i)];
+		KASSERT(m->wire_count > 0,
+		    ("agp_bind_pages: page %p hasn't been wired", m));
+
+		/*
+		 * Install entries in the GATT, making sure that if
+		 * AGP_PAGE_SIZE < PAGE_SIZE and size is not
+		 * aligned to PAGE_SIZE, we don't modify too many GATT 
+		 * entries.
+		 */
+		for (j = 0; j < PAGE_SIZE && i + j < size; j += AGP_PAGE_SIZE) {
+			pa = VM_PAGE_TO_PHYS(m) + j;
+			AGP_DPF("binding offset %#jx to pa %#jx\n",
+				(uintmax_t)offset + i + j, (uintmax_t)pa);
+			error = AGP_BIND_PAGE(dev, offset + i + j, pa);
+			if (error) {
+				/*
+				 * Bail out. Reverse all the mappings.
+				 */
+				for (k = 0; k < i + j; k += AGP_PAGE_SIZE)
+					AGP_UNBIND_PAGE(dev, offset + k);
+
+				mtx_unlock(&sc->as_lock);
+				return (error);
+			}
+		}
+	}
+
+	AGP_FLUSH_TLB(dev);
+
+	mtx_unlock(&sc->as_lock);
+	return (0);
+}
+
+int
+agp_unbind_pages(device_t dev, vm_size_t size, vm_offset_t offset)
+{
+	struct agp_softc *sc;
+	vm_offset_t i;
+
+	if ((size & (AGP_PAGE_SIZE - 1)) != 0 ||
+	    (offset & (AGP_PAGE_SIZE - 1)) != 0)
+		return (EINVAL);
+
+	sc = device_get_softc(dev);
+
+	mtx_lock(&sc->as_lock);
+	for (i = 0; i < size; i += AGP_PAGE_SIZE)
+		AGP_UNBIND_PAGE(dev, offset + i);
+
+	AGP_FLUSH_TLB(dev);
+
+	mtx_unlock(&sc->as_lock);
+	return (0);
+}
diff --git a/sys/dev/agp/agp_amd.c b/sys/dev/agp/agp_amd.c
index 2b58a3e..a39dec5 100644
--- a/sys/dev/agp/agp_amd.c
+++ b/sys/dev/agp/agp_amd.c
@@ -43,6 +43,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pcireg.h>
 
 #include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
 #include <vm/vm_object.h>
 #include <vm/pmap.h>
 #include <machine/bus.h>
@@ -92,34 +94,35 @@ agp_amd_alloc_gatt(device_t dev)
 
 	/*
 	 * The AMD751 uses a page directory to map a non-contiguous
-	 * gatt so we don't need to use contigmalloc.
-	 * Malloc individual gatt pages and map them into the page
+	 * gatt so we don't need to use kmem_alloc_contig.
+	 * Allocate individual GATT pages and map them into the page
 	 * directory.
 	 */
 	gatt->ag_entries = entries;
-	gatt->ag_virtual = malloc(entries * sizeof(u_int32_t),
-				  M_AGP, M_NOWAIT);
+	gatt->ag_virtual = (void *)kmem_alloc_attr(kernel_arena,
+	    entries * sizeof(u_int32_t), M_NOWAIT | M_ZERO, 0, ~0,
+	    VM_MEMATTR_WRITE_COMBINING);
 	if (!gatt->ag_virtual) {
 		if (bootverbose)
 			device_printf(dev, "allocation failed\n");
 		free(gatt, M_AGP);
 		return 0;
 	}
-	bzero(gatt->ag_virtual, entries * sizeof(u_int32_t));
 
 	/*
 	 * Allocate the page directory.
 	 */
-	gatt->ag_vdir = malloc(AGP_PAGE_SIZE, M_AGP, M_NOWAIT);
+	gatt->ag_vdir = (void *)kmem_alloc_attr(kernel_arena, AGP_PAGE_SIZE,
+	    M_NOWAIT | M_ZERO, 0, ~0, VM_MEMATTR_WRITE_COMBINING);
 	if (!gatt->ag_vdir) {
 		if (bootverbose)
 			device_printf(dev,
 				      "failed to allocate page directory\n");
-		free(gatt->ag_virtual, M_AGP);
+		kmem_free(kernel_arena, (vm_offset_t)gatt->ag_virtual,
+		    entries * sizeof(u_int32_t));
 		free(gatt, M_AGP);
 		return 0;
 	}
-	bzero(gatt->ag_vdir, AGP_PAGE_SIZE);
 
 	gatt->ag_pdir = vtophys((vm_offset_t) gatt->ag_vdir);
 	if(bootverbose)
@@ -158,19 +161,15 @@ agp_amd_alloc_gatt(device_t dev)
 		gatt->ag_vdir[i + pdir_offset] = pa | 1;
 	}
 
-	/*
-	 * Make sure the chipset can see everything.
-	 */
-	agp_flush_cache();
-
 	return gatt;
 }
 
 static void
 agp_amd_free_gatt(struct agp_amd_gatt *gatt)
 {
-	free(gatt->ag_virtual, M_AGP);
-	free(gatt->ag_vdir, M_AGP);
+	kmem_free(kernel_arena, (vm_offset_t)gatt->ag_vdir, AGP_PAGE_SIZE);
+	kmem_free(kernel_arena, (vm_offset_t)gatt->ag_virtual,
+	    gatt->ag_entries * sizeof(u_int32_t));
 	free(gatt, M_AGP);
 }
 
@@ -348,9 +347,6 @@ agp_amd_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
 		return EINVAL;
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = physical | 1;
-
-	/* invalidate the cache */
-	AGP_FLUSH_TLB(dev);
 	return 0;
 }
 
diff --git a/sys/dev/agp/agp_amd64.c b/sys/dev/agp/agp_amd64.c
index 5423d5a..56784f0 100644
--- a/sys/dev/agp/agp_amd64.c
+++ b/sys/dev/agp/agp_amd64.c
@@ -241,8 +241,6 @@ agp_amd64_attach(device_t dev)
 		    4);
 	}
 
-	agp_flush_cache();
-
 	return (0);
 }
 
diff --git a/sys/dev/agp/agp_apple.c b/sys/dev/agp/agp_apple.c
index 263631d..cc3723b 100644
--- a/sys/dev/agp/agp_apple.c
+++ b/sys/dev/agp/agp_apple.c
@@ -224,8 +224,6 @@ agp_apple_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
 		return EINVAL;
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = physical;
-	__asm __volatile("dcbst 0,%0; sync" ::
-	    "r"(&sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT]) : "memory");
 	return (0);
 }
 
@@ -238,8 +236,6 @@ agp_apple_unbind_page(device_t dev, vm_offset_t offset)
 		return EINVAL;
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = 0;
-	__asm __volatile("dcbst 0,%0; sync" ::
-	    "r"(&sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT]) : "memory");
 	return (0);
 }
 
diff --git a/sys/dev/agp/agp_ati.c b/sys/dev/agp/agp_ati.c
index c8e35ef..eaf0efa 100644
--- a/sys/dev/agp/agp_ati.c
+++ b/sys/dev/agp/agp_ati.c
@@ -45,6 +45,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pcireg.h>
 
 #include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
 #include <vm/vm_object.h>
 #include <vm/pmap.h>
 #include <machine/bus.h>
@@ -129,20 +131,23 @@ agp_ati_alloc_gatt(device_t dev)
 
 	/* Alloc the GATT -- pointers to pages of AGP memory */
 	sc->ag_entries = entries;
-	sc->ag_virtual = malloc(entries * sizeof(u_int32_t), M_AGP,
-	    M_NOWAIT | M_ZERO);
+	sc->ag_virtual = (void *)kmem_alloc_attr(kernel_arena,
+	    entries * sizeof(u_int32_t), M_NOWAIT | M_ZERO, 0, ~0,
+	    VM_MEMATTR_WRITE_COMBINING);
 	if (sc->ag_virtual == NULL) {
 		if (bootverbose)
-			device_printf(dev, "aperture allocation failed\n");
+			device_printf(dev, "GATT allocation failed\n");
 		return ENOMEM;
 	}
 
 	/* Alloc the page directory -- pointers to each page of the GATT */
-	sc->ag_vdir = malloc(AGP_PAGE_SIZE, M_AGP, M_NOWAIT | M_ZERO);
+	sc->ag_vdir = (void *)kmem_alloc_attr(kernel_arena, AGP_PAGE_SIZE,
+	    M_NOWAIT | M_ZERO, 0, ~0, VM_MEMATTR_WRITE_COMBINING);
 	if (sc->ag_vdir == NULL) {
 		if (bootverbose)
 			device_printf(dev, "pagedir allocation failed\n");
-		free(sc->ag_virtual, M_AGP);
+		kmem_free(kernel_arena, (vm_offset_t)sc->ag_virtual,
+		    entries * sizeof(u_int32_t));
 		return ENOMEM;
 	}
 	sc->ag_pdir = vtophys((vm_offset_t)sc->ag_vdir);
@@ -158,11 +163,6 @@ agp_ati_alloc_gatt(device_t dev)
 		sc->ag_vdir[apbase_offset + i] = pa | 1;
 	}
 
-	/*
-	 * Make sure the chipset can see everything.
-	 */
-	agp_flush_cache();
-
 	return 0;
 }
 
@@ -264,8 +264,9 @@ agp_ati_detach(device_t dev)
 	temp = pci_read_config(dev, apsize_reg, 4);
 	pci_write_config(dev, apsize_reg, temp & ~1, 4);
 
-	free(sc->ag_vdir, M_AGP);
-	free(sc->ag_virtual, M_AGP);
+	kmem_free(kernel_arena, (vm_offset_t)sc->ag_vdir, AGP_PAGE_SIZE);
+	kmem_free(kernel_arena, (vm_offset_t)sc->ag_virtual,
+	    sc->ag_entries * sizeof(u_int32_t));
 
 	bus_release_resource(dev, SYS_RES_MEMORY, ATI_GART_MMADDR, sc->regs);
 	agp_free_res(dev);
diff --git a/sys/dev/agp/agp_i810.c b/sys/dev/agp/agp_i810.c
index 1e2959d..ac8f4cd 100644
--- a/sys/dev/agp/agp_i810.c
+++ b/sys/dev/agp/agp_i810.c
@@ -66,6 +66,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pci_private.h>
 
 #include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
 #include <vm/vm_param.h>
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
@@ -1449,17 +1451,16 @@ agp_i810_install_gatt(device_t dev)
 		sc->dcache_size = 0;
 
 	/* According to the specs the gatt on the i810 must be 64k. */
-	sc->gatt->ag_virtual = contigmalloc(64 * 1024, M_AGP, 0, 0, ~0,
-	    PAGE_SIZE, 0);
+	sc->gatt->ag_virtual = (void *)kmem_alloc_contig(kernel_arena,
+	    64 * 1024, M_NOWAIT | M_ZERO, 0, ~0, PAGE_SIZE,
+	    0, VM_MEMATTR_WRITE_COMBINING);
 	if (sc->gatt->ag_virtual == NULL) {
 		if (bootverbose)
 			device_printf(dev, "contiguous allocation failed\n");
 		return (ENOMEM);
 	}
 
-	bzero(sc->gatt->ag_virtual, sc->gatt->ag_entries * sizeof(u_int32_t));
 	sc->gatt->ag_physical = vtophys((vm_offset_t)sc->gatt->ag_virtual);
-	agp_flush_cache();
 	/* Install the GATT. */
 	bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL,
 	    sc->gatt->ag_physical | 1);
@@ -1591,7 +1592,7 @@ agp_i810_deinstall_gatt(device_t dev)
 
 	sc = device_get_softc(dev);
 	bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, 0);
-	contigfree(sc->gatt->ag_virtual, 64 * 1024, M_AGP);
+	kmem_free(kernel_arena, (vm_offset_t)sc->gatt->ag_virtual, 64 * 1024);
 }
 
 static void
@@ -2146,7 +2147,6 @@ agp_i810_bind_memory(device_t dev, struct agp_memory *mem, vm_offset_t offset)
 			sc->match->driver->install_gtt_pte(dev, (offset + i) >>
 			    AGP_PAGE_SHIFT, mem->am_physical + i, 0);
 		}
-		agp_flush_cache();
 		mem->am_offset = offset;
 		mem->am_is_bound = 1;
 		mtx_unlock(&sc->agp.as_lock);
@@ -2187,7 +2187,6 @@ agp_i810_unbind_memory(device_t dev, struct agp_memory *mem)
 			sc->match->driver->install_gtt_pte(dev,
 			    (mem->am_offset + i) >> AGP_PAGE_SHIFT, 0, 0);
 		}
-		agp_flush_cache();
 		mem->am_is_bound = 0;
 		mtx_unlock(&sc->agp.as_lock);
 		return (0);
diff --git a/sys/dev/agp/agppriv.h b/sys/dev/agp/agppriv.h
index 00e7dc1..2436af2 100644
--- a/sys/dev/agp/agppriv.h
+++ b/sys/dev/agp/agppriv.h
@@ -83,7 +83,6 @@ struct agp_gatt {
 	vm_offset_t	ag_physical;
 };
 
-void			agp_flush_cache(void);
 u_int8_t		agp_find_caps(device_t dev);
 struct agp_gatt	       *agp_alloc_gatt(device_t dev);
 void			agp_set_aperture_resource(device_t dev, int rid);
diff --git a/sys/dev/agp/agpvar.h b/sys/dev/agp/agpvar.h
index 5aeebc9..2c2c87a 100644
--- a/sys/dev/agp/agpvar.h
+++ b/sys/dev/agp/agpvar.h
@@ -122,6 +122,19 @@ int agp_unbind_memory(device_t dev, void *handle);
  */
 void agp_memory_info(device_t dev, void *handle, struct agp_memory_info *mi);
 
+/*
+ * Bind a set of pages at a given offset within the AGP aperture.
+ * Returns EINVAL if the given size or offset is not at an AGP page boundary.
+ */
+int agp_bind_pages(device_t dev, vm_page_t *pages, vm_size_t size,
+		   vm_offset_t offset);
+
+/*
+ * Unbind a set of pages from the AGP aperture.
+ * Returns EINVAL if the given size or offset is not at an AGP page boundary.
+ */
+int agp_unbind_pages(device_t dev, vm_size_t size, vm_offset_t offset);
+
 #define AGP_NORMAL_MEMORY 0
 
 #define AGP_USER_TYPES (1 << 16)