Add support to the virtual memory system for configuring machine-

dependent memory attributes:

Rename vm_cache_mode_t to vm_memattr_t.  The new name reflects the
fact that there are machine-dependent memory attributes that have
nothing to do with controlling the cache's behavior.

Introduce vm_object_set_memattr() for setting the default memory
attributes that will be given to an object's pages.

Introduce and use pmap_page_{get,set}_memattr() for getting and
setting a page's machine-dependent memory attributes.  Add full
support for these functions on amd64 and i386 and stubs for them on
the other architectures.  The function pmap_page_set_memattr() is also
responsible for any other machine-dependent aspects of changing a
page's memory attributes, such as flushing the cache or updating the
direct map.  The uses include kmem_alloc_contig(), vm_page_alloc(),
and the device pager:

  kmem_alloc_contig() can now be used to allocate kernel memory with
  non-default memory attributes on amd64 and i386.

  vm_page_alloc() and the device pager will set the memory attributes
  for the real or fictitious page according to the object's default
  memory attributes.

Update the various pmap functions on amd64 and i386 that map pages to
incorporate each page's memory attributes in the mapping.

Notes: (1) Inherent to this design are safety features that prevent
the specification of inconsistent memory attributes by different
mappings on amd64 and i386.  In addition, the device pager provides a
warning when a device driver creates a fictitious page with memory
attributes that are inconsistent with the real page that the
fictitious page is an alias for. (2) Storing the machine-dependent
memory attributes for amd64 and i386 as a dedicated "int" in "struct
md_page" represents a compromise between space efficiency and the ease
of MFCing these changes to RELENG_7.

In collaboration with: jhb

Approved by:	re (kib)

1 files changed, 43 insertions, 26 deletions

diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
index 69d6dca..7993405 100644
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -70,9 +70,9 @@ static struct mtx dev_pager_mtx;
 
 static uma_zone_t fakepg_zone;
 
-static vm_page_t dev_pager_getfake(vm_paddr_t);
+static vm_page_t dev_pager_getfake(vm_paddr_t, vm_memattr_t);
 static void dev_pager_putfake(vm_page_t);
-static void dev_pager_updatefake(vm_page_t, vm_paddr_t);
+static void dev_pager_updatefake(vm_page_t, vm_paddr_t, vm_memattr_t);
 
 struct pagerops devicepagerops = {
 	.pgo_init =	dev_pager_init,
@@ -210,7 +210,8 @@ dev_pager_getpages(object, m, count, reqpage)
 {
 	vm_pindex_t offset;
 	vm_paddr_t paddr;
-	vm_page_t page;
+	vm_page_t m_paddr, page;
+	vm_memattr_t memattr;
 	struct cdev *dev;
 	int i, ret;
 	int prot;
@@ -222,6 +223,7 @@ dev_pager_getpages(object, m, count, reqpage)
 	dev = object->handle;
 	page = m[reqpage];
 	offset = page->pindex;
+	memattr = object->memattr;
 	VM_OBJECT_UNLOCK(object);
 	csw = dev_refthread(dev);
 	if (csw == NULL)
@@ -235,14 +237,20 @@ dev_pager_getpages(object, m, count, reqpage)
 	KASSERT(ret == 0, ("dev_pager_getpage: map function returns error"));
 	td->td_fpop = fpop;
 	dev_relthread(dev);
-
+	/* If "paddr" is a real page, perform a sanity check on "memattr". */
+	if ((m_paddr = vm_phys_paddr_to_vm_page(paddr)) != NULL &&
+	    pmap_page_get_memattr(m_paddr) != memattr) {
+		memattr = pmap_page_get_memattr(m_paddr);
+		printf(
+	    "WARNING: A device driver has set \"memattr\" inconsistently.\n");
+	}
 	if ((page->flags & PG_FICTITIOUS) != 0) {
 		/*
 		 * If the passed in reqpage page is a fake page, update it with
 		 * the new physical address.
 		 */
 		VM_OBJECT_LOCK(object);
-		dev_pager_updatefake(page, paddr);
+		dev_pager_updatefake(page, paddr, memattr);
 		if (count > 1) {
 			vm_page_lock_queues();
 			for (i = 0; i < count; i++) {
@@ -256,7 +264,7 @@ dev_pager_getpages(object, m, count, reqpage)
 		 * Replace the passed in reqpage page with our own fake page and
 		 * free up the all of the original pages.
 		 */
-		page = dev_pager_getfake(paddr);
+		page = dev_pager_getfake(paddr, memattr);
 		VM_OBJECT_LOCK(object);
 		TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, page, pageq);
 		vm_page_lock_queues();
@@ -296,47 +304,56 @@ dev_pager_haspage(object, pindex, before, after)
 }
 
 /*
- * Instantiate a fictitious page.  Unlike physical memory pages, only
- * the machine-independent fields must be initialized.
+ * Create a fictitious page with the specified physical address and memory
+ * attribute.
  */
 static vm_page_t
-dev_pager_getfake(paddr)
-	vm_paddr_t paddr;
+dev_pager_getfake(vm_paddr_t paddr, vm_memattr_t memattr)
 {
 	vm_page_t m;
 
-	m = uma_zalloc(fakepg_zone, M_WAITOK);
-
+	m = uma_zalloc(fakepg_zone, M_WAITOK | M_ZERO);
+	m->phys_addr = paddr;
+	/* Fictitious pages don't use "segind". */
 	m->flags = PG_FICTITIOUS;
+	/* Fictitious pages don't use "order" or "pool". */
+	pmap_page_init(m);
 	m->oflags = VPO_BUSY;
-	/* Fictitious pages don't use "act_count". */
-	m->dirty = 0;
-	m->busy = 0;
-	m->queue = PQ_NONE;
-	m->object = NULL;
-
 	m->wire_count = 1;
-	m->hold_count = 0;
-	m->phys_addr = paddr;
-
+	if (memattr != VM_MEMATTR_DEFAULT)
+		pmap_page_set_memattr(m, memattr);
 	return (m);
 }
 
+/*
+ * Release a fictitious page.
+ */
 static void
-dev_pager_putfake(m)
-	vm_page_t m;
+dev_pager_putfake(vm_page_t m)
 {
+
 	if (!(m->flags & PG_FICTITIOUS))
 		panic("dev_pager_putfake: bad page");
+	/* Restore the default memory attribute to "phys_addr". */
+	if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
+		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
 	uma_zfree(fakepg_zone, m);
 }
 
+/*
+ * Update the given fictitious page to the specified physical address and
+ * memory attribute.
+ */
 static void
-dev_pager_updatefake(m, paddr)
-	vm_page_t m;
-	vm_paddr_t paddr;
+dev_pager_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
 {
+
 	if (!(m->flags & PG_FICTITIOUS))
 		panic("dev_pager_updatefake: bad page");
+	/* Restore the default memory attribute before changing "phys_addr". */
+	if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
+		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
 	m->phys_addr = paddr;
+	if (memattr != VM_MEMATTR_DEFAULT)
+		pmap_page_set_memattr(m, memattr);
 }