Use the trick of performing the atomic operation on the contained aligned

word to handle the dirty mask updates in vm_page_clear_dirty_mask().
Remove the vm page queue lock around vm_page_dirty() call in vm_fault_hold()
the sole purpose of which was to protect dirty on architectures which
does not provide short or byte-wide atomics.

Reviewed by:	alc, attilio
Tested by:	flo (sparc64)
MFC after:	2 weeks

3 files changed, 50 insertions, 49 deletions

diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 1b8ac2f..8c98b26 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -1090,18 +1090,10 @@ vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
 			 * performed through an unmanaged mapping or by a DMA
 			 * operation.
 			 *
-			 * The object lock is not held here.  Therefore, like
-			 * a pmap operation, the page queues lock may be
-			 * required in order to call vm_page_dirty().  See
-			 * vm_page_clear_dirty_mask().
+			 * The object lock is not held here.
+			 * See vm_page_clear_dirty_mask().
 			 */
-#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
 			vm_page_dirty(*mp);
-#else
-			vm_page_lock_queues();
-			vm_page_dirty(*mp);
-			vm_page_unlock_queues();
-#endif
 		}
 	}
 	if (pmap_failed) {
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 341c238..fe829a3 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -745,9 +745,9 @@ vm_page_sleep(vm_page_t m, const char *msg)
  *
  *	Set all bits in the page's dirty field.
  *
- *	The object containing the specified page must be locked if the call is
- *	made from the machine-independent layer.  If, however, the call is
- *	made from the pmap layer, then the page queues lock may be required.
+ *	The object containing the specified page must be locked if the
+ *	call is made from the machine-independent layer.
+ *
  *	See vm_page_clear_dirty_mask().
  */
 void
@@ -2339,44 +2339,53 @@ vm_page_set_valid(vm_page_t m, int base, int size)
 static __inline void
 vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
 {
+	uintptr_t addr;
+#if PAGE_SIZE < 16384
+	int shift;
+#endif
 
 	/*
 	 * If the object is locked and the page is neither VPO_BUSY nor
 	 * PGA_WRITEABLE, then the page's dirty field cannot possibly be
-	 * set by a concurrent pmap operation. 
+	 * set by a concurrent pmap operation.
+	 *
 	 */
 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 	if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0)
 		m->dirty &= ~pagebits;
 	else {
-#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
 		/*
-		 * On the aforementioned architectures, the page queues lock
-		 * is not required by the following read-modify-write
-		 * operation.  The combination of the object's lock and an
-		 * atomic operation suffice.  Moreover, the pmap layer on
-		 * these architectures can call vm_page_dirty() without
-		 * holding the page queues lock.
+		 * The pmap layer can call vm_page_dirty() without
+		 * holding a distinguished lock.  The combination of
+		 * the object's lock and an atomic operation suffice
+		 * to guarantee consistency of the page dirty field.
+		 *
+		 * For PAGE_SIZE == 32768 case, compiler already
+		 * properly aligns the dirty field, so no forcible
+		 * alignment is needed. Only require existence of
+		 * atomic_clear_64 when page size if 32768.
 		 */
-#if PAGE_SIZE == 4096
-		atomic_clear_char(&m->dirty, pagebits);
-#elif PAGE_SIZE == 8192
-		atomic_clear_short(&m->dirty, pagebits);
+		addr = (uintptr_t)&m->dirty;
+#if PAGE_SIZE == 32768
+#error pagebits too short
+		atomic_clear_64((uint64_t *)addr, pagebits);
 #elif PAGE_SIZE == 16384
-		atomic_clear_int(&m->dirty, pagebits);
-#else
-#error "PAGE_SIZE is not supported."
-#endif
-#else
+		atomic_clear_32((uint32_t *)addr, pagebits);
+#else		/* PAGE_SIZE <= 8192 */
 		/*
-		 * Otherwise, the page queues lock is required to ensure that
-		 * a concurrent pmap operation does not set the page's dirty
-		 * field during the following read-modify-write operation.
+		 * Use a trick to perform an 32bit atomic on the
+		 * contained aligned word, to not depend on existence
+		 * of the atomic_clear_{8, 16}.
 		 */
-		vm_page_lock_queues();
-		m->dirty &= ~pagebits;
-		vm_page_unlock_queues();
+		shift = addr & (sizeof(uint32_t) - 1);
+#if BYTE_ORDER == BIG_ENDIAN
+		shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY;
+#else
+		shift *= NBBY;
 #endif
+		addr &= ~(sizeof(uint32_t) - 1);
+		atomic_clear_32((uint32_t *)addr, pagebits << shift);
+#endif		/* PAGE_SIZE */
 	}
 }
 
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index f7f3aed..1218dc8 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -94,21 +94,21 @@
  *	object that the page belongs to (O), the pool lock for the page (P),
  *	or the lock for either the free or paging queues (Q).  If a field is
  *	annotated below with two of these locks, then holding either lock is
- *	sufficient for read access, but both locks are required for write 
+ *	sufficient for read access, but both locks are required for write
  *	access.
  *
- *	In contrast, the synchronization of accesses to the page's dirty field
- *	is machine dependent (M).  In the machine-independent layer, the lock
- *	on the object that the page belongs to must be held in order to
- *	operate on the field.  However, the pmap layer is permitted to set
- *	all bits within the field without holding that lock.  Therefore, if
- *	the underlying architecture does not support atomic read-modify-write
- *	operations on the field's type, then the machine-independent layer
- *	must also hold the page queues lock when performing read-modify-write
- *	operations and the pmap layer must hold the page queues lock when
- *	setting the field.  In the machine-independent layer, the
- *	implementation of read-modify-write operations on the field is
- *	encapsulated in vm_page_clear_dirty_mask().
+ *	In contrast, the synchronization of accesses to the page's
+ *	dirty field is machine dependent (M).  In the
+ *	machine-independent layer, the lock on the object that the
+ *	page belongs to must be held in order to operate on the field.
+ *	However, the pmap layer is permitted to set all bits within
+ *	the field without holding that lock.  If the underlying
+ *	architecture does not support atomic read-modify-write
+ *	operations on the field's type, then the machine-independent
+ *	layer uses 32bit atomic on the aligned 32bit word that
+ *	contains the dirty field.  In the machine-independent layer,
+ *	the implementation of read-modify-write operations on the
+ *	field is encapsulated in vm_page_clear_dirty_mask().
  */
 
 TAILQ_HEAD(pglist, vm_page);