Don't use atomic operations for page table entries and handle access

and R/W emulation aborts under pmap lock.

There were two reasons for using of atomic operations:
(1) the pmap code is based on i386 one where they are used,
(2) there was an idea that access and R/W emulation aborts should be
    handled as quick as possible, without pmap locking.

However, the atomic operations in i386 pmap code are used only because
page table entries may be modified by hardware. At the beginning, we
were not sure that it's the only reason. So even if arm hardware does
not modify them, we did not risk to not use them at that time. Further,
it turns out after some testing that using of pmap lock for access and
R/W emulation aborts does not bring any extra cost and there was no
measurable difference. Thus, we have decided finally to use pmap lock
for all operations on page table entries and so, there is no reason for
atomic operations on them. This makes the code cleaner and safer.

This decision introduce a question if it's safe to use pmap lock for
access and R/W emulation aborts. Anyhow, there may happen two cases in
general:
(A) Aborts while the pmap lock is locked already - this should not
happen as pmap lock is not recursive. However, under pmap lock only
internal kernel data should be accessed and such data should be mapped
with A bit set and NM bit cleared. If double abort happens, then
a mapping of data which has caused it must be fixed.
(B) Aborts while another lock(s) is/are locked - this already can
happen. There is no difference here if it's either access or R/W
emulation abort, or if it's some other abort.

Reviewed by:	kib

1 files changed, 14 insertions, 32 deletions

diff --git a/sys/arm/include/pmap_var.h b/sys/arm/include/pmap_var.h
index 9387acf..34c101d 100644
--- a/sys/arm/include/pmap_var.h
+++ b/sys/arm/include/pmap_var.h
@@ -143,7 +143,8 @@ static __inline void
 pte1_store(pt1_entry_t *pte1p, pt1_entry_t pte1)
 {
 
-	atomic_store_rel_int(pte1p, pte1);
+	dmb();
+	*pte1p = pte1;
 	pte1_sync(pte1p);
 }
 
@@ -158,22 +159,11 @@ static __inline void
 pte1_clear_bit(pt1_entry_t *pte1p, uint32_t bit)
 {
 
-	atomic_clear_int(pte1p, bit);
+	*pte1p &= ~bit;
 	pte1_sync(pte1p);
 }
 
 static __inline boolean_t
-pte1_cmpset(pt1_entry_t *pte1p, pt1_entry_t opte1, pt1_entry_t npte1)
-{
-	boolean_t ret;
-
-	ret = atomic_cmpset_int(pte1p, opte1, npte1);
-	if (ret) pte1_sync(pte1p);
-
-	return (ret);
-}
-
-static __inline boolean_t
 pte1_is_link(pt1_entry_t pte1)
 {
 
@@ -231,7 +221,8 @@ pte1_load_clear(pt1_entry_t *pte1p)
 {
 	pt1_entry_t opte1;
 
-	opte1 = atomic_readandclear_int(pte1p);
+	opte1 = *pte1p;
+	*pte1p = 0;
 	pte1_sync(pte1p);
 	return (opte1);
 }
@@ -240,7 +231,7 @@ static __inline void
 pte1_set_bit(pt1_entry_t *pte1p, uint32_t bit)
 {
 
-	atomic_set_int(pte1p, bit);
+	*pte1p |= bit;
 	pte1_sync(pte1p);
 }
 
@@ -292,7 +283,8 @@ static __inline void
 pte2_store(pt2_entry_t *pte2p, pt2_entry_t pte2)
 {
 
-	atomic_store_rel_int(pte2p, pte2);
+	dmb();
+	*pte2p = pte2;
 	pte2_sync(pte2p);
 }
 
@@ -307,22 +299,11 @@ static __inline void
 pte2_clear_bit(pt2_entry_t *pte2p, uint32_t bit)
 {
 
-	atomic_clear_int(pte2p, bit);
+	*pte2p &= ~bit;
 	pte2_sync(pte2p);
 }
 
 static __inline boolean_t
-pte2_cmpset(pt2_entry_t *pte2p, pt2_entry_t opte2, pt2_entry_t npte2)
-{
-	boolean_t ret;
-
-	ret = atomic_cmpset_int(pte2p, opte2, npte2);
-	if (ret) pte2_sync(pte2p);
-
-	return (ret);
-}
-
-static __inline boolean_t
 pte2_is_dirty(pt2_entry_t pte2)
 {
 
@@ -364,7 +345,8 @@ pte2_load_clear(pt2_entry_t *pte2p)
 {
 	pt2_entry_t opte2;
 
-	opte2 = atomic_readandclear_int(pte2p);
+	opte2 = *pte2p;
+	*pte2p = 0;
 	pte2_sync(pte2p);
 	return (opte2);
 }
@@ -373,7 +355,7 @@ static __inline void
 pte2_set_bit(pt2_entry_t *pte2p, uint32_t bit)
 {
 
-	atomic_set_int(pte2p, bit);
+	*pte2p |= bit;
 	pte2_sync(pte2p);
 }
 
@@ -386,9 +368,9 @@ pte2_set_wired(pt2_entry_t *pte2p, boolean_t wired)
 	 * so pte2_sync() is not needed.
 	 */
 	if (wired)
-		atomic_set_int(pte2p, PTE2_W);
+		*pte2p |= PTE2_W;
 	else
-		atomic_clear_int(pte2p, PTE2_W);
+		*pte2p &= ~PTE2_W;
 }
 
 static __inline vm_paddr_t