br_prod_tail and br_cons_tail members are used as barrier to

signal bug_ring ownership. However, instructions can be reordered
around members write leading to stale values for ie. br_prod_bufs.

Use correct memory barriers to ensure proper ordering of the
ownership tokens updates.

Sponsored by:	EMC / Isilon storage division
MFC after:	2 weeks

1 files changed, 5 insertions, 5 deletions

diff --git a/sys/sys/buf_ring.h b/sys/sys/buf_ring.h
index 9f99fa2..b33fc47 100644
--- a/sys/sys/buf_ring.h
+++ b/sys/sys/buf_ring.h
@@ -114,10 +114,10 @@ buf_ring_enqueue(struct buf_ring *br, void *buf)
 	 * that preceeded us, we need to wait for them
 	 * to complete 
 	 */   
-	while (br->br_prod_tail != prod_head)
+	while (atomic_load_acq_32(&br->br_prod_tail) != prod_head)
 		cpu_spinwait();
 	br->br_prod_bufs++;
-	br->br_prod_tail = prod_next;
+	atomic_store_rel_32(&br->br_prod_tail, prod_next);
 	critical_exit();
 	return (0);
 }
@@ -161,10 +161,10 @@ buf_ring_dequeue_mc(struct buf_ring *br)
 	 * that preceeded us, we need to wait for them
 	 * to complete 
 	 */   
-	while (br->br_cons_tail != cons_head)
+	while (atomic_load_acq_32(&br->br_cons_tail) != cons_head)
 		cpu_spinwait();
 
-	br->br_cons_tail = cons_next;
+	atomic_store_rel_32(&br->br_cons_tail, cons_next);
 	critical_exit();
 
 	return (buf);
@@ -209,7 +209,7 @@ buf_ring_dequeue_sc(struct buf_ring *br)
 		panic("inconsistent list cons_tail=%d cons_head=%d",
 		    br->br_cons_tail, cons_head);
 #endif
-	br->br_cons_tail = cons_next;
+	atomic_store_rel_32(&br->br_cons_tail, cons_next);
 	return (buf);
 }