MFC: 280375

sfxge: add barriers to BAR write macros

In theory the barriers are required to cope with write combining and
reordering. Two barriers are added (sometimes merged to one):
 1. Before the first write to guarantee that previous writes to the region
    have been done
 2. Before the last write to guarantee that write to the last dword/qword is
    done after previous writes
Barriers are inserted before in the assumption that it is better to
postpone barriers as much as it is possible (more chances that the
operation has already been already done and barrier does not stall CPU).

On x86 and amd64 bus space write barriers are just compiler memory barriers
which are definitely required.

Sponsored by:   Solarflare Communications, Inc.
Original Differential Revision: https://reviews.freebsd.org/D2077

1 files changed, 64 insertions, 0 deletions

diff --git a/sys/dev/sfxge/common/efsys.h b/sys/dev/sfxge/common/efsys.h
index a1e1edf..9e9bdc2 100644
--- a/sys/dev/sfxge/common/efsys.h
+++ b/sys/dev/sfxge/common/efsys.h
@@ -786,6 +786,14 @@ typedef struct efsys_bar_s {
 		EFSYS_PROBE2(bar_writed, unsigned int, (_offset),	\
 		    uint32_t, (_edp)->ed_u32[0]);			\
 									\
+		/*							\
+		 * Make sure that previous writes to the dword have	\
+		 * been done. It should be cheaper than barrier just	\
+		 * after the write below.				\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_dword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_4((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset), (_edp)->ed_u32[0]);			\
@@ -809,6 +817,14 @@ typedef struct efsys_bar_s {
 		    uint32_t, (_eqp)->eq_u32[1],			\
 		    uint32_t, (_eqp)->eq_u32[0]);			\
 									\
+		/*							\
+		 * Make sure that previous writes to the qword have	\
+		 * been done. It should be cheaper than barrier just	\
+		 * after the write below.				\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_qword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_8((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset), (_eqp)->eq_u64[0]);			\
@@ -829,9 +845,25 @@ typedef struct efsys_bar_s {
 		    uint32_t, (_eqp)->eq_u32[1],			\
 		    uint32_t, (_eqp)->eq_u32[0]);			\
 									\
+		/*							\
+		 * Make sure that previous writes to the qword have	\
+		 * been done. It should be cheaper than barrier just	\
+		 * after the last write below.				\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_qword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_4((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset), (_eqp)->eq_u32[0]);			\
+		/*							\
+		 * It should be guaranteed that the last dword comes	\
+		 * the last, so barrier entire qword to be sure that	\
+		 * neither above nor below writes are reordered.	\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_qword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_4((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset) + 4, (_eqp)->eq_u32[1]);			\
@@ -858,9 +890,25 @@ typedef struct efsys_bar_s {
 		    uint32_t, (_eop)->eo_u32[1],			\
 		    uint32_t, (_eop)->eo_u32[0]);			\
 									\
+		/*							\
+		 * Make sure that previous writes to the oword have	\
+		 * been done. It should be cheaper than barrier just	\
+		 * after the last write below.				\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_oword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_8((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset), (_eop)->eo_u64[0]);			\
+		/*							\
+		 * It should be guaranteed that the last qword comes	\
+		 * the last, so barrier entire oword to be sure that	\
+		 * neither above nor below writes are reordered.	\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_oword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_8((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset) + 8, (_eop)->eo_u64[1]);			\
@@ -888,6 +936,14 @@ typedef struct efsys_bar_s {
 		    uint32_t, (_eop)->eo_u32[1],			\
 		    uint32_t, (_eop)->eo_u32[0]);			\
 									\
+		/*							\
+		 * Make sure that previous writes to the oword have	\
+		 * been done. It should be cheaper than barrier just	\
+		 * after the last write below.				\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_oword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_4((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset), (_eop)->eo_u32[0]);			\
@@ -897,6 +953,14 @@ typedef struct efsys_bar_s {
 		bus_space_write_stream_4((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset) + 8, (_eop)->eo_u32[2]);			\
+		/*							\
+		 * It should be guaranteed that the last dword comes	\
+		 * the last, so barrier entire oword to be sure that	\
+		 * neither above nor below writes are reordered.	\
+		 */							\
+		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
+		    (_offset), sizeof (efx_oword_t),			\
+		    BUS_SPACE_BARRIER_WRITE);				\
 		bus_space_write_stream_4((_esbp)->esb_tag,		\
 		    (_esbp)->esb_handle,				\
 		    (_offset) + 12, (_eop)->eo_u32[3]);			\