Clean up fast message ring code for XLR.

Fix message ring send path:
- define msgrng_access_enable() which disables local interrupts
  and enables message ring access. Also define msgrng_restore() which
  restores interrupts
- remove all other msgrng enable/disable macros, no need of critical_enter
  and other locking here.
- message_send() fixup: re-read status until pending bit clears
- message_send_retry() fixup: retry only few times with interrupts disabled
- Fix up message_send/message_send_retry callers - call
  msgrng_access_enable() and msgrng_restore() correctly so that interrupts
  are not disabled for long.
- removed unused and obsolete code from sys/mips/rmi/msgring.h
- some style fixes - more later

rge.c (XLR GMAC driver):
- updated for the message ring changes
- remove unused message_send_block()
- retry on credit failure, this is not a permanent failure when credits
  are configured correctly. Add panic if credits are not available to
  send for a long time.

3 files changed, 116 insertions, 227 deletions

diff --git a/sys/mips/rmi/dev/xlr/rge.c b/sys/mips/rmi/dev/xlr/rge.c
index b021cee..924031a 100644
--- a/sys/mips/rmi/dev/xlr/rge.c
+++ b/sys/mips/rmi/dev/xlr/rge.c
@@ -711,21 +711,27 @@ static __inline__ int
 xlr_mac_send_fr(struct driver_data *priv,
     vm_paddr_t addr, int len)
 {
-	int stid = priv->rfrbucket;
 	struct msgrng_msg msg;
-	int vcpu = xlr_cpu_id();
+	int stid = priv->rfrbucket;
+	int i = 0, code, ret;
+	uint32_t msgrng_flags;
 
 	mac_make_desc_rfr(&msg, addr);
 
 	/* Send the packet to MAC */
 	dbg_msg("mac_%d: Sending free packet %lx to stid %d\n",
 	    priv->instance, (u_long)addr, stid);
-	if (priv->type == XLR_XGMAC) {
-		while (message_send(1, MSGRNG_CODE_XGMAC, stid, &msg));
-	} else {
-		while (message_send(1, MSGRNG_CODE_MAC, stid, &msg));
-		xlr_rge_repl_done[vcpu]++;
-	}
+	if (priv->type == XLR_XGMAC)
+		code = MSGRNG_CODE_XGMAC;        /* WHY? */
+	else
+		code = MSGRNG_CODE_MAC;
+
+	do {
+		msgrng_flags = msgrng_access_enable();
+		ret = message_send_retry(1, code, stid, &msg);
+		msgrng_restore(msgrng_flags);
+		KASSERT(i++ < 100000, ("Too many credit fails\n"));
+	} while (ret != 0);
 
 	return 0;
 }
@@ -1439,37 +1445,16 @@ rmi_xlr_mac_set_duplex(struct driver_data *s,
 #define MAC_TX_PASS 0
 #define MAC_TX_RETRY 1
 
-static __inline__ void
-message_send_block(unsigned int size, unsigned int code,
-    unsigned int stid, struct msgrng_msg *msg)
-{
-	unsigned int dest = 0;
-	unsigned long long status = 0;
-
-	msgrng_load_tx_msg0(msg->msg0);
-	msgrng_load_tx_msg1(msg->msg1);
-	msgrng_load_tx_msg2(msg->msg2);
-	msgrng_load_tx_msg3(msg->msg3);
-
-	dest = ((size - 1) << 16) | (code << 8) | (stid);
-
-	do {
-		msgrng_send(dest);
-		status = msgrng_read_status();
-	} while (status & 0x6);
-
-}
-
 int xlr_dev_queue_xmit_hack = 0;
 
 static int
 mac_xmit(struct mbuf *m, struct rge_softc *sc,
     struct driver_data *priv, int len, struct p2d_tx_desc *tx_desc)
 {
-	struct msgrng_msg msg;
+	struct msgrng_msg msg = {0,0,0,0};
 	int stid = priv->txbucket;
 	uint32_t tx_cycles = 0;
-	unsigned long mflags = 0;
+	uint32_t mflags;
 	int vcpu = xlr_cpu_id();
 	int rv;
 
@@ -1479,17 +1464,17 @@ mac_xmit(struct mbuf *m, struct rge_softc *sc,
 		return MAC_TX_FAIL;
 
 	else {
-		msgrng_access_enable(mflags);
+		mflags = msgrng_access_enable();
 		if ((rv = message_send_retry(1, MSGRNG_CODE_MAC, stid, &msg)) != 0) {
 			msg_snd_failed++;
-			msgrng_access_disable(mflags);
+			msgrng_restore(mflags);
 			release_tx_desc(&msg, 0);
 			xlr_rge_msg_snd_failed[vcpu]++;
 			dbg_msg("Failed packet to cpu %d, rv = %d, stid %d, msg0=%jx\n",
 			    vcpu, rv, stid, (uintmax_t)msg.msg0);
 			return MAC_TX_FAIL;
 		}
-		msgrng_access_disable(mflags);
+		msgrng_restore(mflags);
 		port_inc_counter(priv->instance, PORT_TX);
 	}
 
@@ -1559,7 +1544,6 @@ mac_frin_replenish(void *args /* ignored */ )
 
 		for (i = 0; i < XLR_MAX_MACS; i++) {
 			/* int offset = 0; */
-			unsigned long msgrng_flags;
 			void *m;
 			uint32_t cycles;
 			struct rge_softc *sc;
@@ -1592,14 +1576,11 @@ mac_frin_replenish(void *args /* ignored */ )
 				}
 			}
 			xlr_inc_counter(REPLENISH_FRIN);
-			msgrng_access_enable(msgrng_flags);
 			if (xlr_mac_send_fr(priv, vtophys(m), MAX_FRAME_SIZE)) {
 				free_buf(vtophys(m));
 				printf("[%s]: rx free message_send failed!\n", __FUNCTION__);
-				msgrng_access_disable(msgrng_flags);
 				break;
 			}
-			msgrng_access_disable(msgrng_flags);
 			xlr_set_counter(REPLENISH_CYCLES,
 			    (read_c0_count() - cycles));
 			atomic_subtract_int((&priv->frin_to_be_sent[cpu]), 1);
@@ -2427,7 +2408,6 @@ static int
 rmi_xlr_mac_fill_rxfr(struct rge_softc *sc)
 {
 	struct driver_data *priv = &(sc->priv);
-	unsigned long msgrng_flags;
 	int i;
 	int ret = 0;
 	void *ptr;
@@ -2445,9 +2425,7 @@ rmi_xlr_mac_fill_rxfr(struct rge_softc *sc)
 			break;
 		}
 		/* Send the free Rx desc to the MAC */
-		msgrng_access_enable(msgrng_flags);
 		xlr_mac_send_fr(priv, vtophys(ptr), MAX_FRAME_SIZE);
-		msgrng_access_disable(msgrng_flags);
 	}
 
 	return ret;
diff --git a/sys/mips/rmi/fmn.c b/sys/mips/rmi/fmn.c
index 31cb2f0..f10d3ff 100644
--- a/sys/mips/rmi/fmn.c
+++ b/sys/mips/rmi/fmn.c
@@ -115,7 +115,7 @@ xlr_msgring_cpu_init(void)
 	struct stn_cc *cc_config;
 	struct bucket_size *bucket_sizes;
 	int id;
-	unsigned long flags;
+	uint32_t flags;
 
 	KASSERT(xlr_thr_id() == 0,
 		("xlr_msgring_cpu_init from non-zero thread\n"));
@@ -125,13 +125,14 @@ xlr_msgring_cpu_init(void)
 	bucket_sizes = xlr_board_info.bucket_sizes;
 	cc_config = xlr_board_info.credit_configs[id];
 
-	msgrng_flags_save(flags);
 
 	/*
 	 * Message Stations are shared among all threads in a cpu core
 	 * Assume, thread 0 on all cores are always active when more than 1
 	 * thread is active in a core
 	 */
+	flags = msgrng_access_enable();
+
 	msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]);
 	msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]);
 	msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]);
@@ -158,7 +159,7 @@ xlr_msgring_cpu_init(void)
 	MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters);
 	MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters);
 
-	msgrng_flags_restore(flags);
+	msgrng_restore(flags);
 }
 
 void 
@@ -183,8 +184,7 @@ xlr_msgring_handler(struct trapframe *tf)
 	unsigned int bucket_empty_bm = 0;
 	unsigned int status = 0;
 
-	/* TODO: not necessary to disable preemption */
-	msgrng_flags_save(mflags);
+	mflags = msgrng_access_enable();
 
 	/* First Drain all the high priority messages */
 	for (;;) {
@@ -210,39 +210,37 @@ xlr_msgring_handler(struct trapframe *tf)
 				    __FUNCTION__, tx_stid, bucket, size, (uintmax_t)msg.msg0);
 			} else {
 				//printf("[%s]: rx_stid = %d\n", __FUNCTION__, rx_stid);
-				msgrng_flags_restore(mflags);
+				msgrng_restore(mflags);
 				(*tx_stn_handlers[tx_stid].action) (bucket, size, code, rx_stid,
 				    &msg, tx_stn_handlers[tx_stid].dev_id);
-				msgrng_flags_save(mflags);
+				mflags = msgrng_access_enable();
 			}
 		}
 	}
-	msgrng_flags_restore(mflags);
+	msgrng_restore(mflags);
 }
 
 void 
 enable_msgring_int(void *arg)
 {
-	unsigned long mflags = 0;
+	uint32_t config, mflags;
 
-	msgrng_access_save(&msgrng_lock, mflags);
-	/* enable the message ring interrupts */
-	msgrng_write_config((msgring_watermark_count << 24) | (IRQ_MSGRING << 16)
-	    | (msgring_thread_mask << 8) | msgring_int_type);
-	msgrng_access_restore(&msgrng_lock, mflags);
+	config = (msgring_watermark_count << 24) | (IRQ_MSGRING << 16) |
+	    (msgring_thread_mask << 8) | msgring_int_type;
+	mflags = msgrng_access_enable();
+	msgrng_write_config(config);
+	msgrng_restore(mflags);
 }
 
 void 
 disable_msgring_int(void *arg)
 {
-	unsigned long mflags = 0;
-	uint32_t config;
+	uint32_t config, mflags;
 
-	msgrng_access_save(&msgrng_lock, mflags);
-	config = msgrng_read_config();
-	config &= ~0x3;
+	mflags = msgrng_access_enable();
+	config = msgrng_read_config() & ~0x3;
 	msgrng_write_config(config);
-	msgrng_access_restore(&msgrng_lock, mflags);
+	msgrng_restore(mflags);
 }
 
 static int
diff --git a/sys/mips/rmi/msgring.h b/sys/mips/rmi/msgring.h
index 6986739..d524892 100644
--- a/sys/mips/rmi/msgring.h
+++ b/sys/mips/rmi/msgring.h
@@ -32,54 +32,36 @@
 #ifndef _RMI_MSGRING_H_
 #define _RMI_MSGRING_H_
 
+#include <sys/param.h>
+#include <sys/systm.h>
 #include <sys/types.h>
-#include <mips/rmi/rmi_mips_exts.h>
-
-#define MSGRNG_TX_BUF_REG 0
-#define MSGRNG_RX_BUF_REG 1
-
-#define MSGRNG_MSG_STATUS_REG 2
-#define MSGRNG_MSG_CONFIG_REG 3
-
-#define MSGRNG_MSG_BUCKSIZE_REG 4
-
-#define MSGRNG_CC_0_REG  16
-#define MSGRNG_CC_1_REG  17
-#define MSGRNG_CC_2_REG  18
-#define MSGRNG_CC_3_REG  19
-#define MSGRNG_CC_4_REG  20
-#define MSGRNG_CC_5_REG  21
-#define MSGRNG_CC_6_REG  22
-#define MSGRNG_CC_7_REG  23
-#define MSGRNG_CC_8_REG  24
-#define MSGRNG_CC_9_REG  25
-#define MSGRNG_CC_10_REG 26
-#define MSGRNG_CC_11_REG 27
-#define MSGRNG_CC_12_REG 28
-#define MSGRNG_CC_13_REG 29
-#define MSGRNG_CC_14_REG 30
-#define MSGRNG_CC_15_REG 31
-
-#define msgrng_read_status() read_c2_register32(MSGRNG_MSG_STATUS_REG, 0)
-
-#define msgrng_read_config() read_c2_register32(MSGRNG_MSG_CONFIG_REG, 0)
-#define msgrng_write_config(value) write_c2_register32(MSGRNG_MSG_CONFIG_REG, 0, value)
-
-#define msgrng_read_bucksize(bucket) read_c2_register32(MSGRNG_MSG_BUCKSIZE_REG, bucket)
-#define msgrng_write_bucksize(bucket, value) write_c2_register32(MSGRNG_MSG_BUCKSIZE_REG, bucket, value)
 
-#define msgrng_read_cc(reg, pri) read_c2_register32(reg, pri)
-#define msgrng_write_cc(reg, value, pri) write_c2_register32(reg, pri, value)
+#include <machine/cpuregs.h>
+#include <machine/cpufunc.h>
+#include <mips/rmi/rmi_mips_exts.h>
 
-#define msgrng_load_rx_msg0() read_c2_register64(MSGRNG_RX_BUF_REG, 0)
-#define msgrng_load_rx_msg1() read_c2_register64(MSGRNG_RX_BUF_REG, 1)
-#define msgrng_load_rx_msg2() read_c2_register64(MSGRNG_RX_BUF_REG, 2)
-#define msgrng_load_rx_msg3() read_c2_register64(MSGRNG_RX_BUF_REG, 3)
-
-#define msgrng_load_tx_msg0(value) write_c2_register64(MSGRNG_TX_BUF_REG, 0, value)
-#define msgrng_load_tx_msg1(value) write_c2_register64(MSGRNG_TX_BUF_REG, 1, value)
-#define msgrng_load_tx_msg2(value) write_c2_register64(MSGRNG_TX_BUF_REG, 2, value)
-#define msgrng_load_tx_msg3(value) write_c2_register64(MSGRNG_TX_BUF_REG, 3, value)
+#define	MSGRNG_TX_BUF_REG	0
+#define	MSGRNG_RX_BUF_REG	1
+#define	MSGRNG_MSG_STATUS_REG	2
+#define	MSGRNG_MSG_CONFIG_REG	3
+#define MSGRNG_MSG_BUCKSIZE_REG	4
+
+#define	MSGRNG_CC_0_REG	16
+#define	MSGRNG_CC_1_REG	17
+#define	MSGRNG_CC_2_REG	18
+#define	MSGRNG_CC_3_REG	19
+#define	MSGRNG_CC_4_REG	20
+#define	MSGRNG_CC_5_REG	21
+#define	MSGRNG_CC_6_REG	22
+#define	MSGRNG_CC_7_REG	23
+#define	MSGRNG_CC_8_REG	24
+#define	MSGRNG_CC_9_REG	25
+#define	MSGRNG_CC_10_REG	26
+#define	MSGRNG_CC_11_REG	27
+#define	MSGRNG_CC_12_REG	28
+#define	MSGRNG_CC_13_REG	29
+#define	MSGRNG_CC_14_REG	30
+#define	MSGRNG_CC_15_REG	31
 
 /* Station IDs */
 #define MSGRNG_STNID_CPU0  0x00
@@ -189,54 +171,26 @@
 #define MSGRNG_CODE_SEC 0
 #define MSGRNG_CODE_BOOT_WAKEUP 200
 #define MSGRNG_CODE_SPI4 3
+#define msgrng_read_status() read_c2_register32(MSGRNG_MSG_STATUS_REG, 0)
 
-static inline int 
-msgrng_xgmac_stid_rfr(int id)
-{
-	return !id ? MSGRNG_STNID_XMAC0RFR : MSGRNG_STNID_XMAC1RFR;
-}
-
-static inline int 
-msgrng_xgmac_stid_jfr(int id)
-{
-	return !id ? MSGRNG_STNID_XMAC0JFR : MSGRNG_STNID_XMAC1JFR;
-}
-
-static inline int 
-msgrng_xgmac_stid_tx(int id)
-{
-	return !id ? MSGRNG_STNID_XMAC0_00_TX : MSGRNG_STNID_XMAC1_00_TX;
-}
-
-static inline int 
-msgrng_gmac_stid_rfr(int id)
-{
-	return (MSGRNG_STNID_GMACRFR_0);
-}
+#define msgrng_read_config() read_c2_register32(MSGRNG_MSG_CONFIG_REG, 0)
+#define msgrng_write_config(value) write_c2_register32(MSGRNG_MSG_CONFIG_REG, 0, value)
 
-static inline int 
-msgrng_gmac_stid_rfr_split_mode(int id)
-{
-	return ((id >> 1) ? MSGRNG_STNID_GMACRFR_1 : MSGRNG_STNID_GMACRFR_0);
-}
+#define msgrng_read_bucksize(bucket) read_c2_register32(MSGRNG_MSG_BUCKSIZE_REG, bucket)
+#define msgrng_write_bucksize(bucket, value) write_c2_register32(MSGRNG_MSG_BUCKSIZE_REG, bucket, value)
 
-static inline int 
-msgrng_gmac_stid_jfr(int id)
-{
-	return MSGRNG_STNID_GMACJFR_0;
-}
+#define msgrng_read_cc(reg, pri) read_c2_register32(reg, pri)
+#define msgrng_write_cc(reg, value, pri) write_c2_register32(reg, pri, value)
 
-static inline int 
-msgrng_gmac_stid_jfr_split_mode(int id)
-{
-	return ((id >> 1) ? MSGRNG_STNID_GMACJFR_1 : MSGRNG_STNID_GMACJFR_0);
-}
+#define msgrng_load_rx_msg0() read_c2_register64(MSGRNG_RX_BUF_REG, 0)
+#define msgrng_load_rx_msg1() read_c2_register64(MSGRNG_RX_BUF_REG, 1)
+#define msgrng_load_rx_msg2() read_c2_register64(MSGRNG_RX_BUF_REG, 2)
+#define msgrng_load_rx_msg3() read_c2_register64(MSGRNG_RX_BUF_REG, 3)
 
-static inline int 
-msgrng_gmac_stid_tx(int id)
-{
-	return (MSGRNG_STNID_GMACTX0 + id);
-}
+#define msgrng_load_tx_msg0(value) write_c2_register64(MSGRNG_TX_BUF_REG, 0, value)
+#define msgrng_load_tx_msg1(value) write_c2_register64(MSGRNG_TX_BUF_REG, 1, value)
+#define msgrng_load_tx_msg2(value) write_c2_register64(MSGRNG_TX_BUF_REG, 2, value)
+#define msgrng_load_tx_msg3(value) write_c2_register64(MSGRNG_TX_BUF_REG, 3, value)
 
 static inline void 
 msgrng_send(unsigned int stid)
@@ -280,30 +234,21 @@ msgrng_wait(unsigned int mask)
 	);
 }
 
-#define msgrng_enable(flags)                        \
-do {                                                \
-  __asm__ volatile (                                \
-		    ".set push\n\t"                 \
-		    ".set reorder\n\t"              \
-		    ".set noat\n\t"                 \
-		    "mfc0 %0, $12\n\t"              \
-		    "li  $8, 0x40000001\n\t"        \
-		    "or  $1, %0, $8\n\t"            \
-		    "xori $1, 1\n\t"                \
-		    ".set noreorder\n\t"            \
-		    "mtc0 $1, $12\n\t"              \
-		    ".set\tpop\n\t"                 \
-		    : "=r" (flags)                  \
-		    :                               \
-		    : "$8"                          \
-		    );                              \
-} while (0)
-
-#define msgrng_disable(flags) __asm__ volatile (    \
-                 "mtc0 %0, $12" : : "r" (flags))
-
-#define msgrng_flags_save(flags) msgrng_enable(flags)
-#define msgrng_flags_restore(flags) msgrng_disable(flags)
+static __inline uint32_t
+msgrng_access_enable(void)
+{
+	uint32_t sr = mips_rd_status();
+
+	mips_wr_status((sr & ~MIPS_SR_INT_IE) | MIPS_SR_COP_2_BIT);
+	return (sr);
+}
+
+static __inline void
+msgrng_restore(uint32_t sr)
+{
+
+	mips_wr_status(sr);
+}
 
 struct msgrng_msg {
 	__uint64_t msg0;
@@ -355,7 +300,7 @@ message_send_block_fast(int size, unsigned int code, unsigned int stid,
            _tmp;                                                \
         } )
 
-static __inline__ int 
+static __inline int 
 message_send(unsigned int size, unsigned int code,
     unsigned int stid, struct msgrng_msg *msg)
 {
@@ -369,42 +314,35 @@ message_send(unsigned int size, unsigned int code,
 	msgrng_load_tx_msg3(msg->msg3);
 
 	dest = ((size - 1) << 16) | (code << 8) | (stid);
-
 	msgrng_send(dest);
 
-	for (i = 0; i < 16; i++) {
+	/* Wait for the thread pending to clear */
+	do {
 		status = msgrng_read_status();
+		KASSERT(i++ < 10000, ("Too many fails\n"));
+	} while ((status & 0x2) != 0);
 
-		if (status & 0x6) {
-			continue;
-		} else
-			break;
-	}
-	return msgrng_read_status() & 0x06;
+	/* If there is a credit failure, return error */
+	return status & 0x06;
 }
 
-static __inline__ int 
+static __inline int 
 message_send_retry(unsigned int size, unsigned int code,
     unsigned int stid, struct msgrng_msg *msg)
 {
-	int res = 0;
-	int retry = 0;
-
-	for (;;) {
-		res = message_send(size, code, stid, msg);
-		/* retry a pending fail */
-		if (res & 0x02)
-			continue;
-		/* credit fail */
-		if (res & 0x04)
-			retry++;
-		else
-			break;
-		if (retry == 4)
-			return res & 0x06;
+	int i, ret;
+
+	/*
+	 * we are in with interrupt disabled, retrying too many 
+	 * times is not good
+	 */
+	for (i = 0; i < 16; i++) {
+		ret = message_send(size, code, stid, msg);
+		if (ret == 0)
+			return (0);
 	}
 
-	return 0;
+	return (1);
 }
 
 static __inline__ int 
@@ -457,27 +395,6 @@ extern struct stn_cc xls_cc_table_pcie;
 extern struct stn_cc xls_cc_table_dma;
 extern struct stn_cc xls_cc_table_sec;
 
-
-#define msgrng_access_save(lock, mflags) do {                \
-  mtx_lock_spin(lock);                                       \
-  msgrng_flags_save(mflags);                                 \
- }while(0)
-
-#define msgrng_access_restore(lock, mflags) do {             \
-  msgrng_flags_restore(mflags);                              \
-  mtx_unlock_spin(lock);                                     \
- }while(0)
-
-#define msgrng_access_enable(mflags) do {   \
-  critical_enter();                         \
-  msgrng_flags_save(mflags);                \
-} while(0)
-
-#define msgrng_access_disable(mflags) do {   \
-  msgrng_flags_restore(mflags);              \
-  critical_exit();                           \
-} while(0)
-
 /*
  * NOTE: this is not stationid/8, ie the station numbers below are just
  * for internal use
@@ -504,14 +421,10 @@ enum {
 	MAX_TX_STNS
 };
 
-extern int 
-register_msgring_handler(int major,
+extern int register_msgring_handler(int major,
     void (*action) (int, int, int, int, struct msgrng_msg *, void *),
     void *dev_id);
-	extern void xlr_msgring_cpu_init(void);
-
-	extern void xlr_msgring_config(void);
-
-#define cpu_to_msgring_bucket(cpu) ((((cpu) >> 2)<<3)|((cpu) & 0x03))
+extern void xlr_msgring_cpu_init(void);
+extern void xlr_msgring_config(void);
 
 #endif