diff options
author | arybchik <arybchik@FreeBSD.org> | 2015-03-25 13:18:51 +0000 |
---|---|---|
committer | arybchik <arybchik@FreeBSD.org> | 2015-03-25 13:18:51 +0000 |
commit | ddeba7284fc9139c5460118d137de8ad8742f86d (patch) | |
tree | b3e7ab6fe15f579c6d1514ed60c3cb9ea1cba957 | |
parent | e601483ad6fc9b1299cd92d3eb5d74c30c676cd2 (diff) | |
download | FreeBSD-src-ddeba7284fc9139c5460118d137de8ad8742f86d.zip FreeBSD-src-ddeba7284fc9139c5460118d137de8ad8742f86d.tar.gz |
MFC: 279351
sfxge: expect required init_state on data path and in periodic calls
With the patch applied the number of instruction events is 1% less and
number of mispredicted branch events is 5% less under multistream TCP
traffic load close to line rate.
Sponsored by: Solarflare Communications, Inc.
Approved by: gnn (mentor)
-rw-r--r-- | sys/dev/sfxge/sfxge_ev.c | 10 | ||||
-rw-r--r-- | sys/dev/sfxge/sfxge_intr.c | 2 | ||||
-rw-r--r-- | sys/dev/sfxge/sfxge_port.c | 13 | ||||
-rw-r--r-- | sys/dev/sfxge/sfxge_rx.c | 6 | ||||
-rw-r--r-- | sys/dev/sfxge/sfxge_tx.c | 4 |
5 files changed, 18 insertions, 17 deletions
diff --git a/sys/dev/sfxge/sfxge_ev.c b/sys/dev/sfxge/sfxge_ev.c index 1b1caf7..14ed008 100644 --- a/sys/dev/sfxge/sfxge_ev.c +++ b/sys/dev/sfxge/sfxge_ev.c @@ -98,7 +98,7 @@ sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); - if (rxq->init_state != SFXGE_RXQ_STARTED) + if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) goto done; expected = rxq->pending++ & rxq->ptr_mask; @@ -251,7 +251,7 @@ sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) KASSERT(evq->index == txq->evq_index, ("evq->index != txq->evq_index")); - if (txq->init_state != SFXGE_TXQ_STARTED) + if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) goto done; stop = (id + 1) & txq->ptr_mask; @@ -428,7 +428,7 @@ sfxge_ev_stat_update(struct sfxge_softc *sc) SFXGE_ADAPTER_LOCK(sc); - if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) + if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) goto out; now = ticks; @@ -593,8 +593,8 @@ sfxge_ev_qpoll(struct sfxge_evq *evq) SFXGE_EVQ_LOCK(evq); - if (evq->init_state != SFXGE_EVQ_STARTING && - evq->init_state != SFXGE_EVQ_STARTED) { + if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && + evq->init_state != SFXGE_EVQ_STARTED)) { rc = EINVAL; goto fail; } diff --git a/sys/dev/sfxge/sfxge_intr.c b/sys/dev/sfxge/sfxge_intr.c index e3ea729..acb21d3 100644 --- a/sys/dev/sfxge/sfxge_intr.c +++ b/sys/dev/sfxge/sfxge_intr.c @@ -130,7 +130,7 @@ sfxge_intr_message(void *arg) KASSERT(intr->type == EFX_INTR_MESSAGE, ("intr->type != EFX_INTR_MESSAGE")); - if (intr->state != SFXGE_INTR_STARTED) + if (__predict_false(intr->state != SFXGE_INTR_STARTED)) return; (void)efx_intr_status_message(enp, index, &fatal); diff --git a/sys/dev/sfxge/sfxge_port.c b/sys/dev/sfxge/sfxge_port.c index 95a7efd..e815eb2 100644 --- a/sys/dev/sfxge/sfxge_port.c +++ b/sys/dev/sfxge/sfxge_port.c @@ -50,7 +50,7 @@ sfxge_mac_stat_update(struct sfxge_softc *sc) SFXGE_PORT_LOCK_ASSERT_OWNED(port); - if (port->init_state != SFXGE_PORT_STARTED) { + if (__predict_false(port->init_state != SFXGE_PORT_STARTED)) { rc = 0; goto out; } @@ -179,7 +179,7 @@ sfxge_port_wanted_fc_handler(SYSCTL_HANDLER_ARGS) SFXGE_PORT_LOCK(port); if (port->wanted_fc != fcntl) { - if (port->init_state == SFXGE_PORT_STARTED) + if (__predict_false(port->init_state == SFXGE_PORT_STARTED)) error = efx_mac_fcntl_set(sc->enp, port->wanted_fc, B_TRUE); @@ -210,7 +210,8 @@ sfxge_port_link_fc_handler(SYSCTL_HANDLER_ARGS) port = &sc->port; SFXGE_PORT_LOCK(port); - if (port->init_state == SFXGE_PORT_STARTED && SFXGE_LINK_UP(sc)) + if (__predict_true(port->init_state == SFXGE_PORT_STARTED) && + SFXGE_LINK_UP(sc)) efx_mac_fcntl_get(sc->enp, &wanted_fc, &link_fc); else link_fc = 0; @@ -265,7 +266,7 @@ sfxge_mac_poll_work(void *arg, int npending) SFXGE_PORT_LOCK(port); - if (port->init_state != SFXGE_PORT_STARTED) + if (__predict_false(port->init_state != SFXGE_PORT_STARTED)) goto done; /* This may sleep waiting for MCDI completion */ @@ -332,7 +333,7 @@ sfxge_mac_filter_set(struct sfxge_softc *sc) * lock is held in sleeping thread. Both problems are repeatable * on LAG with LACP proto bring up. */ - if (port->init_state == SFXGE_PORT_STARTED) + if (__predict_true(port->init_state == SFXGE_PORT_STARTED)) rc = sfxge_mac_filter_set_locked(sc); else rc = 0; @@ -456,7 +457,7 @@ sfxge_phy_stat_update(struct sfxge_softc *sc) SFXGE_PORT_LOCK_ASSERT_OWNED(port); - if (port->init_state != SFXGE_PORT_STARTED) { + if (__predict_false(port->init_state != SFXGE_PORT_STARTED)) { rc = 0; goto out; } diff --git a/sys/dev/sfxge/sfxge_rx.c b/sys/dev/sfxge/sfxge_rx.c index fdb7901..6c3b485 100644 --- a/sys/dev/sfxge/sfxge_rx.c +++ b/sys/dev/sfxge/sfxge_rx.c @@ -209,7 +209,7 @@ sfxge_rx_qfill(struct sfxge_rxq *rxq, unsigned int target, boolean_t retrying) SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); - if (rxq->init_state != SFXGE_RXQ_STARTED) + if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) return; rxfill = rxq->added - rxq->completed; @@ -269,7 +269,7 @@ void sfxge_rx_qrefill(struct sfxge_rxq *rxq) { - if (rxq->init_state != SFXGE_RXQ_STARTED) + if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) return; /* Make sure the queue is full */ @@ -760,7 +760,7 @@ sfxge_rx_qcomplete(struct sfxge_rxq *rxq, boolean_t eop) rx_desc = &rxq->queue[id]; m = rx_desc->mbuf; - if (rxq->init_state != SFXGE_RXQ_STARTED) + if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) goto discard; if (rx_desc->flags & (EFX_ADDR_MISMATCH | EFX_DISCARD)) diff --git a/sys/dev/sfxge/sfxge_tx.c b/sys/dev/sfxge/sfxge_tx.c index 72a4b5b..716e64c 100644 --- a/sys/dev/sfxge/sfxge_tx.c +++ b/sys/dev/sfxge/sfxge_tx.c @@ -311,7 +311,7 @@ static int sfxge_tx_queue_mbuf(struct sfxge_txq *txq, struct mbuf *mbuf) if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) prefetch_read_many(mbuf->m_data); - if (txq->init_state != SFXGE_TXQ_STARTED) { + if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) { rc = EINTR; goto reject; } @@ -1142,7 +1142,7 @@ sfxge_tx_qunblock(struct sfxge_txq *txq) SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); - if (txq->init_state != SFXGE_TXQ_STARTED) + if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) return; SFXGE_TXQ_LOCK(txq); |