diff options
-rw-r--r-- | sys/dev/netmap/if_em_netmap.h | 1 | ||||
-rw-r--r-- | sys/dev/netmap/if_igb_netmap.h | 1 | ||||
-rw-r--r-- | sys/dev/netmap/if_lem_netmap.h | 6 | ||||
-rw-r--r-- | sys/dev/netmap/if_re_netmap.h | 1 | ||||
-rw-r--r-- | sys/dev/netmap/ixgbe_netmap.h | 17 | ||||
-rw-r--r-- | sys/dev/netmap/netmap.c | 840 | ||||
-rw-r--r-- | sys/dev/netmap/netmap_kern.h | 12 | ||||
-rw-r--r-- | sys/net/netmap.h | 6 | ||||
-rw-r--r-- | sys/net/netmap_user.h | 8 | ||||
-rw-r--r-- | tools/tools/netmap/pkt-gen.c | 2 |
10 files changed, 444 insertions, 450 deletions
diff --git a/sys/dev/netmap/if_em_netmap.h b/sys/dev/netmap/if_em_netmap.h index 959f52f..6d0b9b2 100644 --- a/sys/dev/netmap/if_em_netmap.h +++ b/sys/dev/netmap/if_em_netmap.h @@ -61,7 +61,6 @@ em_netmap_attach(struct adapter *adapter) na.nm_rxsync = em_netmap_rxsync; na.nm_lock = em_netmap_lock_wrapper; na.nm_register = em_netmap_reg; - na.buff_size = NETMAP_BUF_SIZE; netmap_attach(&na, adapter->num_queues); } diff --git a/sys/dev/netmap/if_igb_netmap.h b/sys/dev/netmap/if_igb_netmap.h index 0e4229b..3a8ad51 100644 --- a/sys/dev/netmap/if_igb_netmap.h +++ b/sys/dev/netmap/if_igb_netmap.h @@ -58,7 +58,6 @@ igb_netmap_attach(struct adapter *adapter) na.nm_rxsync = igb_netmap_rxsync; na.nm_lock = igb_netmap_lock_wrapper; na.nm_register = igb_netmap_reg; - na.buff_size = NETMAP_BUF_SIZE; netmap_attach(&na, adapter->num_queues); } diff --git a/sys/dev/netmap/if_lem_netmap.h b/sys/dev/netmap/if_lem_netmap.h index b9438fb..bc2a1bb 100644 --- a/sys/dev/netmap/if_lem_netmap.h +++ b/sys/dev/netmap/if_lem_netmap.h @@ -62,7 +62,6 @@ lem_netmap_attach(struct adapter *adapter) na.nm_rxsync = lem_netmap_rxsync; na.nm_lock = lem_netmap_lock_wrapper; na.nm_register = lem_netmap_reg; - na.buff_size = NETMAP_BUF_SIZE; netmap_attach(&na, 1); } @@ -247,6 +246,7 @@ lem_netmap_txsync(void *a, u_int ring_nr, int do_lock) ring->avail = kring->nr_hwavail; } } + if (do_lock) EM_TX_UNLOCK(adapter); return 0; @@ -351,9 +351,9 @@ lem_netmap_rxsync(void *a, u_int ring_nr, int do_lock) /* tell userspace that there are new packets */ ring->avail = kring->nr_hwavail ; + if (do_lock) EM_RX_UNLOCK(adapter); return 0; } - - +/* end of file */ diff --git a/sys/dev/netmap/if_re_netmap.h b/sys/dev/netmap/if_re_netmap.h index a92432c..3dcc26b 100644 --- a/sys/dev/netmap/if_re_netmap.h +++ b/sys/dev/netmap/if_re_netmap.h @@ -56,7 +56,6 @@ re_netmap_attach(struct rl_softc *sc) na.nm_rxsync = re_netmap_rxsync; na.nm_lock = re_netmap_lock_wrapper; na.nm_register = re_netmap_reg; - na.buff_size = NETMAP_BUF_SIZE; netmap_attach(&na, 1); } diff --git a/sys/dev/netmap/ixgbe_netmap.h b/sys/dev/netmap/ixgbe_netmap.h index 8e5fe1b..644ab2e 100644 --- a/sys/dev/netmap/ixgbe_netmap.h +++ b/sys/dev/netmap/ixgbe_netmap.h @@ -82,13 +82,6 @@ ixgbe_netmap_attach(struct adapter *adapter) na.nm_rxsync = ixgbe_netmap_rxsync; na.nm_lock = ixgbe_netmap_lock_wrapper; na.nm_register = ixgbe_netmap_reg; - /* - * XXX where do we put this comment ? - * adapter->rx_mbuf_sz is set by SIOCSETMTU, but in netmap mode - * we allocate the buffers on the first register. So we must - * disallow a SIOCSETMTU when if_capenable & IFCAP_NETMAP is set. - */ - na.buff_size = NETMAP_BUF_SIZE; netmap_attach(&na, adapter->num_queues); } @@ -354,7 +347,8 @@ ring_reset: * otherwise we go to sleep (in netmap_poll()) and will be * woken up when slot nr_kflags will be ready. */ - struct ixgbe_legacy_tx_desc *txd = (struct ixgbe_legacy_tx_desc *)txr->tx_base; + struct ixgbe_legacy_tx_desc *txd = + (struct ixgbe_legacy_tx_desc *)txr->tx_base; j = txr->next_to_clean + kring->nkr_num_slots/2; if (j >= kring->nkr_num_slots) @@ -365,9 +359,7 @@ ring_reset: kring->nr_kflags = j; /* the slot to check */ j = txd[j].upper.fields.status & IXGBE_TXD_STAT_DD; } - if (!j) { - netmap_skip_txsync++; - } else { + if (j) { int delta; /* @@ -471,7 +463,7 @@ ixgbe_netmap_rxsync(void *a, u_int ring_nr, int do_lock) if (j > lim) j -= lim + 1; - if (force_update) { + if (netmap_no_pendintr || force_update) { for (n = 0; ; n++) { union ixgbe_adv_rx_desc *curr = &rxr->rx_base[l]; uint32_t staterr = le32toh(curr->wb.upper.status_error); @@ -548,6 +540,7 @@ ixgbe_netmap_rxsync(void *a, u_int ring_nr, int do_lock) } /* tell userspace that there are new packets */ ring->avail = kring->nr_hwavail ; + if (do_lock) IXGBE_RX_UNLOCK(rxr); return 0; diff --git a/sys/dev/netmap/netmap.c b/sys/dev/netmap/netmap.c index 1f282ce..8a5b4f2 100644 --- a/sys/dev/netmap/netmap.c +++ b/sys/dev/netmap/netmap.c @@ -92,6 +92,39 @@ MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map"); */ #define NMA_LOCK() mtx_lock(&netmap_mem_d->nm_mtx); #define NMA_UNLOCK() mtx_unlock(&netmap_mem_d->nm_mtx); +struct netmap_mem_d; +static struct netmap_mem_d *netmap_mem_d; /* Our memory allocator. */ + +u_int netmap_total_buffers; +char *netmap_buffer_base; /* address of an invalid buffer */ + +/* user-controlled variables */ +int netmap_verbose; + +static int netmap_no_timestamp; /* don't timestamp on rxsync */ + +SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args"); +SYSCTL_INT(_dev_netmap, OID_AUTO, verbose, + CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode"); +SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp, + CTLFLAG_RW, &netmap_no_timestamp, 0, "no_timestamp"); +int netmap_buf_size = 2048; +TUNABLE_INT("hw.netmap.buf_size", &netmap_buf_size); +SYSCTL_INT(_dev_netmap, OID_AUTO, buf_size, + CTLFLAG_RD, &netmap_buf_size, 0, "Size of packet buffers"); +int netmap_mitigate = 1; +SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, ""); +int netmap_no_pendintr; +SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr, + CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets."); + + + +/*----- memory allocator -----------------*/ +/* + * Here we have the low level routines for memory allocator + * and its primary users. + */ /* * Default amount of memory pre-allocated by the module. @@ -128,30 +161,13 @@ struct netmap_buf_pool { uint32_t *bitmap; /* one bit per buffer, 1 means free */ }; struct netmap_buf_pool nm_buf_pool; -/* XXX move these two vars back into netmap_buf_pool */ -u_int netmap_total_buffers; -char *netmap_buffer_base; /* address of an invalid buffer */ - -/* user-controlled variables */ -int netmap_verbose; - -static int no_timestamp; /* don't timestamp on rxsync */ - -SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args"); -SYSCTL_INT(_dev_netmap, OID_AUTO, verbose, - CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode"); -SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp, - CTLFLAG_RW, &no_timestamp, 0, "no_timestamp"); SYSCTL_INT(_dev_netmap, OID_AUTO, total_buffers, CTLFLAG_RD, &nm_buf_pool.total_buffers, 0, "total_buffers"); SYSCTL_INT(_dev_netmap, OID_AUTO, free_buffers, CTLFLAG_RD, &nm_buf_pool.free, 0, "free_buffers"); -int netmap_mitigate = 1; -SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, ""); -int netmap_skip_txsync; -SYSCTL_INT(_dev_netmap, OID_AUTO, skip_txsync, CTLFLAG_RW, &netmap_skip_txsync, 0, ""); -int netmap_skip_rxsync; -SYSCTL_INT(_dev_netmap, OID_AUTO, skip_rxsync, CTLFLAG_RW, &netmap_skip_rxsync, 0, ""); + + + /* * Allocate n buffers from the ring, and fill the slot. @@ -239,6 +255,373 @@ struct netmap_mem_d { area. */ }; +/* Shorthand to compute a netmap interface offset. */ +#define netmap_if_offset(v) \ + ((char *) (v) - (char *) netmap_mem_d->nm_buffer) +/* .. and get a physical address given a memory offset */ +#define netmap_ofstophys(o) \ + (vtophys(netmap_mem_d->nm_buffer) + (o)) + + +/*------ netmap memory allocator -------*/ +/* + * Request for a chunk of memory. + * + * Memory objects are arranged into a list, hence we need to walk this + * list until we find an object with the needed amount of data free. + * This sounds like a completely inefficient implementation, but given + * the fact that data allocation is done once, we can handle it + * flawlessly. + * + * Return NULL on failure. + */ +static void * +netmap_malloc(size_t size, __unused const char *msg) +{ + struct netmap_mem_obj *mem_obj, *new_mem_obj; + void *ret = NULL; + + NMA_LOCK(); + TAILQ_FOREACH(mem_obj, &netmap_mem_d->nm_molist, nmo_next) { + if (mem_obj->nmo_used != 0 || mem_obj->nmo_size < size) + continue; + + new_mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP, + M_WAITOK | M_ZERO); + TAILQ_INSERT_BEFORE(mem_obj, new_mem_obj, nmo_next); + + new_mem_obj->nmo_used = 1; + new_mem_obj->nmo_size = size; + new_mem_obj->nmo_data = mem_obj->nmo_data; + memset(new_mem_obj->nmo_data, 0, new_mem_obj->nmo_size); + + mem_obj->nmo_size -= size; + mem_obj->nmo_data = (char *) mem_obj->nmo_data + size; + if (mem_obj->nmo_size == 0) { + TAILQ_REMOVE(&netmap_mem_d->nm_molist, mem_obj, + nmo_next); + free(mem_obj, M_NETMAP); + } + + ret = new_mem_obj->nmo_data; + + break; + } + NMA_UNLOCK(); + ND("%s: %d bytes at %p", msg, size, ret); + + return (ret); +} + +/* + * Return the memory to the allocator. + * + * While freeing a memory object, we try to merge adjacent chunks in + * order to reduce memory fragmentation. + */ +static void +netmap_free(void *addr, const char *msg) +{ + size_t size; + struct netmap_mem_obj *cur, *prev, *next; + + if (addr == NULL) { + D("NULL addr for %s", msg); + return; + } + + NMA_LOCK(); + TAILQ_FOREACH(cur, &netmap_mem_d->nm_molist, nmo_next) { + if (cur->nmo_data == addr && cur->nmo_used) + break; + } + if (cur == NULL) { + NMA_UNLOCK(); + D("invalid addr %s %p", msg, addr); + return; + } + + size = cur->nmo_size; + cur->nmo_used = 0; + + /* merge current chunk of memory with the previous one, + if present. */ + prev = TAILQ_PREV(cur, netmap_mem_obj_h, nmo_next); + if (prev && prev->nmo_used == 0) { + TAILQ_REMOVE(&netmap_mem_d->nm_molist, cur, nmo_next); + prev->nmo_size += cur->nmo_size; + free(cur, M_NETMAP); + cur = prev; + } + + /* merge with the next one */ + next = TAILQ_NEXT(cur, nmo_next); + if (next && next->nmo_used == 0) { + TAILQ_REMOVE(&netmap_mem_d->nm_molist, next, nmo_next); + cur->nmo_size += next->nmo_size; + free(next, M_NETMAP); + } + NMA_UNLOCK(); + ND("freed %s %d bytes at %p", msg, size, addr); +} + + +/* + * Create and return a new ``netmap_if`` object, and possibly also + * rings and packet buffors. + * + * Return NULL on failure. + */ +static void * +netmap_if_new(const char *ifname, struct netmap_adapter *na) +{ + struct netmap_if *nifp; + struct netmap_ring *ring; + char *buff; + u_int i, len, ofs; + u_int n = na->num_queues + 1; /* shorthand, include stack queue */ + + /* + * the descriptor is followed inline by an array of offsets + * to the tx and rx rings in the shared memory region. + */ + len = sizeof(struct netmap_if) + 2 * n * sizeof(ssize_t); + nifp = netmap_if_malloc(len); + if (nifp == NULL) + return (NULL); + + /* initialize base fields */ + *(int *)(uintptr_t)&nifp->ni_num_queues = na->num_queues; + strncpy(nifp->ni_name, ifname, IFNAMSIZ); + + (na->refcount)++; /* XXX atomic ? we are under lock */ + if (na->refcount > 1) + goto final; + + /* + * If this is the first instance, allocate the shadow rings and + * buffers for this card (one for each hw queue, one for the host). + * The rings are contiguous, but have variable size. + * The entire block is reachable at + * na->tx_rings[0].ring + */ + + len = n * (2 * sizeof(struct netmap_ring) + + (na->num_tx_desc + na->num_rx_desc) * + sizeof(struct netmap_slot) ); + buff = netmap_ring_malloc(len); + if (buff == NULL) { + D("failed to allocate %d bytes for %s shadow ring", + len, ifname); +error: + (na->refcount)--; + netmap_if_free(nifp); + return (NULL); + } + /* do we have the bufers ? we are in need of num_tx_desc buffers for + * each tx ring and num_tx_desc buffers for each rx ring. */ + len = n * (na->num_tx_desc + na->num_rx_desc); + NMA_LOCK(); + if (nm_buf_pool.free < len) { + NMA_UNLOCK(); + netmap_free(buff, "not enough bufs"); + goto error; + } + /* + * in the kring, store the pointers to the shared rings + * and initialize the rings. We are under NMA_LOCK(). + */ + ofs = 0; + for (i = 0; i < n; i++) { + struct netmap_kring *kring; + int numdesc; + + /* Transmit rings */ + kring = &na->tx_rings[i]; + numdesc = na->num_tx_desc; + bzero(kring, sizeof(*kring)); + kring->na = na; + + ring = kring->ring = (struct netmap_ring *)(buff + ofs); + *(ssize_t *)(uintptr_t)&ring->buf_ofs = + nm_buf_pool.base - (char *)ring; + ND("txring[%d] at %p ofs %d", i, ring, ring->buf_ofs); + *(uint32_t *)(uintptr_t)&ring->num_slots = + kring->nkr_num_slots = numdesc; + + /* + * IMPORTANT: + * Always keep one slot empty, so we can detect new + * transmissions comparing cur and nr_hwcur (they are + * the same only if there are no new transmissions). + */ + ring->avail = kring->nr_hwavail = numdesc - 1; + ring->cur = kring->nr_hwcur = 0; + *(uint16_t *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE; + netmap_new_bufs(nifp, ring->slot, numdesc); + + ofs += sizeof(struct netmap_ring) + + numdesc * sizeof(struct netmap_slot); + + /* Receive rings */ + kring = &na->rx_rings[i]; + numdesc = na->num_rx_desc; + bzero(kring, sizeof(*kring)); + kring->na = na; + + ring = kring->ring = (struct netmap_ring *)(buff + ofs); + *(ssize_t *)(uintptr_t)&ring->buf_ofs = + nm_buf_pool.base - (char *)ring; + ND("rxring[%d] at %p offset %d", i, ring, ring->buf_ofs); + *(uint32_t *)(uintptr_t)&ring->num_slots = + kring->nkr_num_slots = numdesc; + ring->cur = kring->nr_hwcur = 0; + ring->avail = kring->nr_hwavail = 0; /* empty */ + *(uint16_t *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE; + netmap_new_bufs(nifp, ring->slot, numdesc); + ofs += sizeof(struct netmap_ring) + + numdesc * sizeof(struct netmap_slot); + } + NMA_UNLOCK(); + for (i = 0; i < n+1; i++) { + // XXX initialize the selrecord structs. + } +final: + /* + * fill the slots for the rx and tx queues. They contain the offset + * between the ring and nifp, so the information is usable in + * userspace to reach the ring from the nifp. + */ + for (i = 0; i < n; i++) { + char *base = (char *)nifp; + *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = + (char *)na->tx_rings[i].ring - base; + *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+n] = + (char *)na->rx_rings[i].ring - base; + } + return (nifp); +} + +/* + * Initialize the memory allocator. + * + * Create the descriptor for the memory , allocate the pool of memory + * and initialize the list of memory objects with a single chunk + * containing the whole pre-allocated memory marked as free. + * + * Start with a large size, then halve as needed if we fail to + * allocate the block. While halving, always add one extra page + * because buffers 0 and 1 are used for special purposes. + * Return 0 on success, errno otherwise. + */ +static int +netmap_memory_init(void) +{ + struct netmap_mem_obj *mem_obj; + void *buf = NULL; + int i, n, sz = NETMAP_MEMORY_SIZE; + int extra_sz = 0; // space for rings and two spare buffers + + for (; sz >= 1<<20; sz >>=1) { + extra_sz = sz/200; + extra_sz = (extra_sz + 2*PAGE_SIZE - 1) & ~(PAGE_SIZE-1); + buf = contigmalloc(sz + extra_sz, + M_NETMAP, + M_WAITOK | M_ZERO, + 0, /* low address */ + -1UL, /* high address */ + PAGE_SIZE, /* alignment */ + 0 /* boundary */ + ); + if (buf) + break; + } + if (buf == NULL) + return (ENOMEM); + sz += extra_sz; + netmap_mem_d = malloc(sizeof(struct netmap_mem_d), M_NETMAP, + M_WAITOK | M_ZERO); + mtx_init(&netmap_mem_d->nm_mtx, "netmap memory allocator lock", NULL, + MTX_DEF); + TAILQ_INIT(&netmap_mem_d->nm_molist); + netmap_mem_d->nm_buffer = buf; + netmap_mem_d->nm_totalsize = sz; + + /* + * A buffer takes 2k, a slot takes 8 bytes + ring overhead, + * so the ratio is 200:1. In other words, we can use 1/200 of + * the memory for the rings, and the rest for the buffers, + * and be sure we never run out. + */ + netmap_mem_d->nm_size = sz/200; + netmap_mem_d->nm_buf_start = + (netmap_mem_d->nm_size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1); + netmap_mem_d->nm_buf_len = sz - netmap_mem_d->nm_buf_start; + + nm_buf_pool.base = netmap_mem_d->nm_buffer; + nm_buf_pool.base += netmap_mem_d->nm_buf_start; + netmap_buffer_base = nm_buf_pool.base; + D("netmap_buffer_base %p (offset %d)", + netmap_buffer_base, (int)netmap_mem_d->nm_buf_start); + /* number of buffers, they all start as free */ + + netmap_total_buffers = nm_buf_pool.total_buffers = + netmap_mem_d->nm_buf_len / NETMAP_BUF_SIZE; + nm_buf_pool.bufsize = NETMAP_BUF_SIZE; + + D("Have %d MB, use %dKB for rings, %d buffers at %p", + (sz >> 20), (int)(netmap_mem_d->nm_size >> 10), + nm_buf_pool.total_buffers, nm_buf_pool.base); + + /* allocate and initialize the bitmap. Entry 0 is considered + * always busy (used as default when there are no buffers left). + */ + n = (nm_buf_pool.total_buffers + 31) / 32; + nm_buf_pool.bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP, + M_WAITOK | M_ZERO); + nm_buf_pool.bitmap[0] = ~3; /* slot 0 and 1 always busy */ + for (i = 1; i < n; i++) + nm_buf_pool.bitmap[i] = ~0; + nm_buf_pool.free = nm_buf_pool.total_buffers - 2; + + mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP, + M_WAITOK | M_ZERO); + TAILQ_INSERT_HEAD(&netmap_mem_d->nm_molist, mem_obj, nmo_next); + mem_obj->nmo_used = 0; + mem_obj->nmo_size = netmap_mem_d->nm_size; + mem_obj->nmo_data = netmap_mem_d->nm_buffer; + + return (0); +} + + +/* + * Finalize the memory allocator. + * + * Free all the memory objects contained inside the list, and deallocate + * the pool of memory; finally free the memory allocator descriptor. + */ +static void +netmap_memory_fini(void) +{ + struct netmap_mem_obj *mem_obj; + + while (!TAILQ_EMPTY(&netmap_mem_d->nm_molist)) { + mem_obj = TAILQ_FIRST(&netmap_mem_d->nm_molist); + TAILQ_REMOVE(&netmap_mem_d->nm_molist, mem_obj, nmo_next); + if (mem_obj->nmo_used == 1) { + printf("netmap: leaked %d bytes at %p\n", + (int)mem_obj->nmo_size, + mem_obj->nmo_data); + } + free(mem_obj, M_NETMAP); + } + contigfree(netmap_mem_d->nm_buffer, netmap_mem_d->nm_totalsize, M_NETMAP); + // XXX mutex_destroy(nm_mtx); + free(netmap_mem_d, M_NETMAP); +} +/*------------- end of memory allocator -----------------*/ + /* Structure associated to each thread which registered an interface. */ struct netmap_priv_d { @@ -250,15 +633,8 @@ struct netmap_priv_d { uint16_t np_txpoll; }; -/* Shorthand to compute a netmap interface offset. */ -#define netmap_if_offset(v) \ - ((char *) (v) - (char *) netmap_mem_d->nm_buffer) -/* .. and get a physical address given a memory offset */ -#define netmap_ofstophys(o) \ - (vtophys(netmap_mem_d->nm_buffer) + (o)) static struct cdev *netmap_dev; /* /dev/netmap character device. */ -static struct netmap_mem_d *netmap_mem_d; /* Our memory allocator. */ static d_mmap_t netmap_mmap; @@ -351,6 +727,7 @@ netmap_kqfilter(struct cdev *dev, struct knote *kn) } #endif /* NETMAP_KEVENT */ + /* * File descriptor's private data destructor. * @@ -358,19 +735,13 @@ netmap_kqfilter(struct cdev *dev, struct knote *kn) * revert to normal operation. We expect that np_ifp has not gone. */ static void -netmap_dtor(void *data) +netmap_dtor_locked(void *data) { struct netmap_priv_d *priv = data; struct ifnet *ifp = priv->np_ifp; struct netmap_adapter *na = NA(ifp); struct netmap_if *nifp = priv->np_nifp; - if (0) - printf("%s starting for %p ifp %p\n", __FUNCTION__, priv, - priv ? priv->np_ifp : NULL); - - na->nm_lock(ifp->if_softc, NETMAP_CORE_LOCK, 0); - na->refcount--; if (na->refcount <= 0) { /* last instance */ u_int i; @@ -422,148 +793,23 @@ netmap_dtor(void *data) wakeup(na); } netmap_if_free(nifp); - - na->nm_lock(ifp->if_softc, NETMAP_CORE_UNLOCK, 0); - - if_rele(ifp); - - bzero(priv, sizeof(*priv)); /* XXX for safety */ - free(priv, M_DEVBUF); } -/* - * Create and return a new ``netmap_if`` object, and possibly also - * rings and packet buffors. - * - * Return NULL on failure. - */ -static void * -netmap_if_new(const char *ifname, struct netmap_adapter *na) +static void +netmap_dtor(void *data) { - struct netmap_if *nifp; - struct netmap_ring *ring; - char *buff; - u_int i, len, ofs; - u_int n = na->num_queues + 1; /* shorthand, include stack queue */ - - /* - * the descriptor is followed inline by an array of offsets - * to the tx and rx rings in the shared memory region. - */ - len = sizeof(struct netmap_if) + 2 * n * sizeof(ssize_t); - nifp = netmap_if_malloc(len); - if (nifp == NULL) - return (NULL); - - /* initialize base fields */ - *(int *)(uintptr_t)&nifp->ni_num_queues = na->num_queues; - strncpy(nifp->ni_name, ifname, IFNAMSIZ); - - (na->refcount)++; /* XXX atomic ? we are under lock */ - if (na->refcount > 1) - goto final; - - /* - * If this is the first instance, allocate the shadow rings and - * buffers for this card (one for each hw queue, one for the host). - * The rings are contiguous, but have variable size. - * The entire block is reachable at - * na->tx_rings[0].ring - */ - - len = n * (2 * sizeof(struct netmap_ring) + - (na->num_tx_desc + na->num_rx_desc) * - sizeof(struct netmap_slot) ); - buff = netmap_ring_malloc(len); - if (buff == NULL) { - D("failed to allocate %d bytes for %s shadow ring", - len, ifname); -error: - (na->refcount)--; - netmap_if_free(nifp); - return (NULL); - } - /* do we have the bufers ? we are in need of num_tx_desc buffers for - * each tx ring and num_tx_desc buffers for each rx ring. */ - len = n * (na->num_tx_desc + na->num_rx_desc); - NMA_LOCK(); - if (nm_buf_pool.free < len) { - NMA_UNLOCK(); - netmap_free(buff, "not enough bufs"); - goto error; - } - /* - * in the kring, store the pointers to the shared rings - * and initialize the rings. We are under NMA_LOCK(). - */ - ofs = 0; - for (i = 0; i < n; i++) { - struct netmap_kring *kring; - int numdesc; - - /* Transmit rings */ - kring = &na->tx_rings[i]; - numdesc = na->num_tx_desc; - bzero(kring, sizeof(*kring)); - kring->na = na; - - ring = kring->ring = (struct netmap_ring *)(buff + ofs); - *(ssize_t *)(uintptr_t)&ring->buf_ofs = - nm_buf_pool.base - (char *)ring; - ND("txring[%d] at %p ofs %d", i, ring, ring->buf_ofs); - *(int *)(int *)(uintptr_t)&ring->num_slots = - kring->nkr_num_slots = numdesc; - - /* - * IMPORTANT: - * Always keep one slot empty, so we can detect new - * transmissions comparing cur and nr_hwcur (they are - * the same only if there are no new transmissions). - */ - ring->avail = kring->nr_hwavail = numdesc - 1; - ring->cur = kring->nr_hwcur = 0; - netmap_new_bufs(nifp, ring->slot, numdesc); - - ofs += sizeof(struct netmap_ring) + - numdesc * sizeof(struct netmap_slot); + struct netmap_priv_d *priv = data; + struct ifnet *ifp = priv->np_ifp; + struct netmap_adapter *na = NA(ifp); - /* Receive rings */ - kring = &na->rx_rings[i]; - numdesc = na->num_rx_desc; - bzero(kring, sizeof(*kring)); - kring->na = na; + na->nm_lock(ifp->if_softc, NETMAP_CORE_LOCK, 0); + netmap_dtor_locked(data); + na->nm_lock(ifp->if_softc, NETMAP_CORE_UNLOCK, 0); - ring = kring->ring = (struct netmap_ring *)(buff + ofs); - *(ssize_t *)(uintptr_t)&ring->buf_ofs = - nm_buf_pool.base - (char *)ring; - ND("rxring[%d] at %p offset %d", i, ring, ring->buf_ofs); - *(int *)(int *)(uintptr_t)&ring->num_slots = - kring->nkr_num_slots = numdesc; - ring->cur = kring->nr_hwcur = 0; - ring->avail = kring->nr_hwavail = 0; /* empty */ - netmap_new_bufs(nifp, ring->slot, numdesc); - ofs += sizeof(struct netmap_ring) + - numdesc * sizeof(struct netmap_slot); - } - NMA_UNLOCK(); - for (i = 0; i < n+1; i++) { - // XXX initialize the selrecord structs. - } -final: - /* - * fill the slots for the rx and tx queues. They contain the offset - * between the ring and nifp, so the information is usable in - * userspace to reach the ring from the nifp. - */ - for (i = 0; i < n; i++) { - char *base = (char *)nifp; - *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = - (char *)na->tx_rings[i].ring - base; - *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+n] = - (char *)na->rx_rings[i].ring - base; - } - return (nifp); + if_rele(ifp); + bzero(priv, sizeof(*priv)); /* XXX for safety */ + free(priv, M_DEVBUF); } @@ -894,7 +1140,6 @@ netmap_ioctl(__unused struct cdev *dev, u_long cmd, caddr_t data, break; } - for (i = 10; i > 0; i--) { na->nm_lock(adapter, NETMAP_CORE_LOCK, 0); if (!NETMAP_DELETING(na)) @@ -924,25 +1169,16 @@ netmap_ioctl(__unused struct cdev *dev, u_long cmd, caddr_t data, * and make it use the shared buffers. */ error = na->nm_register(ifp, 1); /* mode on */ - if (error) { - /* - * do something similar to netmap_dtor(). - */ - netmap_free_rings(na); - // XXX tx_rings is inline, must not be freed. - // free(na->tx_rings, M_DEVBUF); // XXX wrong ? - na->tx_rings = na->rx_rings = NULL; - na->refcount--; - netmap_if_free(nifp); - nifp = NULL; - } + if (error) + netmap_dtor_locked(priv); } if (error) { /* reg. failed, release priv and ref */ error: na->nm_lock(adapter, NETMAP_CORE_UNLOCK, 0); - free(priv, M_DEVBUF); if_rele(ifp); /* return the refcount */ + bzero(priv, sizeof(*priv)); + free(priv, M_DEVBUF); break; } @@ -1166,6 +1402,12 @@ netmap_poll(__unused struct cdev *dev, int events, struct thread *td) if (priv->np_txpoll || want_tx) { for (i = priv->np_qfirst; i < priv->np_qlast; i++) { kring = &na->tx_rings[i]; + /* + * Skip the current ring if want_tx == 0 + * (we have already done a successful sync on + * a previous ring) AND kring->cur == kring->hwcur + * (there are no pending transmissions for this ring). + */ if (!want_tx && kring->ring->cur == kring->nr_hwcur) continue; if (core_lock == NEED_CL) { @@ -1181,6 +1423,7 @@ netmap_poll(__unused struct cdev *dev, int events, struct thread *td) if (na->nm_txsync(adapter, i, 0 /* no lock */)) revents |= POLLERR; + /* Check avail/call selrecord only if called with POLLOUT */ if (want_tx) { if (kring->ring->avail > 0) { /* stop at the first ring. We don't risk @@ -1212,9 +1455,10 @@ netmap_poll(__unused struct cdev *dev, int events, struct thread *td) if (na->nm_rxsync(adapter, i, 0 /* no lock */)) revents |= POLLERR; - if (no_timestamp == 0 || - kring->ring->flags & NR_TIMESTAMP) + if (netmap_no_timestamp == 0 || + kring->ring->flags & NR_TIMESTAMP) { microtime(&kring->ring->ts); + } if (kring->ring->avail > 0) revents |= want_rx; @@ -1269,6 +1513,7 @@ netmap_attach(struct netmap_adapter *na, int num_queues) WNA(ifp) = buf; na->tx_rings = (void *)((char *)buf + sizeof(*na)); na->rx_rings = na->tx_rings + n; + na->buff_size = NETMAP_BUF_SIZE; bcopy(na, buf, sizeof(*na)); ifp->if_capabilities |= IFCAP_NETMAP; } @@ -1399,229 +1644,6 @@ netmap_reset(struct netmap_adapter *na, enum txrx tx, int n, } -/*------ netmap memory allocator -------*/ -/* - * Request for a chunk of memory. - * - * Memory objects are arranged into a list, hence we need to walk this - * list until we find an object with the needed amount of data free. - * This sounds like a completely inefficient implementation, but given - * the fact that data allocation is done once, we can handle it - * flawlessly. - * - * Return NULL on failure. - */ -static void * -netmap_malloc(size_t size, __unused const char *msg) -{ - struct netmap_mem_obj *mem_obj, *new_mem_obj; - void *ret = NULL; - - NMA_LOCK(); - TAILQ_FOREACH(mem_obj, &netmap_mem_d->nm_molist, nmo_next) { - if (mem_obj->nmo_used != 0 || mem_obj->nmo_size < size) - continue; - - new_mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP, - M_WAITOK | M_ZERO); - TAILQ_INSERT_BEFORE(mem_obj, new_mem_obj, nmo_next); - - new_mem_obj->nmo_used = 1; - new_mem_obj->nmo_size = size; - new_mem_obj->nmo_data = mem_obj->nmo_data; - memset(new_mem_obj->nmo_data, 0, new_mem_obj->nmo_size); - - mem_obj->nmo_size -= size; - mem_obj->nmo_data = (char *) mem_obj->nmo_data + size; - if (mem_obj->nmo_size == 0) { - TAILQ_REMOVE(&netmap_mem_d->nm_molist, mem_obj, - nmo_next); - free(mem_obj, M_NETMAP); - } - - ret = new_mem_obj->nmo_data; - - break; - } - NMA_UNLOCK(); - ND("%s: %d bytes at %p", msg, size, ret); - - return (ret); -} - -/* - * Return the memory to the allocator. - * - * While freeing a memory object, we try to merge adjacent chunks in - * order to reduce memory fragmentation. - */ -static void -netmap_free(void *addr, const char *msg) -{ - size_t size; - struct netmap_mem_obj *cur, *prev, *next; - - if (addr == NULL) { - D("NULL addr for %s", msg); - return; - } - - NMA_LOCK(); - TAILQ_FOREACH(cur, &netmap_mem_d->nm_molist, nmo_next) { - if (cur->nmo_data == addr && cur->nmo_used) - break; - } - if (cur == NULL) { - NMA_UNLOCK(); - D("invalid addr %s %p", msg, addr); - return; - } - - size = cur->nmo_size; - cur->nmo_used = 0; - - /* merge current chunk of memory with the previous one, - if present. */ - prev = TAILQ_PREV(cur, netmap_mem_obj_h, nmo_next); - if (prev && prev->nmo_used == 0) { - TAILQ_REMOVE(&netmap_mem_d->nm_molist, cur, nmo_next); - prev->nmo_size += cur->nmo_size; - free(cur, M_NETMAP); - cur = prev; - } - - /* merge with the next one */ - next = TAILQ_NEXT(cur, nmo_next); - if (next && next->nmo_used == 0) { - TAILQ_REMOVE(&netmap_mem_d->nm_molist, next, nmo_next); - cur->nmo_size += next->nmo_size; - free(next, M_NETMAP); - } - NMA_UNLOCK(); - ND("freed %s %d bytes at %p", msg, size, addr); -} - - -/* - * Initialize the memory allocator. - * - * Create the descriptor for the memory , allocate the pool of memory - * and initialize the list of memory objects with a single chunk - * containing the whole pre-allocated memory marked as free. - * - * Start with a large size, then halve as needed if we fail to - * allocate the block. While halving, always add one extra page - * because buffers 0 and 1 are used for special purposes. - * Return 0 on success, errno otherwise. - */ -static int -netmap_memory_init(void) -{ - struct netmap_mem_obj *mem_obj; - void *buf = NULL; - int i, n, sz = NETMAP_MEMORY_SIZE; - int extra_sz = 0; // space for rings and two spare buffers - - for (; sz >= 1<<20; sz >>=1) { - extra_sz = sz/200; - extra_sz = (extra_sz + 2*PAGE_SIZE - 1) & ~(PAGE_SIZE-1); - buf = contigmalloc(sz + extra_sz, - M_NETMAP, - M_WAITOK | M_ZERO, - 0, /* low address */ - -1UL, /* high address */ - PAGE_SIZE, /* alignment */ - 0 /* boundary */ - ); - if (buf) - break; - } - if (buf == NULL) - return (ENOMEM); - sz += extra_sz; - netmap_mem_d = malloc(sizeof(struct netmap_mem_d), M_NETMAP, - M_WAITOK | M_ZERO); - mtx_init(&netmap_mem_d->nm_mtx, "netmap memory allocator lock", NULL, - MTX_DEF); - TAILQ_INIT(&netmap_mem_d->nm_molist); - netmap_mem_d->nm_buffer = buf; - netmap_mem_d->nm_totalsize = sz; - - /* - * A buffer takes 2k, a slot takes 8 bytes + ring overhead, - * so the ratio is 200:1. In other words, we can use 1/200 of - * the memory for the rings, and the rest for the buffers, - * and be sure we never run out. - */ - netmap_mem_d->nm_size = sz/200; - netmap_mem_d->nm_buf_start = - (netmap_mem_d->nm_size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1); - netmap_mem_d->nm_buf_len = sz - netmap_mem_d->nm_buf_start; - - nm_buf_pool.base = netmap_mem_d->nm_buffer; - nm_buf_pool.base += netmap_mem_d->nm_buf_start; - netmap_buffer_base = nm_buf_pool.base; - D("netmap_buffer_base %p (offset %d)", - netmap_buffer_base, (int)netmap_mem_d->nm_buf_start); - /* number of buffers, they all start as free */ - - netmap_total_buffers = nm_buf_pool.total_buffers = - netmap_mem_d->nm_buf_len / NETMAP_BUF_SIZE; - nm_buf_pool.bufsize = NETMAP_BUF_SIZE; - - D("Have %d MB, use %dKB for rings, %d buffers at %p", - (sz >> 20), (int)(netmap_mem_d->nm_size >> 10), - nm_buf_pool.total_buffers, nm_buf_pool.base); - - /* allocate and initialize the bitmap. Entry 0 is considered - * always busy (used as default when there are no buffers left). - */ - n = (nm_buf_pool.total_buffers + 31) / 32; - nm_buf_pool.bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP, - M_WAITOK | M_ZERO); - nm_buf_pool.bitmap[0] = ~3; /* slot 0 and 1 always busy */ - for (i = 1; i < n; i++) - nm_buf_pool.bitmap[i] = ~0; - nm_buf_pool.free = nm_buf_pool.total_buffers - 2; - - mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP, - M_WAITOK | M_ZERO); - TAILQ_INSERT_HEAD(&netmap_mem_d->nm_molist, mem_obj, nmo_next); - mem_obj->nmo_used = 0; - mem_obj->nmo_size = netmap_mem_d->nm_size; - mem_obj->nmo_data = netmap_mem_d->nm_buffer; - - return (0); -} - - -/* - * Finalize the memory allocator. - * - * Free all the memory objects contained inside the list, and deallocate - * the pool of memory; finally free the memory allocator descriptor. - */ -static void -netmap_memory_fini(void) -{ - struct netmap_mem_obj *mem_obj; - - while (!TAILQ_EMPTY(&netmap_mem_d->nm_molist)) { - mem_obj = TAILQ_FIRST(&netmap_mem_d->nm_molist); - TAILQ_REMOVE(&netmap_mem_d->nm_molist, mem_obj, nmo_next); - if (mem_obj->nmo_used == 1) { - printf("netmap: leaked %d bytes at %p\n", - (int)mem_obj->nmo_size, - mem_obj->nmo_data); - } - free(mem_obj, M_NETMAP); - } - contigfree(netmap_mem_d->nm_buffer, netmap_mem_d->nm_totalsize, M_NETMAP); - // XXX mutex_destroy(nm_mtx); - free(netmap_mem_d, M_NETMAP); -} - - /* * Module loader. * diff --git a/sys/dev/netmap/netmap_kern.h b/sys/dev/netmap/netmap_kern.h index 08f11fe..04e2e27 100644 --- a/sys/dev/netmap/netmap_kern.h +++ b/sys/dev/netmap/netmap_kern.h @@ -172,8 +172,10 @@ struct netmap_slot *netmap_reset(struct netmap_adapter *na, enum txrx tx, int n, u_int new_cur); int netmap_ring_reinit(struct netmap_kring *); +extern int netmap_buf_size; +#define NETMAP_BUF_SIZE netmap_buf_size extern int netmap_mitigate; -extern int netmap_skip_txsync, netmap_skip_rxsync; +extern int netmap_no_pendintr; extern u_int netmap_total_buffers; extern char *netmap_buffer_base; extern int netmap_verbose; // XXX debugging @@ -236,11 +238,7 @@ NMB(struct netmap_slot *slot) { uint32_t i = slot->buf_idx; return (i >= netmap_total_buffers) ? netmap_buffer_base : -#if NETMAP_BUF_SIZE == 2048 - netmap_buffer_base + (i << 11); -#else netmap_buffer_base + (i *NETMAP_BUF_SIZE); -#endif } static inline void * @@ -248,11 +246,7 @@ PNMB(struct netmap_slot *slot, uint64_t *pp) { uint32_t i = slot->buf_idx; void *ret = (i >= netmap_total_buffers) ? netmap_buffer_base : -#if NETMAP_BUF_SIZE == 2048 - netmap_buffer_base + (i << 11); -#else netmap_buffer_base + (i *NETMAP_BUF_SIZE); -#endif *pp = vtophys(ret); return ret; } diff --git a/sys/net/netmap.h b/sys/net/netmap.h index 4dec1fd..0ba1537 100644 --- a/sys/net/netmap.h +++ b/sys/net/netmap.h @@ -258,12 +258,6 @@ struct nmreq { #define NETMAP_RING_MASK 0xfff /* the ring number */ }; -/* - * default buf size is 2048, but it may make sense to have - * it shorter for better cache usage. - */ - -#define NETMAP_BUF_SIZE (2048) #define NIOCGINFO _IOWR('i', 145, struct nmreq) /* return IF info */ #define NIOCREGIF _IOWR('i', 146, struct nmreq) /* interface register */ #define NIOCUNREGIF _IO('i', 147) /* interface unregister */ diff --git a/sys/net/netmap_user.h b/sys/net/netmap_user.h index c9443b8..6449045 100644 --- a/sys/net/netmap_user.h +++ b/sys/net/netmap_user.h @@ -73,14 +73,8 @@ ((struct netmap_ring *)((char *)(nifp) + \ (nifp)->ring_ofs[index + (nifp)->ni_num_queues+1] ) ) -#if NETMAP_BUF_SIZE != 2048 -#error cannot handle odd size #define NETMAP_BUF(ring, index) \ - ((char *)(ring) + (ring)->buf_ofs + ((index)*NETMAP_BUF_SIZE)) -#else -#define NETMAP_BUF(ring, index) \ - ((char *)(ring) + (ring)->buf_ofs + ((index)<<11)) -#endif + ((char *)(ring) + (ring)->buf_ofs + ((index)*(ring)->nr_buf_size)) #define NETMAP_RING_NEXT(r, i) \ ((i)+1 == (r)->num_slots ? 0 : (i) + 1 ) diff --git a/tools/tools/netmap/pkt-gen.c b/tools/tools/netmap/pkt-gen.c index f2f4e40..5627e9e 100644 --- a/tools/tools/netmap/pkt-gen.c +++ b/tools/tools/netmap/pkt-gen.c @@ -124,7 +124,7 @@ struct pkt { struct ether_header eh; struct ip ip; struct udphdr udp; - uint8_t body[NETMAP_BUF_SIZE]; + uint8_t body[2048]; // XXX hardwired } __attribute__((__packed__)); /* |