1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
|
/*-
* Copyright (c) 2010-2015 Solarflare Communications Inc.
* All rights reserved.
*
* This software was developed in part by Philip Paeps under contract for
* Solarflare Communications, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation are
* those of the authors and should not be interpreted as representing official
* policies, either expressed or implied, of the FreeBSD Project.
*
* $FreeBSD$
*/
#ifndef _SFXGE_H
#define _SFXGE_H
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <sys/sx.h>
#include <vm/uma.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_media.h>
#include <net/if_types.h>
#include "sfxge_ioc.h"
/*
* Debugging
*/
#if 0
#define DBGPRINT(dev, fmt, args...) \
device_printf(dev, "%s: " fmt "\n", __func__, ## args)
#else
#define DBGPRINT(dev, fmt, args...)
#endif
/*
* Backward-compatibility
*/
#ifndef CACHE_LINE_SIZE
/* This should be right on most machines the driver will be used on, and
* we needn't care too much about wasting a few KB per interface.
*/
#define CACHE_LINE_SIZE 128
#endif
#ifndef IFCAP_LINKSTATE
#define IFCAP_LINKSTATE 0
#endif
#ifndef IFCAP_VLAN_HWTSO
#define IFCAP_VLAN_HWTSO 0
#endif
#ifndef IFM_10G_T
#define IFM_10G_T IFM_UNKNOWN
#endif
#ifndef IFM_10G_KX4
#define IFM_10G_KX4 IFM_10G_CX4
#endif
#ifndef IFM_40G_CR4
#define IFM_40G_CR4 IFM_UNKNOWN
#endif
#if (__FreeBSD_version >= 800501 && __FreeBSD_version < 900000) || \
__FreeBSD_version >= 900003
#define SFXGE_HAVE_DESCRIBE_INTR
#endif
#ifdef IFM_ETH_RXPAUSE
#define SFXGE_HAVE_PAUSE_MEDIAOPTS
#endif
#ifndef CTLTYPE_U64
#define CTLTYPE_U64 CTLTYPE_QUAD
#endif
#include "sfxge_rx.h"
#include "sfxge_tx.h"
#define ROUNDUP_POW_OF_TWO(_n) (1ULL << flsl((_n) - 1))
#define SFXGE_IP_ALIGN 2
#define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */
enum sfxge_evq_state {
SFXGE_EVQ_UNINITIALIZED = 0,
SFXGE_EVQ_INITIALIZED,
SFXGE_EVQ_STARTING,
SFXGE_EVQ_STARTED
};
#define SFXGE_EV_BATCH 16384
struct sfxge_evq {
/* Structure members below are sorted by usage order */
struct sfxge_softc *sc;
struct mtx lock;
unsigned int index;
enum sfxge_evq_state init_state;
efsys_mem_t mem;
efx_evq_t *common;
unsigned int read_ptr;
boolean_t exception;
unsigned int rx_done;
unsigned int tx_done;
/* Linked list of TX queues with completions to process */
struct sfxge_txq *txq;
struct sfxge_txq **txqs;
/* Structure members not used on event processing path */
unsigned int buf_base_id;
unsigned int entries;
char lock_name[SFXGE_LOCK_NAME_MAX];
} __aligned(CACHE_LINE_SIZE);
#define SFXGE_NDESCS 1024
#define SFXGE_MODERATION 30
enum sfxge_intr_state {
SFXGE_INTR_UNINITIALIZED = 0,
SFXGE_INTR_INITIALIZED,
SFXGE_INTR_TESTING,
SFXGE_INTR_STARTED
};
struct sfxge_intr_hdl {
int eih_rid;
void *eih_tag;
struct resource *eih_res;
};
struct sfxge_intr {
enum sfxge_intr_state state;
struct resource *msix_res;
struct sfxge_intr_hdl *table;
int n_alloc;
int type;
efsys_mem_t status;
uint32_t zero_count;
};
enum sfxge_mcdi_state {
SFXGE_MCDI_UNINITIALIZED = 0,
SFXGE_MCDI_INITIALIZED,
SFXGE_MCDI_BUSY,
SFXGE_MCDI_COMPLETED
};
struct sfxge_mcdi {
struct mtx lock;
efsys_mem_t mem;
enum sfxge_mcdi_state state;
efx_mcdi_transport_t transport;
/* Only used in debugging output */
char lock_name[SFXGE_LOCK_NAME_MAX];
};
struct sfxge_hw_stats {
clock_t update_time;
efsys_mem_t dma_buf;
void *decode_buf;
};
enum sfxge_port_state {
SFXGE_PORT_UNINITIALIZED = 0,
SFXGE_PORT_INITIALIZED,
SFXGE_PORT_STARTED
};
struct sfxge_port {
struct sfxge_softc *sc;
struct mtx lock;
enum sfxge_port_state init_state;
#ifndef SFXGE_HAVE_PAUSE_MEDIAOPTS
unsigned int wanted_fc;
#endif
struct sfxge_hw_stats phy_stats;
struct sfxge_hw_stats mac_stats;
efx_link_mode_t link_mode;
uint8_t mcast_addrs[EFX_MAC_MULTICAST_LIST_MAX *
EFX_MAC_ADDR_LEN];
unsigned int mcast_count;
/* Only used in debugging output */
char lock_name[SFXGE_LOCK_NAME_MAX];
};
enum sfxge_softc_state {
SFXGE_UNINITIALIZED = 0,
SFXGE_INITIALIZED,
SFXGE_REGISTERED,
SFXGE_STARTED
};
struct sfxge_softc {
device_t dev;
struct sx softc_lock;
char softc_lock_name[SFXGE_LOCK_NAME_MAX];
enum sfxge_softc_state init_state;
struct ifnet *ifnet;
unsigned int if_flags;
struct sysctl_oid *stats_node;
struct sysctl_oid *txqs_node;
struct task task_reset;
efx_family_t family;
caddr_t vpd_data;
size_t vpd_size;
efx_nic_t *enp;
efsys_lock_t enp_lock;
unsigned int rxq_entries;
unsigned int txq_entries;
bus_dma_tag_t parent_dma_tag;
efsys_bar_t bar;
struct sfxge_intr intr;
struct sfxge_mcdi mcdi;
struct sfxge_port port;
uint32_t buffer_table_next;
struct sfxge_evq *evq[SFXGE_RX_SCALE_MAX];
unsigned int ev_moderation;
#if EFSYS_OPT_QSTATS
clock_t ev_stats_update_time;
uint64_t ev_stats[EV_NQSTATS];
#endif
unsigned int max_rss_channels;
uma_zone_t rxq_cache;
struct sfxge_rxq *rxq[SFXGE_RX_SCALE_MAX];
unsigned int rx_indir_table[SFXGE_RX_SCALE_MAX];
struct sfxge_txq *txq[SFXGE_TXQ_NTYPES + SFXGE_RX_SCALE_MAX];
struct ifmedia media;
size_t rx_prefix_size;
size_t rx_buffer_size;
size_t rx_buffer_align;
uma_zone_t rx_buffer_zone;
unsigned int evq_max;
unsigned int evq_count;
unsigned int rxq_count;
unsigned int txq_count;
int tso_fw_assisted;
};
#define SFXGE_LINK_UP(sc) ((sc)->port.link_mode != EFX_LINK_DOWN)
#define SFXGE_RUNNING(sc) ((sc)->ifnet->if_drv_flags & IFF_DRV_RUNNING)
#define SFXGE_PARAM(_name) "hw.sfxge." #_name
SYSCTL_DECL(_hw_sfxge);
/*
* From sfxge.c.
*/
extern void sfxge_schedule_reset(struct sfxge_softc *sc);
extern void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n,
uint32_t *idp);
/*
* From sfxge_dma.c.
*/
extern int sfxge_dma_init(struct sfxge_softc *sc);
extern void sfxge_dma_fini(struct sfxge_softc *sc);
extern int sfxge_dma_alloc(struct sfxge_softc *sc, bus_size_t len,
efsys_mem_t *esmp);
extern void sfxge_dma_free(efsys_mem_t *esmp);
extern int sfxge_dma_map_sg_collapse(bus_dma_tag_t tag, bus_dmamap_t map,
struct mbuf **mp,
bus_dma_segment_t *segs,
int *nsegs, int maxsegs);
/*
* From sfxge_ev.c.
*/
extern int sfxge_ev_init(struct sfxge_softc *sc);
extern void sfxge_ev_fini(struct sfxge_softc *sc);
extern int sfxge_ev_start(struct sfxge_softc *sc);
extern void sfxge_ev_stop(struct sfxge_softc *sc);
extern int sfxge_ev_qpoll(struct sfxge_evq *evq);
/*
* From sfxge_intr.c.
*/
extern int sfxge_intr_init(struct sfxge_softc *sc);
extern void sfxge_intr_fini(struct sfxge_softc *sc);
extern int sfxge_intr_start(struct sfxge_softc *sc);
extern void sfxge_intr_stop(struct sfxge_softc *sc);
/*
* From sfxge_mcdi.c.
*/
extern int sfxge_mcdi_init(struct sfxge_softc *sc);
extern void sfxge_mcdi_fini(struct sfxge_softc *sc);
extern int sfxge_mcdi_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ip);
/*
* From sfxge_nvram.c.
*/
extern int sfxge_nvram_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ip);
/*
* From sfxge_port.c.
*/
extern int sfxge_port_init(struct sfxge_softc *sc);
extern void sfxge_port_fini(struct sfxge_softc *sc);
extern int sfxge_port_start(struct sfxge_softc *sc);
extern void sfxge_port_stop(struct sfxge_softc *sc);
extern void sfxge_mac_link_update(struct sfxge_softc *sc,
efx_link_mode_t mode);
extern int sfxge_mac_filter_set(struct sfxge_softc *sc);
extern int sfxge_port_ifmedia_init(struct sfxge_softc *sc);
#define SFXGE_MAX_MTU (9 * 1024)
#define SFXGE_ADAPTER_LOCK_INIT(_sc, _ifname) \
do { \
struct sfxge_softc *__sc = (_sc); \
\
snprintf((__sc)->softc_lock_name, \
sizeof((__sc)->softc_lock_name), \
"%s:softc", (_ifname)); \
sx_init(&(__sc)->softc_lock, (__sc)->softc_lock_name); \
} while (B_FALSE)
#define SFXGE_ADAPTER_LOCK_DESTROY(_sc) \
sx_destroy(&(_sc)->softc_lock)
#define SFXGE_ADAPTER_LOCK(_sc) \
sx_xlock(&(_sc)->softc_lock)
#define SFXGE_ADAPTER_UNLOCK(_sc) \
sx_xunlock(&(_sc)->softc_lock)
#define SFXGE_ADAPTER_LOCK_ASSERT_OWNED(_sc) \
sx_assert(&(_sc)->softc_lock, LA_XLOCKED)
#define SFXGE_PORT_LOCK_INIT(_port, _ifname) \
do { \
struct sfxge_port *__port = (_port); \
\
snprintf((__port)->lock_name, \
sizeof((__port)->lock_name), \
"%s:port", (_ifname)); \
mtx_init(&(__port)->lock, (__port)->lock_name, \
NULL, MTX_DEF); \
} while (B_FALSE)
#define SFXGE_PORT_LOCK_DESTROY(_port) \
mtx_destroy(&(_port)->lock)
#define SFXGE_PORT_LOCK(_port) \
mtx_lock(&(_port)->lock)
#define SFXGE_PORT_UNLOCK(_port) \
mtx_unlock(&(_port)->lock)
#define SFXGE_PORT_LOCK_ASSERT_OWNED(_port) \
mtx_assert(&(_port)->lock, MA_OWNED)
#define SFXGE_MCDI_LOCK_INIT(_mcdi, _ifname) \
do { \
struct sfxge_mcdi *__mcdi = (_mcdi); \
\
snprintf((__mcdi)->lock_name, \
sizeof((__mcdi)->lock_name), \
"%s:mcdi", (_ifname)); \
mtx_init(&(__mcdi)->lock, (__mcdi)->lock_name, \
NULL, MTX_DEF); \
} while (B_FALSE)
#define SFXGE_MCDI_LOCK_DESTROY(_mcdi) \
mtx_destroy(&(_mcdi)->lock)
#define SFXGE_MCDI_LOCK(_mcdi) \
mtx_lock(&(_mcdi)->lock)
#define SFXGE_MCDI_UNLOCK(_mcdi) \
mtx_unlock(&(_mcdi)->lock)
#define SFXGE_MCDI_LOCK_ASSERT_OWNED(_mcdi) \
mtx_assert(&(_mcdi)->lock, MA_OWNED)
#define SFXGE_EVQ_LOCK_INIT(_evq, _ifname, _evq_index) \
do { \
struct sfxge_evq *__evq = (_evq); \
\
snprintf((__evq)->lock_name, \
sizeof((__evq)->lock_name), \
"%s:evq%u", (_ifname), (_evq_index)); \
mtx_init(&(__evq)->lock, (__evq)->lock_name, \
NULL, MTX_DEF); \
} while (B_FALSE)
#define SFXGE_EVQ_LOCK_DESTROY(_evq) \
mtx_destroy(&(_evq)->lock)
#define SFXGE_EVQ_LOCK(_evq) \
mtx_lock(&(_evq)->lock)
#define SFXGE_EVQ_UNLOCK(_evq) \
mtx_unlock(&(_evq)->lock)
#define SFXGE_EVQ_LOCK_ASSERT_OWNED(_evq) \
mtx_assert(&(_evq)->lock, MA_OWNED)
#endif /* _SFXGE_H */
|