Annotation of src/sys/dev/pci/ixgbe/ixgbe.c, Revision 1.20
1.1 dyoung 1: /******************************************************************************
2:
3: Copyright (c) 2001-2011, Intel Corporation
4: All rights reserved.
5:
6: Redistribution and use in source and binary forms, with or without
7: modification, are permitted provided that the following conditions are met:
8:
9: 1. Redistributions of source code must retain the above copyright notice,
10: this list of conditions and the following disclaimer.
11:
12: 2. Redistributions in binary form must reproduce the above copyright
13: notice, this list of conditions and the following disclaimer in the
14: documentation and/or other materials provided with the distribution.
15:
16: 3. Neither the name of the Intel Corporation nor the names of its
17: contributors may be used to endorse or promote products derived from
18: this software without specific prior written permission.
19:
20: THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21: AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22: IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23: ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24: LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25: CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26: SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27: INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28: CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29: ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30: POSSIBILITY OF SUCH DAMAGE.
31:
32: ******************************************************************************/
33: /*
34: * Copyright (c) 2011 The NetBSD Foundation, Inc.
35: * All rights reserved.
36: *
37: * This code is derived from software contributed to The NetBSD Foundation
38: * by Coyote Point Systems, Inc.
39: *
40: * Redistribution and use in source and binary forms, with or without
41: * modification, are permitted provided that the following conditions
42: * are met:
43: * 1. Redistributions of source code must retain the above copyright
44: * notice, this list of conditions and the following disclaimer.
45: * 2. Redistributions in binary form must reproduce the above copyright
46: * notice, this list of conditions and the following disclaimer in the
47: * documentation and/or other materials provided with the distribution.
48: *
49: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
50: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
51: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
52: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
53: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
54: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
55: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
56: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
57: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
58: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
59: * POSSIBILITY OF SUCH DAMAGE.
60: */
61: /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.51 2011/04/25 23:34:21 jfv Exp $*/
1.20 ! msaitoh 62: /*$NetBSD: ixgbe.c,v 1.19 2015/02/07 00:02:09 christos Exp $*/
1.1 dyoung 63:
64: #include "opt_inet.h"
65:
66: #include "ixgbe.h"
67:
68: /*********************************************************************
69: * Set this to one to display debug statistics
70: *********************************************************************/
71: int ixgbe_display_debug_stats = 0;
72:
73: /*********************************************************************
74: * Driver version
75: *********************************************************************/
76: char ixgbe_driver_version[] = "2.3.10";
77:
78: /*********************************************************************
79: * PCI Device ID Table
80: *
81: * Used by probe to select devices to load on
82: * Last field stores an index into ixgbe_strings
83: * Last entry must be all 0s
84: *
85: * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
86: *********************************************************************/
87:
88: static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
89: {
90: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
91: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
92: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
93: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
94: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
95: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
96: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
97: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
98: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
99: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
100: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
101: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
102: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
103: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
104: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
105: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
106: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
107: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
108: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
109: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
1.20 ! msaitoh 110: {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
1.1 dyoung 111: /* required last entry */
112: {0, 0, 0, 0, 0}
113: };
114:
115: /*********************************************************************
116: * Table of branding strings
117: *********************************************************************/
118:
119: static const char *ixgbe_strings[] = {
120: "Intel(R) PRO/10GbE PCI-Express Network Driver"
121: };
122:
123: /*********************************************************************
124: * Function prototypes
125: *********************************************************************/
126: static int ixgbe_probe(device_t, cfdata_t, void *);
127: static void ixgbe_attach(device_t, device_t, void *);
128: static int ixgbe_detach(device_t, int);
129: #if 0
130: static int ixgbe_shutdown(device_t);
131: #endif
132: static void ixgbe_start(struct ifnet *);
133: static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
134: #if __FreeBSD_version >= 800000
135: static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
136: static int ixgbe_mq_start_locked(struct ifnet *,
137: struct tx_ring *, struct mbuf *);
138: static void ixgbe_qflush(struct ifnet *);
139: #endif
140: static int ixgbe_ioctl(struct ifnet *, u_long, void *);
141: static void ixgbe_ifstop(struct ifnet *, int);
142: static int ixgbe_init(struct ifnet *);
143: static void ixgbe_init_locked(struct adapter *);
144: static void ixgbe_stop(void *);
145: static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
146: static int ixgbe_media_change(struct ifnet *);
147: static void ixgbe_identify_hardware(struct adapter *);
148: static int ixgbe_allocate_pci_resources(struct adapter *,
149: const struct pci_attach_args *);
150: static int ixgbe_allocate_msix(struct adapter *,
151: const struct pci_attach_args *);
152: static int ixgbe_allocate_legacy(struct adapter *,
153: const struct pci_attach_args *);
154: static int ixgbe_allocate_queues(struct adapter *);
155: static int ixgbe_setup_msix(struct adapter *);
156: static void ixgbe_free_pci_resources(struct adapter *);
157: static void ixgbe_local_timer(void *);
158: static int ixgbe_setup_interface(device_t, struct adapter *);
159: static void ixgbe_config_link(struct adapter *);
160:
161: static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
162: static int ixgbe_setup_transmit_structures(struct adapter *);
163: static void ixgbe_setup_transmit_ring(struct tx_ring *);
164: static void ixgbe_initialize_transmit_units(struct adapter *);
165: static void ixgbe_free_transmit_structures(struct adapter *);
166: static void ixgbe_free_transmit_buffers(struct tx_ring *);
167:
168: static int ixgbe_allocate_receive_buffers(struct rx_ring *);
169: static int ixgbe_setup_receive_structures(struct adapter *);
170: static int ixgbe_setup_receive_ring(struct rx_ring *);
171: static void ixgbe_initialize_receive_units(struct adapter *);
172: static void ixgbe_free_receive_structures(struct adapter *);
173: static void ixgbe_free_receive_buffers(struct rx_ring *);
174: static void ixgbe_setup_hw_rsc(struct rx_ring *);
175:
176: static void ixgbe_enable_intr(struct adapter *);
177: static void ixgbe_disable_intr(struct adapter *);
178: static void ixgbe_update_stats_counters(struct adapter *);
179: static bool ixgbe_txeof(struct tx_ring *);
180: static bool ixgbe_rxeof(struct ix_queue *, int);
181: static void ixgbe_rx_checksum(u32, struct mbuf *, u32,
182: struct ixgbe_hw_stats *);
183: static void ixgbe_set_promisc(struct adapter *);
184: static void ixgbe_set_multi(struct adapter *);
185: static void ixgbe_update_link_status(struct adapter *);
186: static void ixgbe_refresh_mbufs(struct rx_ring *, int);
187: static int ixgbe_xmit(struct tx_ring *, struct mbuf *);
188: static int ixgbe_set_flowcntl(SYSCTLFN_PROTO);
189: static int ixgbe_set_advertise(SYSCTLFN_PROTO);
190: static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
191: struct ixgbe_dma_alloc *, int);
192: static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
193: static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
194: const char *, int *, int);
195: static u32 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
196: static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
197: static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
198: static void ixgbe_configure_ivars(struct adapter *);
199: static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
200:
201: static void ixgbe_setup_vlan_hw_support(struct adapter *);
202: #if 0
203: static void ixgbe_register_vlan(void *, struct ifnet *, u16);
204: static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
205: #endif
206:
207: static void ixgbe_add_hw_stats(struct adapter *adapter);
208:
209: static __inline void ixgbe_rx_discard(struct rx_ring *, int);
210: static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
211: struct mbuf *, u32);
212:
213: /* Support for pluggable optic modules */
214: static bool ixgbe_sfp_probe(struct adapter *);
215: static void ixgbe_setup_optics(struct adapter *);
216:
217: /* Legacy (single vector interrupt handler */
218: static int ixgbe_legacy_irq(void *);
219:
220: #if defined(NETBSD_MSI_OR_MSIX)
221: /* The MSI/X Interrupt handlers */
222: static void ixgbe_msix_que(void *);
223: static void ixgbe_msix_link(void *);
224: #endif
225:
226: /* Software interrupts for deferred work */
227: static void ixgbe_handle_que(void *);
228: static void ixgbe_handle_link(void *);
229: static void ixgbe_handle_msf(void *);
230: static void ixgbe_handle_mod(void *);
231:
232: const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *);
233: static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *);
234:
235: #ifdef IXGBE_FDIR
236: static void ixgbe_atr(struct tx_ring *, struct mbuf *);
237: static void ixgbe_reinit_fdir(void *, int);
238: #endif
239:
240: /*********************************************************************
241: * FreeBSD Device Interface Entry Points
242: *********************************************************************/
243:
244: CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter),
245: ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL,
246: DVF_DETACH_SHUTDOWN);
247:
248: #if 0
249: devclass_t ixgbe_devclass;
250: DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
251:
252: MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
253: MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
254: #endif
255:
256: /*
257: ** TUNEABLE PARAMETERS:
258: */
259:
260: /*
261: ** AIM: Adaptive Interrupt Moderation
262: ** which means that the interrupt rate
263: ** is varied over time based on the
264: ** traffic for that interrupt vector
265: */
266: static int ixgbe_enable_aim = TRUE;
267: #define TUNABLE_INT(__x, __y)
268: TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
269:
270: static int ixgbe_max_interrupt_rate = (8000000 / IXGBE_LOW_LATENCY);
271: TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
272:
273: /* How many packets rxeof tries to clean at a time */
274: static int ixgbe_rx_process_limit = 256;
275: TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
276:
277: /* Flow control setting, default to full */
278: static int ixgbe_flow_control = ixgbe_fc_full;
279: TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
280:
281: /*
282: ** Smart speed setting, default to on
283: ** this only works as a compile option
284: ** right now as its during attach, set
285: ** this to 'ixgbe_smart_speed_off' to
286: ** disable.
287: */
288: static int ixgbe_smart_speed = ixgbe_smart_speed_on;
289:
290: /*
291: * MSIX should be the default for best performance,
292: * but this allows it to be forced off for testing.
293: */
294: static int ixgbe_enable_msix = 1;
295: TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
296:
297: /*
298: * Header split: this causes the hardware to DMA
299: * the header into a separate mbuf from the payload,
300: * it can be a performance win in some workloads, but
301: * in others it actually hurts, its off by default.
302: */
303: static bool ixgbe_header_split = FALSE;
304: TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
305:
306: #if defined(NETBSD_MSI_OR_MSIX)
307: /*
308: * Number of Queues, can be set to 0,
309: * it then autoconfigures based on the
310: * number of cpus with a max of 8. This
311: * can be overriden manually here.
312: */
313: static int ixgbe_num_queues = 0;
314: TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
315: #endif
316:
317: /*
318: ** Number of TX descriptors per ring,
319: ** setting higher than RX as this seems
320: ** the better performing choice.
321: */
322: static int ixgbe_txd = PERFORM_TXD;
323: TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
324:
325: /* Number of RX descriptors per ring */
326: static int ixgbe_rxd = PERFORM_RXD;
327: TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
328:
329: /* Keep running tab on them for sanity check */
330: static int ixgbe_total_ports;
331:
332: #ifdef IXGBE_FDIR
333: /*
334: ** For Flow Director: this is the
335: ** number of TX packets we sample
336: ** for the filter pool, this means
337: ** every 20th packet will be probed.
338: **
339: ** This feature can be disabled by
340: ** setting this to 0.
341: */
342: static int atr_sample_rate = 20;
343: /*
344: ** Flow Director actually 'steals'
345: ** part of the packet buffer as its
346: ** filter pool, this variable controls
347: ** how much it uses:
348: ** 0 = 64K, 1 = 128K, 2 = 256K
349: */
350: static int fdir_pballoc = 1;
351: #endif
352:
353: /*********************************************************************
354: * Device identification routine
355: *
356: * ixgbe_probe determines if the driver should be loaded on
357: * adapter based on PCI vendor/device id of the adapter.
358: *
359: * return 1 on success, 0 on failure
360: *********************************************************************/
361:
362: static int
363: ixgbe_probe(device_t dev, cfdata_t cf, void *aux)
364: {
365: const struct pci_attach_args *pa = aux;
366:
367: return (ixgbe_lookup(pa) != NULL) ? 1 : 0;
368: }
369:
370: static ixgbe_vendor_info_t *
371: ixgbe_lookup(const struct pci_attach_args *pa)
372: {
373: pcireg_t subid;
374: ixgbe_vendor_info_t *ent;
375:
376: INIT_DEBUGOUT("ixgbe_probe: begin");
377:
378: if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID)
379: return NULL;
380:
381: subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
382:
383: for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) {
384: if (PCI_VENDOR(pa->pa_id) == ent->vendor_id &&
385: PCI_PRODUCT(pa->pa_id) == ent->device_id &&
386:
387: (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id ||
388: ent->subvendor_id == 0) &&
389:
390: (PCI_SUBSYS_ID(subid) == ent->subdevice_id ||
391: ent->subdevice_id == 0)) {
392: ++ixgbe_total_ports;
393: return ent;
394: }
395: }
396: return NULL;
397: }
398:
399:
400: static void
401: ixgbe_sysctl_attach(struct adapter *adapter)
402: {
403: struct sysctllog **log;
404: const struct sysctlnode *rnode, *cnode;
405: device_t dev;
406:
407: dev = adapter->dev;
408: log = &adapter->sysctllog;
409:
410: if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) {
411: aprint_error_dev(dev, "could not create sysctl root\n");
412: return;
413: }
414:
415: if (sysctl_createv(log, 0, &rnode, &cnode,
416: CTLFLAG_READONLY, CTLTYPE_INT,
417: "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"),
418: NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0)
419: aprint_error_dev(dev, "could not create sysctl\n");
420:
421: if (sysctl_createv(log, 0, &rnode, &cnode,
422: CTLFLAG_READONLY, CTLTYPE_INT,
423: "num_queues", SYSCTL_DESCR("Number of queues"),
424: NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0)
425: aprint_error_dev(dev, "could not create sysctl\n");
426:
427: if (sysctl_createv(log, 0, &rnode, &cnode,
428: CTLFLAG_READWRITE, CTLTYPE_INT,
429: "flow_control", SYSCTL_DESCR("Flow Control"),
1.3 dsl 430: ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0)
1.1 dyoung 431: aprint_error_dev(dev, "could not create sysctl\n");
432:
433: if (sysctl_createv(log, 0, &rnode, &cnode,
434: CTLFLAG_READWRITE, CTLTYPE_INT,
435: "advertise_gig", SYSCTL_DESCR("1G Link"),
1.3 dsl 436: ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0)
1.1 dyoung 437: aprint_error_dev(dev, "could not create sysctl\n");
438:
439: /* XXX This is an *instance* sysctl controlling a *global* variable.
440: * XXX It's that way in the FreeBSD driver that this derives from.
441: */
442: if (sysctl_createv(log, 0, &rnode, &cnode,
443: CTLFLAG_READWRITE, CTLTYPE_INT,
444: "enable_aim", SYSCTL_DESCR("Interrupt Moderation"),
445: NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0)
446: aprint_error_dev(dev, "could not create sysctl\n");
447: }
448:
449: /*********************************************************************
450: * Device initialization routine
451: *
452: * The attach entry point is called when the driver is being loaded.
453: * This routine identifies the type of hardware, allocates all resources
454: * and initializes the hardware.
455: *
456: * return 0 on success, positive on failure
457: *********************************************************************/
458:
459: static void
460: ixgbe_attach(device_t parent, device_t dev, void *aux)
461: {
462: struct adapter *adapter;
463: struct ixgbe_hw *hw;
464: int error = 0;
465: u16 csum;
466: u32 ctrl_ext;
467: ixgbe_vendor_info_t *ent;
468: const struct pci_attach_args *pa = aux;
469:
470: INIT_DEBUGOUT("ixgbe_attach: begin");
471:
472: /* Allocate, clear, and link in our adapter structure */
473: adapter = device_private(dev);
474: adapter->dev = adapter->osdep.dev = dev;
475: hw = &adapter->hw;
476: adapter->osdep.pc = pa->pa_pc;
477: adapter->osdep.tag = pa->pa_tag;
478: adapter->osdep.dmat = pa->pa_dmat;
479:
480: ent = ixgbe_lookup(pa);
481:
482: KASSERT(ent != NULL);
483:
484: aprint_normal(": %s, Version - %s\n",
485: ixgbe_strings[ent->index], ixgbe_driver_version);
486:
487: /* Core Lock Init*/
488: IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev));
489:
490: /* SYSCTL APIs */
491:
492: ixgbe_sysctl_attach(adapter);
493:
494: /* Set up the timer callout */
495: callout_init(&adapter->timer, 0);
496:
497: /* Determine hardware revision */
498: ixgbe_identify_hardware(adapter);
499:
500: /* Do base PCI setup - map BAR0 */
501: if (ixgbe_allocate_pci_resources(adapter, pa)) {
502: aprint_error_dev(dev, "Allocation of PCI resources failed\n");
503: error = ENXIO;
504: goto err_out;
505: }
506:
507: /* Do descriptor calc and sanity checks */
508: if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
509: ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
510: aprint_error_dev(dev, "TXD config issue, using default!\n");
511: adapter->num_tx_desc = DEFAULT_TXD;
512: } else
513: adapter->num_tx_desc = ixgbe_txd;
514:
515: /*
516: ** With many RX rings it is easy to exceed the
517: ** system mbuf allocation. Tuning nmbclusters
518: ** can alleviate this.
519: */
520: if (nmbclusters > 0 ) {
521: int s;
522: s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
523: if (s > nmbclusters) {
524: aprint_error_dev(dev, "RX Descriptors exceed "
525: "system mbuf max, using default instead!\n");
526: ixgbe_rxd = DEFAULT_RXD;
527: }
528: }
529:
530: if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
531: ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
532: aprint_error_dev(dev, "RXD config issue, using default!\n");
533: adapter->num_rx_desc = DEFAULT_RXD;
534: } else
535: adapter->num_rx_desc = ixgbe_rxd;
536:
537: /* Allocate our TX/RX Queues */
538: if (ixgbe_allocate_queues(adapter)) {
539: error = ENOMEM;
540: goto err_out;
541: }
542:
543: /* Allocate multicast array memory. */
544: adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
545: MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
546: if (adapter->mta == NULL) {
547: aprint_error_dev(dev, "Cannot allocate multicast setup array\n");
548: error = ENOMEM;
549: goto err_late;
550: }
551:
552: /* Initialize the shared code */
553: error = ixgbe_init_shared_code(hw);
554: if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
555: /*
556: ** No optics in this port, set up
557: ** so the timer routine will probe
558: ** for later insertion.
559: */
560: adapter->sfp_probe = TRUE;
561: error = 0;
562: } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
563: aprint_error_dev(dev,"Unsupported SFP+ module detected!\n");
564: error = EIO;
565: goto err_late;
566: } else if (error) {
567: aprint_error_dev(dev,"Unable to initialize the shared code\n");
568: error = EIO;
569: goto err_late;
570: }
571:
572: /* Make sure we have a good EEPROM before we read from it */
573: if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
574: aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n");
575: error = EIO;
576: goto err_late;
577: }
578:
579: /* Get Hardware Flow Control setting */
580: hw->fc.requested_mode = ixgbe_fc_full;
581: hw->fc.pause_time = IXGBE_FC_PAUSE;
582: hw->fc.low_water = IXGBE_FC_LO;
583: hw->fc.high_water = IXGBE_FC_HI;
584: hw->fc.send_xon = TRUE;
585:
586: error = ixgbe_init_hw(hw);
587: if (error == IXGBE_ERR_EEPROM_VERSION) {
588: aprint_error_dev(dev, "This device is a pre-production adapter/"
589: "LOM. Please be aware there may be issues associated "
590: "with your hardware.\n If you are experiencing problems "
591: "please contact your Intel or hardware representative "
592: "who provided you with this hardware.\n");
593: } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED)
594: aprint_error_dev(dev,"Unsupported SFP+ Module\n");
595:
596: if (error) {
597: error = EIO;
598: aprint_error_dev(dev,"Hardware Initialization Failure\n");
599: goto err_late;
600: }
601:
602: /* Detect and set physical type */
603: ixgbe_setup_optics(adapter);
604:
605: if ((adapter->msix > 1) && (ixgbe_enable_msix))
606: error = ixgbe_allocate_msix(adapter, pa);
607: else
608: error = ixgbe_allocate_legacy(adapter, pa);
609: if (error)
610: goto err_late;
611:
612: /* Setup OS specific network interface */
613: if (ixgbe_setup_interface(dev, adapter) != 0)
614: goto err_late;
615:
616: /* Sysctl for limiting the amount of work done in software interrupts */
617: ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
618: "max number of rx packets to process", &adapter->rx_process_limit,
619: ixgbe_rx_process_limit);
620:
621: /* Initialize statistics */
622: ixgbe_update_stats_counters(adapter);
623:
624: /* Print PCIE bus type/speed/width info */
625: ixgbe_get_bus_info(hw);
626: aprint_normal_dev(dev,"PCI Express Bus: Speed %s %s\n",
627: ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
628: (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
629: (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
630: (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
631: (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
632: ("Unknown"));
633:
634: if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
635: (hw->bus.speed == ixgbe_bus_speed_2500)) {
636: aprint_error_dev(dev, "PCI-Express bandwidth available"
637: " for this card\n is not sufficient for"
638: " optimal performance.\n");
639: aprint_error_dev(dev, "For optimal performance a x8 "
640: "PCIE, or x4 PCIE 2 slot is required.\n");
641: }
642:
643: /* let hardware know driver is loaded */
644: ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
645: ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
646: IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
647:
648: ixgbe_add_hw_stats(adapter);
649:
650: INIT_DEBUGOUT("ixgbe_attach: end");
651: return;
652: err_late:
653: ixgbe_free_transmit_structures(adapter);
654: ixgbe_free_receive_structures(adapter);
655: err_out:
656: if (adapter->ifp != NULL)
657: if_free(adapter->ifp);
658: ixgbe_free_pci_resources(adapter);
659: if (adapter->mta != NULL)
660: free(adapter->mta, M_DEVBUF);
661: return;
662:
663: }
664:
665: /*********************************************************************
666: * Device removal routine
667: *
668: * The detach entry point is called when the driver is being removed.
669: * This routine stops the adapter and deallocates all the resources
670: * that were allocated for driver operation.
671: *
672: * return 0 on success, positive on failure
673: *********************************************************************/
674:
675: static int
676: ixgbe_detach(device_t dev, int flags)
677: {
678: struct adapter *adapter = device_private(dev);
679: struct tx_ring *txr = adapter->tx_rings;
680: struct rx_ring *rxr = adapter->rx_rings;
681: struct ixgbe_hw_stats *stats = &adapter->stats;
682: struct ix_queue *que = adapter->queues;
683: u32 ctrl_ext;
684:
685: INIT_DEBUGOUT("ixgbe_detach: begin");
686:
687: /* Make sure VLANs are not using driver */
688: if (!VLAN_ATTACHED(&adapter->osdep.ec))
689: ; /* nothing to do: no VLANs */
690: else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0)
691: vlan_ifdetach(adapter->ifp);
692: else {
693: aprint_error_dev(dev, "VLANs in use\n");
694: return EBUSY;
695: }
696:
697: IXGBE_CORE_LOCK(adapter);
698: ixgbe_stop(adapter);
699: IXGBE_CORE_UNLOCK(adapter);
700:
701: for (int i = 0; i < adapter->num_queues; i++, que++) {
702: softint_disestablish(que->que_si);
703: }
704:
705: /* Drain the Link queue */
706: softint_disestablish(adapter->link_si);
707: softint_disestablish(adapter->mod_si);
708: softint_disestablish(adapter->msf_si);
709: #ifdef IXGBE_FDIR
710: softint_disestablish(adapter->fdir_si);
711: #endif
712:
713: /* let hardware know driver is unloading */
714: ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
715: ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
716: IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
717:
718: ether_ifdetach(adapter->ifp);
719: callout_halt(&adapter->timer, NULL);
720: ixgbe_free_pci_resources(adapter);
721: #if 0 /* XXX the NetBSD port is probably missing something here */
722: bus_generic_detach(dev);
723: #endif
724: if_detach(adapter->ifp);
725:
726: sysctl_teardown(&adapter->sysctllog);
727: evcnt_detach(&adapter->handleq);
728: evcnt_detach(&adapter->req);
729: evcnt_detach(&adapter->morerx);
730: evcnt_detach(&adapter->moretx);
731: evcnt_detach(&adapter->txloops);
732: evcnt_detach(&adapter->efbig_tx_dma_setup);
733: evcnt_detach(&adapter->m_defrag_failed);
734: evcnt_detach(&adapter->efbig2_tx_dma_setup);
735: evcnt_detach(&adapter->einval_tx_dma_setup);
736: evcnt_detach(&adapter->other_tx_dma_setup);
737: evcnt_detach(&adapter->eagain_tx_dma_setup);
738: evcnt_detach(&adapter->enomem_tx_dma_setup);
739: evcnt_detach(&adapter->watchdog_events);
740: evcnt_detach(&adapter->tso_err);
741: evcnt_detach(&adapter->tso_tx);
742: evcnt_detach(&adapter->link_irq);
743: for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
744: evcnt_detach(&txr->no_desc_avail);
745: evcnt_detach(&txr->total_packets);
746:
747: if (i < __arraycount(adapter->stats.mpc)) {
748: evcnt_detach(&adapter->stats.mpc[i]);
749: }
750: if (i < __arraycount(adapter->stats.pxontxc)) {
751: evcnt_detach(&adapter->stats.pxontxc[i]);
752: evcnt_detach(&adapter->stats.pxonrxc[i]);
753: evcnt_detach(&adapter->stats.pxofftxc[i]);
754: evcnt_detach(&adapter->stats.pxoffrxc[i]);
755: evcnt_detach(&adapter->stats.pxon2offc[i]);
756: }
757: if (i < __arraycount(adapter->stats.qprc)) {
758: evcnt_detach(&adapter->stats.qprc[i]);
759: evcnt_detach(&adapter->stats.qptc[i]);
760: evcnt_detach(&adapter->stats.qbrc[i]);
761: evcnt_detach(&adapter->stats.qbtc[i]);
762: evcnt_detach(&adapter->stats.qprdc[i]);
763: }
764:
765: evcnt_detach(&rxr->rx_packets);
766: evcnt_detach(&rxr->rx_bytes);
767: evcnt_detach(&rxr->no_jmbuf);
768: evcnt_detach(&rxr->rx_discarded);
769: evcnt_detach(&rxr->rx_split_packets);
770: evcnt_detach(&rxr->rx_irq);
771: }
772: evcnt_detach(&stats->ipcs);
773: evcnt_detach(&stats->l4cs);
774: evcnt_detach(&stats->ipcs_bad);
775: evcnt_detach(&stats->l4cs_bad);
776: evcnt_detach(&stats->intzero);
777: evcnt_detach(&stats->legint);
778: evcnt_detach(&stats->crcerrs);
779: evcnt_detach(&stats->illerrc);
780: evcnt_detach(&stats->errbc);
781: evcnt_detach(&stats->mspdc);
782: evcnt_detach(&stats->mlfc);
783: evcnt_detach(&stats->mrfc);
784: evcnt_detach(&stats->rlec);
785: evcnt_detach(&stats->lxontxc);
786: evcnt_detach(&stats->lxonrxc);
787: evcnt_detach(&stats->lxofftxc);
788: evcnt_detach(&stats->lxoffrxc);
789:
790: /* Packet Reception Stats */
791: evcnt_detach(&stats->tor);
792: evcnt_detach(&stats->gorc);
793: evcnt_detach(&stats->tpr);
794: evcnt_detach(&stats->gprc);
795: evcnt_detach(&stats->mprc);
796: evcnt_detach(&stats->bprc);
797: evcnt_detach(&stats->prc64);
798: evcnt_detach(&stats->prc127);
799: evcnt_detach(&stats->prc255);
800: evcnt_detach(&stats->prc511);
801: evcnt_detach(&stats->prc1023);
802: evcnt_detach(&stats->prc1522);
803: evcnt_detach(&stats->ruc);
804: evcnt_detach(&stats->rfc);
805: evcnt_detach(&stats->roc);
806: evcnt_detach(&stats->rjc);
807: evcnt_detach(&stats->mngprc);
808: evcnt_detach(&stats->xec);
809:
810: /* Packet Transmission Stats */
811: evcnt_detach(&stats->gotc);
812: evcnt_detach(&stats->tpt);
813: evcnt_detach(&stats->gptc);
814: evcnt_detach(&stats->bptc);
815: evcnt_detach(&stats->mptc);
816: evcnt_detach(&stats->mngptc);
817: evcnt_detach(&stats->ptc64);
818: evcnt_detach(&stats->ptc127);
819: evcnt_detach(&stats->ptc255);
820: evcnt_detach(&stats->ptc511);
821: evcnt_detach(&stats->ptc1023);
822: evcnt_detach(&stats->ptc1522);
823:
824: /* FC Stats */
825: evcnt_detach(&stats->fccrc);
826: evcnt_detach(&stats->fclast);
827: if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
828: evcnt_detach(&stats->fcoerpdc);
829: evcnt_detach(&stats->fcoeprc);
830: evcnt_detach(&stats->fcoeptc);
831: evcnt_detach(&stats->fcoedwrc);
832: evcnt_detach(&stats->fcoedwtc);
833: }
834:
835: ixgbe_free_transmit_structures(adapter);
836: ixgbe_free_receive_structures(adapter);
837: free(adapter->mta, M_DEVBUF);
838:
839: IXGBE_CORE_LOCK_DESTROY(adapter);
840: return (0);
841: }
842:
843: /*********************************************************************
844: *
845: * Shutdown entry point
846: *
847: **********************************************************************/
848:
849: #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */
850: static int
851: ixgbe_shutdown(device_t dev)
852: {
853: struct adapter *adapter = device_private(dev);
854: IXGBE_CORE_LOCK(adapter);
855: ixgbe_stop(adapter);
856: IXGBE_CORE_UNLOCK(adapter);
857: return (0);
858: }
859: #endif
860:
861:
862: /*********************************************************************
863: * Transmit entry point
864: *
865: * ixgbe_start is called by the stack to initiate a transmit.
866: * The driver will remain in this routine as long as there are
867: * packets to transmit and transmit resources are available.
868: * In case resources are not available stack is notified and
869: * the packet is requeued.
870: **********************************************************************/
871:
872: static void
873: ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
874: {
875: int rc;
876: struct mbuf *m_head;
877: struct adapter *adapter = txr->adapter;
878:
879: IXGBE_TX_LOCK_ASSERT(txr);
880:
881: if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) !=
882: IFF_RUNNING)
883: return;
884: if (!adapter->link_active)
885: return;
886:
887: while (!IFQ_IS_EMPTY(&ifp->if_snd)) {
888:
889: IFQ_POLL(&ifp->if_snd, m_head);
890: if (m_head == NULL)
891: break;
892:
893: if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) {
894: ifp->if_flags |= IFF_OACTIVE;
895: break;
896: }
897: IFQ_DEQUEUE(&ifp->if_snd, m_head);
898: if (rc == EFBIG) {
899: struct mbuf *mtmp;
900:
901: if ((mtmp = m_defrag(m_head, M_DONTWAIT)) != NULL) {
902: m_head = mtmp;
903: rc = ixgbe_xmit(txr, m_head);
904: if (rc != 0)
905: adapter->efbig2_tx_dma_setup.ev_count++;
906: } else
907: adapter->m_defrag_failed.ev_count++;
908: }
909: if (rc != 0) {
910: m_freem(m_head);
911: continue;
912: }
913:
914: /* Send a copy of the frame to the BPF listener */
915: bpf_mtap(ifp, m_head);
916:
917: /* Set watchdog on */
918: getmicrotime(&txr->watchdog_time);
919: txr->queue_status = IXGBE_QUEUE_WORKING;
920:
921: }
922: return;
923: }
924:
925: /*
926: * Legacy TX start - called by the stack, this
927: * always uses the first tx ring, and should
928: * not be used with multiqueue tx enabled.
929: */
930: static void
931: ixgbe_start(struct ifnet *ifp)
932: {
933: struct adapter *adapter = ifp->if_softc;
934: struct tx_ring *txr = adapter->tx_rings;
935:
936: if (ifp->if_flags & IFF_RUNNING) {
937: IXGBE_TX_LOCK(txr);
938: ixgbe_start_locked(txr, ifp);
939: IXGBE_TX_UNLOCK(txr);
940: }
941: return;
942: }
943:
944: #if __FreeBSD_version >= 800000
945: /*
946: ** Multiqueue Transmit driver
947: **
948: */
949: static int
950: ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
951: {
952: struct adapter *adapter = ifp->if_softc;
953: struct ix_queue *que;
954: struct tx_ring *txr;
955: int i = 0, err = 0;
956:
957: /* Which queue to use */
958: if ((m->m_flags & M_FLOWID) != 0)
959: i = m->m_pkthdr.flowid % adapter->num_queues;
960:
961: txr = &adapter->tx_rings[i];
962: que = &adapter->queues[i];
963:
964: if (IXGBE_TX_TRYLOCK(txr)) {
965: err = ixgbe_mq_start_locked(ifp, txr, m);
966: IXGBE_TX_UNLOCK(txr);
967: } else {
968: err = drbr_enqueue(ifp, txr->br, m);
969: softint_schedule(que->que_si);
970: }
971:
972: return (err);
973: }
974:
975: static int
976: ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
977: {
978: struct adapter *adapter = txr->adapter;
979: struct mbuf *next;
980: int enqueued, err = 0;
981:
982: if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) !=
983: IFF_RUNNING || adapter->link_active == 0) {
984: if (m != NULL)
985: err = drbr_enqueue(ifp, txr->br, m);
986: return (err);
987: }
988:
989: enqueued = 0;
990: if (m == NULL) {
991: next = drbr_dequeue(ifp, txr->br);
992: } else if (drbr_needs_enqueue(ifp, txr->br)) {
993: if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
994: return (err);
995: next = drbr_dequeue(ifp, txr->br);
996: } else
997: next = m;
998:
999: /* Process the queue */
1000: while (next != NULL) {
1001: if ((err = ixgbe_xmit(txr, &next)) != 0) {
1002: if (next != NULL)
1003: err = drbr_enqueue(ifp, txr->br, next);
1004: break;
1005: }
1006: enqueued++;
1007: drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
1008: /* Send a copy of the frame to the BPF listener */
1009: bpf_mtap(ifp, next);
1010: if ((ifp->if_flags & IFF_RUNNING) == 0)
1011: break;
1012: if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
1013: ixgbe_txeof(txr);
1014: if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
1015: ifp->if_flags |= IFF_OACTIVE;
1016: break;
1017: }
1018: next = drbr_dequeue(ifp, txr->br);
1019: }
1020:
1021: if (enqueued > 0) {
1022: /* Set watchdog on */
1023: txr->queue_status = IXGBE_QUEUE_WORKING;
1024: getmicrotime(&txr->watchdog_time);
1025: }
1026:
1027: return (err);
1028: }
1029:
1030: /*
1031: ** Flush all ring buffers
1032: */
1033: static void
1034: ixgbe_qflush(struct ifnet *ifp)
1035: {
1036: struct adapter *adapter = ifp->if_softc;
1037: struct tx_ring *txr = adapter->tx_rings;
1038: struct mbuf *m;
1039:
1040: for (int i = 0; i < adapter->num_queues; i++, txr++) {
1041: IXGBE_TX_LOCK(txr);
1042: while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1043: m_freem(m);
1044: IXGBE_TX_UNLOCK(txr);
1045: }
1046: if_qflush(ifp);
1047: }
1048: #endif /* __FreeBSD_version >= 800000 */
1049:
1050: static int
1051: ixgbe_ifflags_cb(struct ethercom *ec)
1052: {
1053: struct ifnet *ifp = &ec->ec_if;
1054: struct adapter *adapter = ifp->if_softc;
1055: int change = ifp->if_flags ^ adapter->if_flags, rc = 0;
1056:
1057: IXGBE_CORE_LOCK(adapter);
1058:
1059: if (change != 0)
1060: adapter->if_flags = ifp->if_flags;
1061:
1062: if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0)
1063: rc = ENETRESET;
1064: else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0)
1065: ixgbe_set_promisc(adapter);
1066:
1067: IXGBE_CORE_UNLOCK(adapter);
1068:
1069: return rc;
1070: }
1071:
1072: /*********************************************************************
1073: * Ioctl entry point
1074: *
1075: * ixgbe_ioctl is called when the user wants to configure the
1076: * interface.
1077: *
1078: * return 0 on success, positive on failure
1079: **********************************************************************/
1080:
1081: static int
1082: ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data)
1083: {
1084: struct adapter *adapter = ifp->if_softc;
1085: struct ifcapreq *ifcr = data;
1086: struct ifreq *ifr = data;
1087: int error = 0;
1088: int l4csum_en;
1089: const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx|
1090: IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx;
1091:
1092: switch (command) {
1093: case SIOCSIFFLAGS:
1094: IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1095: break;
1096: case SIOCADDMULTI:
1097: case SIOCDELMULTI:
1098: IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1099: break;
1100: case SIOCSIFMEDIA:
1101: case SIOCGIFMEDIA:
1102: IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1103: break;
1104: case SIOCSIFCAP:
1105: IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1106: break;
1107: case SIOCSIFMTU:
1108: IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1109: break;
1110: default:
1111: IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1112: break;
1113: }
1114:
1115: switch (command) {
1116: case SIOCSIFMEDIA:
1117: case SIOCGIFMEDIA:
1118: return ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1119: case SIOCSIFCAP:
1120: /* Layer-4 Rx checksum offload has to be turned on and
1121: * off as a unit.
1122: */
1123: l4csum_en = ifcr->ifcr_capenable & l4csum;
1124: if (l4csum_en != l4csum && l4csum_en != 0)
1125: return EINVAL;
1126: /*FALLTHROUGH*/
1127: case SIOCADDMULTI:
1128: case SIOCDELMULTI:
1129: case SIOCSIFFLAGS:
1130: case SIOCSIFMTU:
1131: default:
1132: if ((error = ether_ioctl(ifp, command, data)) != ENETRESET)
1133: return error;
1134: if ((ifp->if_flags & IFF_RUNNING) == 0)
1135: ;
1136: else if (command == SIOCSIFCAP || command == SIOCSIFMTU) {
1137: IXGBE_CORE_LOCK(adapter);
1138: ixgbe_init_locked(adapter);
1139: IXGBE_CORE_UNLOCK(adapter);
1140: } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) {
1141: /*
1142: * Multicast list has changed; set the hardware filter
1143: * accordingly.
1144: */
1145: IXGBE_CORE_LOCK(adapter);
1146: ixgbe_disable_intr(adapter);
1147: ixgbe_set_multi(adapter);
1148: ixgbe_enable_intr(adapter);
1149: IXGBE_CORE_UNLOCK(adapter);
1150: }
1151: return 0;
1152: }
1153: }
1154:
1155: /*********************************************************************
1156: * Init entry point
1157: *
1158: * This routine is used in two ways. It is used by the stack as
1159: * init entry point in network interface structure. It is also used
1160: * by the driver as a hw/sw initialization routine to get to a
1161: * consistent state.
1162: *
1163: * return 0 on success, positive on failure
1164: **********************************************************************/
1165: #define IXGBE_MHADD_MFS_SHIFT 16
1166:
1167: static void
1168: ixgbe_init_locked(struct adapter *adapter)
1169: {
1170: struct ifnet *ifp = adapter->ifp;
1171: device_t dev = adapter->dev;
1172: struct ixgbe_hw *hw = &adapter->hw;
1173: u32 k, txdctl, mhadd, gpie;
1174: u32 rxdctl, rxctrl;
1175:
1176: /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */
1177:
1178: KASSERT(mutex_owned(&adapter->core_mtx));
1179: INIT_DEBUGOUT("ixgbe_init: begin");
1180: hw->adapter_stopped = FALSE;
1181: ixgbe_stop_adapter(hw);
1182: callout_stop(&adapter->timer);
1183:
1184: /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */
1185: adapter->max_frame_size =
1186: ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1187:
1188: /* reprogram the RAR[0] in case user changed it. */
1189: ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1190:
1191: /* Get the latest mac address, User can use a LAA */
1192: memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl),
1193: IXGBE_ETH_LENGTH_OF_ADDRESS);
1194: ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1195: hw->addr_ctrl.rar_used_count = 1;
1196:
1197: /* Prepare transmit descriptors and buffers */
1198: if (ixgbe_setup_transmit_structures(adapter)) {
1199: device_printf(dev,"Could not setup transmit structures\n");
1200: ixgbe_stop(adapter);
1201: return;
1202: }
1203:
1204: ixgbe_init_hw(hw);
1205: ixgbe_initialize_transmit_units(adapter);
1206:
1207: /* Setup Multicast table */
1208: ixgbe_set_multi(adapter);
1209:
1210: /*
1211: ** Determine the correct mbuf pool
1212: ** for doing jumbo/headersplit
1213: */
1214: if (adapter->max_frame_size <= 2048)
1215: adapter->rx_mbuf_sz = MCLBYTES;
1216: else if (adapter->max_frame_size <= 4096)
1217: adapter->rx_mbuf_sz = MJUMPAGESIZE;
1218: else if (adapter->max_frame_size <= 9216)
1219: adapter->rx_mbuf_sz = MJUM9BYTES;
1220: else
1221: adapter->rx_mbuf_sz = MJUM16BYTES;
1222:
1223: /* Prepare receive descriptors and buffers */
1224: if (ixgbe_setup_receive_structures(adapter)) {
1225: device_printf(dev,"Could not setup receive structures\n");
1226: ixgbe_stop(adapter);
1227: return;
1228: }
1229:
1230: /* Configure RX settings */
1231: ixgbe_initialize_receive_units(adapter);
1232:
1233: gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1234:
1235: /* Enable Fan Failure Interrupt */
1236: gpie |= IXGBE_SDP1_GPIEN;
1237:
1238: /* Add for Thermal detection */
1239: if (hw->mac.type == ixgbe_mac_82599EB)
1240: gpie |= IXGBE_SDP2_GPIEN;
1241:
1242: if (adapter->msix > 1) {
1243: /* Enable Enhanced MSIX mode */
1244: gpie |= IXGBE_GPIE_MSIX_MODE;
1245: gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1246: IXGBE_GPIE_OCD;
1247: }
1248: IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1249:
1250: /* Set MTU size */
1251: if (ifp->if_mtu > ETHERMTU) {
1252: mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1253: mhadd &= ~IXGBE_MHADD_MFS_MASK;
1254: mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1255: IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1256: }
1257:
1258: /* Now enable all the queues */
1259:
1260: for (int i = 0; i < adapter->num_queues; i++) {
1261: txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1262: txdctl |= IXGBE_TXDCTL_ENABLE;
1263: /* Set WTHRESH to 8, burst writeback */
1264: txdctl |= (8 << 16);
1265: IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1266: }
1267:
1268: for (int i = 0; i < adapter->num_queues; i++) {
1269: rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1270: if (hw->mac.type == ixgbe_mac_82598EB) {
1271: /*
1272: ** PTHRESH = 21
1273: ** HTHRESH = 4
1274: ** WTHRESH = 8
1275: */
1276: rxdctl &= ~0x3FFFFF;
1277: rxdctl |= 0x080420;
1278: }
1279: rxdctl |= IXGBE_RXDCTL_ENABLE;
1280: IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1281: /* XXX I don't trust this loop, and I don't trust the
1282: * XXX memory barrier. What is this meant to do? --dyoung
1283: */
1284: for (k = 0; k < 10; k++) {
1285: if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1286: IXGBE_RXDCTL_ENABLE)
1287: break;
1288: else
1289: msec_delay(1);
1290: }
1291: wmb();
1292: IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1293: }
1294:
1295: /* Set up VLAN support and filter */
1296: ixgbe_setup_vlan_hw_support(adapter);
1297:
1298: /* Enable Receive engine */
1299: rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1300: if (hw->mac.type == ixgbe_mac_82598EB)
1301: rxctrl |= IXGBE_RXCTRL_DMBYPS;
1302: rxctrl |= IXGBE_RXCTRL_RXEN;
1303: ixgbe_enable_rx_dma(hw, rxctrl);
1304:
1305: callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1306:
1307: /* Set up MSI/X routing */
1308: if (ixgbe_enable_msix) {
1309: ixgbe_configure_ivars(adapter);
1310: /* Set up auto-mask */
1311: if (hw->mac.type == ixgbe_mac_82598EB)
1312: IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1313: else {
1314: IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1315: IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1316: }
1317: } else { /* Simple settings for Legacy/MSI */
1318: ixgbe_set_ivar(adapter, 0, 0, 0);
1319: ixgbe_set_ivar(adapter, 0, 0, 1);
1320: IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1321: }
1322:
1323: #ifdef IXGBE_FDIR
1324: /* Init Flow director */
1325: if (hw->mac.type != ixgbe_mac_82598EB)
1326: ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1327: #endif
1328:
1329: /*
1330: ** Check on any SFP devices that
1331: ** need to be kick-started
1332: */
1333: if (hw->phy.type == ixgbe_phy_none) {
1334: int err = hw->phy.ops.identify(hw);
1335: if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1336: device_printf(dev,
1337: "Unsupported SFP+ module type was detected.\n");
1338: return;
1339: }
1340: }
1341:
1342: /* Set moderation on the Link interrupt */
1343: IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1344:
1345: /* Config/Enable Link */
1346: ixgbe_config_link(adapter);
1347:
1348: /* And now turn on interrupts */
1349: ixgbe_enable_intr(adapter);
1350:
1351: /* Now inform the stack we're ready */
1352: ifp->if_flags |= IFF_RUNNING;
1353: ifp->if_flags &= ~IFF_OACTIVE;
1354:
1355: return;
1356: }
1357:
1358: static int
1359: ixgbe_init(struct ifnet *ifp)
1360: {
1361: struct adapter *adapter = ifp->if_softc;
1362:
1363: IXGBE_CORE_LOCK(adapter);
1364: ixgbe_init_locked(adapter);
1365: IXGBE_CORE_UNLOCK(adapter);
1366: return 0; /* XXX ixgbe_init_locked cannot fail? really? */
1367: }
1368:
1369:
1370: /*
1371: **
1372: ** MSIX Interrupt Handlers and Tasklets
1373: **
1374: */
1375:
1376: static inline void
1377: ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1378: {
1379: struct ixgbe_hw *hw = &adapter->hw;
1.13 christos 1380: u64 queue = (u64)(1ULL << vector);
1.1 dyoung 1381: u32 mask;
1382:
1383: if (hw->mac.type == ixgbe_mac_82598EB) {
1384: mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1385: IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1386: } else {
1387: mask = (queue & 0xFFFFFFFF);
1388: if (mask)
1389: IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1390: mask = (queue >> 32);
1391: if (mask)
1392: IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1393: }
1394: }
1395:
1.11 joerg 1396: __unused static inline void
1.1 dyoung 1397: ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1398: {
1399: struct ixgbe_hw *hw = &adapter->hw;
1.13 christos 1400: u64 queue = (u64)(1ULL << vector);
1.1 dyoung 1401: u32 mask;
1402:
1403: if (hw->mac.type == ixgbe_mac_82598EB) {
1404: mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1405: IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1406: } else {
1407: mask = (queue & 0xFFFFFFFF);
1408: if (mask)
1409: IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1410: mask = (queue >> 32);
1411: if (mask)
1412: IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1413: }
1414: }
1415:
1416: static inline void
1417: ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1418: {
1419: u32 mask;
1420:
1421: if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1422: mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1423: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1424: } else {
1425: mask = (queues & 0xFFFFFFFF);
1426: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1427: mask = (queues >> 32);
1428: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1429: }
1430: }
1431:
1432:
1433: static void
1434: ixgbe_handle_que(void *context)
1435: {
1436: struct ix_queue *que = context;
1437: struct adapter *adapter = que->adapter;
1438: struct tx_ring *txr = que->txr;
1439: struct ifnet *ifp = adapter->ifp;
1440: bool more;
1441:
1442: adapter->handleq.ev_count++;
1443:
1444: if (ifp->if_flags & IFF_RUNNING) {
1445: more = ixgbe_rxeof(que, adapter->rx_process_limit);
1446: IXGBE_TX_LOCK(txr);
1447: ixgbe_txeof(txr);
1448: #if __FreeBSD_version >= 800000
1449: if (!drbr_empty(ifp, txr->br))
1450: ixgbe_mq_start_locked(ifp, txr, NULL);
1451: #else
1452: if (!IFQ_IS_EMPTY(&ifp->if_snd))
1453: ixgbe_start_locked(txr, ifp);
1454: #endif
1455: IXGBE_TX_UNLOCK(txr);
1456: if (more) {
1457: adapter->req.ev_count++;
1458: softint_schedule(que->que_si);
1459: return;
1460: }
1461: }
1462:
1463: /* Reenable this interrupt */
1464: ixgbe_enable_queue(adapter, que->msix);
1465:
1466: return;
1467: }
1468:
1469:
1470: /*********************************************************************
1471: *
1472: * Legacy Interrupt Service routine
1473: *
1474: **********************************************************************/
1475:
1476: static int
1477: ixgbe_legacy_irq(void *arg)
1478: {
1479: struct ix_queue *que = arg;
1480: struct adapter *adapter = que->adapter;
1.15 msaitoh 1481: struct ifnet *ifp = adapter->ifp;
1.1 dyoung 1482: struct ixgbe_hw *hw = &adapter->hw;
1483: struct tx_ring *txr = adapter->tx_rings;
1.15 msaitoh 1484: bool more_tx = false, more_rx = false;
1.1 dyoung 1485: u32 reg_eicr, loop = MAX_LOOP;
1486:
1487: reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1488:
1489: adapter->stats.legint.ev_count++;
1490: ++que->irqs;
1491: if (reg_eicr == 0) {
1492: adapter->stats.intzero.ev_count++;
1.15 msaitoh 1493: if ((ifp->if_flags & IFF_UP) != 0)
1494: ixgbe_enable_intr(adapter);
1.1 dyoung 1495: return 0;
1496: }
1497:
1.15 msaitoh 1498: if ((ifp->if_flags & IFF_RUNNING) != 0) {
1499: more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1.1 dyoung 1500:
1.15 msaitoh 1501: IXGBE_TX_LOCK(txr);
1502: do {
1503: adapter->txloops.ev_count++;
1504: more_tx = ixgbe_txeof(txr);
1505: } while (loop-- && more_tx);
1506: IXGBE_TX_UNLOCK(txr);
1507: }
1.1 dyoung 1508:
1509: if (more_rx || more_tx) {
1510: if (more_rx)
1511: adapter->morerx.ev_count++;
1512: if (more_tx)
1513: adapter->moretx.ev_count++;
1514: softint_schedule(que->que_si);
1515: }
1516:
1517: /* Check for fan failure */
1518: if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1519: (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1520: device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1521: "REPLACE IMMEDIATELY!!\n");
1522: IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1523: }
1524:
1525: /* Link status change */
1526: if (reg_eicr & IXGBE_EICR_LSC)
1527: softint_schedule(adapter->link_si);
1528:
1529: ixgbe_enable_intr(adapter);
1530: return 1;
1531: }
1532:
1533:
1534: #if defined(NETBSD_MSI_OR_MSIX)
1535: /*********************************************************************
1536: *
1537: * MSI Queue Interrupt Service routine
1538: *
1539: **********************************************************************/
1540: void
1541: ixgbe_msix_que(void *arg)
1542: {
1543: struct ix_queue *que = arg;
1544: struct adapter *adapter = que->adapter;
1545: struct tx_ring *txr = que->txr;
1546: struct rx_ring *rxr = que->rxr;
1547: bool more_tx, more_rx;
1548: u32 newitr = 0;
1549:
1550: ++que->irqs;
1551:
1552: more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1553:
1554: IXGBE_TX_LOCK(txr);
1555: more_tx = ixgbe_txeof(txr);
1556: IXGBE_TX_UNLOCK(txr);
1557:
1558: /* Do AIM now? */
1559:
1560: if (ixgbe_enable_aim == FALSE)
1561: goto no_calc;
1562: /*
1563: ** Do Adaptive Interrupt Moderation:
1564: ** - Write out last calculated setting
1565: ** - Calculate based on average size over
1566: ** the last interval.
1567: */
1568: if (que->eitr_setting)
1569: IXGBE_WRITE_REG(&adapter->hw,
1570: IXGBE_EITR(que->msix), que->eitr_setting);
1571:
1572: que->eitr_setting = 0;
1573:
1574: /* Idle, do nothing */
1575: if ((txr->bytes == 0) && (rxr->bytes == 0))
1576: goto no_calc;
1577:
1578: if ((txr->bytes) && (txr->packets))
1579: newitr = txr->bytes/txr->packets;
1580: if ((rxr->bytes) && (rxr->packets))
1581: newitr = max(newitr,
1582: (rxr->bytes / rxr->packets));
1583: newitr += 24; /* account for hardware frame, crc */
1584:
1585: /* set an upper boundary */
1586: newitr = min(newitr, 3000);
1587:
1588: /* Be nice to the mid range */
1589: if ((newitr > 300) && (newitr < 1200))
1590: newitr = (newitr / 3);
1591: else
1592: newitr = (newitr / 2);
1593:
1594: if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1595: newitr |= newitr << 16;
1596: else
1597: newitr |= IXGBE_EITR_CNT_WDIS;
1598:
1599: /* save for next interrupt */
1600: que->eitr_setting = newitr;
1601:
1602: /* Reset state */
1603: txr->bytes = 0;
1604: txr->packets = 0;
1605: rxr->bytes = 0;
1606: rxr->packets = 0;
1607:
1608: no_calc:
1609: if (more_tx || more_rx)
1610: softint_schedule(que->que_si);
1611: else /* Reenable this interrupt */
1612: ixgbe_enable_queue(adapter, que->msix);
1613: return;
1614: }
1615:
1616:
1617: static void
1618: ixgbe_msix_link(void *arg)
1619: {
1620: struct adapter *adapter = arg;
1621: struct ixgbe_hw *hw = &adapter->hw;
1622: u32 reg_eicr;
1623:
1624: ++adapter->link_irq.ev_count;
1625:
1626: /* First get the cause */
1627: reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1628: /* Clear interrupt with write */
1629: IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1630:
1631: /* Link status change */
1632: if (reg_eicr & IXGBE_EICR_LSC)
1633: softint_schedule(adapter->link_si);
1634:
1635: if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1636: #ifdef IXGBE_FDIR
1637: if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1638: /* This is probably overkill :) */
1639: if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1640: return;
1641: /* Clear the interrupt */
1642: IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR);
1643: /* Turn off the interface */
1644: adapter->ifp->if_flags &= ~IFF_RUNNING;
1645: softint_schedule(adapter->fdir_si);
1646: } else
1647: #endif
1648: if (reg_eicr & IXGBE_EICR_ECC) {
1649: device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1650: "Please Reboot!!\n");
1651: IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1652: } else
1653:
1654: if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1655: /* Clear the interrupt */
1656: IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1657: softint_schedule(adapter->msf_si);
1658: } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1659: /* Clear the interrupt */
1660: IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1661: softint_schedule(adapter->mod_si);
1662: }
1663: }
1664:
1665: /* Check for fan failure */
1666: if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1667: (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1668: device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1669: "REPLACE IMMEDIATELY!!\n");
1670: IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1671: }
1672:
1673: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1674: return;
1675: }
1676: #endif
1677:
1678: /*********************************************************************
1679: *
1680: * Media Ioctl callback
1681: *
1682: * This routine is called whenever the user queries the status of
1683: * the interface using ifconfig.
1684: *
1685: **********************************************************************/
1686: static void
1687: ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1688: {
1689: struct adapter *adapter = ifp->if_softc;
1690:
1691: INIT_DEBUGOUT("ixgbe_media_status: begin");
1692: IXGBE_CORE_LOCK(adapter);
1693: ixgbe_update_link_status(adapter);
1694:
1695: ifmr->ifm_status = IFM_AVALID;
1696: ifmr->ifm_active = IFM_ETHER;
1697:
1698: if (!adapter->link_active) {
1699: IXGBE_CORE_UNLOCK(adapter);
1700: return;
1701: }
1702:
1703: ifmr->ifm_status |= IFM_ACTIVE;
1704:
1705: switch (adapter->link_speed) {
1706: case IXGBE_LINK_SPEED_1GB_FULL:
1707: ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1708: break;
1709: case IXGBE_LINK_SPEED_10GB_FULL:
1710: ifmr->ifm_active |= adapter->optics | IFM_FDX;
1711: break;
1712: }
1713:
1714: IXGBE_CORE_UNLOCK(adapter);
1715:
1716: return;
1717: }
1718:
1719: /*********************************************************************
1720: *
1721: * Media Ioctl callback
1722: *
1723: * This routine is called when the user changes speed/duplex using
1724: * media/mediopt option with ifconfig.
1725: *
1726: **********************************************************************/
1727: static int
1728: ixgbe_media_change(struct ifnet * ifp)
1729: {
1730: struct adapter *adapter = ifp->if_softc;
1731: struct ifmedia *ifm = &adapter->media;
1732:
1733: INIT_DEBUGOUT("ixgbe_media_change: begin");
1734:
1735: if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1736: return (EINVAL);
1737:
1738: switch (IFM_SUBTYPE(ifm->ifm_media)) {
1739: case IFM_AUTO:
1740: adapter->hw.phy.autoneg_advertised =
1741: IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL;
1742: break;
1743: default:
1744: device_printf(adapter->dev, "Only auto media type\n");
1745: return (EINVAL);
1746: }
1747:
1748: return (0);
1749: }
1750:
1751: /*********************************************************************
1752: *
1753: * This routine maps the mbufs to tx descriptors, allowing the
1754: * TX engine to transmit the packets.
1755: * - return 0 on success, positive on failure
1756: *
1757: **********************************************************************/
1758:
1759: static int
1760: ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head)
1761: {
1762: struct m_tag *mtag;
1763: struct adapter *adapter = txr->adapter;
1764: struct ethercom *ec = &adapter->osdep.ec;
1765: u32 olinfo_status = 0, cmd_type_len;
1766: u32 paylen = 0;
1767: int i, j, error;
1768: int first, last = 0;
1769: bus_dmamap_t map;
1.9 skrll 1770: struct ixgbe_tx_buf *txbuf;
1.1 dyoung 1771: union ixgbe_adv_tx_desc *txd = NULL;
1772:
1773: /* Basic descriptor defines */
1774: cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1775: IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1776:
1777: if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL)
1778: cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1779:
1780: /*
1781: * Important to capture the first descriptor
1782: * used because it will contain the index of
1783: * the one we tell the hardware to report back
1784: */
1785: first = txr->next_avail_desc;
1786: txbuf = &txr->tx_buffers[first];
1787: map = txbuf->map;
1788:
1789: /*
1790: * Map the packet for DMA.
1791: */
1792: error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map,
1793: m_head, BUS_DMA_NOWAIT);
1794:
1795: switch (error) {
1796: case EAGAIN:
1797: adapter->eagain_tx_dma_setup.ev_count++;
1798: return EAGAIN;
1799: case ENOMEM:
1800: adapter->enomem_tx_dma_setup.ev_count++;
1801: return EAGAIN;
1802: case EFBIG:
1803: adapter->efbig_tx_dma_setup.ev_count++;
1804: return error;
1805: case EINVAL:
1806: adapter->einval_tx_dma_setup.ev_count++;
1807: return error;
1808: default:
1809: adapter->other_tx_dma_setup.ev_count++;
1810: return error;
1811: case 0:
1812: break;
1813: }
1814:
1815: /* Make certain there are enough descriptors */
1816: if (map->dm_nsegs > txr->tx_avail - 2) {
1817: txr->no_desc_avail.ev_count++;
1818: ixgbe_dmamap_unload(txr->txtag, txbuf->map);
1819: return EAGAIN;
1820: }
1821:
1822: /*
1823: ** Set up the appropriate offload context
1824: ** this becomes the first descriptor of
1825: ** a packet.
1826: */
1827: if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) {
1828: if (ixgbe_tso_setup(txr, m_head, &paylen)) {
1829: cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1830: olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1831: olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1832: olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1833: ++adapter->tso_tx.ev_count;
1834: } else {
1835: ++adapter->tso_err.ev_count;
1836: /* XXX unload DMA map! --dyoung */
1837: return ENXIO;
1838: }
1839: } else
1840: olinfo_status |= ixgbe_tx_ctx_setup(txr, m_head);
1841:
1842: #ifdef IXGBE_IEEE1588
1843: /* This is changing soon to an mtag detection */
1844: if (we detect this mbuf has a TSTAMP mtag)
1845: cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1846: #endif
1847:
1848: #ifdef IXGBE_FDIR
1849: /* Do the flow director magic */
1850: if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1851: ++txr->atr_count;
1852: if (txr->atr_count >= atr_sample_rate) {
1853: ixgbe_atr(txr, m_head);
1854: txr->atr_count = 0;
1855: }
1856: }
1857: #endif
1858: /* Record payload length */
1859: if (paylen == 0)
1860: olinfo_status |= m_head->m_pkthdr.len <<
1861: IXGBE_ADVTXD_PAYLEN_SHIFT;
1862:
1863: i = txr->next_avail_desc;
1864: for (j = 0; j < map->dm_nsegs; j++) {
1865: bus_size_t seglen;
1866: bus_addr_t segaddr;
1867:
1868: txbuf = &txr->tx_buffers[i];
1869: txd = &txr->tx_base[i];
1870: seglen = map->dm_segs[j].ds_len;
1871: segaddr = htole64(map->dm_segs[j].ds_addr);
1872:
1873: txd->read.buffer_addr = segaddr;
1874: txd->read.cmd_type_len = htole32(txr->txd_cmd |
1875: cmd_type_len |seglen);
1876: txd->read.olinfo_status = htole32(olinfo_status);
1877: last = i; /* descriptor that will get completion IRQ */
1878:
1879: if (++i == adapter->num_tx_desc)
1880: i = 0;
1881:
1882: txbuf->m_head = NULL;
1883: txbuf->eop_index = -1;
1884: }
1885:
1886: txd->read.cmd_type_len |=
1887: htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1888: txr->tx_avail -= map->dm_nsegs;
1889: txr->next_avail_desc = i;
1890:
1891: txbuf->m_head = m_head;
1892: /* We exchange the maps instead of copying because otherwise
1893: * we end up with many pointers to the same map and we free
1894: * one map twice in ixgbe_free_transmit_structures(). Who
1895: * knows what other problems this caused. --dyoung
1896: */
1897: txr->tx_buffers[first].map = txbuf->map;
1898: txbuf->map = map;
1899: bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len,
1900: BUS_DMASYNC_PREWRITE);
1901:
1902: /* Set the index of the descriptor that will be marked done */
1903: txbuf = &txr->tx_buffers[first];
1904: txbuf->eop_index = last;
1905:
1906: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1907: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1908: /*
1909: * Advance the Transmit Descriptor Tail (Tdt), this tells the
1910: * hardware that this frame is available to transmit.
1911: */
1912: ++txr->total_packets.ev_count;
1913: IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1914:
1915: return 0;
1916: }
1917:
1918: static void
1919: ixgbe_set_promisc(struct adapter *adapter)
1920: {
1921: u_int32_t reg_rctl;
1922: struct ifnet *ifp = adapter->ifp;
1923:
1924: reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1925: reg_rctl &= (~IXGBE_FCTRL_UPE);
1926: reg_rctl &= (~IXGBE_FCTRL_MPE);
1927: IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1928:
1929: if (ifp->if_flags & IFF_PROMISC) {
1930: reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1931: IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1932: } else if (ifp->if_flags & IFF_ALLMULTI) {
1933: reg_rctl |= IXGBE_FCTRL_MPE;
1934: reg_rctl &= ~IXGBE_FCTRL_UPE;
1935: IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1936: }
1937: return;
1938: }
1939:
1940:
1941: /*********************************************************************
1942: * Multicast Update
1943: *
1944: * This routine is called whenever multicast address list is updated.
1945: *
1946: **********************************************************************/
1947: #define IXGBE_RAR_ENTRIES 16
1948:
1949: static void
1950: ixgbe_set_multi(struct adapter *adapter)
1951: {
1952: struct ether_multi *enm;
1953: struct ether_multistep step;
1954: u32 fctrl;
1955: u8 *mta;
1956: u8 *update_ptr;
1957: int mcnt = 0;
1958: struct ethercom *ec = &adapter->osdep.ec;
1959: struct ifnet *ifp = adapter->ifp;
1960:
1961: IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1962:
1963: mta = adapter->mta;
1964: bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1965: MAX_NUM_MULTICAST_ADDRESSES);
1966:
1967: fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1968: fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1969: if (ifp->if_flags & IFF_PROMISC)
1970: fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1971: else if (ifp->if_flags & IFF_ALLMULTI) {
1972: fctrl |= IXGBE_FCTRL_MPE;
1973: fctrl &= ~IXGBE_FCTRL_UPE;
1974: } else
1975: fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1976:
1977: IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1978:
1979: ETHER_FIRST_MULTI(step, ec, enm);
1980: while (enm != NULL) {
1981: if (memcmp(enm->enm_addrlo, enm->enm_addrhi,
1982: ETHER_ADDR_LEN) != 0) {
1983: fctrl |= IXGBE_FCTRL_MPE;
1984: IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1985: break;
1986: }
1987: bcopy(enm->enm_addrlo,
1988: &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1989: IXGBE_ETH_LENGTH_OF_ADDRESS);
1990: mcnt++;
1991: ETHER_NEXT_MULTI(step, enm);
1992: }
1993:
1994: update_ptr = mta;
1995: ixgbe_update_mc_addr_list(&adapter->hw,
1996: update_ptr, mcnt, ixgbe_mc_array_itr);
1997:
1998: return;
1999: }
2000:
2001: /*
2002: * This is an iterator function now needed by the multicast
2003: * shared code. It simply feeds the shared code routine the
2004: * addresses in the array of ixgbe_set_multi() one by one.
2005: */
2006: static u8 *
2007: ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2008: {
2009: u8 *addr = *update_ptr;
2010: u8 *newptr;
2011: *vmdq = 0;
2012:
2013: newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2014: *update_ptr = newptr;
2015: return addr;
2016: }
2017:
2018:
2019: /*********************************************************************
2020: * Timer routine
2021: *
2022: * This routine checks for link status,updates statistics,
2023: * and runs the watchdog check.
2024: *
2025: **********************************************************************/
2026:
2027: static void
2028: ixgbe_local_timer1(void *arg)
2029: {
2030: struct adapter *adapter = arg;
2031: device_t dev = adapter->dev;
2032: struct tx_ring *txr = adapter->tx_rings;
2033:
2034: KASSERT(mutex_owned(&adapter->core_mtx));
2035:
2036: /* Check for pluggable optics */
2037: if (adapter->sfp_probe)
2038: if (!ixgbe_sfp_probe(adapter))
2039: goto out; /* Nothing to do */
2040:
2041: ixgbe_update_link_status(adapter);
2042: ixgbe_update_stats_counters(adapter);
2043:
2044: /*
2045: * If the interface has been paused
2046: * then don't do the watchdog check
2047: */
2048: if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2049: goto out;
2050:
2051: /*
2052: ** Check status on the TX queues for a hang
2053: */
2054: for (int i = 0; i < adapter->num_queues; i++, txr++)
2055: if (txr->queue_status == IXGBE_QUEUE_HUNG)
2056: goto hung;
2057:
2058: out:
2059: ixgbe_rearm_queues(adapter, adapter->que_mask);
2060: callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2061: return;
2062:
2063: hung:
2064: device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2065: device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2066: IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2067: IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2068: device_printf(dev,"TX(%d) desc avail = %d,"
2069: "Next TX to Clean = %d\n",
2070: txr->me, txr->tx_avail, txr->next_to_clean);
2071: adapter->ifp->if_flags &= ~IFF_RUNNING;
2072: adapter->watchdog_events.ev_count++;
2073: ixgbe_init_locked(adapter);
2074: }
2075:
2076: static void
2077: ixgbe_local_timer(void *arg)
2078: {
2079: struct adapter *adapter = arg;
2080:
2081: IXGBE_CORE_LOCK(adapter);
2082: ixgbe_local_timer1(adapter);
2083: IXGBE_CORE_UNLOCK(adapter);
2084: }
2085:
2086: /*
2087: ** Note: this routine updates the OS on the link state
2088: ** the real check of the hardware only happens with
2089: ** a link interrupt.
2090: */
2091: static void
2092: ixgbe_update_link_status(struct adapter *adapter)
2093: {
2094: struct ifnet *ifp = adapter->ifp;
2095: struct tx_ring *txr = adapter->tx_rings;
2096: device_t dev = adapter->dev;
2097:
2098:
2099: if (adapter->link_up){
2100: if (adapter->link_active == FALSE) {
2101: if (bootverbose)
2102: device_printf(dev,"Link is up %d Gbps %s \n",
2103: ((adapter->link_speed == 128)? 10:1),
2104: "Full Duplex");
2105: adapter->link_active = TRUE;
2106: if_link_state_change(ifp, LINK_STATE_UP);
2107: }
2108: } else { /* Link down */
2109: if (adapter->link_active == TRUE) {
2110: if (bootverbose)
2111: device_printf(dev,"Link is Down\n");
2112: if_link_state_change(ifp, LINK_STATE_DOWN);
2113: adapter->link_active = FALSE;
2114: for (int i = 0; i < adapter->num_queues;
2115: i++, txr++)
2116: txr->queue_status = IXGBE_QUEUE_IDLE;
2117: }
2118: }
2119:
2120: return;
2121: }
2122:
2123:
2124: static void
2125: ixgbe_ifstop(struct ifnet *ifp, int disable)
2126: {
2127: struct adapter *adapter = ifp->if_softc;
2128:
2129: IXGBE_CORE_LOCK(adapter);
2130: ixgbe_stop(adapter);
2131: IXGBE_CORE_UNLOCK(adapter);
2132: }
2133:
2134: /*********************************************************************
2135: *
2136: * This routine disables all traffic on the adapter by issuing a
2137: * global reset on the MAC and deallocates TX/RX buffers.
2138: *
2139: **********************************************************************/
2140:
2141: static void
2142: ixgbe_stop(void *arg)
2143: {
2144: struct ifnet *ifp;
2145: struct adapter *adapter = arg;
2146: struct ixgbe_hw *hw = &adapter->hw;
2147: ifp = adapter->ifp;
2148:
2149: KASSERT(mutex_owned(&adapter->core_mtx));
2150:
2151: INIT_DEBUGOUT("ixgbe_stop: begin\n");
2152: ixgbe_disable_intr(adapter);
2153:
2154: /* Tell the stack that the interface is no longer active */
2155: ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
2156:
2157: ixgbe_reset_hw(hw);
2158: hw->adapter_stopped = FALSE;
2159: ixgbe_stop_adapter(hw);
2160: /* Turn off the laser */
2161: if (hw->phy.multispeed_fiber)
2162: ixgbe_disable_tx_laser(hw);
2163: callout_stop(&adapter->timer);
2164:
2165: /* reprogram the RAR[0] in case user changed it. */
2166: ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2167:
2168: return;
2169: }
2170:
2171:
2172: /*********************************************************************
2173: *
2174: * Determine hardware revision.
2175: *
2176: **********************************************************************/
2177: static void
2178: ixgbe_identify_hardware(struct adapter *adapter)
2179: {
2180: pcitag_t tag;
2181: pci_chipset_tag_t pc;
2182: pcireg_t subid, id;
2183: struct ixgbe_hw *hw = &adapter->hw;
2184:
2185: pc = adapter->osdep.pc;
2186: tag = adapter->osdep.tag;
2187:
2188: id = pci_conf_read(pc, tag, PCI_ID_REG);
2189: subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
2190:
2191: /* Save off the information about this board */
2192: hw->vendor_id = PCI_VENDOR(id);
2193: hw->device_id = PCI_PRODUCT(id);
2194: hw->revision_id =
2195: PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG));
2196: hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid);
2197: hw->subsystem_device_id = PCI_SUBSYS_ID(subid);
2198:
2199: /* We need this here to set the num_segs below */
2200: ixgbe_set_mac_type(hw);
2201:
2202: /* Pick up the 82599 and VF settings */
2203: if (hw->mac.type != ixgbe_mac_82598EB) {
2204: hw->phy.smart_speed = ixgbe_smart_speed;
2205: adapter->num_segs = IXGBE_82599_SCATTER;
2206: } else
2207: adapter->num_segs = IXGBE_82598_SCATTER;
2208:
2209: return;
2210: }
2211:
2212: /*********************************************************************
2213: *
2214: * Determine optic type
2215: *
2216: **********************************************************************/
2217: static void
2218: ixgbe_setup_optics(struct adapter *adapter)
2219: {
2220: struct ixgbe_hw *hw = &adapter->hw;
2221: int layer;
2222:
2223: layer = ixgbe_get_supported_physical_layer(hw);
2224: switch (layer) {
2225: case IXGBE_PHYSICAL_LAYER_10GBASE_T:
2226: adapter->optics = IFM_10G_T;
2227: break;
2228: case IXGBE_PHYSICAL_LAYER_1000BASE_T:
2229: adapter->optics = IFM_1000_T;
2230: break;
2231: case IXGBE_PHYSICAL_LAYER_10GBASE_LR:
2232: case IXGBE_PHYSICAL_LAYER_10GBASE_LRM:
2233: adapter->optics = IFM_10G_LR;
2234: break;
2235: case IXGBE_PHYSICAL_LAYER_10GBASE_SR:
2236: adapter->optics = IFM_10G_SR;
2237: break;
2238: case IXGBE_PHYSICAL_LAYER_10GBASE_KX4:
2239: case IXGBE_PHYSICAL_LAYER_10GBASE_CX4:
2240: adapter->optics = IFM_10G_CX4;
2241: break;
2242: case IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU:
2243: adapter->optics = IFM_10G_TWINAX;
2244: break;
2245: case IXGBE_PHYSICAL_LAYER_1000BASE_KX:
2246: case IXGBE_PHYSICAL_LAYER_10GBASE_KR:
2247: case IXGBE_PHYSICAL_LAYER_10GBASE_XAUI:
2248: case IXGBE_PHYSICAL_LAYER_UNKNOWN:
2249: default:
2250: adapter->optics = IFM_ETHER | IFM_AUTO;
2251: break;
2252: }
2253: return;
2254: }
2255:
2256: /*********************************************************************
2257: *
2258: * Setup the Legacy or MSI Interrupt handler
2259: *
2260: **********************************************************************/
2261: static int
2262: ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa)
2263: {
2264: device_t dev = adapter->dev;
2265: struct ix_queue *que = adapter->queues;
1.9 skrll 2266: char intrbuf[PCI_INTRSTR_LEN];
2267: #if 0
1.1 dyoung 2268: int rid = 0;
2269:
2270: /* MSI RID at 1 */
2271: if (adapter->msix == 1)
2272: rid = 1;
1.9 skrll 2273: #endif
1.1 dyoung 2274:
2275: /* We allocate a single interrupt resource */
2276: if (pci_intr_map(pa, &adapter->osdep.ih) != 0) {
2277: aprint_error_dev(dev, "unable to map interrupt\n");
2278: return ENXIO;
2279: } else {
2280: aprint_normal_dev(dev, "interrupting at %s\n",
1.14 chs 2281: pci_intr_string(adapter->osdep.pc, adapter->osdep.ih,
2282: intrbuf, sizeof(intrbuf)));
1.1 dyoung 2283: }
2284:
2285: /*
2286: * Try allocating a fast interrupt and the associated deferred
2287: * processing contexts.
2288: */
2289: que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que);
2290:
2291: /* Tasklets for Link, SFP and Multispeed Fiber */
2292: adapter->link_si =
2293: softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter);
2294: adapter->mod_si =
2295: softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter);
2296: adapter->msf_si =
2297: softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter);
2298:
2299: #ifdef IXGBE_FDIR
2300: adapter->fdir_si =
2301: softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter);
2302: #endif
2303: if (que->que_si == NULL ||
2304: adapter->link_si == NULL ||
2305: adapter->mod_si == NULL ||
2306: #ifdef IXGBE_FDIR
2307: adapter->fdir_si == NULL ||
2308: #endif
2309: adapter->msf_si == NULL) {
2310: aprint_error_dev(dev,
2311: "could not establish software interrupts\n");
2312: return ENXIO;
2313: }
2314:
2315: adapter->osdep.intr = pci_intr_establish(adapter->osdep.pc,
2316: adapter->osdep.ih, IPL_NET, ixgbe_legacy_irq, que);
2317: if (adapter->osdep.intr == NULL) {
2318: aprint_error_dev(dev, "failed to register interrupt handler\n");
2319: softint_disestablish(que->que_si);
2320: softint_disestablish(adapter->link_si);
2321: softint_disestablish(adapter->mod_si);
2322: softint_disestablish(adapter->msf_si);
2323: #ifdef IXGBE_FDIR
2324: softint_disestablish(adapter->fdir_si);
2325: #endif
2326: return ENXIO;
2327: }
2328: /* For simplicity in the handlers */
2329: adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2330:
2331: return (0);
2332: }
2333:
2334:
2335: /*********************************************************************
2336: *
2337: * Setup MSIX Interrupt resources and handlers
2338: *
2339: **********************************************************************/
2340: static int
2341: ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa)
2342: {
2343: #if !defined(NETBSD_MSI_OR_MSIX)
2344: return 0;
2345: #else
2346: device_t dev = adapter->dev;
2347: struct ix_queue *que = adapter->queues;
2348: int error, rid, vector = 0;
2349:
2350: for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2351: rid = vector + 1;
2352: que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2353: RF_SHAREABLE | RF_ACTIVE);
2354: if (que->res == NULL) {
2355: aprint_error_dev(dev,"Unable to allocate"
2356: " bus resource: que interrupt [%d]\n", vector);
2357: return (ENXIO);
2358: }
2359: /* Set the handler function */
2360: error = bus_setup_intr(dev, que->res,
2361: INTR_TYPE_NET | INTR_MPSAFE, NULL,
2362: ixgbe_msix_que, que, &que->tag);
2363: if (error) {
2364: que->res = NULL;
2365: aprint_error_dev(dev,
2366: "Failed to register QUE handler\n");
2367: return error;
2368: }
2369: #if __FreeBSD_version >= 800504
2370: bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2371: #endif
2372: que->msix = vector;
2373: adapter->que_mask |= (u64)(1 << que->msix);
2374: /*
2375: ** Bind the msix vector, and thus the
2376: ** ring to the corresponding cpu.
2377: */
2378: if (adapter->num_queues > 1)
2379: bus_bind_intr(dev, que->res, i);
2380:
2381: que->que_si = softint_establish(ixgbe_handle_que, que);
2382: if (que->que_si == NULL) {
2383: aprint_error_dev(dev,
2384: "could not establish software interrupt\n");
2385: }
2386: }
2387:
2388: /* and Link */
2389: rid = vector + 1;
2390: adapter->res = bus_alloc_resource_any(dev,
2391: SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2392: if (!adapter->res) {
2393: aprint_error_dev(dev,"Unable to allocate bus resource: "
2394: "Link interrupt [%d]\n", rid);
2395: return (ENXIO);
2396: }
2397: /* Set the link handler function */
2398: error = bus_setup_intr(dev, adapter->res,
2399: INTR_TYPE_NET | INTR_MPSAFE, NULL,
2400: ixgbe_msix_link, adapter, &adapter->tag);
2401: if (error) {
2402: adapter->res = NULL;
2403: aprint_error_dev(dev, "Failed to register LINK handler\n");
2404: return (error);
2405: }
2406: #if __FreeBSD_version >= 800504
2407: bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2408: #endif
2409: adapter->linkvec = vector;
2410: /* Tasklets for Link, SFP and Multispeed Fiber */
2411: adapter->link_si =
2412: softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter);
2413: adapter->mod_si =
2414: softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter);
2415: adapter->msf_si =
2416: softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter);
2417: #ifdef IXGBE_FDIR
2418: adapter->fdir_si =
2419: softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter);
2420: #endif
2421:
2422: return (0);
2423: #endif
2424: }
2425:
2426: /*
2427: * Setup Either MSI/X or MSI
2428: */
2429: static int
2430: ixgbe_setup_msix(struct adapter *adapter)
2431: {
2432: #if !defined(NETBSD_MSI_OR_MSIX)
2433: return 0;
2434: #else
2435: device_t dev = adapter->dev;
2436: int rid, want, queues, msgs;
2437:
2438: /* Override by tuneable */
2439: if (ixgbe_enable_msix == 0)
2440: goto msi;
2441:
2442: /* First try MSI/X */
2443: rid = PCI_BAR(MSIX_82598_BAR);
2444: adapter->msix_mem = bus_alloc_resource_any(dev,
2445: SYS_RES_MEMORY, &rid, RF_ACTIVE);
2446: if (!adapter->msix_mem) {
2447: rid += 4; /* 82599 maps in higher BAR */
2448: adapter->msix_mem = bus_alloc_resource_any(dev,
2449: SYS_RES_MEMORY, &rid, RF_ACTIVE);
2450: }
2451: if (!adapter->msix_mem) {
2452: /* May not be enabled */
2453: device_printf(adapter->dev,
2454: "Unable to map MSIX table \n");
2455: goto msi;
2456: }
2457:
2458: msgs = pci_msix_count(dev);
2459: if (msgs == 0) { /* system has msix disabled */
2460: bus_release_resource(dev, SYS_RES_MEMORY,
2461: rid, adapter->msix_mem);
2462: adapter->msix_mem = NULL;
2463: goto msi;
2464: }
2465:
2466: /* Figure out a reasonable auto config value */
2467: queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2468:
2469: if (ixgbe_num_queues != 0)
2470: queues = ixgbe_num_queues;
2471: /* Set max queues to 8 when autoconfiguring */
2472: else if ((ixgbe_num_queues == 0) && (queues > 8))
2473: queues = 8;
2474:
2475: /*
2476: ** Want one vector (RX/TX pair) per queue
2477: ** plus an additional for Link.
2478: */
2479: want = queues + 1;
2480: if (msgs >= want)
2481: msgs = want;
2482: else {
2483: device_printf(adapter->dev,
2484: "MSIX Configuration Problem, "
2485: "%d vectors but %d queues wanted!\n",
2486: msgs, want);
2487: return (0); /* Will go to Legacy setup */
2488: }
2489: if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2490: device_printf(adapter->dev,
2491: "Using MSIX interrupts with %d vectors\n", msgs);
2492: adapter->num_queues = queues;
2493: return (msgs);
2494: }
2495: msi:
2496: msgs = pci_msi_count(dev);
2497: if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2498: device_printf(adapter->dev,"Using MSI interrupt\n");
2499: return (msgs);
2500: #endif
2501: }
2502:
2503:
2504: static int
2505: ixgbe_allocate_pci_resources(struct adapter *adapter, const struct pci_attach_args *pa)
2506: {
2507: pcireg_t memtype;
2508: device_t dev = adapter->dev;
2509: bus_addr_t addr;
2510: int flags;
2511:
2512: memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0));
2513: switch (memtype) {
2514: case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT:
2515: case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT:
2516: adapter->osdep.mem_bus_space_tag = pa->pa_memt;
2517: if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0),
2518: memtype, &addr, &adapter->osdep.mem_size, &flags) != 0)
2519: goto map_err;
2520: if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
2521: aprint_normal_dev(dev, "clearing prefetchable bit\n");
2522: flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
2523: }
2524: if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr,
2525: adapter->osdep.mem_size, flags,
2526: &adapter->osdep.mem_bus_space_handle) != 0) {
2527: map_err:
2528: adapter->osdep.mem_size = 0;
2529: aprint_error_dev(dev, "unable to map BAR0\n");
2530: return ENXIO;
2531: }
2532: break;
2533: default:
2534: aprint_error_dev(dev, "unexpected type on BAR0\n");
2535: return ENXIO;
2536: }
2537:
2538: /* Legacy defaults */
2539: adapter->num_queues = 1;
2540: adapter->hw.back = &adapter->osdep;
2541:
2542: /*
2543: ** Now setup MSI or MSI/X, should
2544: ** return us the number of supported
2545: ** vectors. (Will be 1 for MSI)
2546: */
2547: adapter->msix = ixgbe_setup_msix(adapter);
2548: return (0);
2549: }
2550:
2551: static void
2552: ixgbe_free_pci_resources(struct adapter * adapter)
2553: {
2554: #if defined(NETBSD_MSI_OR_MSIX)
2555: struct ix_queue *que = adapter->queues;
1.14 chs 2556: device_t dev = adapter->dev;
1.1 dyoung 2557: #endif
1.9 skrll 2558: int rid;
1.1 dyoung 2559:
1.9 skrll 2560: #if defined(NETBSD_MSI_OR_MSIX)
2561: int memrid;
1.1 dyoung 2562: if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2563: memrid = PCI_BAR(MSIX_82598_BAR);
2564: else
2565: memrid = PCI_BAR(MSIX_82599_BAR);
2566:
2567: /*
2568: ** There is a slight possibility of a failure mode
2569: ** in attach that will result in entering this function
2570: ** before interrupt resources have been initialized, and
2571: ** in that case we do not want to execute the loops below
2572: ** We can detect this reliably by the state of the adapter
2573: ** res pointer.
2574: */
2575: if (adapter->res == NULL)
2576: goto mem;
2577:
2578: /*
2579: ** Release all msix queue resources:
2580: */
2581: for (int i = 0; i < adapter->num_queues; i++, que++) {
2582: rid = que->msix + 1;
2583: if (que->tag != NULL) {
2584: bus_teardown_intr(dev, que->res, que->tag);
2585: que->tag = NULL;
2586: }
2587: if (que->res != NULL)
2588: bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2589: }
2590: #endif
2591:
2592: /* Clean the Legacy or Link interrupt last */
2593: if (adapter->linkvec) /* we are doing MSIX */
2594: rid = adapter->linkvec + 1;
2595: else
2596: (adapter->msix != 0) ? (rid = 1):(rid = 0);
2597:
2598: pci_intr_disestablish(adapter->osdep.pc, adapter->osdep.intr);
2599: adapter->osdep.intr = NULL;
2600:
2601: #if defined(NETBSD_MSI_OR_MSIX)
2602: mem:
2603: if (adapter->msix)
2604: pci_release_msi(dev);
2605:
2606: if (adapter->msix_mem != NULL)
2607: bus_release_resource(dev, SYS_RES_MEMORY,
2608: memrid, adapter->msix_mem);
2609: #endif
2610:
2611: if (adapter->osdep.mem_size != 0) {
2612: bus_space_unmap(adapter->osdep.mem_bus_space_tag,
2613: adapter->osdep.mem_bus_space_handle,
2614: adapter->osdep.mem_size);
2615: }
2616:
2617: return;
2618: }
2619:
2620: /*********************************************************************
2621: *
2622: * Setup networking device structure and register an interface.
2623: *
2624: **********************************************************************/
2625: static int
2626: ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2627: {
2628: struct ethercom *ec = &adapter->osdep.ec;
2629: struct ixgbe_hw *hw = &adapter->hw;
2630: struct ifnet *ifp;
2631:
2632: INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2633:
2634: ifp = adapter->ifp = &ec->ec_if;
2635: strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ);
2636: ifp->if_mtu = ETHERMTU;
2637: ifp->if_baudrate = 1000000000;
2638: ifp->if_init = ixgbe_init;
2639: ifp->if_stop = ixgbe_ifstop;
2640: ifp->if_softc = adapter;
2641: ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2642: ifp->if_ioctl = ixgbe_ioctl;
2643: ifp->if_start = ixgbe_start;
2644: #if __FreeBSD_version >= 800000
2645: ifp->if_transmit = ixgbe_mq_start;
2646: ifp->if_qflush = ixgbe_qflush;
2647: #endif
2648: ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2649:
2650: if_attach(ifp);
2651: ether_ifattach(ifp, adapter->hw.mac.addr);
2652: ether_set_ifflags_cb(ec, ixgbe_ifflags_cb);
2653:
2654: adapter->max_frame_size =
2655: ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2656:
2657: /*
2658: * Tell the upper layer(s) we support long frames.
2659: */
2660: ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2661:
2662: ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4;
2663: ifp->if_capenable = 0;
2664:
2665: ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM;
2666: ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU;
2667: ec->ec_capabilities |= ETHERCAP_JUMBO_MTU;
2668: ec->ec_capenable = ec->ec_capabilities;
2669:
2670: /* Don't enable LRO by default */
2671: ifp->if_capabilities |= IFCAP_LRO;
2672:
2673: /*
2674: ** Dont turn this on by default, if vlans are
2675: ** created on another pseudo device (eg. lagg)
2676: ** then vlan events are not passed thru, breaking
2677: ** operation, but with HW FILTER off it works. If
2678: ** using vlans directly on the em driver you can
2679: ** enable this and get full hardware tag filtering.
2680: */
2681: ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER;
2682:
2683: /*
2684: * Specify the media types supported by this adapter and register
2685: * callbacks to update media and link information
2686: */
2687: ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2688: ixgbe_media_status);
2689: ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2690: ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2691: if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2692: ifmedia_add(&adapter->media,
2693: IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2694: ifmedia_add(&adapter->media,
2695: IFM_ETHER | IFM_1000_T, 0, NULL);
2696: }
2697: ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2698: ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2699:
2700: return (0);
2701: }
2702:
2703: static void
2704: ixgbe_config_link(struct adapter *adapter)
2705: {
2706: struct ixgbe_hw *hw = &adapter->hw;
2707: u32 autoneg, err = 0;
2708: bool sfp, negotiate;
2709:
2710: sfp = ixgbe_is_sfp(hw);
2711:
2712: if (sfp) {
1.19 christos 2713: void *ip;
2714:
1.1 dyoung 2715: if (hw->phy.multispeed_fiber) {
2716: hw->mac.ops.setup_sfp(hw);
2717: ixgbe_enable_tx_laser(hw);
1.19 christos 2718: ip = adapter->msf_si;
1.1 dyoung 2719: } else {
1.19 christos 2720: ip = adapter->mod_si;
1.1 dyoung 2721: }
1.19 christos 2722:
2723: kpreempt_disable();
2724: softint_schedule(ip);
2725: kpreempt_enable();
1.1 dyoung 2726: } else {
2727: if (hw->mac.ops.check_link)
2728: err = ixgbe_check_link(hw, &autoneg,
2729: &adapter->link_up, FALSE);
2730: if (err)
2731: goto out;
2732: autoneg = hw->phy.autoneg_advertised;
2733: if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2734: err = hw->mac.ops.get_link_capabilities(hw,
2735: &autoneg, &negotiate);
1.13 christos 2736: else
2737: negotiate = 0;
1.1 dyoung 2738: if (err)
2739: goto out;
2740: if (hw->mac.ops.setup_link)
2741: err = hw->mac.ops.setup_link(hw, autoneg,
2742: negotiate, adapter->link_up);
2743: }
2744: out:
2745: return;
2746: }
2747:
2748: /********************************************************************
2749: * Manage DMA'able memory.
2750: *******************************************************************/
2751:
2752: static int
2753: ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size,
2754: struct ixgbe_dma_alloc *dma, const int mapflags)
2755: {
2756: device_t dev = adapter->dev;
2757: int r, rsegs;
2758:
2759: r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
2760: DBA_ALIGN, 0, /* alignment, bounds */
2761: size, /* maxsize */
2762: 1, /* nsegments */
2763: size, /* maxsegsize */
2764: BUS_DMA_ALLOCNOW, /* flags */
2765: &dma->dma_tag);
2766: if (r != 0) {
2767: aprint_error_dev(dev,
2768: "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r);
2769: goto fail_0;
2770: }
2771:
2772: r = bus_dmamem_alloc(dma->dma_tag->dt_dmat,
2773: size,
2774: dma->dma_tag->dt_alignment,
2775: dma->dma_tag->dt_boundary,
2776: &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT);
2777: if (r != 0) {
2778: aprint_error_dev(dev,
2779: "%s: bus_dmamem_alloc failed; error %d\n", __func__, r);
2780: goto fail_1;
2781: }
2782:
2783: r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs,
2784: size, &dma->dma_vaddr, BUS_DMA_NOWAIT);
2785: if (r != 0) {
2786: aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n",
2787: __func__, r);
2788: goto fail_2;
2789: }
2790:
2791: r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map);
2792: if (r != 0) {
2793: aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n",
2794: __func__, r);
2795: goto fail_3;
2796: }
2797:
2798: r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr,
2799: size,
2800: NULL,
2801: mapflags | BUS_DMA_NOWAIT);
2802: if (r != 0) {
2803: aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n",
2804: __func__, r);
2805: goto fail_4;
2806: }
2807: dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
2808: dma->dma_size = size;
2809: return 0;
2810: fail_4:
2811: ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map);
2812: fail_3:
2813: bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size);
2814: fail_2:
2815: bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs);
2816: fail_1:
2817: ixgbe_dma_tag_destroy(dma->dma_tag);
2818: fail_0:
2819: return r;
2820: }
2821:
2822: static void
2823: ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2824: {
2825: bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size,
2826: BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2827: ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map);
2828: bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1);
2829: ixgbe_dma_tag_destroy(dma->dma_tag);
2830: }
2831:
2832:
2833: /*********************************************************************
2834: *
2835: * Allocate memory for the transmit and receive rings, and then
2836: * the descriptors associated with each, called only once at attach.
2837: *
2838: **********************************************************************/
2839: static int
2840: ixgbe_allocate_queues(struct adapter *adapter)
2841: {
2842: device_t dev = adapter->dev;
2843: struct ix_queue *que;
2844: struct tx_ring *txr;
2845: struct rx_ring *rxr;
2846: int rsize, tsize, error = IXGBE_SUCCESS;
2847: int txconf = 0, rxconf = 0;
2848:
2849: /* First allocate the top level queue structs */
2850: if (!(adapter->queues =
2851: (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2852: adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2853: aprint_error_dev(dev, "Unable to allocate queue memory\n");
2854: error = ENOMEM;
2855: goto fail;
2856: }
2857:
2858: /* First allocate the TX ring struct memory */
2859: if (!(adapter->tx_rings =
2860: (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2861: adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2862: aprint_error_dev(dev, "Unable to allocate TX ring memory\n");
2863: error = ENOMEM;
2864: goto tx_fail;
2865: }
2866:
2867: /* Next allocate the RX */
2868: if (!(adapter->rx_rings =
2869: (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2870: adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2871: aprint_error_dev(dev, "Unable to allocate RX ring memory\n");
2872: error = ENOMEM;
2873: goto rx_fail;
2874: }
2875:
2876: /* For the ring itself */
2877: tsize = roundup2(adapter->num_tx_desc *
2878: sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2879:
2880: /*
2881: * Now set up the TX queues, txconf is needed to handle the
2882: * possibility that things fail midcourse and we need to
2883: * undo memory gracefully
2884: */
2885: for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2886: /* Set up some basics */
2887: txr = &adapter->tx_rings[i];
2888: txr->adapter = adapter;
2889: txr->me = i;
2890:
2891: /* Initialize the TX side lock */
2892: snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2893: device_xname(dev), txr->me);
2894: mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET);
2895:
2896: if (ixgbe_dma_malloc(adapter, tsize,
2897: &txr->txdma, BUS_DMA_NOWAIT)) {
2898: aprint_error_dev(dev,
2899: "Unable to allocate TX Descriptor memory\n");
2900: error = ENOMEM;
2901: goto err_tx_desc;
2902: }
2903: txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2904: bzero((void *)txr->tx_base, tsize);
2905:
2906: /* Now allocate transmit buffers for the ring */
2907: if (ixgbe_allocate_transmit_buffers(txr)) {
2908: aprint_error_dev(dev,
2909: "Critical Failure setting up transmit buffers\n");
2910: error = ENOMEM;
2911: goto err_tx_desc;
2912: }
2913: #if __FreeBSD_version >= 800000
2914: /* Allocate a buf ring */
2915: txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2916: M_WAITOK, &txr->tx_mtx);
2917: if (txr->br == NULL) {
2918: aprint_error_dev(dev,
2919: "Critical Failure setting up buf ring\n");
2920: error = ENOMEM;
2921: goto err_tx_desc;
2922: }
2923: #endif
2924: }
2925:
2926: /*
2927: * Next the RX queues...
2928: */
2929: rsize = roundup2(adapter->num_rx_desc *
2930: sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2931: for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2932: rxr = &adapter->rx_rings[i];
2933: /* Set up some basics */
2934: rxr->adapter = adapter;
2935: rxr->me = i;
2936:
2937: /* Initialize the RX side lock */
2938: snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2939: device_xname(dev), rxr->me);
2940: mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET);
2941:
2942: if (ixgbe_dma_malloc(adapter, rsize,
2943: &rxr->rxdma, BUS_DMA_NOWAIT)) {
2944: aprint_error_dev(dev,
2945: "Unable to allocate RxDescriptor memory\n");
2946: error = ENOMEM;
2947: goto err_rx_desc;
2948: }
2949: rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2950: bzero((void *)rxr->rx_base, rsize);
2951:
2952: /* Allocate receive buffers for the ring*/
2953: if (ixgbe_allocate_receive_buffers(rxr)) {
2954: aprint_error_dev(dev,
2955: "Critical Failure setting up receive buffers\n");
2956: error = ENOMEM;
2957: goto err_rx_desc;
2958: }
2959: }
2960:
2961: /*
2962: ** Finally set up the queue holding structs
2963: */
2964: for (int i = 0; i < adapter->num_queues; i++) {
2965: que = &adapter->queues[i];
2966: que->adapter = adapter;
2967: que->txr = &adapter->tx_rings[i];
2968: que->rxr = &adapter->rx_rings[i];
2969: }
2970:
2971: return (0);
2972:
2973: err_rx_desc:
2974: for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2975: ixgbe_dma_free(adapter, &rxr->rxdma);
2976: err_tx_desc:
2977: for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2978: ixgbe_dma_free(adapter, &txr->txdma);
2979: free(adapter->rx_rings, M_DEVBUF);
2980: rx_fail:
2981: free(adapter->tx_rings, M_DEVBUF);
2982: tx_fail:
2983: free(adapter->queues, M_DEVBUF);
2984: fail:
2985: return (error);
2986: }
2987:
2988: /*********************************************************************
2989: *
2990: * Allocate memory for tx_buffer structures. The tx_buffer stores all
2991: * the information needed to transmit a packet on the wire. This is
2992: * called only once at attach, setup is done every reset.
2993: *
2994: **********************************************************************/
2995: static int
2996: ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2997: {
2998: struct adapter *adapter = txr->adapter;
2999: device_t dev = adapter->dev;
3000: struct ixgbe_tx_buf *txbuf;
3001: int error, i;
3002:
3003: /*
3004: * Setup DMA descriptor areas.
3005: */
3006: if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
3007: 1, 0, /* alignment, bounds */
3008: IXGBE_TSO_SIZE, /* maxsize */
3009: adapter->num_segs, /* nsegments */
3010: PAGE_SIZE, /* maxsegsize */
3011: 0, /* flags */
3012: &txr->txtag))) {
3013: aprint_error_dev(dev,"Unable to allocate TX DMA tag\n");
3014: goto fail;
3015: }
3016:
3017: if (!(txr->tx_buffers =
3018: (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
3019: adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3020: aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n");
3021: error = ENOMEM;
3022: goto fail;
3023: }
3024:
3025: /* Create the descriptor buffer dma maps */
3026: txbuf = txr->tx_buffers;
3027: for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3028: error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map);
3029: if (error != 0) {
3030: aprint_error_dev(dev, "Unable to create TX DMA map\n");
3031: goto fail;
3032: }
3033: }
3034:
3035: return 0;
3036: fail:
3037: /* We free all, it handles case where we are in the middle */
3038: ixgbe_free_transmit_structures(adapter);
3039: return (error);
3040: }
3041:
3042: /*********************************************************************
3043: *
3044: * Initialize a transmit ring.
3045: *
3046: **********************************************************************/
3047: static void
3048: ixgbe_setup_transmit_ring(struct tx_ring *txr)
3049: {
3050: struct adapter *adapter = txr->adapter;
3051: struct ixgbe_tx_buf *txbuf;
3052: int i;
3053:
3054: /* Clear the old ring contents */
3055: IXGBE_TX_LOCK(txr);
3056: bzero((void *)txr->tx_base,
3057: (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3058: /* Reset indices */
3059: txr->next_avail_desc = 0;
3060: txr->next_to_clean = 0;
3061:
3062: /* Free any existing tx buffers. */
3063: txbuf = txr->tx_buffers;
3064: for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3065: if (txbuf->m_head != NULL) {
3066: bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map,
3067: 0, txbuf->m_head->m_pkthdr.len,
3068: BUS_DMASYNC_POSTWRITE);
3069: ixgbe_dmamap_unload(txr->txtag, txbuf->map);
3070: m_freem(txbuf->m_head);
3071: txbuf->m_head = NULL;
3072: }
3073: /* Clear the EOP index */
3074: txbuf->eop_index = -1;
3075: }
3076:
3077: #ifdef IXGBE_FDIR
3078: /* Set the rate at which we sample packets */
3079: if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3080: txr->atr_sample = atr_sample_rate;
3081: #endif
3082:
3083: /* Set number of descriptors available */
3084: txr->tx_avail = adapter->num_tx_desc;
3085:
3086: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3087: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3088: IXGBE_TX_UNLOCK(txr);
3089: }
3090:
3091: /*********************************************************************
3092: *
3093: * Initialize all transmit rings.
3094: *
3095: **********************************************************************/
3096: static int
3097: ixgbe_setup_transmit_structures(struct adapter *adapter)
3098: {
3099: struct tx_ring *txr = adapter->tx_rings;
3100:
3101: for (int i = 0; i < adapter->num_queues; i++, txr++)
3102: ixgbe_setup_transmit_ring(txr);
3103:
3104: return (0);
3105: }
3106:
3107: /*********************************************************************
3108: *
3109: * Enable transmit unit.
3110: *
3111: **********************************************************************/
3112: static void
3113: ixgbe_initialize_transmit_units(struct adapter *adapter)
3114: {
3115: struct tx_ring *txr = adapter->tx_rings;
3116: struct ixgbe_hw *hw = &adapter->hw;
3117:
3118: /* Setup the Base and Length of the Tx Descriptor Ring */
3119:
3120: for (int i = 0; i < adapter->num_queues; i++, txr++) {
3121: u64 tdba = txr->txdma.dma_paddr;
3122: u32 txctrl;
3123:
3124: IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3125: (tdba & 0x00000000ffffffffULL));
3126: IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3127: IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3128: adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3129:
3130: /* Setup the HW Tx Head and Tail descriptor pointers */
3131: IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3132: IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3133:
3134: /* Setup Transmit Descriptor Cmd Settings */
3135: txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3136: txr->queue_status = IXGBE_QUEUE_IDLE;
3137:
3138: /* Disable Head Writeback */
3139: switch (hw->mac.type) {
3140: case ixgbe_mac_82598EB:
3141: txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3142: break;
3143: case ixgbe_mac_82599EB:
3144: default:
3145: txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3146: break;
3147: }
3148: txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
3149: switch (hw->mac.type) {
3150: case ixgbe_mac_82598EB:
3151: IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3152: break;
3153: case ixgbe_mac_82599EB:
3154: default:
3155: IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3156: break;
3157: }
3158:
3159: }
3160:
3161: if (hw->mac.type != ixgbe_mac_82598EB) {
3162: u32 dmatxctl, rttdcs;
3163: dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3164: dmatxctl |= IXGBE_DMATXCTL_TE;
3165: IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3166: /* Disable arbiter to set MTQC */
3167: rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3168: rttdcs |= IXGBE_RTTDCS_ARBDIS;
3169: IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3170: IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3171: rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3172: IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3173: }
3174:
3175: return;
3176: }
3177:
3178: /*********************************************************************
3179: *
3180: * Free all transmit rings.
3181: *
3182: **********************************************************************/
3183: static void
3184: ixgbe_free_transmit_structures(struct adapter *adapter)
3185: {
3186: struct tx_ring *txr = adapter->tx_rings;
3187:
3188: for (int i = 0; i < adapter->num_queues; i++, txr++) {
3189: ixgbe_free_transmit_buffers(txr);
3190: ixgbe_dma_free(adapter, &txr->txdma);
3191: IXGBE_TX_LOCK_DESTROY(txr);
3192: }
3193: free(adapter->tx_rings, M_DEVBUF);
3194: }
3195:
3196: /*********************************************************************
3197: *
3198: * Free transmit ring related data structures.
3199: *
3200: **********************************************************************/
3201: static void
3202: ixgbe_free_transmit_buffers(struct tx_ring *txr)
3203: {
3204: struct adapter *adapter = txr->adapter;
3205: struct ixgbe_tx_buf *tx_buffer;
3206: int i;
3207:
3208: INIT_DEBUGOUT("free_transmit_ring: begin");
3209:
3210: if (txr->tx_buffers == NULL)
3211: return;
3212:
3213: tx_buffer = txr->tx_buffers;
3214: for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3215: if (tx_buffer->m_head != NULL) {
3216: bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map,
3217: 0, tx_buffer->m_head->m_pkthdr.len,
3218: BUS_DMASYNC_POSTWRITE);
3219: ixgbe_dmamap_unload(txr->txtag, tx_buffer->map);
3220: m_freem(tx_buffer->m_head);
3221: tx_buffer->m_head = NULL;
3222: if (tx_buffer->map != NULL) {
3223: ixgbe_dmamap_destroy(txr->txtag,
3224: tx_buffer->map);
3225: tx_buffer->map = NULL;
3226: }
3227: } else if (tx_buffer->map != NULL) {
3228: ixgbe_dmamap_unload(txr->txtag, tx_buffer->map);
3229: ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map);
3230: tx_buffer->map = NULL;
3231: }
3232: }
3233: #if __FreeBSD_version >= 800000
3234: if (txr->br != NULL)
3235: buf_ring_free(txr->br, M_DEVBUF);
3236: #endif
3237: if (txr->tx_buffers != NULL) {
3238: free(txr->tx_buffers, M_DEVBUF);
3239: txr->tx_buffers = NULL;
3240: }
3241: if (txr->txtag != NULL) {
3242: ixgbe_dma_tag_destroy(txr->txtag);
3243: txr->txtag = NULL;
3244: }
3245: return;
3246: }
3247:
3248: /*********************************************************************
3249: *
3250: * Advanced Context Descriptor setup for VLAN or L4 CSUM
3251: *
3252: **********************************************************************/
3253:
3254: static u32
3255: ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3256: {
3257: struct m_tag *mtag;
3258: struct adapter *adapter = txr->adapter;
3259: struct ethercom *ec = &adapter->osdep.ec;
3260: struct ixgbe_adv_tx_context_desc *TXD;
3261: struct ixgbe_tx_buf *tx_buffer;
3262: u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3263: struct ether_vlan_header *eh;
3264: struct ip ip;
3265: struct ip6_hdr ip6;
3266: int ehdrlen, ip_hlen = 0;
3267: u16 etype;
1.12 hannken 3268: u8 ipproto __diagused = 0;
1.1 dyoung 3269: bool offload;
3270: int ctxd = txr->next_avail_desc;
3271: u16 vtag = 0;
3272:
3273: offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0);
3274:
3275: tx_buffer = &txr->tx_buffers[ctxd];
3276: TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3277:
3278: /*
3279: ** In advanced descriptors the vlan tag must
3280: ** be placed into the descriptor itself.
3281: */
3282: if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) {
3283: vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff);
3284: vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3285: } else if (!offload)
3286: return 0;
3287:
3288: /*
3289: * Determine where frame payload starts.
3290: * Jump over vlan headers if already present,
3291: * helpful for QinQ too.
3292: */
3293: KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag));
3294: eh = mtod(mp, struct ether_vlan_header *);
3295: if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3296: KASSERT(mp->m_len >= sizeof(struct ether_vlan_header));
3297: etype = ntohs(eh->evl_proto);
3298: ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3299: } else {
3300: etype = ntohs(eh->evl_encap_proto);
3301: ehdrlen = ETHER_HDR_LEN;
3302: }
3303:
3304: /* Set the ether header length */
3305: vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3306:
3307: switch (etype) {
3308: case ETHERTYPE_IP:
3309: m_copydata(mp, ehdrlen, sizeof(ip), &ip);
3310: ip_hlen = ip.ip_hl << 2;
3311: ipproto = ip.ip_p;
3312: #if 0
3313: ip.ip_sum = 0;
3314: m_copyback(mp, ehdrlen, sizeof(ip), &ip);
3315: #else
3316: KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 ||
3317: ip.ip_sum == 0);
3318: #endif
3319: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3320: break;
3321: case ETHERTYPE_IPV6:
3322: m_copydata(mp, ehdrlen, sizeof(ip6), &ip6);
3323: ip_hlen = sizeof(ip6);
3324: ipproto = ip6.ip6_nxt;
3325: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3326: break;
3327: default:
3328: break;
3329: }
3330:
3331: if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0)
3332: olinfo |= IXGBE_TXD_POPTS_IXSM << 8;
3333:
3334: vlan_macip_lens |= ip_hlen;
3335: type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3336:
3337: if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) {
3338: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3339: olinfo |= IXGBE_TXD_POPTS_TXSM << 8;
3340: KASSERT(ipproto == IPPROTO_TCP);
3341: } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) {
3342: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3343: olinfo |= IXGBE_TXD_POPTS_TXSM << 8;
3344: KASSERT(ipproto == IPPROTO_UDP);
3345: }
3346:
3347: /* Now copy bits into descriptor */
3348: TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3349: TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3350: TXD->seqnum_seed = htole32(0);
3351: TXD->mss_l4len_idx = htole32(0);
3352:
3353: tx_buffer->m_head = NULL;
3354: tx_buffer->eop_index = -1;
3355:
3356: /* We've consumed the first desc, adjust counters */
3357: if (++ctxd == adapter->num_tx_desc)
3358: ctxd = 0;
3359: txr->next_avail_desc = ctxd;
3360: --txr->tx_avail;
3361:
3362: return olinfo;
3363: }
3364:
3365: /**********************************************************************
3366: *
3367: * Setup work for hardware segmentation offload (TSO) on
3368: * adapters using advanced tx descriptors
3369: *
3370: **********************************************************************/
3371: static bool
3372: ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
3373: {
3374: struct m_tag *mtag;
3375: struct adapter *adapter = txr->adapter;
3376: struct ethercom *ec = &adapter->osdep.ec;
3377: struct ixgbe_adv_tx_context_desc *TXD;
3378: struct ixgbe_tx_buf *tx_buffer;
3379: u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3380: u32 mss_l4len_idx = 0;
3381: u16 vtag = 0;
3382: int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen;
3383: struct ether_vlan_header *eh;
3384: struct ip *ip;
3385: struct tcphdr *th;
3386:
3387:
3388: /*
3389: * Determine where frame payload starts.
3390: * Jump over vlan headers if already present
3391: */
3392: eh = mtod(mp, struct ether_vlan_header *);
3393: if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3394: ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3395: else
3396: ehdrlen = ETHER_HDR_LEN;
3397:
3398: /* Ensure we have at least the IP+TCP header in the first mbuf. */
3399: if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3400: return FALSE;
3401:
3402: ctxd = txr->next_avail_desc;
3403: tx_buffer = &txr->tx_buffers[ctxd];
3404: TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3405:
3406: ip = (struct ip *)(mp->m_data + ehdrlen);
3407: if (ip->ip_p != IPPROTO_TCP)
3408: return FALSE; /* 0 */
3409: ip->ip_sum = 0;
3410: ip_hlen = ip->ip_hl << 2;
3411: th = (struct tcphdr *)((char *)ip + ip_hlen);
3412: /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */
3413: th->th_sum = in_cksum_phdr(ip->ip_src.s_addr,
3414: ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3415: tcp_hlen = th->th_off << 2;
3416: hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3417:
3418: /* This is used in the transmit desc in encap */
3419: *paylen = mp->m_pkthdr.len - hdrlen;
3420:
3421: /* VLAN MACLEN IPLEN */
3422: if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) {
3423: vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff);
3424: vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3425: }
3426:
3427: vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3428: vlan_macip_lens |= ip_hlen;
3429: TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3430:
3431: /* ADV DTYPE TUCMD */
3432: type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3433: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3434: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3435: TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3436:
3437:
3438: /* MSS L4LEN IDX */
3439: mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT);
3440: mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3441: TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3442:
3443: TXD->seqnum_seed = htole32(0);
3444: tx_buffer->m_head = NULL;
3445: tx_buffer->eop_index = -1;
3446:
3447: if (++ctxd == adapter->num_tx_desc)
3448: ctxd = 0;
3449:
3450: txr->tx_avail--;
3451: txr->next_avail_desc = ctxd;
3452: return TRUE;
3453: }
3454:
3455: #ifdef IXGBE_FDIR
3456: /*
3457: ** This routine parses packet headers so that Flow
3458: ** Director can make a hashed filter table entry
3459: ** allowing traffic flows to be identified and kept
3460: ** on the same cpu. This would be a performance
3461: ** hit, but we only do it at IXGBE_FDIR_RATE of
3462: ** packets.
3463: */
3464: static void
3465: ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3466: {
3467: struct adapter *adapter = txr->adapter;
3468: struct ix_queue *que;
3469: struct ip *ip;
3470: struct tcphdr *th;
3471: struct udphdr *uh;
3472: struct ether_vlan_header *eh;
3473: union ixgbe_atr_hash_dword input = {.dword = 0};
3474: union ixgbe_atr_hash_dword common = {.dword = 0};
3475: int ehdrlen, ip_hlen;
3476: u16 etype;
3477:
3478: eh = mtod(mp, struct ether_vlan_header *);
3479: if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3480: ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3481: etype = eh->evl_proto;
3482: } else {
3483: ehdrlen = ETHER_HDR_LEN;
3484: etype = eh->evl_encap_proto;
3485: }
3486:
3487: /* Only handling IPv4 */
3488: if (etype != htons(ETHERTYPE_IP))
3489: return;
3490:
3491: ip = (struct ip *)(mp->m_data + ehdrlen);
3492: ip_hlen = ip->ip_hl << 2;
3493:
3494: /* check if we're UDP or TCP */
3495: switch (ip->ip_p) {
3496: case IPPROTO_TCP:
3497: th = (struct tcphdr *)((char *)ip + ip_hlen);
3498: /* src and dst are inverted */
3499: common.port.dst ^= th->th_sport;
3500: common.port.src ^= th->th_dport;
3501: input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3502: break;
3503: case IPPROTO_UDP:
3504: uh = (struct udphdr *)((char *)ip + ip_hlen);
3505: /* src and dst are inverted */
3506: common.port.dst ^= uh->uh_sport;
3507: common.port.src ^= uh->uh_dport;
3508: input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3509: break;
3510: default:
3511: return;
3512: }
3513:
3514: input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3515: if (mp->m_pkthdr.ether_vtag)
3516: common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3517: else
3518: common.flex_bytes ^= etype;
3519: common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3520:
3521: que = &adapter->queues[txr->me];
3522: /*
3523: ** This assumes the Rx queue and Tx
3524: ** queue are bound to the same CPU
3525: */
3526: ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3527: input, common, que->msix);
3528: }
3529: #endif /* IXGBE_FDIR */
3530:
3531: /**********************************************************************
3532: *
3533: * Examine each tx_buffer in the used queue. If the hardware is done
3534: * processing the packet then free associated resources. The
3535: * tx_buffer is put back on the free queue.
3536: *
3537: **********************************************************************/
3538: static bool
3539: ixgbe_txeof(struct tx_ring *txr)
3540: {
3541: struct adapter *adapter = txr->adapter;
3542: struct ifnet *ifp = adapter->ifp;
3543: u32 first, last, done, processed;
3544: struct ixgbe_tx_buf *tx_buffer;
3545: struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3546: struct timeval now, elapsed;
3547:
3548: KASSERT(mutex_owned(&txr->tx_mtx));
3549:
3550: if (txr->tx_avail == adapter->num_tx_desc) {
3551: txr->queue_status = IXGBE_QUEUE_IDLE;
3552: return false;
3553: }
3554:
3555: processed = 0;
3556: first = txr->next_to_clean;
3557: tx_buffer = &txr->tx_buffers[first];
3558: /* For cleanup we just use legacy struct */
3559: tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3560: last = tx_buffer->eop_index;
3561: if (last == -1)
3562: return false;
3563: eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3564:
3565: /*
3566: ** Get the index of the first descriptor
3567: ** BEYOND the EOP and call that 'done'.
3568: ** I do this so the comparison in the
3569: ** inner while loop below can be simple
3570: */
3571: if (++last == adapter->num_tx_desc) last = 0;
3572: done = last;
3573:
3574: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3575: BUS_DMASYNC_POSTREAD);
3576: /*
3577: ** Only the EOP descriptor of a packet now has the DD
3578: ** bit set, this is what we look for...
3579: */
3580: while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3581: /* We clean the range of the packet */
3582: while (first != done) {
3583: tx_desc->upper.data = 0;
3584: tx_desc->lower.data = 0;
3585: tx_desc->buffer_addr = 0;
3586: ++txr->tx_avail;
3587: ++processed;
3588:
3589: if (tx_buffer->m_head) {
3590: txr->bytes +=
3591: tx_buffer->m_head->m_pkthdr.len;
3592: bus_dmamap_sync(txr->txtag->dt_dmat,
3593: tx_buffer->map,
3594: 0, tx_buffer->m_head->m_pkthdr.len,
3595: BUS_DMASYNC_POSTWRITE);
3596: ixgbe_dmamap_unload(txr->txtag, tx_buffer->map);
3597: m_freem(tx_buffer->m_head);
3598: tx_buffer->m_head = NULL;
3599: }
3600: tx_buffer->eop_index = -1;
3601: getmicrotime(&txr->watchdog_time);
3602:
3603: if (++first == adapter->num_tx_desc)
3604: first = 0;
3605:
3606: tx_buffer = &txr->tx_buffers[first];
3607: tx_desc =
3608: (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3609: }
3610: ++txr->packets;
3611: ++ifp->if_opackets;
3612: /* See if there is more work now */
3613: last = tx_buffer->eop_index;
3614: if (last != -1) {
3615: eop_desc =
3616: (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3617: /* Get next done point */
3618: if (++last == adapter->num_tx_desc) last = 0;
3619: done = last;
3620: } else
3621: break;
3622: }
3623: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3624: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3625:
3626: txr->next_to_clean = first;
3627:
3628: /*
3629: ** Watchdog calculation, we know there's
3630: ** work outstanding or the first return
3631: ** would have been taken, so none processed
3632: ** for too long indicates a hang.
3633: */
3634: getmicrotime(&now);
3635: timersub(&now, &txr->watchdog_time, &elapsed);
3636: if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG)
3637: txr->queue_status = IXGBE_QUEUE_HUNG;
3638:
3639: /*
3640: * If we have enough room, clear IFF_OACTIVE to tell the stack that
3641: * it is OK to send packets. If there are no pending descriptors,
3642: * clear the timeout. Otherwise, if some descriptors have been freed,
3643: * restart the timeout.
3644: */
3645: if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) {
3646: ifp->if_flags &= ~IFF_OACTIVE;
3647: if (txr->tx_avail == adapter->num_tx_desc) {
3648: txr->queue_status = IXGBE_QUEUE_IDLE;
3649: return false;
3650: }
3651: }
3652:
3653: return true;
3654: }
3655:
3656: /*********************************************************************
3657: *
3658: * Refresh mbuf buffers for RX descriptor rings
3659: * - now keeps its own state so discards due to resource
3660: * exhaustion are unnecessary, if an mbuf cannot be obtained
3661: * it just returns, keeping its placeholder, thus it can simply
3662: * be recalled to try again.
3663: *
3664: **********************************************************************/
3665: static void
3666: ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3667: {
3668: struct adapter *adapter = rxr->adapter;
3669: struct ixgbe_rx_buf *rxbuf;
3670: struct mbuf *mh, *mp;
3671: int i, j, error;
3672: bool refreshed = false;
3673:
3674: i = j = rxr->next_to_refresh;
3675: /* Control the loop with one beyond */
3676: if (++j == adapter->num_rx_desc)
3677: j = 0;
3678:
3679: while (j != limit) {
3680: rxbuf = &rxr->rx_buffers[i];
3681: if (rxr->hdr_split == FALSE)
3682: goto no_split;
3683:
3684: if (rxbuf->m_head == NULL) {
3685: mh = m_gethdr(M_DONTWAIT, MT_DATA);
3686: if (mh == NULL)
3687: goto update;
3688: } else
3689: mh = rxbuf->m_head;
3690:
3691: mh->m_pkthdr.len = mh->m_len = MHLEN;
3692: mh->m_len = MHLEN;
3693: mh->m_flags |= M_PKTHDR;
3694: /* Get the memory mapping */
3695: error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat,
3696: rxbuf->hmap, mh, BUS_DMA_NOWAIT);
3697: if (error != 0) {
3698: printf("Refresh mbufs: hdr dmamap load"
3699: " failure - %d\n", error);
3700: m_free(mh);
3701: rxbuf->m_head = NULL;
3702: goto update;
3703: }
3704: rxbuf->m_head = mh;
3705: ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD);
3706: rxr->rx_base[i].read.hdr_addr =
3707: htole64(rxbuf->hmap->dm_segs[0].ds_addr);
3708:
3709: no_split:
3710: if (rxbuf->m_pack == NULL) {
3711: mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT,
3712: MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz);
3713: if (mp == NULL) {
3714: rxr->no_jmbuf.ev_count++;
3715: goto update;
3716: }
3717: } else
3718: mp = rxbuf->m_pack;
3719:
3720: mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3721: /* Get the memory mapping */
3722: error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
3723: rxbuf->pmap, mp, BUS_DMA_NOWAIT);
3724: if (error != 0) {
3725: printf("Refresh mbufs: payload dmamap load"
3726: " failure - %d\n", error);
3727: m_free(mp);
3728: rxbuf->m_pack = NULL;
3729: goto update;
3730: }
3731: rxbuf->m_pack = mp;
3732: bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
3733: 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD);
3734: rxr->rx_base[i].read.pkt_addr =
3735: htole64(rxbuf->pmap->dm_segs[0].ds_addr);
3736:
3737: refreshed = true;
3738: /* Next is precalculated */
3739: i = j;
3740: rxr->next_to_refresh = i;
3741: if (++j == adapter->num_rx_desc)
3742: j = 0;
3743: }
3744: update:
3745: if (refreshed) /* Update hardware tail index */
3746: IXGBE_WRITE_REG(&adapter->hw,
3747: IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3748: return;
3749: }
3750:
3751: /*********************************************************************
3752: *
3753: * Allocate memory for rx_buffer structures. Since we use one
3754: * rx_buffer per received packet, the maximum number of rx_buffer's
3755: * that we'll need is equal to the number of receive descriptors
3756: * that we've allocated.
3757: *
3758: **********************************************************************/
3759: static int
3760: ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3761: {
3762: struct adapter *adapter = rxr->adapter;
3763: device_t dev = adapter->dev;
3764: struct ixgbe_rx_buf *rxbuf;
3765: int i, bsize, error;
3766:
3767: bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3768: if (!(rxr->rx_buffers =
3769: (struct ixgbe_rx_buf *) malloc(bsize,
3770: M_DEVBUF, M_NOWAIT | M_ZERO))) {
3771: aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n");
3772: error = ENOMEM;
3773: goto fail;
3774: }
3775:
3776: if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
3777: 1, 0, /* alignment, bounds */
3778: MSIZE, /* maxsize */
3779: 1, /* nsegments */
3780: MSIZE, /* maxsegsize */
3781: 0, /* flags */
3782: &rxr->htag))) {
3783: aprint_error_dev(dev, "Unable to create RX DMA tag\n");
3784: goto fail;
3785: }
3786:
3787: if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
3788: 1, 0, /* alignment, bounds */
3789: MJUM16BYTES, /* maxsize */
3790: 1, /* nsegments */
3791: MJUM16BYTES, /* maxsegsize */
3792: 0, /* flags */
3793: &rxr->ptag))) {
3794: aprint_error_dev(dev, "Unable to create RX DMA tag\n");
3795: goto fail;
3796: }
3797:
3798: for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3799: rxbuf = &rxr->rx_buffers[i];
3800: error = ixgbe_dmamap_create(rxr->htag,
3801: BUS_DMA_NOWAIT, &rxbuf->hmap);
3802: if (error) {
3803: aprint_error_dev(dev, "Unable to create RX head map\n");
3804: goto fail;
3805: }
3806: error = ixgbe_dmamap_create(rxr->ptag,
3807: BUS_DMA_NOWAIT, &rxbuf->pmap);
3808: if (error) {
3809: aprint_error_dev(dev, "Unable to create RX pkt map\n");
3810: goto fail;
3811: }
3812: }
3813:
3814: return (0);
3815:
3816: fail:
3817: /* Frees all, but can handle partial completion */
3818: ixgbe_free_receive_structures(adapter);
3819: return (error);
3820: }
3821:
3822: /*
3823: ** Used to detect a descriptor that has
3824: ** been merged by Hardware RSC.
3825: */
3826: static inline u32
3827: ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3828: {
3829: return (le32toh(rx->wb.lower.lo_dword.data) &
3830: IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3831: }
3832:
3833: /*********************************************************************
3834: *
3835: * Initialize Hardware RSC (LRO) feature on 82599
3836: * for an RX ring, this is toggled by the LRO capability
3837: * even though it is transparent to the stack.
3838: *
3839: **********************************************************************/
3840: static void
3841: ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3842: {
3843: struct adapter *adapter = rxr->adapter;
3844: struct ixgbe_hw *hw = &adapter->hw;
3845: u32 rscctrl, rdrxctl;
3846:
3847: rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3848: rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3849: rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3850: rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3851: IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3852:
3853: rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3854: rscctrl |= IXGBE_RSCCTL_RSCEN;
3855: /*
3856: ** Limit the total number of descriptors that
3857: ** can be combined, so it does not exceed 64K
3858: */
3859: if (adapter->rx_mbuf_sz == MCLBYTES)
3860: rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3861: else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3862: rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3863: else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3864: rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3865: else /* Using 16K cluster */
3866: rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3867:
3868: IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3869:
3870: /* Enable TCP header recognition */
3871: IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3872: (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3873: IXGBE_PSRTYPE_TCPHDR));
3874:
3875: /* Disable RSC for ACK packets */
3876: IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3877: (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3878:
3879: rxr->hw_rsc = TRUE;
3880: }
3881:
3882:
3883: static void
3884: ixgbe_free_receive_ring(struct rx_ring *rxr)
3885: {
3886: struct adapter *adapter;
3887: struct ixgbe_rx_buf *rxbuf;
3888: int i;
3889:
3890: adapter = rxr->adapter;
3891: for (i = 0; i < adapter->num_rx_desc; i++) {
3892: rxbuf = &rxr->rx_buffers[i];
3893: if (rxbuf->m_head != NULL) {
3894: ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap,
3895: BUS_DMASYNC_POSTREAD);
3896: ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap);
3897: rxbuf->m_head->m_flags |= M_PKTHDR;
3898: m_freem(rxbuf->m_head);
3899: }
3900: if (rxbuf->m_pack != NULL) {
3901: bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
3902: 0, rxbuf->m_pack->m_pkthdr.len,
3903: BUS_DMASYNC_POSTREAD);
3904: ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
3905: rxbuf->m_pack->m_flags |= M_PKTHDR;
3906: m_freem(rxbuf->m_pack);
3907: }
3908: rxbuf->m_head = NULL;
3909: rxbuf->m_pack = NULL;
3910: }
3911: }
3912:
3913:
3914: /*********************************************************************
3915: *
3916: * Initialize a receive ring and its buffers.
3917: *
3918: **********************************************************************/
3919: static int
3920: ixgbe_setup_receive_ring(struct rx_ring *rxr)
3921: {
3922: struct adapter *adapter;
3923: struct ifnet *ifp;
3924: struct ixgbe_rx_buf *rxbuf;
3925: #ifdef LRO
3926: struct lro_ctrl *lro = &rxr->lro;
3927: #endif /* LRO */
3928: int rsize, error = 0;
3929:
3930: adapter = rxr->adapter;
3931: ifp = adapter->ifp;
3932:
3933: /* Clear the ring contents */
3934: IXGBE_RX_LOCK(rxr);
3935: rsize = roundup2(adapter->num_rx_desc *
3936: sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3937: bzero((void *)rxr->rx_base, rsize);
3938:
3939: /* Free current RX buffer structs and their mbufs */
3940: ixgbe_free_receive_ring(rxr);
3941:
1.18 msaitoh 3942: IXGBE_RX_UNLOCK(rxr);
3943:
1.1 dyoung 3944: /* Now reinitialize our supply of jumbo mbufs. The number
3945: * or size of jumbo mbufs may have changed.
3946: */
3947: ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat,
3948: 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz);
3949:
1.18 msaitoh 3950: IXGBE_RX_LOCK(rxr);
3951:
1.1 dyoung 3952: /* Configure header split? */
3953: if (ixgbe_header_split)
3954: rxr->hdr_split = TRUE;
3955:
3956: /* Now replenish the mbufs */
3957: for (int j = 0; j != adapter->num_rx_desc; ++j) {
3958: struct mbuf *mh, *mp;
3959:
3960: rxbuf = &rxr->rx_buffers[j];
3961: /*
3962: ** Don't allocate mbufs if not
3963: ** doing header split, its wasteful
3964: */
3965: if (rxr->hdr_split == FALSE)
3966: goto skip_head;
3967:
3968: /* First the header */
3969: rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA);
3970: if (rxbuf->m_head == NULL) {
3971: error = ENOBUFS;
3972: goto fail;
3973: }
3974: m_adj(rxbuf->m_head, ETHER_ALIGN);
3975: mh = rxbuf->m_head;
3976: mh->m_len = mh->m_pkthdr.len = MHLEN;
3977: mh->m_flags |= M_PKTHDR;
3978: /* Get the memory mapping */
3979: error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat,
3980: rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT);
3981: if (error != 0) /* Nothing elegant to do here */
3982: goto fail;
3983: bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap,
3984: 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD);
3985: /* Update descriptor */
3986: rxr->rx_base[j].read.hdr_addr =
3987: htole64(rxbuf->hmap->dm_segs[0].ds_addr);
3988:
3989: skip_head:
3990: /* Now the payload cluster */
3991: rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT,
3992: MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz);
3993: if (rxbuf->m_pack == NULL) {
3994: error = ENOBUFS;
3995: goto fail;
3996: }
3997: mp = rxbuf->m_pack;
3998: mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3999: /* Get the memory mapping */
4000: error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
4001: rxbuf->pmap, mp, BUS_DMA_NOWAIT);
4002: if (error != 0)
4003: goto fail;
4004: bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
4005: 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD);
4006: /* Update descriptor */
4007: rxr->rx_base[j].read.pkt_addr =
4008: htole64(rxbuf->pmap->dm_segs[0].ds_addr);
4009: }
4010:
4011:
4012: /* Setup our descriptor indices */
4013: rxr->next_to_check = 0;
4014: rxr->next_to_refresh = 0;
4015: rxr->lro_enabled = FALSE;
4016: rxr->rx_split_packets.ev_count = 0;
4017: rxr->rx_bytes.ev_count = 0;
4018: rxr->discard = FALSE;
4019:
4020: ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4021: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4022:
4023: /*
4024: ** Now set up the LRO interface:
4025: ** 82598 uses software LRO, the
4026: ** 82599 uses a hardware assist.
4027: */
4028: if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4029: (ifp->if_capenable & IFCAP_RXCSUM) &&
4030: (ifp->if_capenable & IFCAP_LRO))
4031: ixgbe_setup_hw_rsc(rxr);
4032: #ifdef LRO
4033: else if (ifp->if_capenable & IFCAP_LRO) {
1.9 skrll 4034: device_t dev = adapter->dev;
1.1 dyoung 4035: int err = tcp_lro_init(lro);
4036: if (err) {
4037: device_printf(dev, "LRO Initialization failed!\n");
4038: goto fail;
4039: }
4040: INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4041: rxr->lro_enabled = TRUE;
4042: lro->ifp = adapter->ifp;
4043: }
4044: #endif /* LRO */
4045:
4046: IXGBE_RX_UNLOCK(rxr);
4047: return (0);
4048:
4049: fail:
4050: ixgbe_free_receive_ring(rxr);
4051: IXGBE_RX_UNLOCK(rxr);
4052: return (error);
4053: }
4054:
4055: /*********************************************************************
4056: *
4057: * Initialize all receive rings.
4058: *
4059: **********************************************************************/
4060: static int
4061: ixgbe_setup_receive_structures(struct adapter *adapter)
4062: {
4063: struct rx_ring *rxr = adapter->rx_rings;
4064: int j;
4065:
4066: for (j = 0; j < adapter->num_queues; j++, rxr++)
4067: if (ixgbe_setup_receive_ring(rxr))
4068: goto fail;
4069:
4070: return (0);
4071: fail:
4072: /*
4073: * Free RX buffers allocated so far, we will only handle
4074: * the rings that completed, the failing case will have
4075: * cleaned up for itself. 'j' failed, so its the terminus.
4076: */
4077: for (int i = 0; i < j; ++i) {
4078: rxr = &adapter->rx_rings[i];
4079: ixgbe_free_receive_ring(rxr);
4080: }
4081:
4082: return (ENOBUFS);
4083: }
4084:
4085: /*********************************************************************
4086: *
4087: * Setup receive registers and features.
4088: *
4089: **********************************************************************/
4090: #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4091:
4092: static void
4093: ixgbe_initialize_receive_units(struct adapter *adapter)
4094: {
4095: int i;
4096: struct rx_ring *rxr = adapter->rx_rings;
4097: struct ixgbe_hw *hw = &adapter->hw;
4098: struct ifnet *ifp = adapter->ifp;
4099: u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
4100: u32 reta, mrqc = 0, hlreg, r[10];
4101:
4102:
4103: /*
4104: * Make sure receives are disabled while
4105: * setting up the descriptor ring
4106: */
4107: rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4108: IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4109: rxctrl & ~IXGBE_RXCTRL_RXEN);
4110:
4111: /* Enable broadcasts */
4112: fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4113: fctrl |= IXGBE_FCTRL_BAM;
4114: fctrl |= IXGBE_FCTRL_DPF;
4115: fctrl |= IXGBE_FCTRL_PMCF;
4116: IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4117:
4118: /* Set for Jumbo Frames? */
4119: hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4120: if (ifp->if_mtu > ETHERMTU)
4121: hlreg |= IXGBE_HLREG0_JUMBOEN;
4122: else
4123: hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4124: IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4125:
4126: bufsz = adapter->rx_mbuf_sz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4127:
4128: for (i = 0; i < adapter->num_queues; i++, rxr++) {
4129: u64 rdba = rxr->rxdma.dma_paddr;
4130:
4131: /* Setup the Base and Length of the Rx Descriptor Ring */
4132: IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4133: (rdba & 0x00000000ffffffffULL));
4134: IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4135: IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4136: adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4137:
4138: /* Set up the SRRCTL register */
4139: srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4140: srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4141: srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4142: srrctl |= bufsz;
4143: if (rxr->hdr_split) {
4144: /* Use a standard mbuf for the header */
4145: srrctl |= ((IXGBE_RX_HDR <<
4146: IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4147: & IXGBE_SRRCTL_BSIZEHDR_MASK);
4148: srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4149: } else
4150: srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4151: IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4152:
4153: /* Setup the HW Rx Head and Tail Descriptor Pointers */
4154: IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4155: IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4156: }
4157:
4158: if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4159: u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4160: IXGBE_PSRTYPE_UDPHDR |
4161: IXGBE_PSRTYPE_IPV4HDR |
4162: IXGBE_PSRTYPE_IPV6HDR;
4163: IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4164: }
4165:
4166: rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4167:
4168: /* Setup RSS */
4169: if (adapter->num_queues > 1) {
4170: int j;
4171: reta = 0;
4172:
4173: /* set up random bits */
1.2 tls 4174: cprng_fast(&r, sizeof(r));
1.1 dyoung 4175:
4176: /* Set up the redirection table */
4177: for (i = 0, j = 0; i < 128; i++, j++) {
4178: if (j == adapter->num_queues) j = 0;
4179: reta = (reta << 8) | (j * 0x11);
4180: if ((i & 3) == 3)
4181: IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4182: }
4183:
4184: /* Now fill our hash function seeds */
4185: for (i = 0; i < 10; i++)
4186: IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), r[i]);
4187:
4188: /* Perform hash on these packet types */
4189: mrqc = IXGBE_MRQC_RSSEN
4190: | IXGBE_MRQC_RSS_FIELD_IPV4
4191: | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4192: | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4193: | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4194: | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4195: | IXGBE_MRQC_RSS_FIELD_IPV6
4196: | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4197: | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4198: | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4199: IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4200:
4201: /* RSS and RX IPP Checksum are mutually exclusive */
4202: rxcsum |= IXGBE_RXCSUM_PCSD;
4203: }
4204:
4205: if (ifp->if_capenable & IFCAP_RXCSUM)
4206: rxcsum |= IXGBE_RXCSUM_PCSD;
4207:
4208: if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4209: rxcsum |= IXGBE_RXCSUM_IPPCSE;
4210:
4211: IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4212:
4213: return;
4214: }
4215:
4216: /*********************************************************************
4217: *
4218: * Free all receive rings.
4219: *
4220: **********************************************************************/
4221: static void
4222: ixgbe_free_receive_structures(struct adapter *adapter)
4223: {
4224: struct rx_ring *rxr = adapter->rx_rings;
4225:
4226: for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4227: #ifdef LRO
4228: struct lro_ctrl *lro = &rxr->lro;
4229: #endif /* LRO */
4230: ixgbe_free_receive_buffers(rxr);
4231: #ifdef LRO
4232: /* Free LRO memory */
4233: tcp_lro_free(lro);
4234: #endif /* LRO */
4235: /* Free the ring memory as well */
4236: ixgbe_dma_free(adapter, &rxr->rxdma);
1.16 msaitoh 4237: IXGBE_RX_LOCK_DESTROY(rxr);
1.1 dyoung 4238: }
4239:
4240: free(adapter->rx_rings, M_DEVBUF);
4241: }
4242:
4243:
4244: /*********************************************************************
4245: *
4246: * Free receive ring data structures
4247: *
4248: **********************************************************************/
4249: static void
4250: ixgbe_free_receive_buffers(struct rx_ring *rxr)
4251: {
4252: struct adapter *adapter = rxr->adapter;
4253: struct ixgbe_rx_buf *rxbuf;
4254:
4255: INIT_DEBUGOUT("free_receive_structures: begin");
4256:
4257: /* Cleanup any existing buffers */
4258: if (rxr->rx_buffers != NULL) {
4259: for (int i = 0; i < adapter->num_rx_desc; i++) {
4260: rxbuf = &rxr->rx_buffers[i];
4261: if (rxbuf->m_head != NULL) {
4262: ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap,
4263: BUS_DMASYNC_POSTREAD);
4264: ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap);
4265: rxbuf->m_head->m_flags |= M_PKTHDR;
4266: m_freem(rxbuf->m_head);
4267: }
4268: if (rxbuf->m_pack != NULL) {
4269: bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
4270: 0, rxbuf->m_pack->m_pkthdr.len,
4271: BUS_DMASYNC_POSTREAD);
4272: ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
4273: rxbuf->m_pack->m_flags |= M_PKTHDR;
4274: m_freem(rxbuf->m_pack);
4275: }
4276: rxbuf->m_head = NULL;
4277: rxbuf->m_pack = NULL;
4278: if (rxbuf->hmap != NULL) {
4279: ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap);
4280: rxbuf->hmap = NULL;
4281: }
4282: if (rxbuf->pmap != NULL) {
4283: ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4284: rxbuf->pmap = NULL;
4285: }
4286: }
4287: if (rxr->rx_buffers != NULL) {
4288: free(rxr->rx_buffers, M_DEVBUF);
4289: rxr->rx_buffers = NULL;
4290: }
4291: }
4292:
4293: if (rxr->htag != NULL) {
4294: ixgbe_dma_tag_destroy(rxr->htag);
4295: rxr->htag = NULL;
4296: }
4297: if (rxr->ptag != NULL) {
4298: ixgbe_dma_tag_destroy(rxr->ptag);
4299: rxr->ptag = NULL;
4300: }
4301:
4302: return;
4303: }
4304:
4305: static __inline void
4306: ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4307: {
4308: int s;
4309:
1.9 skrll 4310: #ifdef LRO
4311: struct adapter *adapter = ifp->if_softc;
4312: struct ethercom *ec = &adapter->osdep.ec;
1.1 dyoung 4313:
4314: /*
4315: * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4316: * should be computed by hardware. Also it should not have VLAN tag in
4317: * ethernet header.
4318: */
4319: if (rxr->lro_enabled &&
4320: (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 &&
4321: (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4322: (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4323: (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) &&
4324: (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4325: (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4326: /*
4327: * Send to the stack if:
4328: ** - LRO not enabled, or
4329: ** - no LRO resources, or
4330: ** - lro enqueue fails
4331: */
4332: if (rxr->lro.lro_cnt != 0)
4333: if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4334: return;
4335: }
4336: #endif /* LRO */
4337:
4338: IXGBE_RX_UNLOCK(rxr);
4339:
4340: s = splnet();
4341: /* Pass this up to any BPF listeners. */
4342: bpf_mtap(ifp, m);
4343: (*ifp->if_input)(ifp, m);
4344: splx(s);
4345:
4346: IXGBE_RX_LOCK(rxr);
4347: }
4348:
4349: static __inline void
4350: ixgbe_rx_discard(struct rx_ring *rxr, int i)
4351: {
4352: struct ixgbe_rx_buf *rbuf;
4353:
4354: rbuf = &rxr->rx_buffers[i];
4355:
4356: if (rbuf->fmp != NULL) {/* Partial chain ? */
4357: rbuf->fmp->m_flags |= M_PKTHDR;
4358: m_freem(rbuf->fmp);
4359: rbuf->fmp = NULL;
4360: }
4361:
4362: /*
4363: ** With advanced descriptors the writeback
4364: ** clobbers the buffer addrs, so its easier
4365: ** to just free the existing mbufs and take
4366: ** the normal refresh path to get new buffers
4367: ** and mapping.
4368: */
4369: if (rbuf->m_head) {
4370: m_free(rbuf->m_head);
4371: rbuf->m_head = NULL;
4372: }
4373:
4374: if (rbuf->m_pack) {
4375: m_free(rbuf->m_pack);
4376: rbuf->m_pack = NULL;
4377: }
4378:
4379: return;
4380: }
4381:
4382:
4383: /*********************************************************************
4384: *
4385: * This routine executes in interrupt context. It replenishes
4386: * the mbufs in the descriptor and sends data which has been
4387: * dma'ed into host memory to upper layer.
4388: *
4389: * We loop at most count times if count is > 0, or until done if
4390: * count < 0.
4391: *
4392: * Return TRUE for more work, FALSE for all clean.
4393: *********************************************************************/
4394: static bool
4395: ixgbe_rxeof(struct ix_queue *que, int count)
4396: {
4397: struct adapter *adapter = que->adapter;
4398: struct rx_ring *rxr = que->rxr;
4399: struct ifnet *ifp = adapter->ifp;
4400: #ifdef LRO
4401: struct lro_ctrl *lro = &rxr->lro;
4402: struct lro_entry *queued;
4403: #endif /* LRO */
4404: int i, nextp, processed = 0;
4405: u32 staterr = 0;
4406: union ixgbe_adv_rx_desc *cur;
4407: struct ixgbe_rx_buf *rbuf, *nbuf;
4408:
4409: IXGBE_RX_LOCK(rxr);
4410:
4411: for (i = rxr->next_to_check; count != 0;) {
4412: struct mbuf *sendmp, *mh, *mp;
4413: u32 rsc, ptype;
4414: u16 hlen, plen, hdr, vtag;
4415: bool eop;
4416:
4417: /* Sync the ring. */
4418: ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4419: BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4420:
4421: cur = &rxr->rx_base[i];
4422: staterr = le32toh(cur->wb.upper.status_error);
4423:
4424: if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4425: break;
4426: if ((ifp->if_flags & IFF_RUNNING) == 0)
4427: break;
4428:
4429: count--;
4430: sendmp = NULL;
4431: nbuf = NULL;
4432: rsc = 0;
4433: cur->wb.upper.status_error = 0;
4434: rbuf = &rxr->rx_buffers[i];
4435: mh = rbuf->m_head;
4436: mp = rbuf->m_pack;
4437:
4438: plen = le16toh(cur->wb.upper.length);
4439: ptype = le32toh(cur->wb.lower.lo_dword.data) &
4440: IXGBE_RXDADV_PKTTYPE_MASK;
4441: hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4442: vtag = le16toh(cur->wb.upper.vlan);
4443: eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4444:
4445: /* Make sure bad packets are discarded */
4446: if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4447: (rxr->discard)) {
4448: ifp->if_ierrors++;
4449: rxr->rx_discarded.ev_count++;
4450: if (eop)
4451: rxr->discard = FALSE;
4452: else
4453: rxr->discard = TRUE;
4454: ixgbe_rx_discard(rxr, i);
4455: goto next_desc;
4456: }
4457:
4458: /*
4459: ** On 82599 which supports a hardware
4460: ** LRO (called HW RSC), packets need
4461: ** not be fragmented across sequential
4462: ** descriptors, rather the next descriptor
4463: ** is indicated in bits of the descriptor.
4464: ** This also means that we might proceses
4465: ** more than one packet at a time, something
4466: ** that has never been true before, it
4467: ** required eliminating global chain pointers
4468: ** in favor of what we are doing here. -jfv
4469: */
4470: if (!eop) {
4471: /*
4472: ** Figure out the next descriptor
4473: ** of this frame.
4474: */
4475: if (rxr->hw_rsc == TRUE) {
4476: rsc = ixgbe_rsc_count(cur);
4477: rxr->rsc_num += (rsc - 1);
4478: }
4479: if (rsc) { /* Get hardware index */
4480: nextp = ((staterr &
4481: IXGBE_RXDADV_NEXTP_MASK) >>
4482: IXGBE_RXDADV_NEXTP_SHIFT);
4483: } else { /* Just sequential */
4484: nextp = i + 1;
4485: if (nextp == adapter->num_rx_desc)
4486: nextp = 0;
4487: }
4488: nbuf = &rxr->rx_buffers[nextp];
4489: prefetch(nbuf);
4490: }
4491: /*
4492: ** The header mbuf is ONLY used when header
4493: ** split is enabled, otherwise we get normal
4494: ** behavior, ie, both header and payload
4495: ** are DMA'd into the payload buffer.
4496: **
4497: ** Rather than using the fmp/lmp global pointers
4498: ** we now keep the head of a packet chain in the
4499: ** buffer struct and pass this along from one
4500: ** descriptor to the next, until we get EOP.
4501: */
4502: if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4503: /* This must be an initial descriptor */
4504: hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4505: IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4506: if (hlen > IXGBE_RX_HDR)
4507: hlen = IXGBE_RX_HDR;
4508: mh->m_len = hlen;
4509: mh->m_flags |= M_PKTHDR;
4510: mh->m_next = NULL;
4511: mh->m_pkthdr.len = mh->m_len;
4512: /* Null buf pointer so it is refreshed */
4513: rbuf->m_head = NULL;
4514: /*
4515: ** Check the payload length, this
4516: ** could be zero if its a small
4517: ** packet.
4518: */
4519: if (plen > 0) {
4520: mp->m_len = plen;
4521: mp->m_next = NULL;
4522: mp->m_flags &= ~M_PKTHDR;
4523: mh->m_next = mp;
4524: mh->m_pkthdr.len += mp->m_len;
4525: /* Null buf pointer so it is refreshed */
4526: rbuf->m_pack = NULL;
4527: rxr->rx_split_packets.ev_count++;
4528: }
4529: /*
4530: ** Now create the forward
4531: ** chain so when complete
4532: ** we wont have to.
4533: */
4534: if (eop == 0) {
4535: /* stash the chain head */
4536: nbuf->fmp = mh;
4537: /* Make forward chain */
4538: if (plen)
4539: mp->m_next = nbuf->m_pack;
4540: else
4541: mh->m_next = nbuf->m_pack;
4542: } else {
4543: /* Singlet, prepare to send */
4544: sendmp = mh;
4545: if (VLAN_ATTACHED(&adapter->osdep.ec) &&
4546: (staterr & IXGBE_RXD_STAT_VP)) {
4547: /* XXX Do something reasonable on
4548: * error.
4549: */
4550: #if 0
4551: printf("%s.%d: VLAN_INPUT_TAG\n",
4552: __func__, __LINE__);
4553: Debugger();
4554: #endif
4555: VLAN_INPUT_TAG(ifp, sendmp, vtag,
4556: printf("%s: could not apply VLAN "
4557: "tag", __func__));
4558: }
4559: }
4560: } else {
4561: /*
4562: ** Either no header split, or a
4563: ** secondary piece of a fragmented
4564: ** split packet.
4565: */
4566: mp->m_len = plen;
4567: /*
4568: ** See if there is a stored head
4569: ** that determines what we are
4570: */
4571: sendmp = rbuf->fmp;
4572: rbuf->m_pack = rbuf->fmp = NULL;
4573:
4574: if (sendmp != NULL) /* secondary frag */
4575: sendmp->m_pkthdr.len += mp->m_len;
4576: else {
4577: /* first desc of a non-ps chain */
4578: sendmp = mp;
4579: sendmp->m_flags |= M_PKTHDR;
4580: sendmp->m_pkthdr.len = mp->m_len;
4581: if (staterr & IXGBE_RXD_STAT_VP) {
4582: /* XXX Do something reasonable on
4583: * error.
4584: */
4585: #if 0
4586: printf("%s.%d: VLAN_INPUT_TAG\n",
4587: __func__, __LINE__);
4588: Debugger();
4589: #endif
4590: VLAN_INPUT_TAG(ifp, sendmp, vtag,
4591: printf("%s: could not apply VLAN "
4592: "tag", __func__));
4593: }
4594: }
4595: /* Pass the head pointer on */
4596: if (eop == 0) {
4597: nbuf->fmp = sendmp;
4598: sendmp = NULL;
4599: mp->m_next = nbuf->m_pack;
4600: }
4601: }
4602: ++processed;
4603: /* Sending this frame? */
4604: if (eop) {
4605: sendmp->m_pkthdr.rcvif = ifp;
4606: ifp->if_ipackets++;
4607: rxr->rx_packets.ev_count++;
4608: /* capture data for AIM */
4609: rxr->bytes += sendmp->m_pkthdr.len;
4610: rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len;
4611: if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
4612: ixgbe_rx_checksum(staterr, sendmp, ptype,
4613: &adapter->stats);
4614: }
4615: #if __FreeBSD_version >= 800000
4616: sendmp->m_pkthdr.flowid = que->msix;
4617: sendmp->m_flags |= M_FLOWID;
4618: #endif
4619: }
4620: next_desc:
4621: ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4622: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4623:
4624: /* Advance our pointers to the next descriptor. */
4625: if (++i == adapter->num_rx_desc)
4626: i = 0;
4627:
4628: /* Now send to the stack or do LRO */
4629: if (sendmp != NULL) {
4630: rxr->next_to_check = i;
4631: ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4632: i = rxr->next_to_check;
4633: }
4634:
4635: /* Every 8 descriptors we go to refresh mbufs */
4636: if (processed == 8) {
4637: ixgbe_refresh_mbufs(rxr, i);
4638: processed = 0;
4639: }
4640: }
4641:
4642: /* Refresh any remaining buf structs */
4643: if (ixgbe_rx_unrefreshed(rxr))
4644: ixgbe_refresh_mbufs(rxr, i);
4645:
4646: rxr->next_to_check = i;
4647:
4648: #ifdef LRO
4649: /*
4650: * Flush any outstanding LRO work
4651: */
4652: while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4653: SLIST_REMOVE_HEAD(&lro->lro_active, next);
4654: tcp_lro_flush(lro, queued);
4655: }
4656: #endif /* LRO */
4657:
4658: IXGBE_RX_UNLOCK(rxr);
4659:
4660: /*
4661: ** We still have cleaning to do?
4662: ** Schedule another interrupt if so.
4663: */
4664: if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
1.13 christos 4665: ixgbe_rearm_queues(adapter, (u64)(1ULL << que->msix));
1.1 dyoung 4666: return true;
4667: }
4668:
4669: return false;
4670: }
4671:
4672:
4673: /*********************************************************************
4674: *
4675: * Verify that the hardware indicated that the checksum is valid.
4676: * Inform the stack about the status of checksum so that stack
4677: * doesn't spend time verifying the checksum.
4678: *
4679: *********************************************************************/
4680: static void
4681: ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype,
4682: struct ixgbe_hw_stats *stats)
4683: {
4684: u16 status = (u16) staterr;
4685: u8 errors = (u8) (staterr >> 24);
1.9 skrll 4686: #if 0
1.1 dyoung 4687: bool sctp = FALSE;
4688:
4689: if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4690: (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4691: sctp = TRUE;
1.9 skrll 4692: #endif
1.1 dyoung 4693:
4694: if (status & IXGBE_RXD_STAT_IPCS) {
4695: stats->ipcs.ev_count++;
4696: if (!(errors & IXGBE_RXD_ERR_IPE)) {
4697: /* IP Checksum Good */
4698: mp->m_pkthdr.csum_flags = M_CSUM_IPv4;
4699:
4700: } else {
4701: stats->ipcs_bad.ev_count++;
4702: mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD;
4703: }
4704: }
4705: if (status & IXGBE_RXD_STAT_L4CS) {
4706: stats->l4cs.ev_count++;
4707: u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6;
4708: if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4709: mp->m_pkthdr.csum_flags |= type;
4710: } else {
4711: stats->l4cs_bad.ev_count++;
4712: mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD;
4713: }
4714: }
4715: return;
4716: }
4717:
4718:
4719: #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */
4720: /*
4721: ** This routine is run via an vlan config EVENT,
4722: ** it enables us to use the HW Filter table since
4723: ** we can get the vlan id. This just creates the
4724: ** entry in the soft version of the VFTA, init will
4725: ** repopulate the real table.
4726: */
4727: static void
4728: ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4729: {
4730: struct adapter *adapter = ifp->if_softc;
4731: u16 index, bit;
4732:
4733: if (ifp->if_softc != arg) /* Not our event */
4734: return;
4735:
4736: if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4737: return;
4738:
4739: IXGBE_CORE_LOCK(adapter);
4740: index = (vtag >> 5) & 0x7F;
4741: bit = vtag & 0x1F;
4742: adapter->shadow_vfta[index] |= (1 << bit);
4743: ixgbe_init_locked(adapter);
4744: IXGBE_CORE_UNLOCK(adapter);
4745: }
4746:
4747: /*
4748: ** This routine is run via an vlan
4749: ** unconfig EVENT, remove our entry
4750: ** in the soft vfta.
4751: */
4752: static void
4753: ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4754: {
4755: struct adapter *adapter = ifp->if_softc;
4756: u16 index, bit;
4757:
4758: if (ifp->if_softc != arg)
4759: return;
4760:
4761: if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4762: return;
4763:
4764: IXGBE_CORE_LOCK(adapter);
4765: index = (vtag >> 5) & 0x7F;
4766: bit = vtag & 0x1F;
4767: adapter->shadow_vfta[index] &= ~(1 << bit);
4768: /* Re-init to load the changes */
4769: ixgbe_init_locked(adapter);
4770: IXGBE_CORE_UNLOCK(adapter);
4771: }
4772: #endif
4773:
4774: static void
4775: ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4776: {
4777: struct ethercom *ec = &adapter->osdep.ec;
4778: struct ixgbe_hw *hw = &adapter->hw;
4779: u32 ctrl;
4780:
4781: /*
4782: ** We get here thru init_locked, meaning
4783: ** a soft reset, this has already cleared
4784: ** the VFTA and other state, so if there
4785: ** have been no vlan's registered do nothing.
4786: */
4787: if (!VLAN_ATTACHED(&adapter->osdep.ec)) {
4788: return;
4789: }
4790:
4791: /*
4792: ** A soft reset zero's out the VFTA, so
4793: ** we need to repopulate it now.
4794: */
4795: for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4796: if (adapter->shadow_vfta[i] != 0)
4797: IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4798: adapter->shadow_vfta[i]);
4799:
4800: ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4801: /* Enable the Filter Table if enabled */
4802: if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) {
4803: ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4804: ctrl |= IXGBE_VLNCTRL_VFE;
4805: }
4806: if (hw->mac.type == ixgbe_mac_82598EB)
4807: ctrl |= IXGBE_VLNCTRL_VME;
4808: IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4809:
4810: /* On 82599 the VLAN enable is per/queue in RXDCTL */
4811: if (hw->mac.type != ixgbe_mac_82598EB)
4812: for (int i = 0; i < adapter->num_queues; i++) {
4813: ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4814: ctrl |= IXGBE_RXDCTL_VME;
4815: IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4816: }
4817: }
4818:
4819: static void
4820: ixgbe_enable_intr(struct adapter *adapter)
4821: {
4822: struct ixgbe_hw *hw = &adapter->hw;
4823: struct ix_queue *que = adapter->queues;
4824: u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4825:
4826:
4827: /* Enable Fan Failure detection */
4828: if (hw->device_id == IXGBE_DEV_ID_82598AT)
4829: mask |= IXGBE_EIMS_GPI_SDP1;
4830: else {
4831: mask |= IXGBE_EIMS_ECC;
4832: mask |= IXGBE_EIMS_GPI_SDP1;
4833: mask |= IXGBE_EIMS_GPI_SDP2;
4834: #ifdef IXGBE_FDIR
4835: mask |= IXGBE_EIMS_FLOW_DIR;
4836: #endif
4837: }
4838:
4839: IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4840:
4841: /* With RSS we use auto clear */
4842: if (adapter->msix_mem) {
4843: mask = IXGBE_EIMS_ENABLE_MASK;
4844: /* Don't autoclear Link */
4845: mask &= ~IXGBE_EIMS_OTHER;
4846: mask &= ~IXGBE_EIMS_LSC;
4847: IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4848: }
4849:
4850: /*
4851: ** Now enable all queues, this is done separately to
4852: ** allow for handling the extended (beyond 32) MSIX
4853: ** vectors that can be used by 82599
4854: */
4855: for (int i = 0; i < adapter->num_queues; i++, que++)
4856: ixgbe_enable_queue(adapter, que->msix);
4857:
4858: IXGBE_WRITE_FLUSH(hw);
4859:
4860: return;
4861: }
4862:
4863: static void
4864: ixgbe_disable_intr(struct adapter *adapter)
4865: {
4866: if (adapter->msix_mem)
4867: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4868: if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4869: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4870: } else {
4871: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4872: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4873: IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4874: }
4875: IXGBE_WRITE_FLUSH(&adapter->hw);
4876: return;
4877: }
4878:
4879: u16
4880: ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4881: {
4882: switch (reg % 4) {
4883: case 0:
4884: return pci_conf_read(hw->back->pc, hw->back->tag, reg) &
4885: __BITS(15, 0);
4886: case 2:
4887: return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag,
4888: reg - 2), __BITS(31, 16));
4889: default:
4890: panic("%s: invalid register (%" PRIx32, __func__, reg);
4891: break;
4892: }
4893: }
4894:
4895: void
4896: ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4897: {
4898: pcireg_t old;
4899:
4900: switch (reg % 4) {
4901: case 0:
4902: old = pci_conf_read(hw->back->pc, hw->back->tag, reg) &
4903: __BITS(31, 16);
4904: pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old);
4905: break;
4906: case 2:
4907: old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) &
4908: __BITS(15, 0);
4909: pci_conf_write(hw->back->pc, hw->back->tag, reg - 2,
4910: __SHIFTIN(value, __BITS(31, 16)) | old);
4911: break;
4912: default:
4913: panic("%s: invalid register (%" PRIx32, __func__, reg);
4914: break;
4915: }
4916:
4917: return;
4918: }
4919:
4920: /*
4921: ** Setup the correct IVAR register for a particular MSIX interrupt
4922: ** (yes this is all very magic and confusing :)
4923: ** - entry is the register array entry
4924: ** - vector is the MSIX vector for this queue
4925: ** - type is RX/TX/MISC
4926: */
4927: static void
4928: ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4929: {
4930: struct ixgbe_hw *hw = &adapter->hw;
4931: u32 ivar, index;
4932:
4933: vector |= IXGBE_IVAR_ALLOC_VAL;
4934:
4935: switch (hw->mac.type) {
4936:
4937: case ixgbe_mac_82598EB:
4938: if (type == -1)
4939: entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4940: else
4941: entry += (type * 64);
4942: index = (entry >> 2) & 0x1F;
4943: ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4944: ivar &= ~(0xFF << (8 * (entry & 0x3)));
4945: ivar |= (vector << (8 * (entry & 0x3)));
4946: IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4947: break;
4948:
4949: case ixgbe_mac_82599EB:
4950: if (type == -1) { /* MISC IVAR */
4951: index = (entry & 1) * 8;
4952: ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4953: ivar &= ~(0xFF << index);
4954: ivar |= (vector << index);
4955: IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4956: } else { /* RX/TX IVARS */
4957: index = (16 * (entry & 1)) + (8 * type);
4958: ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4959: ivar &= ~(0xFF << index);
4960: ivar |= (vector << index);
4961: IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4962: }
4963:
4964: default:
4965: break;
4966: }
4967: }
4968:
4969: static void
4970: ixgbe_configure_ivars(struct adapter *adapter)
4971: {
4972: struct ix_queue *que = adapter->queues;
4973: u32 newitr;
4974:
4975: if (ixgbe_max_interrupt_rate > 0)
4976: newitr = (8000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
4977: else
4978: newitr = 0;
4979:
4980: for (int i = 0; i < adapter->num_queues; i++, que++) {
4981: /* First the RX queue entry */
4982: ixgbe_set_ivar(adapter, i, que->msix, 0);
4983: /* ... and the TX */
4984: ixgbe_set_ivar(adapter, i, que->msix, 1);
4985: /* Set an Initial EITR value */
4986: IXGBE_WRITE_REG(&adapter->hw,
4987: IXGBE_EITR(que->msix), newitr);
4988: }
4989:
4990: /* For the Link interrupt */
4991: ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4992: }
4993:
4994: /*
4995: ** ixgbe_sfp_probe - called in the local timer to
4996: ** determine if a port had optics inserted.
4997: */
4998: static bool ixgbe_sfp_probe(struct adapter *adapter)
4999: {
5000: struct ixgbe_hw *hw = &adapter->hw;
5001: device_t dev = adapter->dev;
5002: bool result = FALSE;
5003:
5004: if ((hw->phy.type == ixgbe_phy_nl) &&
5005: (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5006: s32 ret = hw->phy.ops.identify_sfp(hw);
5007: if (ret)
5008: goto out;
5009: ret = hw->phy.ops.reset(hw);
5010: if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5011: device_printf(dev,"Unsupported SFP+ module detected!");
5012: device_printf(dev, "Reload driver with supported module.\n");
5013: adapter->sfp_probe = FALSE;
5014: goto out;
5015: } else
5016: device_printf(dev,"SFP+ module detected!\n");
5017: /* We now have supported optics */
5018: adapter->sfp_probe = FALSE;
5019: /* Set the optics type so system reports correctly */
5020: ixgbe_setup_optics(adapter);
5021: result = TRUE;
5022: }
5023: out:
5024: return (result);
5025: }
5026:
5027: /*
5028: ** Tasklet handler for MSIX Link interrupts
5029: ** - do outside interrupt since it might sleep
5030: */
5031: static void
5032: ixgbe_handle_link(void *context)
5033: {
5034: struct adapter *adapter = context;
5035:
1.13 christos 5036: if (ixgbe_check_link(&adapter->hw,
5037: &adapter->link_speed, &adapter->link_up, 0) == 0)
5038: ixgbe_update_link_status(adapter);
1.1 dyoung 5039: }
5040:
5041: /*
5042: ** Tasklet for handling SFP module interrupts
5043: */
5044: static void
5045: ixgbe_handle_mod(void *context)
5046: {
5047: struct adapter *adapter = context;
5048: struct ixgbe_hw *hw = &adapter->hw;
5049: device_t dev = adapter->dev;
5050: u32 err;
5051:
5052: err = hw->phy.ops.identify_sfp(hw);
5053: if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5054: device_printf(dev,
5055: "Unsupported SFP+ module type was detected.\n");
5056: return;
5057: }
5058: err = hw->mac.ops.setup_sfp(hw);
5059: if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5060: device_printf(dev,
5061: "Setup failure - unsupported SFP+ module type.\n");
5062: return;
5063: }
5064: softint_schedule(adapter->msf_si);
5065: return;
5066: }
5067:
5068:
5069: /*
5070: ** Tasklet for handling MSF (multispeed fiber) interrupts
5071: */
5072: static void
5073: ixgbe_handle_msf(void *context)
5074: {
5075: struct adapter *adapter = context;
5076: struct ixgbe_hw *hw = &adapter->hw;
5077: u32 autoneg;
5078: bool negotiate;
5079:
5080: autoneg = hw->phy.autoneg_advertised;
5081: if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5082: hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
1.13 christos 5083: else
5084: negotiate = 0;
1.1 dyoung 5085: if (hw->mac.ops.setup_link)
5086: hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5087: return;
5088: }
5089:
5090: #ifdef IXGBE_FDIR
5091: /*
5092: ** Tasklet for reinitializing the Flow Director filter table
5093: */
5094: static void
5095: ixgbe_reinit_fdir(void *context)
5096: {
5097: struct adapter *adapter = context;
5098: struct ifnet *ifp = adapter->ifp;
5099:
5100: if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5101: return;
5102: ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5103: adapter->fdir_reinit = 0;
5104: /* Restart the interface */
5105: ifp->if_flags |= IFF_RUNNING;
5106: return;
5107: }
5108: #endif
5109:
5110: /**********************************************************************
5111: *
5112: * Update the board statistics counters.
5113: *
5114: **********************************************************************/
5115: static void
5116: ixgbe_update_stats_counters(struct adapter *adapter)
5117: {
5118: struct ifnet *ifp = adapter->ifp;
5119: struct ixgbe_hw *hw = &adapter->hw;
5120: u32 missed_rx = 0, bprc, lxon, lxoff, total;
5121: u64 total_missed_rx = 0;
5122:
5123: adapter->stats.crcerrs.ev_count += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5124: adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5125: adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5126: adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5127:
5128: for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) {
5129: int j = i % adapter->num_queues;
5130: u32 mp;
5131: mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5132: /* missed_rx tallies misses for the gprc workaround */
5133: missed_rx += mp;
5134: /* global total per queue */
5135: adapter->stats.mpc[j].ev_count += mp;
5136: /* Running comprehensive total for stats display */
5137: total_missed_rx += adapter->stats.mpc[j].ev_count;
5138: if (hw->mac.type == ixgbe_mac_82598EB)
5139: adapter->stats.rnbc[j] +=
5140: IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5141: adapter->stats.pxontxc[j].ev_count +=
5142: IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5143: adapter->stats.pxonrxc[j].ev_count +=
5144: IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5145: adapter->stats.pxofftxc[j].ev_count +=
5146: IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5147: adapter->stats.pxoffrxc[j].ev_count +=
5148: IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5149: adapter->stats.pxon2offc[j].ev_count +=
5150: IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5151: }
5152: for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) {
5153: int j = i % adapter->num_queues;
5154: adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5155: adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5156: adapter->stats.qbrc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5157: adapter->stats.qbrc[j].ev_count +=
5158: ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5159: adapter->stats.qbtc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5160: adapter->stats.qbtc[j].ev_count +=
5161: ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5162: adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5163: }
5164: adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC);
5165: adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC);
5166: adapter->stats.rlec.ev_count += IXGBE_READ_REG(hw, IXGBE_RLEC);
5167:
5168: /* Hardware workaround, gprc counts missed packets */
5169: adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx;
5170:
5171: lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5172: adapter->stats.lxontxc.ev_count += lxon;
5173: lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5174: adapter->stats.lxofftxc.ev_count += lxoff;
5175: total = lxon + lxoff;
5176:
5177: if (hw->mac.type != ixgbe_mac_82598EB) {
5178: adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5179: ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5180: adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5181: ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN;
5182: adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) +
5183: ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5184: adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5185: adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5186: } else {
5187: adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5188: adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5189: /* 82598 only has a counter in the high register */
5190: adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH);
5191: adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN;
5192: adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH);
5193: }
5194:
5195: /*
5196: * Workaround: mprc hardware is incorrectly counting
5197: * broadcasts, so for now we subtract those.
5198: */
5199: bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5200: adapter->stats.bprc.ev_count += bprc;
5201: adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0);
5202:
5203: adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64);
5204: adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127);
5205: adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255);
5206: adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511);
5207: adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5208: adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5209:
5210: adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total;
5211: adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total;
5212: adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total;
5213:
5214: adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC);
5215: adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC);
5216: adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC);
5217: adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC);
5218: adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5219: adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5220: adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5221: adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR);
5222: adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT);
5223: adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127);
5224: adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255);
5225: adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511);
5226: adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5227: adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5228: adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC);
5229: adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC);
5230: adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5231: adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5232:
5233: /* Only read FCOE on 82599 */
5234: if (hw->mac.type != ixgbe_mac_82598EB) {
5235: adapter->stats.fcoerpdc.ev_count +=
5236: IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5237: adapter->stats.fcoeprc.ev_count +=
5238: IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5239: adapter->stats.fcoeptc.ev_count +=
5240: IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5241: adapter->stats.fcoedwrc.ev_count +=
5242: IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5243: adapter->stats.fcoedwtc.ev_count +=
5244: IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5245: }
5246:
5247: /* Fill out the OS statistics structure */
5248: ifp->if_ipackets = adapter->stats.gprc.ev_count;
5249: ifp->if_opackets = adapter->stats.gptc.ev_count;
5250: ifp->if_ibytes = adapter->stats.gorc.ev_count;
5251: ifp->if_obytes = adapter->stats.gotc.ev_count;
5252: ifp->if_imcasts = adapter->stats.mprc.ev_count;
5253: ifp->if_collisions = 0;
5254:
5255: /* Rx Errors */
5256: ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs.ev_count +
5257: adapter->stats.rlec.ev_count;
5258: }
5259:
5260: /** ixgbe_sysctl_tdh_handler - Handler function
5261: * Retrieves the TDH value from the hardware
5262: */
5263: static int
5264: ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS)
5265: {
5266: struct sysctlnode node;
5267: uint32_t val;
5268: struct tx_ring *txr;
5269:
5270: node = *rnode;
5271: txr = (struct tx_ring *)node.sysctl_data;
5272: if (txr == NULL)
5273: return 0;
5274: val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5275: node.sysctl_data = &val;
5276: return sysctl_lookup(SYSCTLFN_CALL(&node));
5277: }
5278:
5279: /** ixgbe_sysctl_tdt_handler - Handler function
5280: * Retrieves the TDT value from the hardware
5281: */
5282: static int
5283: ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS)
5284: {
5285: struct sysctlnode node;
5286: uint32_t val;
5287: struct tx_ring *txr;
5288:
5289: node = *rnode;
5290: txr = (struct tx_ring *)node.sysctl_data;
5291: if (txr == NULL)
5292: return 0;
5293: val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5294: node.sysctl_data = &val;
5295: return sysctl_lookup(SYSCTLFN_CALL(&node));
5296: }
5297:
5298: /** ixgbe_sysctl_rdh_handler - Handler function
5299: * Retrieves the RDH value from the hardware
5300: */
5301: static int
5302: ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS)
5303: {
5304: struct sysctlnode node;
5305: uint32_t val;
5306: struct rx_ring *rxr;
5307:
5308: node = *rnode;
5309: rxr = (struct rx_ring *)node.sysctl_data;
5310: if (rxr == NULL)
5311: return 0;
5312: val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5313: node.sysctl_data = &val;
5314: return sysctl_lookup(SYSCTLFN_CALL(&node));
5315: }
5316:
5317: /** ixgbe_sysctl_rdt_handler - Handler function
5318: * Retrieves the RDT value from the hardware
5319: */
5320: static int
5321: ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS)
5322: {
5323: struct sysctlnode node;
5324: uint32_t val;
5325: struct rx_ring *rxr;
5326:
5327: node = *rnode;
5328: rxr = (struct rx_ring *)node.sysctl_data;
5329: if (rxr == NULL)
5330: return 0;
5331: val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5332: node.sysctl_data = &val;
5333: return sysctl_lookup(SYSCTLFN_CALL(&node));
5334: }
5335:
5336: static int
5337: ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS)
5338: {
5339: struct sysctlnode node;
5340: struct ix_queue *que;
5341: uint32_t reg, usec, rate;
5342:
5343: node = *rnode;
5344: que = (struct ix_queue *)node.sysctl_data;
5345: if (que == NULL)
5346: return 0;
5347: reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5348: usec = ((reg & 0x0FF8) >> 3);
5349: if (usec > 0)
5350: rate = 1000000 / usec;
5351: else
5352: rate = 0;
5353: node.sysctl_data = &rate;
5354: return sysctl_lookup(SYSCTLFN_CALL(&node));
5355: }
5356:
5357: const struct sysctlnode *
5358: ixgbe_sysctl_instance(struct adapter *adapter)
5359: {
5360: const char *dvname;
5361: struct sysctllog **log;
5362: int rc;
5363: const struct sysctlnode *rnode;
5364:
5365: log = &adapter->sysctllog;
5366: dvname = device_xname(adapter->dev);
5367:
5368: if ((rc = sysctl_createv(log, 0, NULL, &rnode,
5369: 0, CTLTYPE_NODE, dvname,
5370: SYSCTL_DESCR("ixgbe information and settings"),
1.7 pooka 5371: NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0)
1.1 dyoung 5372: goto err;
5373:
5374: return rnode;
5375: err:
5376: printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc);
5377: return NULL;
5378: }
5379:
5380: /*
5381: * Add sysctl variables, one per statistic, to the system.
5382: */
5383: static void
5384: ixgbe_add_hw_stats(struct adapter *adapter)
5385: {
5386: device_t dev = adapter->dev;
5387: const struct sysctlnode *rnode, *cnode;
5388: struct sysctllog **log = &adapter->sysctllog;
5389: struct tx_ring *txr = adapter->tx_rings;
5390: struct rx_ring *rxr = adapter->rx_rings;
5391: struct ixgbe_hw *hw = &adapter->hw;
5392:
5393: struct ixgbe_hw_stats *stats = &adapter->stats;
5394:
5395: /* Driver Statistics */
5396: #if 0
5397: /* These counters are not updated by the software */
5398: SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5399: CTLFLAG_RD, &adapter->dropped_pkts,
5400: "Driver dropped packets");
5401: SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed",
5402: CTLFLAG_RD, &adapter->mbuf_header_failed,
5403: "???");
5404: SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed",
5405: CTLFLAG_RD, &adapter->mbuf_packet_failed,
5406: "???");
5407: SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail",
5408: CTLFLAG_RD, &adapter->no_tx_map_avail,
5409: "???");
5410: #endif
5411: evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC,
5412: NULL, device_xname(dev), "Handled queue in softint");
5413: evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC,
5414: NULL, device_xname(dev), "Requeued in softint");
5415: evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC,
5416: NULL, device_xname(dev), "Interrupt handler more rx");
5417: evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC,
5418: NULL, device_xname(dev), "Interrupt handler more tx");
5419: evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC,
5420: NULL, device_xname(dev), "Interrupt handler tx loops");
5421: evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC,
5422: NULL, device_xname(dev), "Driver tx dma soft fail EFBIG");
5423: evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC,
5424: NULL, device_xname(dev), "m_defrag() failed");
5425: evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC,
5426: NULL, device_xname(dev), "Driver tx dma hard fail EFBIG");
5427: evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC,
5428: NULL, device_xname(dev), "Driver tx dma hard fail EINVAL");
5429: evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC,
5430: NULL, device_xname(dev), "Driver tx dma hard fail other");
5431: evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC,
5432: NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN");
5433: evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC,
5434: NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM");
5435: evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC,
5436: NULL, device_xname(dev), "Watchdog timeouts");
5437: evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC,
5438: NULL, device_xname(dev), "TSO errors");
5439: evcnt_attach_dynamic(&adapter->tso_tx, EVCNT_TYPE_MISC,
5440: NULL, device_xname(dev), "TSO");
5441: evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC,
5442: NULL, device_xname(dev), "Link MSIX IRQ Handled");
5443:
5444: for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5445: snprintf(adapter->queues[i].evnamebuf,
5446: sizeof(adapter->queues[i].evnamebuf), "%s queue%d",
5447: device_xname(dev), i);
5448: snprintf(adapter->queues[i].namebuf,
5449: sizeof(adapter->queues[i].namebuf), "queue%d", i);
5450:
5451: if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) {
5452: aprint_error_dev(dev, "could not create sysctl root\n");
5453: break;
5454: }
5455:
5456: if (sysctl_createv(log, 0, &rnode, &rnode,
5457: 0, CTLTYPE_NODE,
5458: adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"),
5459: NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0)
5460: break;
5461:
5462: if (sysctl_createv(log, 0, &rnode, &cnode,
5463: CTLFLAG_READONLY, CTLTYPE_INT,
5464: "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"),
1.5 dsl 5465: ixgbe_sysctl_interrupt_rate_handler, 0,
5466: (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0)
1.1 dyoung 5467: break;
5468:
5469: if (sysctl_createv(log, 0, &rnode, &cnode,
5470: CTLFLAG_READONLY, CTLTYPE_INT,
5471: "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"),
1.4 dsl 5472: ixgbe_sysctl_tdh_handler, 0, (void *)txr,
1.1 dyoung 5473: 0, CTL_CREATE, CTL_EOL) != 0)
5474: break;
5475:
5476: if (sysctl_createv(log, 0, &rnode, &cnode,
5477: CTLFLAG_READONLY, CTLTYPE_INT,
5478: "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"),
1.4 dsl 5479: ixgbe_sysctl_tdt_handler, 0, (void *)txr,
1.1 dyoung 5480: 0, CTL_CREATE, CTL_EOL) != 0)
5481: break;
5482:
5483: evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC,
5484: NULL, adapter->queues[i].evnamebuf,
5485: "Queue No Descriptor Available");
5486: evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC,
5487: NULL, adapter->queues[i].evnamebuf,
5488: "Queue Packets Transmitted");
5489:
5490: #ifdef LRO
5491: struct lro_ctrl *lro = &rxr->lro;
5492: #endif /* LRO */
5493:
5494: if (sysctl_createv(log, 0, &rnode, &cnode,
5495: CTLFLAG_READONLY,
5496: CTLTYPE_INT,
5497: "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"),
1.4 dsl 5498: ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0,
1.1 dyoung 5499: CTL_CREATE, CTL_EOL) != 0)
5500: break;
5501:
5502: if (sysctl_createv(log, 0, &rnode, &cnode,
5503: CTLFLAG_READONLY,
5504: CTLTYPE_INT,
5505: "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"),
1.4 dsl 5506: ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0,
1.1 dyoung 5507: CTL_CREATE, CTL_EOL) != 0)
5508: break;
5509:
5510: if (i < __arraycount(adapter->stats.mpc)) {
5511: evcnt_attach_dynamic(&adapter->stats.mpc[i],
5512: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5513: "Missed Packet Count");
5514: }
5515: if (i < __arraycount(adapter->stats.pxontxc)) {
5516: evcnt_attach_dynamic(&adapter->stats.pxontxc[i],
5517: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5518: "pxontxc");
5519: evcnt_attach_dynamic(&adapter->stats.pxonrxc[i],
5520: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5521: "pxonrxc");
5522: evcnt_attach_dynamic(&adapter->stats.pxofftxc[i],
5523: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5524: "pxofftxc");
5525: evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i],
5526: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5527: "pxoffrxc");
5528: evcnt_attach_dynamic(&adapter->stats.pxon2offc[i],
5529: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5530: "pxon2offc");
5531: }
5532: if (i < __arraycount(adapter->stats.qprc)) {
5533: evcnt_attach_dynamic(&adapter->stats.qprc[i],
5534: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5535: "qprc");
5536: evcnt_attach_dynamic(&adapter->stats.qptc[i],
5537: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5538: "qptc");
5539: evcnt_attach_dynamic(&adapter->stats.qbrc[i],
5540: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5541: "qbrc");
5542: evcnt_attach_dynamic(&adapter->stats.qbtc[i],
5543: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5544: "qbtc");
5545: evcnt_attach_dynamic(&adapter->stats.qprdc[i],
5546: EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf,
5547: "qprdc");
5548: }
5549:
5550: evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC,
5551: NULL, adapter->queues[i].evnamebuf, "Queue Packets Received");
5552: evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC,
5553: NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received");
5554: evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC,
5555: NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf");
5556: evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC,
5557: NULL, adapter->queues[i].evnamebuf, "Rx discarded");
5558: evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC,
5559: NULL, adapter->queues[i].evnamebuf, "Rx split packets");
5560: evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC,
5561: NULL, adapter->queues[i].evnamebuf, "Rx interrupts");
5562: #ifdef LRO
5563: SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5564: CTLFLAG_RD, &lro->lro_queued, 0,
5565: "LRO Queued");
5566: SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5567: CTLFLAG_RD, &lro->lro_flushed, 0,
5568: "LRO Flushed");
5569: #endif /* LRO */
5570: }
5571:
5572: /* MAC stats get the own sub node */
5573:
5574:
5575: snprintf(stats->namebuf,
5576: sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev));
5577:
5578: evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL,
5579: stats->namebuf, "rx csum offload - IP");
5580: evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL,
5581: stats->namebuf, "rx csum offload - L4");
5582: evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL,
5583: stats->namebuf, "rx csum offload - IP bad");
5584: evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL,
5585: stats->namebuf, "rx csum offload - L4 bad");
5586: evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL,
5587: stats->namebuf, "Interrupt conditions zero");
5588: evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL,
5589: stats->namebuf, "Legacy interrupts");
5590: evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL,
5591: stats->namebuf, "CRC Errors");
5592: evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL,
5593: stats->namebuf, "Illegal Byte Errors");
5594: evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL,
5595: stats->namebuf, "Byte Errors");
5596: evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL,
5597: stats->namebuf, "MAC Short Packets Discarded");
5598: evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL,
5599: stats->namebuf, "MAC Local Faults");
5600: evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL,
5601: stats->namebuf, "MAC Remote Faults");
5602: evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL,
5603: stats->namebuf, "Receive Length Errors");
5604: evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL,
5605: stats->namebuf, "Link XON Transmitted");
5606: evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL,
5607: stats->namebuf, "Link XON Received");
5608: evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL,
5609: stats->namebuf, "Link XOFF Transmitted");
5610: evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL,
5611: stats->namebuf, "Link XOFF Received");
5612:
5613: /* Packet Reception Stats */
5614: evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL,
5615: stats->namebuf, "Total Octets Received");
5616: evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL,
5617: stats->namebuf, "Good Octets Received");
5618: evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL,
5619: stats->namebuf, "Total Packets Received");
5620: evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL,
5621: stats->namebuf, "Good Packets Received");
5622: evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL,
5623: stats->namebuf, "Multicast Packets Received");
5624: evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL,
5625: stats->namebuf, "Broadcast Packets Received");
5626: evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL,
5627: stats->namebuf, "64 byte frames received ");
5628: evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL,
5629: stats->namebuf, "65-127 byte frames received");
5630: evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL,
5631: stats->namebuf, "128-255 byte frames received");
5632: evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL,
5633: stats->namebuf, "256-511 byte frames received");
5634: evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL,
5635: stats->namebuf, "512-1023 byte frames received");
5636: evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL,
5637: stats->namebuf, "1023-1522 byte frames received");
5638: evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL,
5639: stats->namebuf, "Receive Undersized");
5640: evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL,
5641: stats->namebuf, "Fragmented Packets Received ");
5642: evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL,
5643: stats->namebuf, "Oversized Packets Received");
5644: evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL,
5645: stats->namebuf, "Received Jabber");
5646: evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL,
5647: stats->namebuf, "Management Packets Received");
5648: evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL,
5649: stats->namebuf, "Checksum Errors");
5650:
5651: /* Packet Transmission Stats */
5652: evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL,
5653: stats->namebuf, "Good Octets Transmitted");
5654: evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL,
5655: stats->namebuf, "Total Packets Transmitted");
5656: evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL,
5657: stats->namebuf, "Good Packets Transmitted");
5658: evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL,
5659: stats->namebuf, "Broadcast Packets Transmitted");
5660: evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL,
5661: stats->namebuf, "Multicast Packets Transmitted");
5662: evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL,
5663: stats->namebuf, "Management Packets Transmitted");
5664: evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL,
5665: stats->namebuf, "64 byte frames transmitted ");
5666: evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL,
5667: stats->namebuf, "65-127 byte frames transmitted");
5668: evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL,
5669: stats->namebuf, "128-255 byte frames transmitted");
5670: evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL,
5671: stats->namebuf, "256-511 byte frames transmitted");
5672: evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL,
5673: stats->namebuf, "512-1023 byte frames transmitted");
5674: evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL,
5675: stats->namebuf, "1024-1522 byte frames transmitted");
5676:
5677: /* FC Stats */
5678: evcnt_attach_dynamic(&stats->fccrc, EVCNT_TYPE_MISC, NULL,
5679: stats->namebuf, "FC CRC Errors");
5680: evcnt_attach_dynamic(&stats->fclast, EVCNT_TYPE_MISC, NULL,
5681: stats->namebuf, "FC Last Error");
5682: if (hw->mac.type != ixgbe_mac_82598EB) {
5683: evcnt_attach_dynamic(&stats->fcoerpdc, EVCNT_TYPE_MISC, NULL,
5684: stats->namebuf, "FCoE Packets Dropped");
5685: evcnt_attach_dynamic(&stats->fcoeprc, EVCNT_TYPE_MISC, NULL,
5686: stats->namebuf, "FCoE Packets Received");
5687: evcnt_attach_dynamic(&stats->fcoeptc, EVCNT_TYPE_MISC, NULL,
5688: stats->namebuf, "FCoE Packets Transmitted");
5689: evcnt_attach_dynamic(&stats->fcoedwrc, EVCNT_TYPE_MISC, NULL,
5690: stats->namebuf, "FCoE DWords Received");
5691: evcnt_attach_dynamic(&stats->fcoedwtc, EVCNT_TYPE_MISC, NULL,
5692: stats->namebuf, "FCoE DWords Transmitted");
5693: }
5694: }
5695:
5696: /*
5697: ** Set flow control using sysctl:
5698: ** Flow control values:
5699: ** 0 - off
5700: ** 1 - rx pause
5701: ** 2 - tx pause
5702: ** 3 - full
5703: */
5704: static int
5705: ixgbe_set_flowcntl(SYSCTLFN_ARGS)
5706: {
5707: struct sysctlnode node;
5708: int error;
5709: int last = ixgbe_flow_control;
5710: struct adapter *adapter;
5711:
5712: node = *rnode;
5713: adapter = (struct adapter *)node.sysctl_data;
5714: node.sysctl_data = &ixgbe_flow_control;
5715: error = sysctl_lookup(SYSCTLFN_CALL(&node));
5716: if (error != 0 || newp == NULL)
5717: return error;
5718:
5719: /* Don't bother if it's not changed */
5720: if (ixgbe_flow_control == last)
5721: return (0);
5722:
5723: switch (ixgbe_flow_control) {
5724: case ixgbe_fc_rx_pause:
5725: case ixgbe_fc_tx_pause:
5726: case ixgbe_fc_full:
5727: adapter->hw.fc.requested_mode = ixgbe_flow_control;
5728: break;
5729: case ixgbe_fc_none:
5730: default:
5731: adapter->hw.fc.requested_mode = ixgbe_fc_none;
5732: }
5733:
5734: ixgbe_fc_enable(&adapter->hw, 0);
5735: return 0;
5736: }
5737:
5738: static void
5739: ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5740: const char *description, int *limit, int value)
5741: {
5742: const struct sysctlnode *rnode, *cnode;
5743: struct sysctllog **log = &adapter->sysctllog;
5744:
5745: *limit = value;
5746:
5747: if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL)
5748: aprint_error_dev(adapter->dev,
5749: "could not create sysctl root\n");
5750: else if (sysctl_createv(log, 0, &rnode, &cnode,
5751: CTLFLAG_READWRITE,
5752: CTLTYPE_INT,
5753: name, SYSCTL_DESCR(description),
5754: NULL, 0, limit, 0,
5755: CTL_CREATE, CTL_EOL) != 0) {
5756: aprint_error_dev(adapter->dev, "%s: could not create sysctl",
5757: __func__);
5758: }
5759: }
5760:
5761: /*
5762: ** Control link advertise speed:
5763: ** 0 - normal
5764: ** 1 - advertise only 1G
5765: */
5766: static int
5767: ixgbe_set_advertise(SYSCTLFN_ARGS)
5768: {
5769: struct sysctlnode node;
5770: int t, error;
5771: struct adapter *adapter;
5772: struct ixgbe_hw *hw;
5773: ixgbe_link_speed speed, last;
5774:
5775: node = *rnode;
5776: adapter = (struct adapter *)node.sysctl_data;
5777: t = adapter->advertise;
5778: node.sysctl_data = &t;
5779: error = sysctl_lookup(SYSCTLFN_CALL(&node));
5780: if (error != 0 || newp == NULL)
5781: return error;
5782:
5783: if (t == -1)
5784: return 0;
5785:
5786: adapter->advertise = t;
5787:
5788: hw = &adapter->hw;
5789: last = hw->phy.autoneg_advertised;
5790:
5791: if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5792: (hw->phy.multispeed_fiber)))
5793: return 0;
5794:
5795: if (adapter->advertise == 1)
5796: speed = IXGBE_LINK_SPEED_1GB_FULL;
5797: else
5798: speed = IXGBE_LINK_SPEED_1GB_FULL |
5799: IXGBE_LINK_SPEED_10GB_FULL;
5800:
5801: if (speed == last) /* no change */
5802: return 0;
5803:
5804: hw->mac.autotry_restart = TRUE;
5805: hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5806:
5807: return 0;
5808: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to ixgbe.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / dev / pci / ixgbe