Annotation of src/sys/dev/pci/ixgbe/ix_txrx.c, Revision 1.41
1.41 ! msaitoh 1: /* $NetBSD: ix_txrx.c,v 1.40 2018/04/17 08:38:05 msaitoh Exp $ */
1.28 msaitoh 2:
1.1 msaitoh 3: /******************************************************************************
4:
1.28 msaitoh 5: Copyright (c) 2001-2017, Intel Corporation
1.1 msaitoh 6: All rights reserved.
1.28 msaitoh 7:
8: Redistribution and use in source and binary forms, with or without
1.1 msaitoh 9: modification, are permitted provided that the following conditions are met:
1.28 msaitoh 10:
11: 1. Redistributions of source code must retain the above copyright notice,
1.1 msaitoh 12: this list of conditions and the following disclaimer.
1.28 msaitoh 13:
14: 2. Redistributions in binary form must reproduce the above copyright
15: notice, this list of conditions and the following disclaimer in the
1.1 msaitoh 16: documentation and/or other materials provided with the distribution.
1.28 msaitoh 17:
18: 3. Neither the name of the Intel Corporation nor the names of its
19: contributors may be used to endorse or promote products derived from
1.1 msaitoh 20: this software without specific prior written permission.
1.28 msaitoh 21:
1.1 msaitoh 22: THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1.28 msaitoh 23: AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24: IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25: ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26: LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27: CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28: SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29: INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30: CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
1.1 msaitoh 31: ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32: POSSIBILITY OF SUCH DAMAGE.
33:
34: ******************************************************************************/
1.39 msaitoh 35: /*$FreeBSD: head/sys/dev/ixgbe/ix_txrx.c 327031 2017-12-20 18:15:06Z erj $*/
1.28 msaitoh 36:
1.1 msaitoh 37: /*
38: * Copyright (c) 2011 The NetBSD Foundation, Inc.
39: * All rights reserved.
40: *
41: * This code is derived from software contributed to The NetBSD Foundation
42: * by Coyote Point Systems, Inc.
43: *
44: * Redistribution and use in source and binary forms, with or without
45: * modification, are permitted provided that the following conditions
46: * are met:
47: * 1. Redistributions of source code must retain the above copyright
48: * notice, this list of conditions and the following disclaimer.
49: * 2. Redistributions in binary form must reproduce the above copyright
50: * notice, this list of conditions and the following disclaimer in the
51: * documentation and/or other materials provided with the distribution.
52: *
53: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
54: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
55: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
56: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
57: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
58: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
59: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
60: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
61: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
62: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
63: * POSSIBILITY OF SUCH DAMAGE.
64: */
65:
1.8 msaitoh 66: #include "opt_inet.h"
67: #include "opt_inet6.h"
68:
1.1 msaitoh 69: #include "ixgbe.h"
70:
71: /*
1.28 msaitoh 72: * HW RSC control:
73: * this feature only works with
74: * IPv4, and only on 82599 and later.
75: * Also this will cause IP forwarding to
76: * fail and that can't be controlled by
77: * the stack as LRO can. For all these
78: * reasons I've deemed it best to leave
79: * this off and not bother with a tuneable
80: * interface, this would need to be compiled
81: * to enable.
82: */
1.1 msaitoh 83: static bool ixgbe_rsc_enable = FALSE;
84:
1.3 msaitoh 85: /*
1.28 msaitoh 86: * For Flow Director: this is the
87: * number of TX packets we sample
88: * for the filter pool, this means
89: * every 20th packet will be probed.
90: *
91: * This feature can be disabled by
92: * setting this to 0.
93: */
1.3 msaitoh 94: static int atr_sample_rate = 20;
95:
1.28 msaitoh 96: /************************************************************************
1.3 msaitoh 97: * Local Function prototypes
1.28 msaitoh 98: ************************************************************************/
99: static void ixgbe_setup_transmit_ring(struct tx_ring *);
100: static void ixgbe_free_transmit_buffers(struct tx_ring *);
101: static int ixgbe_setup_receive_ring(struct rx_ring *);
102: static void ixgbe_free_receive_buffers(struct rx_ring *);
103: static void ixgbe_rx_checksum(u32, struct mbuf *, u32,
104: struct ixgbe_hw_stats *);
105: static void ixgbe_refresh_mbufs(struct rx_ring *, int);
1.38 knakahar 106: static void ixgbe_drain(struct ifnet *, struct tx_ring *);
1.28 msaitoh 107: static int ixgbe_xmit(struct tx_ring *, struct mbuf *);
108: static int ixgbe_tx_ctx_setup(struct tx_ring *,
109: struct mbuf *, u32 *, u32 *);
110: static int ixgbe_tso_setup(struct tx_ring *,
111: struct mbuf *, u32 *, u32 *);
1.1 msaitoh 112: static __inline void ixgbe_rx_discard(struct rx_ring *, int);
113: static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
1.28 msaitoh 114: struct mbuf *, u32);
115: static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
116: struct ixgbe_dma_alloc *, int);
117: static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
1.1 msaitoh 118:
119: static void ixgbe_setup_hw_rsc(struct rx_ring *);
120:
1.28 msaitoh 121: /************************************************************************
122: * ixgbe_legacy_start_locked - Transmit entry point
1.1 msaitoh 123: *
1.28 msaitoh 124: * Called by the stack to initiate a transmit.
125: * The driver will remain in this routine as long as there are
126: * packets to transmit and transmit resources are available.
127: * In case resources are not available, the stack is notified
128: * and the packet is requeued.
129: ************************************************************************/
130: int
131: ixgbe_legacy_start_locked(struct ifnet *ifp, struct tx_ring *txr)
1.1 msaitoh 132: {
133: int rc;
134: struct mbuf *m_head;
135: struct adapter *adapter = txr->adapter;
136:
137: IXGBE_TX_LOCK_ASSERT(txr);
138:
1.38 knakahar 139: if (!adapter->link_active) {
140: /*
141: * discard all packets buffered in IFQ to avoid
142: * sending old packets at next link up timing.
143: */
144: ixgbe_drain(ifp, txr);
145: return (ENETDOWN);
146: }
1.1 msaitoh 147: if ((ifp->if_flags & IFF_RUNNING) == 0)
1.28 msaitoh 148: return (ENETDOWN);
1.1 msaitoh 149:
150: while (!IFQ_IS_EMPTY(&ifp->if_snd)) {
151: if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
152: break;
153:
154: IFQ_POLL(&ifp->if_snd, m_head);
155: if (m_head == NULL)
156: break;
157:
158: if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) {
159: break;
160: }
161: IFQ_DEQUEUE(&ifp->if_snd, m_head);
162: if (rc != 0) {
163: m_freem(m_head);
164: continue;
165: }
166:
167: /* Send a copy of the frame to the BPF listener */
168: bpf_mtap(ifp, m_head);
169: }
170:
1.28 msaitoh 171: return IXGBE_SUCCESS;
172: } /* ixgbe_legacy_start_locked */
173:
174: /************************************************************************
175: * ixgbe_legacy_start
176: *
177: * Called by the stack, this always uses the first tx ring,
178: * and should not be used with multiqueue tx enabled.
179: ************************************************************************/
1.1 msaitoh 180: void
1.28 msaitoh 181: ixgbe_legacy_start(struct ifnet *ifp)
1.1 msaitoh 182: {
183: struct adapter *adapter = ifp->if_softc;
1.28 msaitoh 184: struct tx_ring *txr = adapter->tx_rings;
1.1 msaitoh 185:
186: if (ifp->if_flags & IFF_RUNNING) {
187: IXGBE_TX_LOCK(txr);
1.28 msaitoh 188: ixgbe_legacy_start_locked(ifp, txr);
1.1 msaitoh 189: IXGBE_TX_UNLOCK(txr);
190: }
1.28 msaitoh 191: } /* ixgbe_legacy_start */
1.1 msaitoh 192:
1.28 msaitoh 193: /************************************************************************
194: * ixgbe_mq_start - Multiqueue Transmit Entry Point
195: *
196: * (if_transmit function)
197: ************************************************************************/
1.1 msaitoh 198: int
199: ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
200: {
201: struct adapter *adapter = ifp->if_softc;
202: struct tx_ring *txr;
203: int i, err = 0;
1.28 msaitoh 204: #ifdef RSS
1.1 msaitoh 205: uint32_t bucket_id;
206: #endif
207:
208: /*
209: * When doing RSS, map it to the same outbound queue
210: * as the incoming flow would be mapped to.
211: *
212: * If everything is setup correctly, it should be the
213: * same bucket that the current CPU we're on is.
214: */
1.28 msaitoh 215: #ifdef RSS
1.1 msaitoh 216: if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
1.28 msaitoh 217: if ((adapter->feat_en & IXGBE_FEATURE_RSS) &&
218: (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
219: &bucket_id) == 0)) {
1.1 msaitoh 220: i = bucket_id % adapter->num_queues;
1.8 msaitoh 221: #ifdef IXGBE_DEBUG
222: if (bucket_id > adapter->num_queues)
1.28 msaitoh 223: if_printf(ifp,
224: "bucket_id (%d) > num_queues (%d)\n",
225: bucket_id, adapter->num_queues);
1.8 msaitoh 226: #endif
227: } else
1.1 msaitoh 228: i = m->m_pkthdr.flowid % adapter->num_queues;
1.3 msaitoh 229: } else
1.28 msaitoh 230: #endif /* 0 */
1.18 msaitoh 231: i = cpu_index(curcpu()) % adapter->num_queues;
1.3 msaitoh 232:
233: /* Check for a hung queue and pick alternative */
234: if (((1 << i) & adapter->active_queues) == 0)
1.18 msaitoh 235: i = ffs64(adapter->active_queues);
1.1 msaitoh 236:
237: txr = &adapter->tx_rings[i];
238:
1.18 msaitoh 239: err = pcq_put(txr->txr_interq, m);
240: if (err == false) {
241: m_freem(m);
242: txr->pcq_drops.ev_count++;
1.1 msaitoh 243: return (err);
1.18 msaitoh 244: }
1.1 msaitoh 245: if (IXGBE_TX_TRYLOCK(txr)) {
246: ixgbe_mq_start_locked(ifp, txr);
247: IXGBE_TX_UNLOCK(txr);
1.34 knakahar 248: } else {
249: if (adapter->txrx_use_workqueue) {
250: /*
251: * This function itself is not called in interrupt
252: * context, however it can be called in fast softint
253: * context right after receiving forwarding packets.
254: * So, it is required to protect workqueue from twice
255: * enqueuing when the machine uses both spontaneous
256: * packets and forwarding packets.
257: */
258: u_int *enqueued = percpu_getref(adapter->txr_wq_enqueued);
259: if (*enqueued == 0) {
260: *enqueued = 1;
261: percpu_putref(adapter->txr_wq_enqueued);
262: workqueue_enqueue(adapter->txr_wq, &txr->wq_cookie, curcpu());
263: } else
264: percpu_putref(adapter->txr_wq_enqueued);
265: } else
266: softint_schedule(txr->txr_si);
267: }
1.1 msaitoh 268:
269: return (0);
1.28 msaitoh 270: } /* ixgbe_mq_start */
1.1 msaitoh 271:
1.28 msaitoh 272: /************************************************************************
273: * ixgbe_mq_start_locked
274: ************************************************************************/
1.1 msaitoh 275: int
276: ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
277: {
1.28 msaitoh 278: struct mbuf *next;
279: int enqueued = 0, err = 0;
1.1 msaitoh 280:
1.38 knakahar 281: if (!txr->adapter->link_active) {
282: /*
283: * discard all packets buffered in txr_interq to avoid
284: * sending old packets at next link up timing.
285: */
286: ixgbe_drain(ifp, txr);
287: return (ENETDOWN);
288: }
1.28 msaitoh 289: if ((ifp->if_flags & IFF_RUNNING) == 0)
290: return (ENETDOWN);
1.1 msaitoh 291:
292: /* Process the queue */
1.18 msaitoh 293: while ((next = pcq_get(txr->txr_interq)) != NULL) {
294: if ((err = ixgbe_xmit(txr, next)) != 0) {
295: m_freem(next);
296: /* All errors are counted in ixgbe_xmit() */
1.1 msaitoh 297: break;
298: }
299: enqueued++;
1.3 msaitoh 300: #if __FreeBSD_version >= 1100036
1.4 msaitoh 301: /*
302: * Since we're looking at the tx ring, we can check
303: * to see if we're a VF by examing our tail register
304: * address.
305: */
1.28 msaitoh 306: if ((txr->adapter->feat_en & IXGBE_FEATURE_VF) &&
307: (next->m_flags & M_MCAST))
1.3 msaitoh 308: if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
309: #endif
1.1 msaitoh 310: /* Send a copy of the frame to the BPF listener */
311: bpf_mtap(ifp, next);
312: if ((ifp->if_flags & IFF_RUNNING) == 0)
313: break;
314: }
315:
1.28 msaitoh 316: if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD(txr->adapter))
1.1 msaitoh 317: ixgbe_txeof(txr);
318:
319: return (err);
1.28 msaitoh 320: } /* ixgbe_mq_start_locked */
1.1 msaitoh 321:
1.28 msaitoh 322: /************************************************************************
323: * ixgbe_deferred_mq_start
324: *
1.34 knakahar 325: * Called from a softint and workqueue (indirectly) to drain queued
326: * transmit packets.
1.28 msaitoh 327: ************************************************************************/
1.1 msaitoh 328: void
1.18 msaitoh 329: ixgbe_deferred_mq_start(void *arg)
1.1 msaitoh 330: {
331: struct tx_ring *txr = arg;
332: struct adapter *adapter = txr->adapter;
1.28 msaitoh 333: struct ifnet *ifp = adapter->ifp;
1.1 msaitoh 334:
335: IXGBE_TX_LOCK(txr);
1.18 msaitoh 336: if (pcq_peek(txr->txr_interq) != NULL)
1.1 msaitoh 337: ixgbe_mq_start_locked(ifp, txr);
338: IXGBE_TX_UNLOCK(txr);
1.28 msaitoh 339: } /* ixgbe_deferred_mq_start */
1.3 msaitoh 340:
1.28 msaitoh 341: /************************************************************************
1.34 knakahar 342: * ixgbe_deferred_mq_start_work
343: *
344: * Called from a workqueue to drain queued transmit packets.
345: ************************************************************************/
346: void
347: ixgbe_deferred_mq_start_work(struct work *wk, void *arg)
348: {
349: struct tx_ring *txr = container_of(wk, struct tx_ring, wq_cookie);
350: struct adapter *adapter = txr->adapter;
351: u_int *enqueued = percpu_getref(adapter->txr_wq_enqueued);
352: *enqueued = 0;
353: percpu_putref(adapter->txr_wq_enqueued);
354:
355: ixgbe_deferred_mq_start(txr);
356: } /* ixgbe_deferred_mq_start */
357:
1.38 knakahar 358: /************************************************************************
359: * ixgbe_drain_all
360: ************************************************************************/
361: void
362: ixgbe_drain_all(struct adapter *adapter)
363: {
364: struct ifnet *ifp = adapter->ifp;
365: struct ix_queue *que = adapter->queues;
366:
367: for (int i = 0; i < adapter->num_queues; i++, que++) {
368: struct tx_ring *txr = que->txr;
369:
370: IXGBE_TX_LOCK(txr);
371: ixgbe_drain(ifp, txr);
372: IXGBE_TX_UNLOCK(txr);
373: }
374: }
1.34 knakahar 375:
376: /************************************************************************
1.28 msaitoh 377: * ixgbe_xmit
1.1 msaitoh 378: *
1.28 msaitoh 379: * Maps the mbufs to tx descriptors, allowing the
380: * TX engine to transmit the packets.
1.1 msaitoh 381: *
1.28 msaitoh 382: * Return 0 on success, positive on failure
383: ************************************************************************/
1.1 msaitoh 384: static int
385: ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head)
386: {
1.28 msaitoh 387: struct adapter *adapter = txr->adapter;
388: struct ixgbe_tx_buf *txbuf;
1.1 msaitoh 389: union ixgbe_adv_tx_desc *txd = NULL;
1.28 msaitoh 390: struct ifnet *ifp = adapter->ifp;
391: int i, j, error;
392: int first;
393: u32 olinfo_status = 0, cmd_type_len;
394: bool remap = TRUE;
395: bus_dmamap_t map;
1.1 msaitoh 396:
397: /* Basic descriptor defines */
1.28 msaitoh 398: cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1.1 msaitoh 399: IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
400:
1.29 knakahar 401: if (vlan_has_tag(m_head))
1.28 msaitoh 402: cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1.1 msaitoh 403:
1.28 msaitoh 404: /*
405: * Important to capture the first descriptor
406: * used because it will contain the index of
407: * the one we tell the hardware to report back
408: */
409: first = txr->next_avail_desc;
1.1 msaitoh 410: txbuf = &txr->tx_buffers[first];
411: map = txbuf->map;
412:
413: /*
414: * Map the packet for DMA.
415: */
1.22 msaitoh 416: retry:
1.28 msaitoh 417: error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, m_head,
418: BUS_DMA_NOWAIT);
1.1 msaitoh 419:
420: if (__predict_false(error)) {
1.22 msaitoh 421: struct mbuf *m;
1.1 msaitoh 422:
423: switch (error) {
424: case EAGAIN:
1.35 msaitoh 425: txr->q_eagain_tx_dma_setup++;
1.1 msaitoh 426: return EAGAIN;
427: case ENOMEM:
1.35 msaitoh 428: txr->q_enomem_tx_dma_setup++;
1.1 msaitoh 429: return EAGAIN;
430: case EFBIG:
1.22 msaitoh 431: /* Try it again? - one try */
432: if (remap == TRUE) {
433: remap = FALSE;
434: /*
435: * XXX: m_defrag will choke on
436: * non-MCLBYTES-sized clusters
437: */
1.35 msaitoh 438: txr->q_efbig_tx_dma_setup++;
1.22 msaitoh 439: m = m_defrag(m_head, M_NOWAIT);
440: if (m == NULL) {
1.35 msaitoh 441: txr->q_mbuf_defrag_failed++;
1.22 msaitoh 442: return ENOBUFS;
443: }
444: m_head = m;
445: goto retry;
446: } else {
1.35 msaitoh 447: txr->q_efbig2_tx_dma_setup++;
1.22 msaitoh 448: return error;
449: }
1.1 msaitoh 450: case EINVAL:
1.35 msaitoh 451: txr->q_einval_tx_dma_setup++;
1.1 msaitoh 452: return error;
453: default:
1.35 msaitoh 454: txr->q_other_tx_dma_setup++;
1.1 msaitoh 455: return error;
456: }
457: }
458:
459: /* Make certain there are enough descriptors */
1.10 msaitoh 460: if (txr->tx_avail < (map->dm_nsegs + 2)) {
1.1 msaitoh 461: txr->no_desc_avail.ev_count++;
462: ixgbe_dmamap_unload(txr->txtag, txbuf->map);
463: return EAGAIN;
464: }
465:
466: /*
1.4 msaitoh 467: * Set up the appropriate offload context
468: * this will consume the first descriptor
469: */
1.1 msaitoh 470: error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
471: if (__predict_false(error)) {
472: return (error);
473: }
474:
475: /* Do the flow director magic */
1.28 msaitoh 476: if ((adapter->feat_en & IXGBE_FEATURE_FDIR) &&
477: (txr->atr_sample) && (!adapter->fdir_reinit)) {
1.1 msaitoh 478: ++txr->atr_count;
479: if (txr->atr_count >= atr_sample_rate) {
480: ixgbe_atr(txr, m_head);
481: txr->atr_count = 0;
482: }
483: }
484:
1.8 msaitoh 485: olinfo_status |= IXGBE_ADVTXD_CC;
1.1 msaitoh 486: i = txr->next_avail_desc;
487: for (j = 0; j < map->dm_nsegs; j++) {
488: bus_size_t seglen;
489: bus_addr_t segaddr;
490:
491: txbuf = &txr->tx_buffers[i];
492: txd = &txr->tx_base[i];
493: seglen = map->dm_segs[j].ds_len;
494: segaddr = htole64(map->dm_segs[j].ds_addr);
495:
496: txd->read.buffer_addr = segaddr;
1.40 msaitoh 497: txd->read.cmd_type_len = htole32(cmd_type_len | seglen);
1.1 msaitoh 498: txd->read.olinfo_status = htole32(olinfo_status);
499:
500: if (++i == txr->num_desc)
501: i = 0;
502: }
503:
1.28 msaitoh 504: txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1.1 msaitoh 505: txr->tx_avail -= map->dm_nsegs;
506: txr->next_avail_desc = i;
507:
508: txbuf->m_head = m_head;
509: /*
1.4 msaitoh 510: * Here we swap the map so the last descriptor,
511: * which gets the completion interrupt has the
512: * real map, and the first descriptor gets the
513: * unused map from this descriptor.
514: */
1.1 msaitoh 515: txr->tx_buffers[first].map = txbuf->map;
516: txbuf->map = map;
517: bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len,
518: BUS_DMASYNC_PREWRITE);
519:
1.28 msaitoh 520: /* Set the EOP descriptor that will be marked done */
521: txbuf = &txr->tx_buffers[first];
1.1 msaitoh 522: txbuf->eop = txd;
523:
1.28 msaitoh 524: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1.1 msaitoh 525: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
526: /*
527: * Advance the Transmit Descriptor Tail (Tdt), this tells the
528: * hardware that this frame is available to transmit.
529: */
530: ++txr->total_packets.ev_count;
1.3 msaitoh 531: IXGBE_WRITE_REG(&adapter->hw, txr->tail, i);
532:
1.23 msaitoh 533: /*
534: * XXXX NOMPSAFE: ifp->if_data should be percpu.
535: */
536: ifp->if_obytes += m_head->m_pkthdr.len;
537: if (m_head->m_flags & M_MCAST)
538: ifp->if_omcasts++;
539:
1.3 msaitoh 540: /* Mark queue as having work */
541: if (txr->busy == 0)
542: txr->busy = 1;
1.1 msaitoh 543:
1.28 msaitoh 544: return (0);
545: } /* ixgbe_xmit */
1.1 msaitoh 546:
1.38 knakahar 547: /************************************************************************
548: * ixgbe_drain
549: ************************************************************************/
550: static void
551: ixgbe_drain(struct ifnet *ifp, struct tx_ring *txr)
552: {
553: struct mbuf *m;
554:
555: IXGBE_TX_LOCK_ASSERT(txr);
556:
557: if (txr->me == 0) {
558: while (!IFQ_IS_EMPTY(&ifp->if_snd)) {
559: IFQ_DEQUEUE(&ifp->if_snd, m);
560: m_freem(m);
561: IF_DROP(&ifp->if_snd);
562: }
563: }
564:
565: while ((m = pcq_get(txr->txr_interq)) != NULL) {
566: m_freem(m);
567: txr->pcq_drops.ev_count++;
568: }
569: }
1.16 msaitoh 570:
1.28 msaitoh 571: /************************************************************************
572: * ixgbe_allocate_transmit_buffers
1.1 msaitoh 573: *
1.28 msaitoh 574: * Allocate memory for tx_buffer structures. The tx_buffer stores all
575: * the information needed to transmit a packet on the wire. This is
576: * called only once at attach, setup is done every reset.
577: ************************************************************************/
578: static int
1.1 msaitoh 579: ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
580: {
1.28 msaitoh 581: struct adapter *adapter = txr->adapter;
582: device_t dev = adapter->dev;
1.1 msaitoh 583: struct ixgbe_tx_buf *txbuf;
1.28 msaitoh 584: int error, i;
1.1 msaitoh 585:
586: /*
587: * Setup DMA descriptor areas.
588: */
1.28 msaitoh 589: error = ixgbe_dma_tag_create(
590: /* parent */ adapter->osdep.dmat,
591: /* alignment */ 1,
592: /* bounds */ 0,
593: /* maxsize */ IXGBE_TSO_SIZE,
594: /* nsegments */ adapter->num_segs,
595: /* maxsegsize */ PAGE_SIZE,
596: /* flags */ 0,
597: &txr->txtag);
598: if (error != 0) {
1.1 msaitoh 599: aprint_error_dev(dev,"Unable to allocate TX DMA tag\n");
600: goto fail;
601: }
602:
1.28 msaitoh 603: txr->tx_buffers =
1.1 msaitoh 604: (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
1.28 msaitoh 605: adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
606: if (txr->tx_buffers == NULL) {
1.1 msaitoh 607: aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n");
608: error = ENOMEM;
609: goto fail;
610: }
611:
1.28 msaitoh 612: /* Create the descriptor buffer dma maps */
1.1 msaitoh 613: txbuf = txr->tx_buffers;
614: for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
615: error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map);
616: if (error != 0) {
617: aprint_error_dev(dev,
618: "Unable to create TX DMA map (%d)\n", error);
619: goto fail;
620: }
621: }
622:
623: return 0;
624: fail:
625: /* We free all, it handles case where we are in the middle */
1.15 msaitoh 626: #if 0 /* XXX was FreeBSD */
1.1 msaitoh 627: ixgbe_free_transmit_structures(adapter);
1.15 msaitoh 628: #else
629: ixgbe_free_transmit_buffers(txr);
630: #endif
1.1 msaitoh 631: return (error);
1.28 msaitoh 632: } /* ixgbe_allocate_transmit_buffers */
1.1 msaitoh 633:
1.28 msaitoh 634: /************************************************************************
635: * ixgbe_setup_transmit_ring - Initialize a transmit ring.
636: ************************************************************************/
1.1 msaitoh 637: static void
638: ixgbe_setup_transmit_ring(struct tx_ring *txr)
639: {
1.28 msaitoh 640: struct adapter *adapter = txr->adapter;
641: struct ixgbe_tx_buf *txbuf;
1.1 msaitoh 642: #ifdef DEV_NETMAP
643: struct netmap_adapter *na = NA(adapter->ifp);
1.28 msaitoh 644: struct netmap_slot *slot;
1.1 msaitoh 645: #endif /* DEV_NETMAP */
646:
647: /* Clear the old ring contents */
648: IXGBE_TX_LOCK(txr);
1.28 msaitoh 649:
1.1 msaitoh 650: #ifdef DEV_NETMAP
1.28 msaitoh 651: if (adapter->feat_en & IXGBE_FEATURE_NETMAP) {
652: /*
653: * (under lock): if in netmap mode, do some consistency
654: * checks and set slot to entry 0 of the netmap ring.
655: */
656: slot = netmap_reset(na, NR_TX, txr->me, 0);
657: }
1.1 msaitoh 658: #endif /* DEV_NETMAP */
1.28 msaitoh 659:
1.1 msaitoh 660: bzero((void *)txr->tx_base,
1.28 msaitoh 661: (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
1.1 msaitoh 662: /* Reset indices */
663: txr->next_avail_desc = 0;
664: txr->next_to_clean = 0;
665:
666: /* Free any existing tx buffers. */
1.28 msaitoh 667: txbuf = txr->tx_buffers;
1.5 msaitoh 668: for (int i = 0; i < txr->num_desc; i++, txbuf++) {
1.1 msaitoh 669: if (txbuf->m_head != NULL) {
670: bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map,
671: 0, txbuf->m_head->m_pkthdr.len,
672: BUS_DMASYNC_POSTWRITE);
673: ixgbe_dmamap_unload(txr->txtag, txbuf->map);
674: m_freem(txbuf->m_head);
675: txbuf->m_head = NULL;
676: }
1.28 msaitoh 677:
1.1 msaitoh 678: #ifdef DEV_NETMAP
679: /*
680: * In netmap mode, set the map for the packet buffer.
681: * NOTE: Some drivers (not this one) also need to set
682: * the physical buffer address in the NIC ring.
683: * Slots in the netmap ring (indexed by "si") are
684: * kring->nkr_hwofs positions "ahead" wrt the
685: * corresponding slot in the NIC ring. In some drivers
686: * (not here) nkr_hwofs can be negative. Function
687: * netmap_idx_n2k() handles wraparounds properly.
688: */
1.28 msaitoh 689: if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) {
1.1 msaitoh 690: int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
1.5 msaitoh 691: netmap_load_map(na, txr->txtag,
692: txbuf->map, NMB(na, slot + si));
1.1 msaitoh 693: }
694: #endif /* DEV_NETMAP */
1.28 msaitoh 695:
1.1 msaitoh 696: /* Clear the EOP descriptor pointer */
697: txbuf->eop = NULL;
1.28 msaitoh 698: }
1.1 msaitoh 699:
700: /* Set the rate at which we sample packets */
1.28 msaitoh 701: if (adapter->feat_en & IXGBE_FEATURE_FDIR)
1.1 msaitoh 702: txr->atr_sample = atr_sample_rate;
703:
704: /* Set number of descriptors available */
705: txr->tx_avail = adapter->num_tx_desc;
706:
707: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
708: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
709: IXGBE_TX_UNLOCK(txr);
1.28 msaitoh 710: } /* ixgbe_setup_transmit_ring */
1.1 msaitoh 711:
1.28 msaitoh 712: /************************************************************************
713: * ixgbe_setup_transmit_structures - Initialize all transmit rings.
714: ************************************************************************/
1.1 msaitoh 715: int
716: ixgbe_setup_transmit_structures(struct adapter *adapter)
717: {
718: struct tx_ring *txr = adapter->tx_rings;
719:
720: for (int i = 0; i < adapter->num_queues; i++, txr++)
721: ixgbe_setup_transmit_ring(txr);
722:
723: return (0);
1.28 msaitoh 724: } /* ixgbe_setup_transmit_structures */
1.1 msaitoh 725:
1.28 msaitoh 726: /************************************************************************
727: * ixgbe_free_transmit_structures - Free all transmit rings.
728: ************************************************************************/
1.1 msaitoh 729: void
730: ixgbe_free_transmit_structures(struct adapter *adapter)
731: {
732: struct tx_ring *txr = adapter->tx_rings;
733:
734: for (int i = 0; i < adapter->num_queues; i++, txr++) {
735: ixgbe_free_transmit_buffers(txr);
736: ixgbe_dma_free(adapter, &txr->txdma);
737: IXGBE_TX_LOCK_DESTROY(txr);
738: }
739: free(adapter->tx_rings, M_DEVBUF);
1.28 msaitoh 740: } /* ixgbe_free_transmit_structures */
1.1 msaitoh 741:
1.28 msaitoh 742: /************************************************************************
743: * ixgbe_free_transmit_buffers
1.1 msaitoh 744: *
1.28 msaitoh 745: * Free transmit ring related data structures.
746: ************************************************************************/
1.1 msaitoh 747: static void
748: ixgbe_free_transmit_buffers(struct tx_ring *txr)
749: {
1.28 msaitoh 750: struct adapter *adapter = txr->adapter;
1.1 msaitoh 751: struct ixgbe_tx_buf *tx_buffer;
1.28 msaitoh 752: int i;
1.1 msaitoh 753:
1.14 msaitoh 754: INIT_DEBUGOUT("ixgbe_free_transmit_buffers: begin");
1.1 msaitoh 755:
756: if (txr->tx_buffers == NULL)
757: return;
758:
759: tx_buffer = txr->tx_buffers;
760: for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
761: if (tx_buffer->m_head != NULL) {
762: bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map,
763: 0, tx_buffer->m_head->m_pkthdr.len,
764: BUS_DMASYNC_POSTWRITE);
765: ixgbe_dmamap_unload(txr->txtag, tx_buffer->map);
766: m_freem(tx_buffer->m_head);
767: tx_buffer->m_head = NULL;
768: if (tx_buffer->map != NULL) {
769: ixgbe_dmamap_destroy(txr->txtag,
770: tx_buffer->map);
771: tx_buffer->map = NULL;
772: }
773: } else if (tx_buffer->map != NULL) {
774: ixgbe_dmamap_unload(txr->txtag, tx_buffer->map);
775: ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map);
776: tx_buffer->map = NULL;
777: }
778: }
1.18 msaitoh 779: if (txr->txr_interq != NULL) {
780: struct mbuf *m;
781:
782: while ((m = pcq_get(txr->txr_interq)) != NULL)
783: m_freem(m);
784: pcq_destroy(txr->txr_interq);
785: }
1.1 msaitoh 786: if (txr->tx_buffers != NULL) {
787: free(txr->tx_buffers, M_DEVBUF);
788: txr->tx_buffers = NULL;
789: }
790: if (txr->txtag != NULL) {
791: ixgbe_dma_tag_destroy(txr->txtag);
792: txr->txtag = NULL;
793: }
1.28 msaitoh 794: } /* ixgbe_free_transmit_buffers */
1.1 msaitoh 795:
1.28 msaitoh 796: /************************************************************************
797: * ixgbe_tx_ctx_setup
1.1 msaitoh 798: *
1.28 msaitoh 799: * Advanced Context Descriptor setup for VLAN, CSUM or TSO
800: ************************************************************************/
1.1 msaitoh 801: static int
802: ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
803: u32 *cmd_type_len, u32 *olinfo_status)
804: {
1.28 msaitoh 805: struct adapter *adapter = txr->adapter;
1.1 msaitoh 806: struct ixgbe_adv_tx_context_desc *TXD;
1.28 msaitoh 807: struct ether_vlan_header *eh;
1.8 msaitoh 808: #ifdef INET
1.28 msaitoh 809: struct ip *ip;
1.8 msaitoh 810: #endif
811: #ifdef INET6
1.28 msaitoh 812: struct ip6_hdr *ip6;
1.8 msaitoh 813: #endif
1.28 msaitoh 814: int ehdrlen, ip_hlen = 0;
815: int offload = TRUE;
816: int ctxd = txr->next_avail_desc;
817: u32 vlan_macip_lens = 0;
818: u32 type_tucmd_mlhl = 0;
819: u16 vtag = 0;
820: u16 etype;
821: u8 ipproto = 0;
822: char *l3d;
1.8 msaitoh 823:
1.1 msaitoh 824:
825: /* First check if TSO is to be used */
1.28 msaitoh 826: if (mp->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
1.17 msaitoh 827: int rv = ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status);
828:
1.21 msaitoh 829: if (rv != 0)
1.17 msaitoh 830: ++adapter->tso_err.ev_count;
1.21 msaitoh 831: return rv;
1.17 msaitoh 832: }
1.1 msaitoh 833:
834: if ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) == 0)
835: offload = FALSE;
836:
837: /* Indicate the whole packet as payload when not doing TSO */
1.28 msaitoh 838: *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
1.1 msaitoh 839:
840: /* Now ready a context descriptor */
1.28 msaitoh 841: TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd];
1.1 msaitoh 842:
843: /*
1.28 msaitoh 844: * In advanced descriptors the vlan tag must
845: * be placed into the context descriptor. Hence
846: * we need to make one even if not doing offloads.
847: */
1.29 knakahar 848: if (vlan_has_tag(mp)) {
849: vtag = htole16(vlan_get_tag(mp));
1.1 msaitoh 850: vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
1.28 msaitoh 851: } else if (!(txr->adapter->feat_en & IXGBE_FEATURE_NEEDS_CTXD) &&
852: (offload == FALSE))
1.4 msaitoh 853: return (0);
1.1 msaitoh 854:
855: /*
856: * Determine where frame payload starts.
857: * Jump over vlan headers if already present,
858: * helpful for QinQ too.
859: */
860: KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag));
861: eh = mtod(mp, struct ether_vlan_header *);
862: if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
863: KASSERT(mp->m_len >= sizeof(struct ether_vlan_header));
864: etype = ntohs(eh->evl_proto);
865: ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
866: } else {
867: etype = ntohs(eh->evl_encap_proto);
868: ehdrlen = ETHER_HDR_LEN;
869: }
870:
871: /* Set the ether header length */
872: vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
873:
1.3 msaitoh 874: if (offload == FALSE)
875: goto no_offloads;
876:
1.8 msaitoh 877: /*
1.28 msaitoh 878: * If the first mbuf only includes the ethernet header,
879: * jump to the next one
880: * XXX: This assumes the stack splits mbufs containing headers
881: * on header boundaries
1.8 msaitoh 882: * XXX: And assumes the entire IP header is contained in one mbuf
883: */
884: if (mp->m_len == ehdrlen && mp->m_next)
885: l3d = mtod(mp->m_next, char *);
886: else
887: l3d = mtod(mp, char *) + ehdrlen;
888:
1.1 msaitoh 889: switch (etype) {
1.9 msaitoh 890: #ifdef INET
1.1 msaitoh 891: case ETHERTYPE_IP:
1.8 msaitoh 892: ip = (struct ip *)(l3d);
893: ip_hlen = ip->ip_hl << 2;
894: ipproto = ip->ip_p;
895: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
1.1 msaitoh 896: KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 ||
1.8 msaitoh 897: ip->ip_sum == 0);
1.1 msaitoh 898: break;
1.9 msaitoh 899: #endif
900: #ifdef INET6
1.1 msaitoh 901: case ETHERTYPE_IPV6:
1.8 msaitoh 902: ip6 = (struct ip6_hdr *)(l3d);
903: ip_hlen = sizeof(struct ip6_hdr);
904: ipproto = ip6->ip6_nxt;
1.1 msaitoh 905: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
906: break;
1.9 msaitoh 907: #endif
1.1 msaitoh 908: default:
1.11 msaitoh 909: offload = false;
1.1 msaitoh 910: break;
911: }
912:
913: if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0)
914: *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
915:
916: vlan_macip_lens |= ip_hlen;
917:
1.8 msaitoh 918: /* No support for offloads for non-L4 next headers */
919: switch (ipproto) {
1.36 msaitoh 920: case IPPROTO_TCP:
921: if (mp->m_pkthdr.csum_flags &
922: (M_CSUM_TCPv4 | M_CSUM_TCPv6))
923: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
924: else
925: offload = false;
926: break;
927: case IPPROTO_UDP:
928: if (mp->m_pkthdr.csum_flags &
929: (M_CSUM_UDPv4 | M_CSUM_UDPv6))
930: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
931: else
1.11 msaitoh 932: offload = false;
1.36 msaitoh 933: break;
934: default:
935: offload = false;
936: break;
1.8 msaitoh 937: }
938:
939: if (offload) /* Insert L4 checksum into data descriptors */
1.1 msaitoh 940: *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
941:
1.3 msaitoh 942: no_offloads:
943: type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
944:
1.1 msaitoh 945: /* Now copy bits into descriptor */
946: TXD->vlan_macip_lens = htole32(vlan_macip_lens);
947: TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
948: TXD->seqnum_seed = htole32(0);
949: TXD->mss_l4len_idx = htole32(0);
950:
951: /* We've consumed the first desc, adjust counters */
952: if (++ctxd == txr->num_desc)
953: ctxd = 0;
954: txr->next_avail_desc = ctxd;
955: --txr->tx_avail;
956:
1.28 msaitoh 957: return (0);
958: } /* ixgbe_tx_ctx_setup */
1.1 msaitoh 959:
1.28 msaitoh 960: /************************************************************************
961: * ixgbe_tso_setup
1.1 msaitoh 962: *
1.28 msaitoh 963: * Setup work for hardware segmentation offload (TSO) on
964: * adapters using advanced tx descriptors
965: ************************************************************************/
1.1 msaitoh 966: static int
1.28 msaitoh 967: ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *cmd_type_len,
968: u32 *olinfo_status)
1.1 msaitoh 969: {
970: struct ixgbe_adv_tx_context_desc *TXD;
1.28 msaitoh 971: struct ether_vlan_header *eh;
1.1 msaitoh 972: #ifdef INET6
1.28 msaitoh 973: struct ip6_hdr *ip6;
1.1 msaitoh 974: #endif
975: #ifdef INET
1.28 msaitoh 976: struct ip *ip;
1.1 msaitoh 977: #endif
1.28 msaitoh 978: struct tcphdr *th;
979: int ctxd, ehdrlen, ip_hlen, tcp_hlen;
980: u32 vlan_macip_lens = 0;
981: u32 type_tucmd_mlhl = 0;
982: u32 mss_l4len_idx = 0, paylen;
983: u16 vtag = 0, eh_type;
1.1 msaitoh 984:
985: /*
986: * Determine where frame payload starts.
987: * Jump over vlan headers if already present
988: */
989: eh = mtod(mp, struct ether_vlan_header *);
990: if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
991: ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
992: eh_type = eh->evl_proto;
993: } else {
994: ehdrlen = ETHER_HDR_LEN;
995: eh_type = eh->evl_encap_proto;
996: }
997:
998: switch (ntohs(eh_type)) {
999: #ifdef INET
1000: case ETHERTYPE_IP:
1001: ip = (struct ip *)(mp->m_data + ehdrlen);
1002: if (ip->ip_p != IPPROTO_TCP)
1003: return (ENXIO);
1004: ip->ip_sum = 0;
1005: ip_hlen = ip->ip_hl << 2;
1006: th = (struct tcphdr *)((char *)ip + ip_hlen);
1007: th->th_sum = in_cksum_phdr(ip->ip_src.s_addr,
1008: ip->ip_dst.s_addr, htons(IPPROTO_TCP));
1009: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
1010: /* Tell transmit desc to also do IPv4 checksum. */
1011: *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1012: break;
1013: #endif
1.28 msaitoh 1014: #ifdef INET6
1015: case ETHERTYPE_IPV6:
1016: ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
1017: /* XXX-BZ For now we do not pretend to support ext. hdrs. */
1018: if (ip6->ip6_nxt != IPPROTO_TCP)
1019: return (ENXIO);
1020: ip_hlen = sizeof(struct ip6_hdr);
1021: ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
1022: th = (struct tcphdr *)((char *)ip6 + ip_hlen);
1023: th->th_sum = in6_cksum_phdr(&ip6->ip6_src,
1024: &ip6->ip6_dst, 0, htonl(IPPROTO_TCP));
1025: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
1026: break;
1027: #endif
1.1 msaitoh 1028: default:
1029: panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
1030: __func__, ntohs(eh_type));
1031: break;
1032: }
1033:
1034: ctxd = txr->next_avail_desc;
1.28 msaitoh 1035: TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd];
1.1 msaitoh 1036:
1037: tcp_hlen = th->th_off << 2;
1038:
1039: /* This is used in the transmit desc in encap */
1040: paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
1041:
1042: /* VLAN MACLEN IPLEN */
1.29 knakahar 1043: if (vlan_has_tag(mp)) {
1044: vtag = htole16(vlan_get_tag(mp));
1.28 msaitoh 1045: vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
1.1 msaitoh 1046: }
1047:
1048: vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
1049: vlan_macip_lens |= ip_hlen;
1050: TXD->vlan_macip_lens = htole32(vlan_macip_lens);
1051:
1052: /* ADV DTYPE TUCMD */
1053: type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
1054: type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
1055: TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
1056:
1057: /* MSS L4LEN IDX */
1058: mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT);
1059: mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
1060: TXD->mss_l4len_idx = htole32(mss_l4len_idx);
1061:
1062: TXD->seqnum_seed = htole32(0);
1063:
1064: if (++ctxd == txr->num_desc)
1065: ctxd = 0;
1066:
1067: txr->tx_avail--;
1068: txr->next_avail_desc = ctxd;
1069: *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1070: *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1071: *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1072: ++txr->tso_tx.ev_count;
1.28 msaitoh 1073:
1.1 msaitoh 1074: return (0);
1.28 msaitoh 1075: } /* ixgbe_tso_setup */
1.1 msaitoh 1076:
1.3 msaitoh 1077:
1.28 msaitoh 1078: /************************************************************************
1079: * ixgbe_txeof
1.1 msaitoh 1080: *
1.28 msaitoh 1081: * Examine each tx_buffer in the used queue. If the hardware is done
1082: * processing the packet then free associated resources. The
1083: * tx_buffer is put back on the free queue.
1084: ************************************************************************/
1.32 msaitoh 1085: bool
1.1 msaitoh 1086: ixgbe_txeof(struct tx_ring *txr)
1087: {
1088: struct adapter *adapter = txr->adapter;
1089: struct ifnet *ifp = adapter->ifp;
1.28 msaitoh 1090: struct ixgbe_tx_buf *buf;
1091: union ixgbe_adv_tx_desc *txd;
1.1 msaitoh 1092: u32 work, processed = 0;
1.7 msaitoh 1093: u32 limit = adapter->tx_process_limit;
1.1 msaitoh 1094:
1095: KASSERT(mutex_owned(&txr->tx_mtx));
1096:
1097: #ifdef DEV_NETMAP
1.28 msaitoh 1098: if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) &&
1099: (adapter->ifp->if_capenable & IFCAP_NETMAP)) {
1100: struct netmap_adapter *na = NA(adapter->ifp);
1.1 msaitoh 1101: struct netmap_kring *kring = &na->tx_rings[txr->me];
1102: txd = txr->tx_base;
1103: bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1104: BUS_DMASYNC_POSTREAD);
1105: /*
1106: * In netmap mode, all the work is done in the context
1107: * of the client thread. Interrupt handlers only wake up
1108: * clients, which may be sleeping on individual rings
1109: * or on a global resource for all rings.
1110: * To implement tx interrupt mitigation, we wake up the client
1111: * thread roughly every half ring, even if the NIC interrupts
1112: * more frequently. This is implemented as follows:
1113: * - ixgbe_txsync() sets kring->nr_kflags with the index of
1114: * the slot that should wake up the thread (nkr_num_slots
1115: * means the user thread should not be woken up);
1116: * - the driver ignores tx interrupts unless netmap_mitigate=0
1117: * or the slot has the DD bit set.
1118: */
1119: if (!netmap_mitigate ||
1120: (kring->nr_kflags < kring->nkr_num_slots &&
1.28 msaitoh 1121: txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
1.1 msaitoh 1122: netmap_tx_irq(ifp, txr->me);
1123: }
1.32 msaitoh 1124: return false;
1.1 msaitoh 1125: }
1126: #endif /* DEV_NETMAP */
1127:
1128: if (txr->tx_avail == txr->num_desc) {
1.3 msaitoh 1129: txr->busy = 0;
1.32 msaitoh 1130: return false;
1.1 msaitoh 1131: }
1132:
1133: /* Get work starting point */
1134: work = txr->next_to_clean;
1135: buf = &txr->tx_buffers[work];
1136: txd = &txr->tx_base[work];
1137: work -= txr->num_desc; /* The distance to ring end */
1.28 msaitoh 1138: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1.1 msaitoh 1139: BUS_DMASYNC_POSTREAD);
1.8 msaitoh 1140:
1.1 msaitoh 1141: do {
1.8 msaitoh 1142: union ixgbe_adv_tx_desc *eop = buf->eop;
1.1 msaitoh 1143: if (eop == NULL) /* No work */
1144: break;
1145:
1146: if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
1147: break; /* I/O not complete */
1148:
1149: if (buf->m_head) {
1.28 msaitoh 1150: txr->bytes += buf->m_head->m_pkthdr.len;
1151: bus_dmamap_sync(txr->txtag->dt_dmat, buf->map,
1.1 msaitoh 1152: 0, buf->m_head->m_pkthdr.len,
1153: BUS_DMASYNC_POSTWRITE);
1.28 msaitoh 1154: ixgbe_dmamap_unload(txr->txtag, buf->map);
1.1 msaitoh 1155: m_freem(buf->m_head);
1156: buf->m_head = NULL;
1157: }
1158: buf->eop = NULL;
1159: ++txr->tx_avail;
1160:
1161: /* We clean the range if multi segment */
1162: while (txd != eop) {
1163: ++txd;
1164: ++buf;
1165: ++work;
1166: /* wrap the ring? */
1167: if (__predict_false(!work)) {
1168: work -= txr->num_desc;
1169: buf = txr->tx_buffers;
1170: txd = txr->tx_base;
1171: }
1172: if (buf->m_head) {
1173: txr->bytes +=
1174: buf->m_head->m_pkthdr.len;
1175: bus_dmamap_sync(txr->txtag->dt_dmat,
1176: buf->map,
1177: 0, buf->m_head->m_pkthdr.len,
1178: BUS_DMASYNC_POSTWRITE);
1179: ixgbe_dmamap_unload(txr->txtag,
1180: buf->map);
1181: m_freem(buf->m_head);
1182: buf->m_head = NULL;
1183: }
1184: ++txr->tx_avail;
1185: buf->eop = NULL;
1186:
1187: }
1188: ++txr->packets;
1189: ++processed;
1190: ++ifp->if_opackets;
1191:
1192: /* Try the next packet */
1193: ++txd;
1194: ++buf;
1195: ++work;
1196: /* reset with a wrap */
1197: if (__predict_false(!work)) {
1198: work -= txr->num_desc;
1199: buf = txr->tx_buffers;
1200: txd = txr->tx_base;
1201: }
1202: prefetch(txd);
1203: } while (__predict_true(--limit));
1204:
1205: ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1206: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1207:
1208: work += txr->num_desc;
1209: txr->next_to_clean = work;
1210:
1211: /*
1.28 msaitoh 1212: * Queue Hang detection, we know there's
1213: * work outstanding or the first return
1214: * would have been taken, so increment busy
1215: * if nothing managed to get cleaned, then
1216: * in local_timer it will be checked and
1217: * marked as HUNG if it exceeds a MAX attempt.
1218: */
1.3 msaitoh 1219: if ((processed == 0) && (txr->busy != IXGBE_QUEUE_HUNG))
1220: ++txr->busy;
1221: /*
1.28 msaitoh 1222: * If anything gets cleaned we reset state to 1,
1223: * note this will turn off HUNG if its set.
1224: */
1.3 msaitoh 1225: if (processed)
1226: txr->busy = 1;
1.1 msaitoh 1227:
1228: if (txr->tx_avail == txr->num_desc)
1.3 msaitoh 1229: txr->busy = 0;
1.1 msaitoh 1230:
1.32 msaitoh 1231: return ((limit > 0) ? false : true);
1.28 msaitoh 1232: } /* ixgbe_txeof */
1.1 msaitoh 1233:
1.28 msaitoh 1234: /************************************************************************
1235: * ixgbe_rsc_count
1236: *
1237: * Used to detect a descriptor that has been merged by Hardware RSC.
1238: ************************************************************************/
1.1 msaitoh 1239: static inline u32
1240: ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
1241: {
1242: return (le32toh(rx->wb.lower.lo_dword.data) &
1243: IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
1.28 msaitoh 1244: } /* ixgbe_rsc_count */
1.1 msaitoh 1245:
1.28 msaitoh 1246: /************************************************************************
1247: * ixgbe_setup_hw_rsc
1.1 msaitoh 1248: *
1.28 msaitoh 1249: * Initialize Hardware RSC (LRO) feature on 82599
1250: * for an RX ring, this is toggled by the LRO capability
1251: * even though it is transparent to the stack.
1252: *
1253: * NOTE: Since this HW feature only works with IPv4 and
1254: * testing has shown soft LRO to be as effective,
1255: * this feature will be disabled by default.
1256: ************************************************************************/
1.1 msaitoh 1257: static void
1258: ixgbe_setup_hw_rsc(struct rx_ring *rxr)
1259: {
1.28 msaitoh 1260: struct adapter *adapter = rxr->adapter;
1261: struct ixgbe_hw *hw = &adapter->hw;
1262: u32 rscctrl, rdrxctl;
1.1 msaitoh 1263:
1264: /* If turning LRO/RSC off we need to disable it */
1265: if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
1266: rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
1267: rscctrl &= ~IXGBE_RSCCTL_RSCEN;
1268: return;
1269: }
1270:
1271: rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
1272: rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
1.28 msaitoh 1273: #ifdef DEV_NETMAP
1274: /* Always strip CRC unless Netmap disabled it */
1275: if (!(adapter->feat_en & IXGBE_FEATURE_NETMAP) ||
1276: !(adapter->ifp->if_capenable & IFCAP_NETMAP) ||
1277: ix_crcstrip)
1.1 msaitoh 1278: #endif /* DEV_NETMAP */
1.28 msaitoh 1279: rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
1.1 msaitoh 1280: rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
1281: IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
1282:
1283: rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
1284: rscctrl |= IXGBE_RSCCTL_RSCEN;
1285: /*
1.28 msaitoh 1286: * Limit the total number of descriptors that
1287: * can be combined, so it does not exceed 64K
1288: */
1.1 msaitoh 1289: if (rxr->mbuf_sz == MCLBYTES)
1290: rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
1291: else if (rxr->mbuf_sz == MJUMPAGESIZE)
1292: rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
1293: else if (rxr->mbuf_sz == MJUM9BYTES)
1294: rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
1295: else /* Using 16K cluster */
1296: rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
1297:
1298: IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
1299:
1300: /* Enable TCP header recognition */
1301: IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
1.28 msaitoh 1302: (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | IXGBE_PSRTYPE_TCPHDR));
1.1 msaitoh 1303:
1304: /* Disable RSC for ACK packets */
1305: IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
1306: (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
1307:
1308: rxr->hw_rsc = TRUE;
1.28 msaitoh 1309: } /* ixgbe_setup_hw_rsc */
1.8 msaitoh 1310:
1.28 msaitoh 1311: /************************************************************************
1312: * ixgbe_refresh_mbufs
1.1 msaitoh 1313: *
1.28 msaitoh 1314: * Refresh mbuf buffers for RX descriptor rings
1315: * - now keeps its own state so discards due to resource
1316: * exhaustion are unnecessary, if an mbuf cannot be obtained
1317: * it just returns, keeping its placeholder, thus it can simply
1318: * be recalled to try again.
1319: ************************************************************************/
1.1 msaitoh 1320: static void
1321: ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
1322: {
1.28 msaitoh 1323: struct adapter *adapter = rxr->adapter;
1324: struct ixgbe_rx_buf *rxbuf;
1325: struct mbuf *mp;
1326: int i, j, error;
1327: bool refreshed = false;
1.1 msaitoh 1328:
1329: i = j = rxr->next_to_refresh;
1330: /* Control the loop with one beyond */
1331: if (++j == rxr->num_desc)
1332: j = 0;
1333:
1334: while (j != limit) {
1335: rxbuf = &rxr->rx_buffers[i];
1336: if (rxbuf->buf == NULL) {
1337: mp = ixgbe_getjcl(&adapter->jcl_head, M_NOWAIT,
1338: MT_DATA, M_PKTHDR, rxr->mbuf_sz);
1339: if (mp == NULL) {
1340: rxr->no_jmbuf.ev_count++;
1341: goto update;
1342: }
1343: if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
1344: m_adj(mp, ETHER_ALIGN);
1345: } else
1346: mp = rxbuf->buf;
1347:
1348: mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1349:
1350: /* If we're dealing with an mbuf that was copied rather
1351: * than replaced, there's no need to go through busdma.
1352: */
1353: if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
1354: /* Get the memory mapping */
1.4 msaitoh 1355: ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
1.1 msaitoh 1356: error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
1357: rxbuf->pmap, mp, BUS_DMA_NOWAIT);
1358: if (error != 0) {
1.28 msaitoh 1359: printf("Refresh mbufs: payload dmamap load failure - %d\n", error);
1.1 msaitoh 1360: m_free(mp);
1361: rxbuf->buf = NULL;
1362: goto update;
1363: }
1364: rxbuf->buf = mp;
1365: bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
1366: 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD);
1367: rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
1368: htole64(rxbuf->pmap->dm_segs[0].ds_addr);
1369: } else {
1370: rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
1371: rxbuf->flags &= ~IXGBE_RX_COPY;
1372: }
1373:
1374: refreshed = true;
1375: /* Next is precalculated */
1376: i = j;
1377: rxr->next_to_refresh = i;
1378: if (++j == rxr->num_desc)
1379: j = 0;
1380: }
1.28 msaitoh 1381:
1.1 msaitoh 1382: update:
1383: if (refreshed) /* Update hardware tail index */
1.28 msaitoh 1384: IXGBE_WRITE_REG(&adapter->hw, rxr->tail, rxr->next_to_refresh);
1385:
1.1 msaitoh 1386: return;
1.28 msaitoh 1387: } /* ixgbe_refresh_mbufs */
1.1 msaitoh 1388:
1.28 msaitoh 1389: /************************************************************************
1390: * ixgbe_allocate_receive_buffers
1.1 msaitoh 1391: *
1.28 msaitoh 1392: * Allocate memory for rx_buffer structures. Since we use one
1393: * rx_buffer per received packet, the maximum number of rx_buffer's
1394: * that we'll need is equal to the number of receive descriptors
1395: * that we've allocated.
1396: ************************************************************************/
1397: static int
1.1 msaitoh 1398: ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
1399: {
1.28 msaitoh 1400: struct adapter *adapter = rxr->adapter;
1401: device_t dev = adapter->dev;
1402: struct ixgbe_rx_buf *rxbuf;
1403: int bsize, error;
1.1 msaitoh 1404:
1405: bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
1.28 msaitoh 1406: rxr->rx_buffers = (struct ixgbe_rx_buf *)malloc(bsize, M_DEVBUF,
1407: M_NOWAIT | M_ZERO);
1408: if (rxr->rx_buffers == NULL) {
1.1 msaitoh 1409: aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n");
1410: error = ENOMEM;
1411: goto fail;
1412: }
1413:
1.28 msaitoh 1414: error = ixgbe_dma_tag_create(
1415: /* parent */ adapter->osdep.dmat,
1416: /* alignment */ 1,
1417: /* bounds */ 0,
1418: /* maxsize */ MJUM16BYTES,
1419: /* nsegments */ 1,
1420: /* maxsegsize */ MJUM16BYTES,
1421: /* flags */ 0,
1422: &rxr->ptag);
1423: if (error != 0) {
1.1 msaitoh 1424: aprint_error_dev(dev, "Unable to create RX DMA tag\n");
1425: goto fail;
1426: }
1427:
1.5 msaitoh 1428: for (int i = 0; i < rxr->num_desc; i++, rxbuf++) {
1.1 msaitoh 1429: rxbuf = &rxr->rx_buffers[i];
1.4 msaitoh 1430: error = ixgbe_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
1.1 msaitoh 1431: if (error) {
1432: aprint_error_dev(dev, "Unable to create RX dma map\n");
1433: goto fail;
1434: }
1435: }
1436:
1437: return (0);
1438:
1439: fail:
1440: /* Frees all, but can handle partial completion */
1441: ixgbe_free_receive_structures(adapter);
1.28 msaitoh 1442:
1.1 msaitoh 1443: return (error);
1.28 msaitoh 1444: } /* ixgbe_allocate_receive_buffers */
1.1 msaitoh 1445:
1.28 msaitoh 1446: /************************************************************************
1.30 msaitoh 1447: * ixgbe_free_receive_ring
1.28 msaitoh 1448: ************************************************************************/
1449: static void
1.1 msaitoh 1450: ixgbe_free_receive_ring(struct rx_ring *rxr)
1.27 msaitoh 1451: {
1.5 msaitoh 1452: for (int i = 0; i < rxr->num_desc; i++) {
1.27 msaitoh 1453: ixgbe_rx_discard(rxr, i);
1.1 msaitoh 1454: }
1.28 msaitoh 1455: } /* ixgbe_free_receive_ring */
1.1 msaitoh 1456:
1.28 msaitoh 1457: /************************************************************************
1458: * ixgbe_setup_receive_ring
1.1 msaitoh 1459: *
1.28 msaitoh 1460: * Initialize a receive ring and its buffers.
1461: ************************************************************************/
1.1 msaitoh 1462: static int
1463: ixgbe_setup_receive_ring(struct rx_ring *rxr)
1464: {
1.28 msaitoh 1465: struct adapter *adapter;
1466: struct ixgbe_rx_buf *rxbuf;
1.1 msaitoh 1467: #ifdef LRO
1.28 msaitoh 1468: struct ifnet *ifp;
1469: struct lro_ctrl *lro = &rxr->lro;
1.1 msaitoh 1470: #endif /* LRO */
1471: #ifdef DEV_NETMAP
1472: struct netmap_adapter *na = NA(rxr->adapter->ifp);
1.28 msaitoh 1473: struct netmap_slot *slot;
1.1 msaitoh 1474: #endif /* DEV_NETMAP */
1.28 msaitoh 1475: int rsize, error = 0;
1.1 msaitoh 1476:
1477: adapter = rxr->adapter;
1478: #ifdef LRO
1479: ifp = adapter->ifp;
1480: #endif /* LRO */
1481:
1482: /* Clear the ring contents */
1483: IXGBE_RX_LOCK(rxr);
1.28 msaitoh 1484:
1.1 msaitoh 1485: #ifdef DEV_NETMAP
1.28 msaitoh 1486: if (adapter->feat_en & IXGBE_FEATURE_NETMAP)
1487: slot = netmap_reset(na, NR_RX, rxr->me, 0);
1.1 msaitoh 1488: #endif /* DEV_NETMAP */
1.28 msaitoh 1489:
1.1 msaitoh 1490: rsize = roundup2(adapter->num_rx_desc *
1491: sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
1492: bzero((void *)rxr->rx_base, rsize);
1493: /* Cache the size */
1494: rxr->mbuf_sz = adapter->rx_mbuf_sz;
1495:
1496: /* Free current RX buffer structs and their mbufs */
1497: ixgbe_free_receive_ring(rxr);
1498:
1499: /* Now replenish the mbufs */
1500: for (int j = 0; j != rxr->num_desc; ++j) {
1.28 msaitoh 1501: struct mbuf *mp;
1.1 msaitoh 1502:
1503: rxbuf = &rxr->rx_buffers[j];
1.28 msaitoh 1504:
1.1 msaitoh 1505: #ifdef DEV_NETMAP
1506: /*
1507: * In netmap mode, fill the map and set the buffer
1508: * address in the NIC ring, considering the offset
1509: * between the netmap and NIC rings (see comment in
1510: * ixgbe_setup_transmit_ring() ). No need to allocate
1511: * an mbuf, so end the block with a continue;
1512: */
1.28 msaitoh 1513: if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) {
1.1 msaitoh 1514: int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
1515: uint64_t paddr;
1516: void *addr;
1517:
1518: addr = PNMB(na, slot + sj, &paddr);
1519: netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
1520: /* Update descriptor and the cached value */
1521: rxr->rx_base[j].read.pkt_addr = htole64(paddr);
1522: rxbuf->addr = htole64(paddr);
1523: continue;
1524: }
1525: #endif /* DEV_NETMAP */
1.28 msaitoh 1526:
1527: rxbuf->flags = 0;
1.1 msaitoh 1528: rxbuf->buf = ixgbe_getjcl(&adapter->jcl_head, M_NOWAIT,
1529: MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz);
1530: if (rxbuf->buf == NULL) {
1531: error = ENOBUFS;
1.28 msaitoh 1532: goto fail;
1.1 msaitoh 1533: }
1534: mp = rxbuf->buf;
1535: mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1536: /* Get the memory mapping */
1.28 msaitoh 1537: error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, rxbuf->pmap,
1538: mp, BUS_DMA_NOWAIT);
1.1 msaitoh 1539: if (error != 0)
1540: goto fail;
1541: bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
1542: 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD);
1543: /* Update the descriptor and the cached value */
1544: rxr->rx_base[j].read.pkt_addr =
1545: htole64(rxbuf->pmap->dm_segs[0].ds_addr);
1546: rxbuf->addr = htole64(rxbuf->pmap->dm_segs[0].ds_addr);
1547: }
1548:
1549:
1550: /* Setup our descriptor indices */
1551: rxr->next_to_check = 0;
1552: rxr->next_to_refresh = 0;
1553: rxr->lro_enabled = FALSE;
1554: rxr->rx_copies.ev_count = 0;
1.13 msaitoh 1555: #if 0 /* NetBSD */
1.1 msaitoh 1556: rxr->rx_bytes.ev_count = 0;
1.13 msaitoh 1557: #if 1 /* Fix inconsistency */
1558: rxr->rx_packets.ev_count = 0;
1559: #endif
1560: #endif
1.1 msaitoh 1561: rxr->vtag_strip = FALSE;
1562:
1563: ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
1564: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1565:
1566: /*
1.28 msaitoh 1567: * Now set up the LRO interface
1568: */
1.1 msaitoh 1569: if (ixgbe_rsc_enable)
1570: ixgbe_setup_hw_rsc(rxr);
1571: #ifdef LRO
1572: else if (ifp->if_capenable & IFCAP_LRO) {
1573: device_t dev = adapter->dev;
1574: int err = tcp_lro_init(lro);
1575: if (err) {
1576: device_printf(dev, "LRO Initialization failed!\n");
1577: goto fail;
1578: }
1579: INIT_DEBUGOUT("RX Soft LRO Initialized\n");
1580: rxr->lro_enabled = TRUE;
1581: lro->ifp = adapter->ifp;
1582: }
1583: #endif /* LRO */
1584:
1585: IXGBE_RX_UNLOCK(rxr);
1.28 msaitoh 1586:
1.1 msaitoh 1587: return (0);
1588:
1589: fail:
1590: ixgbe_free_receive_ring(rxr);
1591: IXGBE_RX_UNLOCK(rxr);
1.28 msaitoh 1592:
1.1 msaitoh 1593: return (error);
1.28 msaitoh 1594: } /* ixgbe_setup_receive_ring */
1.1 msaitoh 1595:
1.28 msaitoh 1596: /************************************************************************
1597: * ixgbe_setup_receive_structures - Initialize all receive rings.
1598: ************************************************************************/
1.1 msaitoh 1599: int
1600: ixgbe_setup_receive_structures(struct adapter *adapter)
1601: {
1602: struct rx_ring *rxr = adapter->rx_rings;
1.28 msaitoh 1603: int j;
1.1 msaitoh 1604:
1.30 msaitoh 1605: /*
1606: * Now reinitialize our supply of jumbo mbufs. The number
1607: * or size of jumbo mbufs may have changed.
1608: * Assume all of rxr->ptag are the same.
1609: */
1.41 ! msaitoh 1610: ixgbe_jcl_reinit(adapter, rxr->ptag->dt_dmat,
1.30 msaitoh 1611: (2 * adapter->num_rx_desc) * adapter->num_queues,
1612: adapter->rx_mbuf_sz);
1613:
1.1 msaitoh 1614: for (j = 0; j < adapter->num_queues; j++, rxr++)
1615: if (ixgbe_setup_receive_ring(rxr))
1616: goto fail;
1617:
1618: return (0);
1619: fail:
1620: /*
1621: * Free RX buffers allocated so far, we will only handle
1622: * the rings that completed, the failing case will have
1623: * cleaned up for itself. 'j' failed, so its the terminus.
1624: */
1625: for (int i = 0; i < j; ++i) {
1626: rxr = &adapter->rx_rings[i];
1.27 msaitoh 1627: IXGBE_RX_LOCK(rxr);
1.1 msaitoh 1628: ixgbe_free_receive_ring(rxr);
1.27 msaitoh 1629: IXGBE_RX_UNLOCK(rxr);
1.1 msaitoh 1630: }
1631:
1632: return (ENOBUFS);
1.28 msaitoh 1633: } /* ixgbe_setup_receive_structures */
1.1 msaitoh 1634:
1.3 msaitoh 1635:
1.28 msaitoh 1636: /************************************************************************
1637: * ixgbe_free_receive_structures - Free all receive rings.
1638: ************************************************************************/
1.1 msaitoh 1639: void
1640: ixgbe_free_receive_structures(struct adapter *adapter)
1641: {
1642: struct rx_ring *rxr = adapter->rx_rings;
1643:
1644: INIT_DEBUGOUT("ixgbe_free_receive_structures: begin");
1645:
1646: for (int i = 0; i < adapter->num_queues; i++, rxr++) {
1647: ixgbe_free_receive_buffers(rxr);
1648: #ifdef LRO
1649: /* Free LRO memory */
1.28 msaitoh 1650: tcp_lro_free(&rxr->lro);
1.1 msaitoh 1651: #endif /* LRO */
1652: /* Free the ring memory as well */
1653: ixgbe_dma_free(adapter, &rxr->rxdma);
1654: IXGBE_RX_LOCK_DESTROY(rxr);
1655: }
1656:
1657: free(adapter->rx_rings, M_DEVBUF);
1.28 msaitoh 1658: } /* ixgbe_free_receive_structures */
1.1 msaitoh 1659:
1660:
1.28 msaitoh 1661: /************************************************************************
1662: * ixgbe_free_receive_buffers - Free receive ring data structures
1663: ************************************************************************/
1.1 msaitoh 1664: static void
1665: ixgbe_free_receive_buffers(struct rx_ring *rxr)
1666: {
1.28 msaitoh 1667: struct adapter *adapter = rxr->adapter;
1668: struct ixgbe_rx_buf *rxbuf;
1.1 msaitoh 1669:
1670: INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin");
1671:
1672: /* Cleanup any existing buffers */
1673: if (rxr->rx_buffers != NULL) {
1674: for (int i = 0; i < adapter->num_rx_desc; i++) {
1675: rxbuf = &rxr->rx_buffers[i];
1.27 msaitoh 1676: ixgbe_rx_discard(rxr, i);
1.1 msaitoh 1677: if (rxbuf->pmap != NULL) {
1678: ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap);
1679: rxbuf->pmap = NULL;
1680: }
1681: }
1682: if (rxr->rx_buffers != NULL) {
1683: free(rxr->rx_buffers, M_DEVBUF);
1684: rxr->rx_buffers = NULL;
1685: }
1686: }
1687:
1688: if (rxr->ptag != NULL) {
1689: ixgbe_dma_tag_destroy(rxr->ptag);
1690: rxr->ptag = NULL;
1691: }
1692:
1693: return;
1.28 msaitoh 1694: } /* ixgbe_free_receive_buffers */
1.1 msaitoh 1695:
1.28 msaitoh 1696: /************************************************************************
1697: * ixgbe_rx_input
1698: ************************************************************************/
1.1 msaitoh 1699: static __inline void
1.28 msaitoh 1700: ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m,
1701: u32 ptype)
1.1 msaitoh 1702: {
1.20 msaitoh 1703: struct adapter *adapter = ifp->if_softc;
1.1 msaitoh 1704:
1705: #ifdef LRO
1706: struct ethercom *ec = &adapter->osdep.ec;
1707:
1.28 msaitoh 1708: /*
1709: * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
1710: * should be computed by hardware. Also it should not have VLAN tag in
1711: * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
1712: */
1.1 msaitoh 1713: if (rxr->lro_enabled &&
1714: (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 &&
1715: (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
1716: ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
1717: (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
1718: (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
1719: (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
1720: (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
1721: (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
1722: /*
1723: * Send to the stack if:
1724: ** - LRO not enabled, or
1725: ** - no LRO resources, or
1726: ** - lro enqueue fails
1727: */
1728: if (rxr->lro.lro_cnt != 0)
1729: if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
1730: return;
1731: }
1732: #endif /* LRO */
1733:
1.20 msaitoh 1734: if_percpuq_enqueue(adapter->ipq, m);
1.28 msaitoh 1735: } /* ixgbe_rx_input */
1.1 msaitoh 1736:
1.28 msaitoh 1737: /************************************************************************
1738: * ixgbe_rx_discard
1739: ************************************************************************/
1.1 msaitoh 1740: static __inline void
1741: ixgbe_rx_discard(struct rx_ring *rxr, int i)
1742: {
1.28 msaitoh 1743: struct ixgbe_rx_buf *rbuf;
1.1 msaitoh 1744:
1745: rbuf = &rxr->rx_buffers[i];
1746:
1747: /*
1.28 msaitoh 1748: * With advanced descriptors the writeback
1749: * clobbers the buffer addrs, so its easier
1750: * to just free the existing mbufs and take
1751: * the normal refresh path to get new buffers
1752: * and mapping.
1753: */
1.1 msaitoh 1754:
1.26 msaitoh 1755: if (rbuf->fmp != NULL) {/* Partial chain ? */
1.27 msaitoh 1756: bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0,
1757: rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD);
1.1 msaitoh 1758: m_freem(rbuf->fmp);
1759: rbuf->fmp = NULL;
1760: rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */
1761: } else if (rbuf->buf) {
1.27 msaitoh 1762: bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0,
1763: rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD);
1.1 msaitoh 1764: m_free(rbuf->buf);
1765: rbuf->buf = NULL;
1766: }
1.4 msaitoh 1767: ixgbe_dmamap_unload(rxr->ptag, rbuf->pmap);
1.1 msaitoh 1768:
1769: rbuf->flags = 0;
1770:
1771: return;
1.28 msaitoh 1772: } /* ixgbe_rx_discard */
1.1 msaitoh 1773:
1774:
1.28 msaitoh 1775: /************************************************************************
1776: * ixgbe_rxeof
1.1 msaitoh 1777: *
1.28 msaitoh 1778: * Executes in interrupt context. It replenishes the
1779: * mbufs in the descriptor and sends data which has
1780: * been dma'ed into host memory to upper layer.
1.1 msaitoh 1781: *
1.28 msaitoh 1782: * Return TRUE for more work, FALSE for all clean.
1783: ************************************************************************/
1.1 msaitoh 1784: bool
1785: ixgbe_rxeof(struct ix_queue *que)
1786: {
1787: struct adapter *adapter = que->adapter;
1788: struct rx_ring *rxr = que->rxr;
1789: struct ifnet *ifp = adapter->ifp;
1790: #ifdef LRO
1791: struct lro_ctrl *lro = &rxr->lro;
1792: #endif /* LRO */
1.28 msaitoh 1793: union ixgbe_adv_rx_desc *cur;
1794: struct ixgbe_rx_buf *rbuf, *nbuf;
1.1 msaitoh 1795: int i, nextp, processed = 0;
1796: u32 staterr = 0;
1.7 msaitoh 1797: u32 count = adapter->rx_process_limit;
1.1 msaitoh 1798: #ifdef RSS
1799: u16 pkt_info;
1800: #endif
1801:
1802: IXGBE_RX_LOCK(rxr);
1803:
1804: #ifdef DEV_NETMAP
1.28 msaitoh 1805: if (adapter->feat_en & IXGBE_FEATURE_NETMAP) {
1806: /* Same as the txeof routine: wakeup clients on intr. */
1807: if (netmap_rx_irq(ifp, rxr->me, &processed)) {
1808: IXGBE_RX_UNLOCK(rxr);
1809: return (FALSE);
1810: }
1.1 msaitoh 1811: }
1812: #endif /* DEV_NETMAP */
1813:
1814: for (i = rxr->next_to_check; count != 0;) {
1.28 msaitoh 1815: struct mbuf *sendmp, *mp;
1816: u32 rsc, ptype;
1817: u16 len;
1818: u16 vtag = 0;
1819: bool eop;
1.1 msaitoh 1820:
1821: /* Sync the ring. */
1822: ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
1823: BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1824:
1825: cur = &rxr->rx_base[i];
1826: staterr = le32toh(cur->wb.upper.status_error);
1827: #ifdef RSS
1828: pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
1829: #endif
1830:
1831: if ((staterr & IXGBE_RXD_STAT_DD) == 0)
1832: break;
1833: if ((ifp->if_flags & IFF_RUNNING) == 0)
1834: break;
1835:
1836: count--;
1837: sendmp = NULL;
1838: nbuf = NULL;
1839: rsc = 0;
1840: cur->wb.upper.status_error = 0;
1841: rbuf = &rxr->rx_buffers[i];
1842: mp = rbuf->buf;
1843:
1844: len = le16toh(cur->wb.upper.length);
1845: ptype = le32toh(cur->wb.lower.lo_dword.data) &
1846: IXGBE_RXDADV_PKTTYPE_MASK;
1847: eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
1848:
1849: /* Make sure bad packets are discarded */
1850: if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) {
1.3 msaitoh 1851: #if __FreeBSD_version >= 1100036
1.28 msaitoh 1852: if (adapter->feat_en & IXGBE_FEATURE_VF)
1.4 msaitoh 1853: if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
1.3 msaitoh 1854: #endif
1.1 msaitoh 1855: rxr->rx_discarded.ev_count++;
1856: ixgbe_rx_discard(rxr, i);
1857: goto next_desc;
1858: }
1859:
1.27 msaitoh 1860: bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0,
1861: rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD);
1862:
1.1 msaitoh 1863: /*
1.28 msaitoh 1864: * On 82599 which supports a hardware
1865: * LRO (called HW RSC), packets need
1866: * not be fragmented across sequential
1867: * descriptors, rather the next descriptor
1868: * is indicated in bits of the descriptor.
1869: * This also means that we might proceses
1870: * more than one packet at a time, something
1871: * that has never been true before, it
1872: * required eliminating global chain pointers
1873: * in favor of what we are doing here. -jfv
1874: */
1.1 msaitoh 1875: if (!eop) {
1876: /*
1.28 msaitoh 1877: * Figure out the next descriptor
1878: * of this frame.
1879: */
1.1 msaitoh 1880: if (rxr->hw_rsc == TRUE) {
1881: rsc = ixgbe_rsc_count(cur);
1882: rxr->rsc_num += (rsc - 1);
1883: }
1884: if (rsc) { /* Get hardware index */
1.28 msaitoh 1885: nextp = ((staterr & IXGBE_RXDADV_NEXTP_MASK) >>
1.1 msaitoh 1886: IXGBE_RXDADV_NEXTP_SHIFT);
1887: } else { /* Just sequential */
1888: nextp = i + 1;
1889: if (nextp == adapter->num_rx_desc)
1890: nextp = 0;
1891: }
1892: nbuf = &rxr->rx_buffers[nextp];
1893: prefetch(nbuf);
1894: }
1895: /*
1.28 msaitoh 1896: * Rather than using the fmp/lmp global pointers
1897: * we now keep the head of a packet chain in the
1898: * buffer struct and pass this along from one
1899: * descriptor to the next, until we get EOP.
1900: */
1.1 msaitoh 1901: mp->m_len = len;
1902: /*
1.28 msaitoh 1903: * See if there is a stored head
1904: * that determines what we are
1905: */
1.1 msaitoh 1906: sendmp = rbuf->fmp;
1907: if (sendmp != NULL) { /* secondary frag */
1908: rbuf->buf = rbuf->fmp = NULL;
1909: mp->m_flags &= ~M_PKTHDR;
1910: sendmp->m_pkthdr.len += mp->m_len;
1911: } else {
1912: /*
1913: * Optimize. This might be a small packet,
1914: * maybe just a TCP ACK. Do a fast copy that
1915: * is cache aligned into a new mbuf, and
1916: * leave the old mbuf+cluster for re-use.
1917: */
1918: if (eop && len <= IXGBE_RX_COPY_LEN) {
1919: sendmp = m_gethdr(M_NOWAIT, MT_DATA);
1920: if (sendmp != NULL) {
1.28 msaitoh 1921: sendmp->m_data += IXGBE_RX_COPY_ALIGN;
1922: ixgbe_bcopy(mp->m_data, sendmp->m_data,
1923: len);
1.1 msaitoh 1924: sendmp->m_len = len;
1925: rxr->rx_copies.ev_count++;
1926: rbuf->flags |= IXGBE_RX_COPY;
1927: }
1928: }
1929: if (sendmp == NULL) {
1930: rbuf->buf = rbuf->fmp = NULL;
1931: sendmp = mp;
1932: }
1933:
1934: /* first desc of a non-ps chain */
1935: sendmp->m_flags |= M_PKTHDR;
1936: sendmp->m_pkthdr.len = mp->m_len;
1937: }
1938: ++processed;
1939:
1940: /* Pass the head pointer on */
1941: if (eop == 0) {
1942: nbuf->fmp = sendmp;
1943: sendmp = NULL;
1944: mp->m_next = nbuf->buf;
1945: } else { /* Sending this frame */
1946: m_set_rcvif(sendmp, ifp);
1.31 msaitoh 1947: ++rxr->packets;
1.1 msaitoh 1948: rxr->rx_packets.ev_count++;
1949: /* capture data for AIM */
1950: rxr->bytes += sendmp->m_pkthdr.len;
1951: rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len;
1952: /* Process vlan info */
1.28 msaitoh 1953: if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
1.1 msaitoh 1954: vtag = le16toh(cur->wb.upper.vlan);
1955: if (vtag) {
1.29 knakahar 1956: vlan_set_tag(sendmp, vtag);
1.1 msaitoh 1957: }
1958: if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
1959: ixgbe_rx_checksum(staterr, sendmp, ptype,
1.3 msaitoh 1960: &adapter->stats.pf);
1.1 msaitoh 1961: }
1.8 msaitoh 1962:
1.6 msaitoh 1963: #if 0 /* FreeBSD */
1.28 msaitoh 1964: /*
1965: * In case of multiqueue, we have RXCSUM.PCSD bit set
1966: * and never cleared. This means we have RSS hash
1967: * available to be used.
1968: */
1969: if (adapter->num_queues > 1) {
1970: sendmp->m_pkthdr.flowid =
1971: le32toh(cur->wb.lower.hi_dword.rss);
1972: switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) {
1973: case IXGBE_RXDADV_RSSTYPE_IPV4:
1974: M_HASHTYPE_SET(sendmp,
1975: M_HASHTYPE_RSS_IPV4);
1976: break;
1977: case IXGBE_RXDADV_RSSTYPE_IPV4_TCP:
1978: M_HASHTYPE_SET(sendmp,
1979: M_HASHTYPE_RSS_TCP_IPV4);
1980: break;
1981: case IXGBE_RXDADV_RSSTYPE_IPV6:
1982: M_HASHTYPE_SET(sendmp,
1983: M_HASHTYPE_RSS_IPV6);
1984: break;
1985: case IXGBE_RXDADV_RSSTYPE_IPV6_TCP:
1986: M_HASHTYPE_SET(sendmp,
1987: M_HASHTYPE_RSS_TCP_IPV6);
1988: break;
1989: case IXGBE_RXDADV_RSSTYPE_IPV6_EX:
1990: M_HASHTYPE_SET(sendmp,
1991: M_HASHTYPE_RSS_IPV6_EX);
1992: break;
1993: case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX:
1994: M_HASHTYPE_SET(sendmp,
1995: M_HASHTYPE_RSS_TCP_IPV6_EX);
1996: break;
1.6 msaitoh 1997: #if __FreeBSD_version > 1100000
1.28 msaitoh 1998: case IXGBE_RXDADV_RSSTYPE_IPV4_UDP:
1999: M_HASHTYPE_SET(sendmp,
2000: M_HASHTYPE_RSS_UDP_IPV4);
2001: break;
2002: case IXGBE_RXDADV_RSSTYPE_IPV6_UDP:
2003: M_HASHTYPE_SET(sendmp,
2004: M_HASHTYPE_RSS_UDP_IPV6);
2005: break;
2006: case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX:
2007: M_HASHTYPE_SET(sendmp,
2008: M_HASHTYPE_RSS_UDP_IPV6_EX);
2009: break;
2010: #endif
2011: default:
2012: M_HASHTYPE_SET(sendmp,
2013: M_HASHTYPE_OPAQUE_HASH);
2014: }
2015: } else {
2016: sendmp->m_pkthdr.flowid = que->msix;
1.1 msaitoh 2017: M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
2018: }
1.8 msaitoh 2019: #endif
1.1 msaitoh 2020: }
2021: next_desc:
2022: ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
2023: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2024:
2025: /* Advance our pointers to the next descriptor. */
2026: if (++i == rxr->num_desc)
2027: i = 0;
2028:
2029: /* Now send to the stack or do LRO */
2030: if (sendmp != NULL) {
2031: rxr->next_to_check = i;
1.28 msaitoh 2032: IXGBE_RX_UNLOCK(rxr);
1.1 msaitoh 2033: ixgbe_rx_input(rxr, ifp, sendmp, ptype);
1.28 msaitoh 2034: IXGBE_RX_LOCK(rxr);
1.1 msaitoh 2035: i = rxr->next_to_check;
2036: }
2037:
1.28 msaitoh 2038: /* Every 8 descriptors we go to refresh mbufs */
1.1 msaitoh 2039: if (processed == 8) {
2040: ixgbe_refresh_mbufs(rxr, i);
2041: processed = 0;
2042: }
2043: }
2044:
2045: /* Refresh any remaining buf structs */
2046: if (ixgbe_rx_unrefreshed(rxr))
2047: ixgbe_refresh_mbufs(rxr, i);
2048:
2049: rxr->next_to_check = i;
2050:
1.28 msaitoh 2051: IXGBE_RX_UNLOCK(rxr);
2052:
1.1 msaitoh 2053: #ifdef LRO
2054: /*
2055: * Flush any outstanding LRO work
2056: */
1.10 msaitoh 2057: tcp_lro_flush_all(lro);
1.1 msaitoh 2058: #endif /* LRO */
2059:
2060: /*
1.28 msaitoh 2061: * Still have cleaning to do?
2062: */
1.1 msaitoh 2063: if ((staterr & IXGBE_RXD_STAT_DD) != 0)
1.28 msaitoh 2064: return (TRUE);
2065:
2066: return (FALSE);
2067: } /* ixgbe_rxeof */
1.1 msaitoh 2068:
2069:
1.28 msaitoh 2070: /************************************************************************
2071: * ixgbe_rx_checksum
1.1 msaitoh 2072: *
1.28 msaitoh 2073: * Verify that the hardware indicated that the checksum is valid.
2074: * Inform the stack about the status of checksum so that stack
2075: * doesn't spend time verifying the checksum.
2076: ************************************************************************/
1.1 msaitoh 2077: static void
2078: ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype,
2079: struct ixgbe_hw_stats *stats)
2080: {
1.28 msaitoh 2081: u16 status = (u16)staterr;
2082: u8 errors = (u8)(staterr >> 24);
1.1 msaitoh 2083: #if 0
1.28 msaitoh 2084: bool sctp = false;
1.1 msaitoh 2085:
2086: if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
2087: (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
1.8 msaitoh 2088: sctp = true;
1.1 msaitoh 2089: #endif
2090:
1.8 msaitoh 2091: /* IPv4 checksum */
1.1 msaitoh 2092: if (status & IXGBE_RXD_STAT_IPCS) {
2093: stats->ipcs.ev_count++;
2094: if (!(errors & IXGBE_RXD_ERR_IPE)) {
2095: /* IP Checksum Good */
2096: mp->m_pkthdr.csum_flags = M_CSUM_IPv4;
2097: } else {
2098: stats->ipcs_bad.ev_count++;
2099: mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD;
2100: }
2101: }
1.8 msaitoh 2102: /* TCP/UDP/SCTP checksum */
1.1 msaitoh 2103: if (status & IXGBE_RXD_STAT_L4CS) {
2104: stats->l4cs.ev_count++;
2105: int type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6;
2106: if (!(errors & IXGBE_RXD_ERR_TCPE)) {
2107: mp->m_pkthdr.csum_flags |= type;
2108: } else {
2109: stats->l4cs_bad.ev_count++;
2110: mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD;
2111: }
2112: }
1.28 msaitoh 2113: } /* ixgbe_rx_checksum */
1.1 msaitoh 2114:
1.28 msaitoh 2115: /************************************************************************
2116: * ixgbe_dma_malloc
2117: ************************************************************************/
1.1 msaitoh 2118: int
2119: ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size,
2120: struct ixgbe_dma_alloc *dma, const int mapflags)
2121: {
2122: device_t dev = adapter->dev;
1.28 msaitoh 2123: int r, rsegs;
1.1 msaitoh 2124:
1.28 msaitoh 2125: r = ixgbe_dma_tag_create(
2126: /* parent */ adapter->osdep.dmat,
2127: /* alignment */ DBA_ALIGN,
2128: /* bounds */ 0,
2129: /* maxsize */ size,
2130: /* nsegments */ 1,
2131: /* maxsegsize */ size,
2132: /* flags */ BUS_DMA_ALLOCNOW,
1.1 msaitoh 2133: &dma->dma_tag);
2134: if (r != 0) {
2135: aprint_error_dev(dev,
2136: "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r);
2137: goto fail_0;
2138: }
2139:
1.28 msaitoh 2140: r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, size,
2141: dma->dma_tag->dt_alignment, dma->dma_tag->dt_boundary,
2142: &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT);
1.1 msaitoh 2143: if (r != 0) {
2144: aprint_error_dev(dev,
2145: "%s: bus_dmamem_alloc failed; error %d\n", __func__, r);
2146: goto fail_1;
2147: }
2148:
2149: r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs,
2150: size, &dma->dma_vaddr, BUS_DMA_NOWAIT);
2151: if (r != 0) {
2152: aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n",
2153: __func__, r);
2154: goto fail_2;
2155: }
2156:
2157: r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map);
2158: if (r != 0) {
2159: aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n",
2160: __func__, r);
2161: goto fail_3;
2162: }
2163:
1.28 msaitoh 2164: r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map,
2165: dma->dma_vaddr, size, NULL, mapflags | BUS_DMA_NOWAIT);
1.1 msaitoh 2166: if (r != 0) {
2167: aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n",
2168: __func__, r);
2169: goto fail_4;
2170: }
2171: dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
2172: dma->dma_size = size;
2173: return 0;
2174: fail_4:
2175: ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map);
2176: fail_3:
2177: bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size);
2178: fail_2:
2179: bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs);
2180: fail_1:
2181: ixgbe_dma_tag_destroy(dma->dma_tag);
2182: fail_0:
2183:
1.28 msaitoh 2184: return (r);
2185: } /* ixgbe_dma_malloc */
2186:
2187: /************************************************************************
2188: * ixgbe_dma_free
2189: ************************************************************************/
1.3 msaitoh 2190: void
1.1 msaitoh 2191: ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2192: {
2193: bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size,
2194: BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2195: ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map);
2196: bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1);
2197: ixgbe_dma_tag_destroy(dma->dma_tag);
1.28 msaitoh 2198: } /* ixgbe_dma_free */
1.1 msaitoh 2199:
2200:
1.28 msaitoh 2201: /************************************************************************
2202: * ixgbe_allocate_queues
1.1 msaitoh 2203: *
1.28 msaitoh 2204: * Allocate memory for the transmit and receive rings, and then
2205: * the descriptors associated with each, called only once at attach.
2206: ************************************************************************/
1.1 msaitoh 2207: int
2208: ixgbe_allocate_queues(struct adapter *adapter)
2209: {
2210: device_t dev = adapter->dev;
2211: struct ix_queue *que;
2212: struct tx_ring *txr;
2213: struct rx_ring *rxr;
1.28 msaitoh 2214: int rsize, tsize, error = IXGBE_SUCCESS;
2215: int txconf = 0, rxconf = 0;
1.1 msaitoh 2216:
1.28 msaitoh 2217: /* First, allocate the top level queue structs */
2218: adapter->queues = (struct ix_queue *)malloc(sizeof(struct ix_queue) *
2219: adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO);
2220: if (adapter->queues == NULL) {
2221: aprint_error_dev(dev, "Unable to allocate queue memory\n");
1.1 msaitoh 2222: error = ENOMEM;
2223: goto fail;
2224: }
2225:
1.28 msaitoh 2226: /* Second, allocate the TX ring struct memory */
2227: adapter->tx_rings = (struct tx_ring *)malloc(sizeof(struct tx_ring) *
2228: adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO);
2229: if (adapter->tx_rings == NULL) {
1.1 msaitoh 2230: aprint_error_dev(dev, "Unable to allocate TX ring memory\n");
2231: error = ENOMEM;
2232: goto tx_fail;
2233: }
2234:
1.28 msaitoh 2235: /* Third, allocate the RX ring */
2236: adapter->rx_rings = (struct rx_ring *)malloc(sizeof(struct rx_ring) *
2237: adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO);
2238: if (adapter->rx_rings == NULL) {
1.1 msaitoh 2239: aprint_error_dev(dev, "Unable to allocate RX ring memory\n");
2240: error = ENOMEM;
2241: goto rx_fail;
2242: }
2243:
2244: /* For the ring itself */
1.28 msaitoh 2245: tsize = roundup2(adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc),
2246: DBA_ALIGN);
1.1 msaitoh 2247:
2248: /*
2249: * Now set up the TX queues, txconf is needed to handle the
2250: * possibility that things fail midcourse and we need to
2251: * undo memory gracefully
1.28 msaitoh 2252: */
1.1 msaitoh 2253: for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2254: /* Set up some basics */
2255: txr = &adapter->tx_rings[i];
2256: txr->adapter = adapter;
1.28 msaitoh 2257: txr->txr_interq = NULL;
2258: /* In case SR-IOV is enabled, align the index properly */
1.5 msaitoh 2259: #ifdef PCI_IOV
1.28 msaitoh 2260: txr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool,
2261: i);
1.5 msaitoh 2262: #else
1.1 msaitoh 2263: txr->me = i;
1.5 msaitoh 2264: #endif
1.1 msaitoh 2265: txr->num_desc = adapter->num_tx_desc;
2266:
2267: /* Initialize the TX side lock */
2268: mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET);
2269:
1.28 msaitoh 2270: if (ixgbe_dma_malloc(adapter, tsize, &txr->txdma,
2271: BUS_DMA_NOWAIT)) {
1.1 msaitoh 2272: aprint_error_dev(dev,
2273: "Unable to allocate TX Descriptor memory\n");
2274: error = ENOMEM;
2275: goto err_tx_desc;
2276: }
2277: txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2278: bzero((void *)txr->tx_base, tsize);
2279:
1.28 msaitoh 2280: /* Now allocate transmit buffers for the ring */
2281: if (ixgbe_allocate_transmit_buffers(txr)) {
1.1 msaitoh 2282: aprint_error_dev(dev,
2283: "Critical Failure setting up transmit buffers\n");
2284: error = ENOMEM;
2285: goto err_tx_desc;
2286: }
1.28 msaitoh 2287: if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) {
2288: /* Allocate a buf ring */
2289: txr->txr_interq = pcq_create(IXGBE_BR_SIZE, KM_SLEEP);
2290: if (txr->txr_interq == NULL) {
2291: aprint_error_dev(dev,
2292: "Critical Failure setting up buf ring\n");
2293: error = ENOMEM;
2294: goto err_tx_desc;
2295: }
2296: }
1.1 msaitoh 2297: }
2298:
2299: /*
2300: * Next the RX queues...
2301: */
1.28 msaitoh 2302: rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc),
2303: DBA_ALIGN);
1.1 msaitoh 2304: for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2305: rxr = &adapter->rx_rings[i];
2306: /* Set up some basics */
2307: rxr->adapter = adapter;
1.5 msaitoh 2308: #ifdef PCI_IOV
1.28 msaitoh 2309: /* In case SR-IOV is enabled, align the index properly */
2310: rxr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool,
2311: i);
1.5 msaitoh 2312: #else
1.1 msaitoh 2313: rxr->me = i;
1.5 msaitoh 2314: #endif
1.1 msaitoh 2315: rxr->num_desc = adapter->num_rx_desc;
2316:
2317: /* Initialize the RX side lock */
2318: mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET);
2319:
1.28 msaitoh 2320: if (ixgbe_dma_malloc(adapter, rsize, &rxr->rxdma,
2321: BUS_DMA_NOWAIT)) {
1.1 msaitoh 2322: aprint_error_dev(dev,
2323: "Unable to allocate RxDescriptor memory\n");
2324: error = ENOMEM;
2325: goto err_rx_desc;
2326: }
2327: rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2328: bzero((void *)rxr->rx_base, rsize);
2329:
1.28 msaitoh 2330: /* Allocate receive buffers for the ring */
1.1 msaitoh 2331: if (ixgbe_allocate_receive_buffers(rxr)) {
2332: aprint_error_dev(dev,
2333: "Critical Failure setting up receive buffers\n");
2334: error = ENOMEM;
2335: goto err_rx_desc;
2336: }
2337: }
2338:
2339: /*
1.28 msaitoh 2340: * Finally set up the queue holding structs
2341: */
1.1 msaitoh 2342: for (int i = 0; i < adapter->num_queues; i++) {
2343: que = &adapter->queues[i];
2344: que->adapter = adapter;
1.3 msaitoh 2345: que->me = i;
1.1 msaitoh 2346: que->txr = &adapter->tx_rings[i];
2347: que->rxr = &adapter->rx_rings[i];
1.33 knakahar 2348:
1.37 knakahar 2349: mutex_init(&que->dc_mtx, MUTEX_DEFAULT, IPL_NET);
2350: que->disabled_count = 0;
1.1 msaitoh 2351: }
2352:
2353: return (0);
2354:
2355: err_rx_desc:
2356: for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2357: ixgbe_dma_free(adapter, &rxr->rxdma);
2358: err_tx_desc:
2359: for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2360: ixgbe_dma_free(adapter, &txr->txdma);
2361: free(adapter->rx_rings, M_DEVBUF);
2362: rx_fail:
2363: free(adapter->tx_rings, M_DEVBUF);
2364: tx_fail:
2365: free(adapter->queues, M_DEVBUF);
2366: fail:
2367: return (error);
1.28 msaitoh 2368: } /* ixgbe_allocate_queues */
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to ix_txrx.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / dev / pci / ixgbe