Annotation of src/sys/dev/pci/if_vioif.c, Revision 1.56
1.56 ! yamaguch 1: /* $NetBSD: if_vioif.c,v 1.55 2020/05/25 08:16:23 yamaguchi Exp $ */
1.1 hannken 2:
3: /*
4: * Copyright (c) 2010 Minoura Makoto.
5: * All rights reserved.
6: *
7: * Redistribution and use in source and binary forms, with or without
8: * modification, are permitted provided that the following conditions
9: * are met:
10: * 1. Redistributions of source code must retain the above copyright
11: * notice, this list of conditions and the following disclaimer.
12: * 2. Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in the
14: * documentation and/or other materials provided with the distribution.
15: *
16: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26: */
27:
28: #include <sys/cdefs.h>
1.56 ! yamaguch 29: __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.55 2020/05/25 08:16:23 yamaguchi Exp $");
1.15 ozaki-r 30:
31: #ifdef _KERNEL_OPT
32: #include "opt_net_mpsafe.h"
33: #endif
1.1 hannken 34:
35: #include <sys/param.h>
36: #include <sys/systm.h>
37: #include <sys/kernel.h>
1.55 yamaguch 38: #include <sys/atomic.h>
1.1 hannken 39: #include <sys/bus.h>
40: #include <sys/condvar.h>
41: #include <sys/device.h>
42: #include <sys/intr.h>
43: #include <sys/kmem.h>
44: #include <sys/mbuf.h>
45: #include <sys/mutex.h>
46: #include <sys/sockio.h>
1.12 ozaki-r 47: #include <sys/cpu.h>
1.26 pgoyette 48: #include <sys/module.h>
1.46 yamaguch 49: #include <sys/pcq.h>
1.55 yamaguch 50: #include <sys/workqueue.h>
1.1 hannken 51:
52: #include <dev/pci/virtioreg.h>
53: #include <dev/pci/virtiovar.h>
54:
55: #include <net/if.h>
56: #include <net/if_media.h>
57: #include <net/if_ether.h>
58:
59: #include <net/bpf.h>
60:
1.26 pgoyette 61: #include "ioconf.h"
1.1 hannken 62:
1.7 ozaki-r 63: #ifdef NET_MPSAFE
64: #define VIOIF_MPSAFE 1
1.46 yamaguch 65: #define VIOIF_MULTIQ 1
1.7 ozaki-r 66: #endif
67:
1.1 hannken 68: /*
69: * if_vioifreg.h:
70: */
71: /* Configuration registers */
72: #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
73: #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
1.46 yamaguch 74: #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
1.1 hannken 75:
76: /* Feature bits */
1.46 yamaguch 77: #define VIRTIO_NET_F_CSUM __BIT(0)
78: #define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
79: #define VIRTIO_NET_F_MAC __BIT(5)
80: #define VIRTIO_NET_F_GSO __BIT(6)
81: #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
82: #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
83: #define VIRTIO_NET_F_GUEST_ECN __BIT(9)
84: #define VIRTIO_NET_F_GUEST_UFO __BIT(10)
85: #define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
86: #define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
87: #define VIRTIO_NET_F_HOST_ECN __BIT(13)
88: #define VIRTIO_NET_F_HOST_UFO __BIT(14)
89: #define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
90: #define VIRTIO_NET_F_STATUS __BIT(16)
91: #define VIRTIO_NET_F_CTRL_VQ __BIT(17)
92: #define VIRTIO_NET_F_CTRL_RX __BIT(18)
93: #define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
94: #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
95: #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
96: #define VIRTIO_NET_F_MQ __BIT(22)
1.1 hannken 97:
1.18 christos 98: #define VIRTIO_NET_FLAG_BITS \
99: VIRTIO_COMMON_FLAG_BITS \
1.46 yamaguch 100: "\x17""MQ" \
101: "\x16""GUEST_ANNOUNCE" \
102: "\x15""CTRL_RX_EXTRA" \
1.18 christos 103: "\x14""CTRL_VLAN" \
104: "\x13""CTRL_RX" \
105: "\x12""CTRL_VQ" \
106: "\x11""STATUS" \
107: "\x10""MRG_RXBUF" \
108: "\x0f""HOST_UFO" \
109: "\x0e""HOST_ECN" \
110: "\x0d""HOST_TSO6" \
111: "\x0c""HOST_TSO4" \
112: "\x0b""GUEST_UFO" \
113: "\x0a""GUEST_ECN" \
114: "\x09""GUEST_TSO6" \
115: "\x08""GUEST_TSO4" \
116: "\x07""GSO" \
117: "\x06""MAC" \
118: "\x02""GUEST_CSUM" \
119: "\x01""CSUM"
120:
1.1 hannken 121: /* Status */
122: #define VIRTIO_NET_S_LINK_UP 1
123:
124: /* Packet header structure */
125: struct virtio_net_hdr {
126: uint8_t flags;
127: uint8_t gso_type;
128: uint16_t hdr_len;
129: uint16_t gso_size;
130: uint16_t csum_start;
131: uint16_t csum_offset;
132: #if 0
133: uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */
134: #endif
135: } __packed;
136:
137: #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
138: #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
139: #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
140: #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
141: #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
142: #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
143:
144: #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN)
145:
146: /* Control virtqueue */
147: struct virtio_net_ctrl_cmd {
148: uint8_t class;
149: uint8_t command;
150: } __packed;
151: #define VIRTIO_NET_CTRL_RX 0
152: # define VIRTIO_NET_CTRL_RX_PROMISC 0
153: # define VIRTIO_NET_CTRL_RX_ALLMULTI 1
154:
155: #define VIRTIO_NET_CTRL_MAC 1
156: # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
157:
158: #define VIRTIO_NET_CTRL_VLAN 2
159: # define VIRTIO_NET_CTRL_VLAN_ADD 0
160: # define VIRTIO_NET_CTRL_VLAN_DEL 1
161:
1.46 yamaguch 162: #define VIRTIO_NET_CTRL_MQ 4
163: # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
164: # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
165: # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
166:
1.1 hannken 167: struct virtio_net_ctrl_status {
168: uint8_t ack;
169: } __packed;
170: #define VIRTIO_NET_OK 0
171: #define VIRTIO_NET_ERR 1
172:
173: struct virtio_net_ctrl_rx {
174: uint8_t onoff;
175: } __packed;
176:
177: struct virtio_net_ctrl_mac_tbl {
178: uint32_t nentries;
179: uint8_t macs[][ETHER_ADDR_LEN];
180: } __packed;
181:
182: struct virtio_net_ctrl_vlan {
183: uint16_t id;
184: } __packed;
185:
1.46 yamaguch 186: struct virtio_net_ctrl_mq {
187: uint16_t virtqueue_pairs;
188: } __packed;
189:
1.44 yamaguch 190: struct vioif_ctrl_cmdspec {
191: bus_dmamap_t dmamap;
192: void *buf;
193: bus_size_t bufsize;
194: };
1.1 hannken 195:
196: /*
197: * if_vioifvar.h:
198: */
1.43 yamaguch 199:
200: /*
201: * Locking notes:
202: * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and
203: * a filds in vioif_rxqueue is protected by rxq_lock (a spin mutex).
204: * - more than one lock cannot be held at onece
205: * + ctrlq_inuse is protected by ctrlq_wait_lock.
206: * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
207: * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock
208: */
209:
1.55 yamaguch 210: struct vioif_work {
211: struct work cookie;
212: void (*func)(void *);
213: void *arg;
214: unsigned int added;
215: };
216:
1.43 yamaguch 217: struct vioif_txqueue {
218: kmutex_t *txq_lock; /* lock for tx operations */
219:
220: struct virtqueue *txq_vq;
221: bool txq_stopping;
222: bool txq_link_active;
1.46 yamaguch 223: pcq_t *txq_intrq;
1.43 yamaguch 224:
225: struct virtio_net_hdr *txq_hdrs;
226: bus_dmamap_t *txq_hdr_dmamaps;
227:
228: struct mbuf **txq_mbufs;
229: bus_dmamap_t *txq_dmamaps;
1.46 yamaguch 230:
231: void *txq_deferred_transmit;
1.55 yamaguch 232: void *txq_handle_si;
233: struct vioif_work txq_work;
234: bool txq_workqueue;
235: bool txq_active;
1.43 yamaguch 236: };
237:
238: struct vioif_rxqueue {
239: kmutex_t *rxq_lock; /* lock for rx operations */
240:
241: struct virtqueue *rxq_vq;
242: bool rxq_stopping;
243:
244: struct virtio_net_hdr *rxq_hdrs;
245: bus_dmamap_t *rxq_hdr_dmamaps;
246:
247: struct mbuf **rxq_mbufs;
248: bus_dmamap_t *rxq_dmamaps;
249:
250: void *rxq_softint;
1.55 yamaguch 251: void *rxq_handle_si;
252: struct vioif_work rxq_work;
253: bool rxq_workqueue;
254: bool rxq_active;
1.43 yamaguch 255: };
256:
257: struct vioif_ctrlqueue {
258: struct virtqueue *ctrlq_vq;
259: enum {
260: FREE, INUSE, DONE
261: } ctrlq_inuse;
262: kcondvar_t ctrlq_wait;
263: kmutex_t ctrlq_wait_lock;
1.44 yamaguch 264: struct lwp *ctrlq_owner;
1.43 yamaguch 265:
266: struct virtio_net_ctrl_cmd *ctrlq_cmd;
267: struct virtio_net_ctrl_status *ctrlq_status;
268: struct virtio_net_ctrl_rx *ctrlq_rx;
269: struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
270: struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
1.46 yamaguch 271: struct virtio_net_ctrl_mq *ctrlq_mq;
1.43 yamaguch 272:
273: bus_dmamap_t ctrlq_cmd_dmamap;
274: bus_dmamap_t ctrlq_status_dmamap;
275: bus_dmamap_t ctrlq_rx_dmamap;
276: bus_dmamap_t ctrlq_tbl_uc_dmamap;
277: bus_dmamap_t ctrlq_tbl_mc_dmamap;
1.46 yamaguch 278: bus_dmamap_t ctrlq_mq_dmamap;
1.43 yamaguch 279: };
280:
1.1 hannken 281: struct vioif_softc {
282: device_t sc_dev;
1.55 yamaguch 283: struct sysctllog *sc_sysctllog;
1.1 hannken 284:
285: struct virtio_softc *sc_virtio;
1.46 yamaguch 286: struct virtqueue *sc_vqs;
287:
288: int sc_max_nvq_pairs;
289: int sc_req_nvq_pairs;
290: int sc_act_nvq_pairs;
1.1 hannken 291:
292: uint8_t sc_mac[ETHER_ADDR_LEN];
293: struct ethercom sc_ethercom;
1.8 pooka 294: short sc_deferred_init_done;
1.34 ozaki-r 295: bool sc_link_active;
1.1 hannken 296:
1.46 yamaguch 297: struct vioif_txqueue *sc_txq;
298: struct vioif_rxqueue *sc_rxq;
1.43 yamaguch 299:
300: bool sc_has_ctrl;
301: struct vioif_ctrlqueue sc_ctrlq;
302:
1.1 hannken 303: bus_dma_segment_t sc_hdr_segs[1];
1.43 yamaguch 304: void *sc_dmamem;
305: void *sc_kmem;
1.1 hannken 306:
1.34 ozaki-r 307: void *sc_ctl_softint;
1.55 yamaguch 308:
309: struct workqueue *sc_txrx_workqueue;
310: bool sc_txrx_workqueue_sysctl;
311: u_int sc_tx_intr_process_limit;
312: u_int sc_tx_process_limit;
313: u_int sc_rx_intr_process_limit;
314: u_int sc_rx_process_limit;
1.1 hannken 315: };
316: #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
317: #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
318:
1.55 yamaguch 319: #define VIOIF_TX_INTR_PROCESS_LIMIT 256
320: #define VIOIF_TX_PROCESS_LIMIT 256
321: #define VIOIF_RX_INTR_PROCESS_LIMIT 0U
322: #define VIOIF_RX_PROCESS_LIMIT 256
323:
324: #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET
325:
1.1 hannken 326: /* cfattach interface functions */
327: static int vioif_match(device_t, cfdata_t, void *);
328: static void vioif_attach(device_t, device_t, void *);
329: static void vioif_deferred_init(device_t);
1.55 yamaguch 330: static int vioif_finalize_teardown(device_t);
1.1 hannken 331:
332: /* ifnet interface functions */
333: static int vioif_init(struct ifnet *);
334: static void vioif_stop(struct ifnet *, int);
335: static void vioif_start(struct ifnet *);
1.46 yamaguch 336: static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
337: static int vioif_transmit(struct ifnet *, struct mbuf *);
338: static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
1.1 hannken 339: static int vioif_ioctl(struct ifnet *, u_long, void *);
340: static void vioif_watchdog(struct ifnet *);
341:
342: /* rx */
1.46 yamaguch 343: static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
344: static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
345: static void vioif_populate_rx_mbufs(struct vioif_rxqueue *);
346: static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *);
1.55 yamaguch 347: static void vioif_rx_queue_clear(struct vioif_rxqueue *);
348: static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *,
349: struct vioif_rxqueue *, u_int);
1.54 yamaguch 350: static int vioif_rx_intr(void *);
1.55 yamaguch 351: static void vioif_rx_handle(void *);
352: static void vioif_rx_sched_handle(struct vioif_softc *,
353: struct vioif_rxqueue *);
1.1 hannken 354: static void vioif_rx_softint(void *);
1.46 yamaguch 355: static void vioif_rx_drain(struct vioif_rxqueue *);
1.1 hannken 356:
357: /* tx */
1.54 yamaguch 358: static int vioif_tx_intr(void *);
1.55 yamaguch 359: static void vioif_tx_handle(void *);
360: static void vioif_tx_sched_handle(struct vioif_softc *,
361: struct vioif_txqueue *);
362: static void vioif_tx_queue_clear(struct vioif_txqueue *);
363: static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *,
364: struct vioif_txqueue *, u_int);
1.46 yamaguch 365: static void vioif_tx_drain(struct vioif_txqueue *);
366: static void vioif_deferred_transmit(void *);
1.1 hannken 367:
1.55 yamaguch 368: /* workqueue */
369: static struct workqueue*
370: vioif_workq_create(const char *, pri_t, int, int);
371: static void vioif_workq_destroy(struct workqueue *);
372: static void vioif_workq_work(struct work *, void *);
373: static void vioif_work_set(struct vioif_work *, void(*)(void *), void *);
374: static void vioif_work_add(struct workqueue *, struct vioif_work *);
375: static void vioif_work_wait(struct workqueue *, struct vioif_work *);
376:
1.1 hannken 377: /* other control */
1.34 ozaki-r 378: static bool vioif_is_link_up(struct vioif_softc *);
379: static void vioif_update_link_status(struct vioif_softc *);
1.1 hannken 380: static int vioif_ctrl_rx(struct vioif_softc *, int, bool);
381: static int vioif_set_promisc(struct vioif_softc *, bool);
382: static int vioif_set_allmulti(struct vioif_softc *, bool);
383: static int vioif_set_rx_filter(struct vioif_softc *);
384: static int vioif_rx_filter(struct vioif_softc *);
1.54 yamaguch 385: static int vioif_ctrl_intr(void *);
1.34 ozaki-r 386: static int vioif_config_change(struct virtio_softc *);
387: static void vioif_ctl_softint(void *);
1.46 yamaguch 388: static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
389: static void vioif_enable_interrupt_vqpairs(struct vioif_softc *);
390: static void vioif_disable_interrupt_vqpairs(struct vioif_softc *);
1.55 yamaguch 391: static int vioif_setup_sysctl(struct vioif_softc *);
1.1 hannken 392:
393: CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
394: vioif_match, vioif_attach, NULL, NULL);
395:
396: static int
397: vioif_match(device_t parent, cfdata_t match, void *aux)
398: {
1.32 jdolecek 399: struct virtio_attach_args *va = aux;
1.1 hannken 400:
401: if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK)
402: return 1;
403:
404: return 0;
405: }
406:
1.51 chs 407: static void
1.46 yamaguch 408: vioif_alloc_queues(struct vioif_softc *sc)
409: {
410: int nvq_pairs = sc->sc_max_nvq_pairs;
411: int nvqs = nvq_pairs * 2;
412: int i;
413:
414: KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
415:
416: sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
1.51 chs 417: KM_SLEEP);
1.46 yamaguch 418: sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
1.51 chs 419: KM_SLEEP);
1.46 yamaguch 420:
421: if (sc->sc_has_ctrl)
422: nvqs++;
423:
1.51 chs 424: sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
1.46 yamaguch 425: nvqs = 0;
426: for (i = 0; i < nvq_pairs; i++) {
427: sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
428: sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
429: }
430:
431: if (sc->sc_has_ctrl)
432: sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
433: }
434:
435: static void
436: vioif_free_queues(struct vioif_softc *sc)
437: {
438: int nvq_pairs = sc->sc_max_nvq_pairs;
439: int nvqs = nvq_pairs * 2;
440:
441: if (sc->sc_ctrlq.ctrlq_vq)
442: nvqs++;
443:
444: if (sc->sc_txq) {
445: kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
446: sc->sc_txq = NULL;
447: }
448:
449: if (sc->sc_rxq) {
450: kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
451: sc->sc_rxq = NULL;
452: }
453:
454: if (sc->sc_vqs) {
455: kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
456: sc->sc_vqs = NULL;
457: }
458: }
459:
1.1 hannken 460: /* allocate memory */
461: /*
462: * dma memory is used for:
1.43 yamaguch 463: * rxq_hdrs[slot]: metadata array for received frames (READ)
464: * txq_hdrs[slot]: metadata array for frames to be sent (WRITE)
465: * ctrlq_cmd: command to be sent via ctrl vq (WRITE)
466: * ctrlq_status: return value for a command via ctrl vq (READ)
467: * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command
1.1 hannken 468: * (WRITE)
1.43 yamaguch 469: * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1.1 hannken 470: * class command (WRITE)
1.43 yamaguch 471: * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1.1 hannken 472: * class command (WRITE)
1.43 yamaguch 473: * ctrlq_* structures are allocated only one each; they are protected by
474: * ctrlq_inuse variable and ctrlq_wait condvar.
1.1 hannken 475: */
476: /*
477: * dynamically allocated memory is used for:
1.43 yamaguch 478: * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot]
479: * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot]
480: * rxq_dmamaps[slot]: bus_dmamap_t array for received payload
481: * txq_dmamaps[slot]: bus_dmamap_t array for sent payload
482: * rxq_mbufs[slot]: mbuf pointer array for received frames
483: * txq_mbufs[slot]: mbuf pointer array for sent frames
1.1 hannken 484: */
485: static int
486: vioif_alloc_mems(struct vioif_softc *sc)
487: {
488: struct virtio_softc *vsc = sc->sc_virtio;
1.46 yamaguch 489: struct vioif_txqueue *txq;
490: struct vioif_rxqueue *rxq;
1.43 yamaguch 491: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46 yamaguch 492: int allocsize, allocsize2, r, rsegs, i, qid;
1.1 hannken 493: void *vaddr;
494: intptr_t p;
495:
1.46 yamaguch 496: allocsize = 0;
497: for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
498: rxq = &sc->sc_rxq[qid];
499: txq = &sc->sc_txq[qid];
500:
501: allocsize +=
502: sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num;
503: allocsize +=
504: sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num;
505: }
1.32 jdolecek 506: if (sc->sc_has_ctrl) {
1.1 hannken 507: allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1;
508: allocsize += sizeof(struct virtio_net_ctrl_status) * 1;
509: allocsize += sizeof(struct virtio_net_ctrl_rx) * 1;
510: allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
1.50 christos 511: + sizeof(struct virtio_net_ctrl_mac_tbl)
512: + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
1.46 yamaguch 513: allocsize += sizeof(struct virtio_net_ctrl_mq) * 1;
1.1 hannken 514: }
1.32 jdolecek 515: r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0,
1.50 christos 516: &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
1.1 hannken 517: if (r != 0) {
518: aprint_error_dev(sc->sc_dev,
1.50 christos 519: "DMA memory allocation failed, size %d, "
520: "error code %d\n", allocsize, r);
1.1 hannken 521: goto err_none;
522: }
1.32 jdolecek 523: r = bus_dmamem_map(virtio_dmat(vsc),
1.50 christos 524: &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT);
1.1 hannken 525: if (r != 0) {
526: aprint_error_dev(sc->sc_dev,
1.50 christos 527: "DMA memory map failed, error code %d\n", r);
1.1 hannken 528: goto err_dmamem_alloc;
529: }
1.42 yamaguch 530:
531: #define P(p, p0, p0size) do { p0 = (void *) p; \
532: p += p0size; } while (0)
1.1 hannken 533: memset(vaddr, 0, allocsize);
1.43 yamaguch 534: sc->sc_dmamem = vaddr;
1.1 hannken 535: p = (intptr_t) vaddr;
1.42 yamaguch 536:
1.46 yamaguch 537: for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
538: rxq = &sc->sc_rxq[qid];
539: txq = &sc->sc_txq[qid];
540:
541: P(p, rxq->rxq_hdrs,
542: sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num);
543: P(p, txq->txq_hdrs,
544: sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num);
545: }
1.32 jdolecek 546: if (sc->sc_has_ctrl) {
1.43 yamaguch 547: P(p, ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd));
548: P(p, ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status));
549: P(p, ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx));
1.50 christos 550: P(p, ctrlq->ctrlq_mac_tbl_uc, sizeof(*ctrlq->ctrlq_mac_tbl_uc));
551: P(p, ctrlq->ctrlq_mac_tbl_mc, sizeof(*ctrlq->ctrlq_mac_tbl_mc)
552: + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
1.46 yamaguch 553: P(p, ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq));
1.1 hannken 554: }
555:
1.46 yamaguch 556: allocsize2 = 0;
557: for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
558: int rxqsize, txqsize;
559:
560: rxq = &sc->sc_rxq[qid];
561: txq = &sc->sc_txq[qid];
562: rxqsize = rxq->rxq_vq->vq_num;
563: txqsize = txq->txq_vq->vq_num;
564:
565: allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize;
566: allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize;
567: allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize;
568:
569: allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize;
570: allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize;
571: allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize;
572: }
1.42 yamaguch 573: vaddr = kmem_zalloc(allocsize2, KM_SLEEP);
1.43 yamaguch 574: sc->sc_kmem = vaddr;
1.42 yamaguch 575: p = (intptr_t) vaddr;
576:
1.46 yamaguch 577: for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
578: int rxqsize, txqsize;
579: rxq = &sc->sc_rxq[qid];
580: txq = &sc->sc_txq[qid];
581: rxqsize = rxq->rxq_vq->vq_num;
582: txqsize = txq->txq_vq->vq_num;
583:
1.48 msaitoh 584: P(p, rxq->rxq_hdr_dmamaps,
585: sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize);
586: P(p, txq->txq_hdr_dmamaps,
587: sizeof(txq->txq_hdr_dmamaps[0]) * txqsize);
1.46 yamaguch 588: P(p, rxq->rxq_dmamaps, sizeof(rxq->rxq_dmamaps[0]) * rxqsize);
589: P(p, txq->txq_dmamaps, sizeof(txq->txq_dmamaps[0]) * txqsize);
590: P(p, rxq->rxq_mbufs, sizeof(rxq->rxq_mbufs[0]) * rxqsize);
591: P(p, txq->txq_mbufs, sizeof(txq->txq_mbufs[0]) * txqsize);
592: }
1.42 yamaguch 593: #undef P
1.1 hannken 594:
1.48 msaitoh 595: #define C(map, size, nsegs, usage) \
596: do { \
597: r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \
598: BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, \
599: &map); \
600: if (r != 0) { \
601: aprint_error_dev(sc->sc_dev, \
1.50 christos 602: usage " dmamap creation failed, " \
603: "error code %d\n", r); \
1.48 msaitoh 604: goto err_reqs; \
605: } \
1.1 hannken 606: } while (0)
1.42 yamaguch 607: #define C_L(map, buf, size, nsegs, rw, usage) \
608: C(map, size, nsegs, usage); \
1.1 hannken 609: do { \
1.42 yamaguch 610: r = bus_dmamap_load(virtio_dmat(vsc), map, \
611: buf, size, NULL, \
612: rw | BUS_DMA_NOWAIT); \
1.1 hannken 613: if (r != 0) { \
614: aprint_error_dev(sc->sc_dev, \
1.50 christos 615: usage " dmamap load failed, " \
616: "error code %d\n", r); \
1.1 hannken 617: goto err_reqs; \
618: } \
619: } while (0)
620:
1.46 yamaguch 621: for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
622: rxq = &sc->sc_rxq[qid];
623: txq = &sc->sc_txq[qid];
624:
625: for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
626: C_L(rxq->rxq_hdr_dmamaps[i], &rxq->rxq_hdrs[i],
627: sizeof(rxq->rxq_hdrs[0]), 1,
628: BUS_DMA_READ, "rx header");
629: C(rxq->rxq_dmamaps[i], MCLBYTES, 1, "rx payload");
630: }
631:
632: for (i = 0; i < txq->txq_vq->vq_num; i++) {
633: C_L(txq->txq_hdr_dmamaps[i], &txq->txq_hdrs[i],
634: sizeof(txq->txq_hdrs[0]), 1,
635: BUS_DMA_READ, "tx header");
636: C(txq->txq_dmamaps[i], ETHER_MAX_LEN,
637: VIRTIO_NET_TX_MAXNSEGS, "tx payload");
638: }
1.1 hannken 639: }
640:
1.32 jdolecek 641: if (sc->sc_has_ctrl) {
1.1 hannken 642: /* control vq class & command */
1.43 yamaguch 643: C_L(ctrlq->ctrlq_cmd_dmamap,
644: ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
1.42 yamaguch 645: BUS_DMA_WRITE, "control command");
1.43 yamaguch 646: C_L(ctrlq->ctrlq_status_dmamap,
647: ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
1.42 yamaguch 648: BUS_DMA_READ, "control status");
1.1 hannken 649:
650: /* control vq rx mode command parameter */
1.43 yamaguch 651: C_L(ctrlq->ctrlq_rx_dmamap,
652: ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
1.42 yamaguch 653: BUS_DMA_WRITE, "rx mode control command");
1.1 hannken 654:
1.46 yamaguch 655: /* multiqueue set command */
656: C_L(ctrlq->ctrlq_mq_dmamap,
657: ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
658: BUS_DMA_WRITE, "multiqueue set command");
659:
1.1 hannken 660: /* control vq MAC filter table for unicast */
661: /* do not load now since its length is variable */
1.43 yamaguch 662: C(ctrlq->ctrlq_tbl_uc_dmamap,
663: sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1,
1.42 yamaguch 664: "unicast MAC address filter command");
1.1 hannken 665:
666: /* control vq MAC filter table for multicast */
1.43 yamaguch 667: C(ctrlq->ctrlq_tbl_mc_dmamap,
668: sizeof(*ctrlq->ctrlq_mac_tbl_mc)
1.42 yamaguch 669: + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
670: "multicast MAC address filter command");
1.1 hannken 671: }
1.42 yamaguch 672: #undef C_L
1.1 hannken 673: #undef C
674:
675: return 0;
676:
677: err_reqs:
678: #define D(map) \
679: do { \
1.42 yamaguch 680: if (map) { \
681: bus_dmamap_destroy(virtio_dmat(vsc), map); \
682: map = NULL; \
1.1 hannken 683: } \
684: } while (0)
1.43 yamaguch 685: D(ctrlq->ctrlq_tbl_mc_dmamap);
686: D(ctrlq->ctrlq_tbl_uc_dmamap);
687: D(ctrlq->ctrlq_rx_dmamap);
688: D(ctrlq->ctrlq_status_dmamap);
689: D(ctrlq->ctrlq_cmd_dmamap);
1.46 yamaguch 690: for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
691: rxq = &sc->sc_rxq[qid];
692: txq = &sc->sc_txq[qid];
693:
694: for (i = 0; i < txq->txq_vq->vq_num; i++) {
695: D(txq->txq_dmamaps[i]);
696: D(txq->txq_hdr_dmamaps[i]);
697: }
698: for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
699: D(rxq->rxq_dmamaps[i]);
700: D(rxq->rxq_hdr_dmamaps[i]);
701: }
1.1 hannken 702: }
703: #undef D
1.43 yamaguch 704: if (sc->sc_kmem) {
705: kmem_free(sc->sc_kmem, allocsize2);
706: sc->sc_kmem = NULL;
1.1 hannken 707: }
1.43 yamaguch 708: bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize);
1.1 hannken 709: err_dmamem_alloc:
1.32 jdolecek 710: bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
1.1 hannken 711: err_none:
712: return -1;
713: }
714:
715: static void
716: vioif_attach(device_t parent, device_t self, void *aux)
717: {
718: struct vioif_softc *sc = device_private(self);
719: struct virtio_softc *vsc = device_private(parent);
1.43 yamaguch 720: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46 yamaguch 721: struct vioif_txqueue *txq;
722: struct vioif_rxqueue *rxq;
723: uint32_t features, req_features;
1.1 hannken 724: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.43 yamaguch 725: u_int softint_flags;
1.46 yamaguch 726: int r, i, nvqs=0, req_flags;
1.55 yamaguch 727: char xnamebuf[MAXCOMLEN];
1.1 hannken 728:
1.32 jdolecek 729: if (virtio_child(vsc) != NULL) {
1.1 hannken 730: aprint_normal(": child already attached for %s; "
1.50 christos 731: "something wrong...\n", device_xname(parent));
1.1 hannken 732: return;
733: }
734:
735: sc->sc_dev = self;
736: sc->sc_virtio = vsc;
1.34 ozaki-r 737: sc->sc_link_active = false;
1.1 hannken 738:
1.46 yamaguch 739: sc->sc_max_nvq_pairs = 1;
740: sc->sc_req_nvq_pairs = 1;
741: sc->sc_act_nvq_pairs = 1;
1.55 yamaguch 742: sc->sc_txrx_workqueue_sysctl = true;
743: sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
744: sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
745: sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
746: sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
747:
748: snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
749: sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
750: IPL_NET, WQ_PERCPU | WQ_MPSAFE);
751: if (sc->sc_txrx_workqueue == NULL)
752: goto err;
1.46 yamaguch 753:
1.32 jdolecek 754: req_flags = 0;
1.1 hannken 755:
1.7 ozaki-r 756: #ifdef VIOIF_MPSAFE
1.32 jdolecek 757: req_flags |= VIRTIO_F_PCI_INTR_MPSAFE;
1.7 ozaki-r 758: #endif
1.32 jdolecek 759: req_flags |= VIRTIO_F_PCI_INTR_MSIX;
760:
1.46 yamaguch 761: req_features =
762: VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
763: VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
764: #ifdef VIOIF_MULTIQ
765: req_features |= VIRTIO_NET_F_MQ;
766: #endif
767: virtio_child_attach_start(vsc, self, IPL_NET, NULL,
1.54 yamaguch 768: vioif_config_change, virtio_vq_intrhand, req_flags,
1.46 yamaguch 769: req_features, VIRTIO_NET_FLAG_BITS);
1.32 jdolecek 770:
771: features = virtio_features(vsc);
1.7 ozaki-r 772:
1.1 hannken 773: if (features & VIRTIO_NET_F_MAC) {
1.50 christos 774: for (i = 0; i < __arraycount(sc->sc_mac); i++) {
775: sc->sc_mac[i] = virtio_read_device_config_1(vsc,
776: VIRTIO_NET_CONFIG_MAC + i);
777: }
1.1 hannken 778: } else {
779: /* code stolen from sys/net/if_tap.c */
780: struct timeval tv;
781: uint32_t ui;
782: getmicrouptime(&tv);
783: ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
784: memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
1.50 christos 785: for (i = 0; i < __arraycount(sc->sc_mac); i++) {
786: virtio_write_device_config_1(vsc,
787: VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
788: }
1.1 hannken 789: }
1.32 jdolecek 790:
1.50 christos 791: aprint_normal_dev(self, "Ethernet address %s\n",
792: ether_sprintf(sc->sc_mac));
1.1 hannken 793:
1.46 yamaguch 794: if ((features & VIRTIO_NET_F_CTRL_VQ) &&
795: (features & VIRTIO_NET_F_CTRL_RX)) {
796: sc->sc_has_ctrl = true;
797:
798: cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
799: mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
800: ctrlq->ctrlq_inuse = FREE;
801: } else {
802: sc->sc_has_ctrl = false;
803: }
804:
805: if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
806: sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
807: VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
808:
809: if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
810: goto err;
811:
812: /* Limit the number of queue pairs to use */
813: sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
814: }
815:
1.51 chs 816: vioif_alloc_queues(sc);
1.46 yamaguch 817: virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
818:
1.43 yamaguch 819: #ifdef VIOIF_MPSAFE
820: softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
821: #else
822: softint_flags = SOFTINT_NET;
823: #endif
1.7 ozaki-r 824:
1.17 ozaki-r 825: /*
1.46 yamaguch 826: * Allocating a virtqueues
1.17 ozaki-r 827: */
1.46 yamaguch 828: for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
829: rxq = &sc->sc_rxq[i];
830: txq = &sc->sc_txq[i];
831: char qname[32];
832:
833: rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
834:
1.50 christos 835: rxq->rxq_softint = softint_establish(softint_flags,
836: vioif_rx_softint, rxq);
1.46 yamaguch 837: if (rxq->rxq_softint == NULL) {
838: aprint_error_dev(self, "cannot establish rx softint\n");
839: goto err;
840: }
1.55 yamaguch 841: rxq->rxq_handle_si = softint_establish(softint_flags,
842: vioif_rx_handle, rxq);
843: if (rxq->rxq_handle_si == NULL) {
844: aprint_error_dev(self, "cannot establish rx softint\n");
845: goto err;
846: }
847:
1.46 yamaguch 848: snprintf(qname, sizeof(qname), "rx%d", i);
849: r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs,
1.56 ! yamaguch 850: MCLBYTES+sizeof(struct virtio_net_hdr), 2, qname);
1.46 yamaguch 851: if (r != 0)
852: goto err;
853: nvqs++;
1.54 yamaguch 854: rxq->rxq_vq->vq_intrhand = vioif_rx_intr;
855: rxq->rxq_vq->vq_intrhand_arg = (void *)rxq;
1.46 yamaguch 856: rxq->rxq_stopping = true;
1.55 yamaguch 857: vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq);
1.46 yamaguch 858:
859: txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
1.55 yamaguch 860:
1.46 yamaguch 861: txq->txq_deferred_transmit = softint_establish(softint_flags,
862: vioif_deferred_transmit, txq);
863: if (txq->txq_deferred_transmit == NULL) {
864: aprint_error_dev(self, "cannot establish tx softint\n");
865: goto err;
866: }
1.55 yamaguch 867: txq->txq_handle_si = softint_establish(softint_flags,
868: vioif_tx_handle, txq);
869: if (txq->txq_handle_si == NULL) {
870: aprint_error_dev(self, "cannot establish tx softint\n");
871: goto err;
872: }
873:
1.46 yamaguch 874: snprintf(qname, sizeof(qname), "tx%d", i);
875: r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs,
1.50 christos 876: sizeof(struct virtio_net_hdr)
877: + (ETHER_MAX_LEN - ETHER_HDR_LEN),
1.46 yamaguch 878: VIRTIO_NET_TX_MAXNSEGS + 1, qname);
879: if (r != 0)
880: goto err;
881: nvqs++;
1.54 yamaguch 882: txq->txq_vq->vq_intrhand = vioif_tx_intr;
883: txq->txq_vq->vq_intrhand_arg = (void *)txq;
1.46 yamaguch 884: txq->txq_link_active = sc->sc_link_active;
885: txq->txq_stopping = false;
1.51 chs 886: txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP);
1.55 yamaguch 887: vioif_work_set(&txq->txq_work, vioif_tx_handle, txq);
1.43 yamaguch 888: }
1.17 ozaki-r 889:
1.46 yamaguch 890: if (sc->sc_has_ctrl) {
1.17 ozaki-r 891: /*
892: * Allocating a virtqueue for control channel
893: */
1.46 yamaguch 894: r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs,
1.17 ozaki-r 895: NBPG, 1, "control");
896: if (r != 0) {
897: aprint_error_dev(self, "failed to allocate "
1.50 christos 898: "a virtqueue for control channel, error code %d\n",
899: r);
1.46 yamaguch 900:
901: sc->sc_has_ctrl = false;
902: cv_destroy(&ctrlq->ctrlq_wait);
903: mutex_destroy(&ctrlq->ctrlq_wait_lock);
904: } else {
905: nvqs++;
1.54 yamaguch 906: ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
907: ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
1.1 hannken 908: }
909: }
1.34 ozaki-r 910:
1.48 msaitoh 911: sc->sc_ctl_softint = softint_establish(softint_flags,
912: vioif_ctl_softint, sc);
1.34 ozaki-r 913: if (sc->sc_ctl_softint == NULL) {
914: aprint_error_dev(self, "cannot establish ctl softint\n");
1.1 hannken 915: goto err;
916: }
917:
918: if (vioif_alloc_mems(sc) < 0)
919: goto err;
920:
1.32 jdolecek 921: if (virtio_child_attach_finish(vsc) != 0)
922: goto err;
923:
1.55 yamaguch 924: if (vioif_setup_sysctl(sc) != 0) {
925: aprint_error_dev(self, "unable to create sysctl node\n");
926: /* continue */
927: }
928:
1.1 hannken 929: strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
930: ifp->if_softc = sc;
931: ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1.45 yamaguch 932: #ifdef VIOIF_MPSAFE
933: ifp->if_extflags = IFEF_MPSAFE;
934: #endif
1.1 hannken 935: ifp->if_start = vioif_start;
1.46 yamaguch 936: if (sc->sc_req_nvq_pairs > 1)
937: ifp->if_transmit = vioif_transmit;
1.1 hannken 938: ifp->if_ioctl = vioif_ioctl;
939: ifp->if_init = vioif_init;
940: ifp->if_stop = vioif_stop;
941: ifp->if_capabilities = 0;
942: ifp->if_watchdog = vioif_watchdog;
1.46 yamaguch 943: txq = &sc->sc_txq[0];
1.43 yamaguch 944: IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN));
1.36 jdolecek 945: IFQ_SET_READY(&ifp->if_snd);
1.1 hannken 946:
1.11 ozaki-r 947: sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
948:
1.1 hannken 949: if_attach(ifp);
1.28 ozaki-r 950: if_deferred_start_init(ifp, NULL);
1.1 hannken 951: ether_ifattach(ifp, sc->sc_mac);
952:
953: return;
954:
955: err:
1.46 yamaguch 956: for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
957: rxq = &sc->sc_rxq[i];
958: txq = &sc->sc_txq[i];
959:
960: if (rxq->rxq_lock) {
961: mutex_obj_free(rxq->rxq_lock);
962: rxq->rxq_lock = NULL;
963: }
964:
965: if (rxq->rxq_softint) {
966: softint_disestablish(rxq->rxq_softint);
967: rxq->rxq_softint = NULL;
968: }
969:
1.55 yamaguch 970: if (rxq->rxq_handle_si) {
971: softint_disestablish(rxq->rxq_handle_si);
972: rxq->rxq_handle_si = NULL;
973: }
974:
1.46 yamaguch 975: if (txq->txq_lock) {
976: mutex_obj_free(txq->txq_lock);
977: txq->txq_lock = NULL;
978: }
1.43 yamaguch 979:
1.55 yamaguch 980: if (txq->txq_handle_si) {
981: softint_disestablish(txq->txq_handle_si);
982: txq->txq_handle_si = NULL;
983: }
984:
1.46 yamaguch 985: if (txq->txq_deferred_transmit) {
986: softint_disestablish(txq->txq_deferred_transmit);
987: txq->txq_deferred_transmit = NULL;
988: }
1.43 yamaguch 989:
1.46 yamaguch 990: if (txq->txq_intrq) {
991: pcq_destroy(txq->txq_intrq);
992: txq->txq_intrq = NULL;
993: }
1.43 yamaguch 994: }
1.7 ozaki-r 995:
1.32 jdolecek 996: if (sc->sc_has_ctrl) {
1.43 yamaguch 997: cv_destroy(&ctrlq->ctrlq_wait);
998: mutex_destroy(&ctrlq->ctrlq_wait_lock);
1.1 hannken 999: }
1.20 christos 1000:
1.32 jdolecek 1001: while (nvqs > 0)
1.46 yamaguch 1002: virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]);
1003:
1004: vioif_free_queues(sc);
1.55 yamaguch 1005: virtio_child_attach_failed(vsc);
1006: config_finalize_register(self, vioif_finalize_teardown);
1.20 christos 1007:
1.1 hannken 1008: return;
1009: }
1010:
1.55 yamaguch 1011: static int
1012: vioif_finalize_teardown(device_t self)
1013: {
1014: struct vioif_softc *sc = device_private(self);
1015:
1016: if (sc->sc_txrx_workqueue != NULL) {
1017: vioif_workq_destroy(sc->sc_txrx_workqueue);
1018: sc->sc_txrx_workqueue = NULL;
1019: }
1020:
1021: return 0;
1022: }
1023:
1.1 hannken 1024: /* we need interrupts to make promiscuous mode off */
1025: static void
1026: vioif_deferred_init(device_t self)
1027: {
1028: struct vioif_softc *sc = device_private(self);
1029: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1030: int r;
1031:
1.9 ozaki-r 1032: if (ifp->if_flags & IFF_PROMISC)
1.10 ozaki-r 1033: return;
1.9 ozaki-r 1034:
1.1 hannken 1035: r = vioif_set_promisc(sc, false);
1036: if (r != 0)
1037: aprint_error_dev(self, "resetting promisc mode failed, "
1.50 christos 1038: "error code %d\n", r);
1.1 hannken 1039: }
1040:
1.46 yamaguch 1041: static void
1042: vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
1043: {
1044: struct virtio_softc *vsc = sc->sc_virtio;
1045: struct vioif_txqueue *txq;
1046: struct vioif_rxqueue *rxq;
1047: int i;
1048:
1049: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1050: txq = &sc->sc_txq[i];
1051: rxq = &sc->sc_rxq[i];
1052:
1053: virtio_start_vq_intr(vsc, txq->txq_vq);
1054: virtio_start_vq_intr(vsc, rxq->rxq_vq);
1055: }
1056: }
1057:
1058: static void
1059: vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
1060: {
1061: struct virtio_softc *vsc = sc->sc_virtio;
1062: struct vioif_txqueue *txq;
1063: struct vioif_rxqueue *rxq;
1064: int i;
1065:
1066: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1067: txq = &sc->sc_txq[i];
1068: rxq = &sc->sc_rxq[i];
1069:
1070: virtio_stop_vq_intr(vsc, txq->txq_vq);
1071: virtio_stop_vq_intr(vsc, rxq->rxq_vq);
1072: }
1073: }
1074:
1.1 hannken 1075: /*
1076: * Interface functions for ifnet
1077: */
1078: static int
1079: vioif_init(struct ifnet *ifp)
1080: {
1081: struct vioif_softc *sc = ifp->if_softc;
1.33 ozaki-r 1082: struct virtio_softc *vsc = sc->sc_virtio;
1.46 yamaguch 1083: struct vioif_rxqueue *rxq;
1.43 yamaguch 1084: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46 yamaguch 1085: int r, i;
1.1 hannken 1086:
1087: vioif_stop(ifp, 0);
1.7 ozaki-r 1088:
1.33 ozaki-r 1089: virtio_reinit_start(vsc);
1090: virtio_negotiate_features(vsc, virtio_features(vsc));
1.46 yamaguch 1091:
1092: for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
1093: rxq = &sc->sc_rxq[i];
1094:
1095: /* Have to set false before vioif_populate_rx_mbufs */
1096: rxq->rxq_stopping = false;
1097: vioif_populate_rx_mbufs(rxq);
1098: }
1099:
1100: virtio_reinit_end(vsc);
1.47 yamaguch 1101:
1102: if (sc->sc_has_ctrl)
1.43 yamaguch 1103: virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
1.46 yamaguch 1104:
1105: r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
1106: if (r == 0)
1107: sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
1108: else
1109: sc->sc_act_nvq_pairs = 1;
1110:
1111: for (i = 0; i < sc->sc_act_nvq_pairs; i++)
1112: sc->sc_txq[i].txq_stopping = false;
1113:
1114: vioif_enable_interrupt_vqpairs(sc);
1.33 ozaki-r 1115:
1.8 pooka 1116: if (!sc->sc_deferred_init_done) {
1117: sc->sc_deferred_init_done = 1;
1.32 jdolecek 1118: if (sc->sc_has_ctrl)
1.8 pooka 1119: vioif_deferred_init(sc->sc_dev);
1120: }
1121:
1.34 ozaki-r 1122: vioif_update_link_status(sc);
1.1 hannken 1123: ifp->if_flags |= IFF_RUNNING;
1124: ifp->if_flags &= ~IFF_OACTIVE;
1125: vioif_rx_filter(sc);
1126:
1127: return 0;
1128: }
1129:
1130: static void
1131: vioif_stop(struct ifnet *ifp, int disable)
1132: {
1133: struct vioif_softc *sc = ifp->if_softc;
1134: struct virtio_softc *vsc = sc->sc_virtio;
1.46 yamaguch 1135: struct vioif_txqueue *txq;
1136: struct vioif_rxqueue *rxq;
1.43 yamaguch 1137: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46 yamaguch 1138: int i;
1.1 hannken 1139:
1.13 ozaki-r 1140: /* Take the locks to ensure that ongoing TX/RX finish */
1.46 yamaguch 1141: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1142: txq = &sc->sc_txq[i];
1143: rxq = &sc->sc_rxq[i];
1144:
1145: mutex_enter(txq->txq_lock);
1146: txq->txq_stopping = true;
1147: mutex_exit(txq->txq_lock);
1148:
1149: mutex_enter(rxq->rxq_lock);
1150: rxq->rxq_stopping = true;
1151: mutex_exit(rxq->rxq_lock);
1152: }
1.7 ozaki-r 1153:
1.33 ozaki-r 1154: /* disable interrupts */
1.46 yamaguch 1155: vioif_disable_interrupt_vqpairs(sc);
1156:
1.33 ozaki-r 1157: if (sc->sc_has_ctrl)
1.43 yamaguch 1158: virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
1.33 ozaki-r 1159:
1.1 hannken 1160: /* only way to stop I/O and DMA is resetting... */
1161: virtio_reset(vsc);
1.55 yamaguch 1162:
1163: /* rendezvous for finish of handlers */
1164: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1165: txq = &sc->sc_txq[i];
1166: rxq = &sc->sc_rxq[i];
1167:
1168: mutex_enter(txq->txq_lock);
1169: mutex_exit(txq->txq_lock);
1170:
1171: mutex_enter(rxq->rxq_lock);
1172: mutex_exit(rxq->rxq_lock);
1173:
1174: vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work);
1175: vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work);
1176: }
1177:
1178: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1179: vioif_rx_queue_clear(&sc->sc_rxq[i]);
1180: vioif_tx_queue_clear(&sc->sc_txq[i]);
1181: }
1.46 yamaguch 1182:
1.1 hannken 1183: ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1.34 ozaki-r 1184: sc->sc_link_active = false;
1.1 hannken 1185:
1.46 yamaguch 1186: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1187: txq = &sc->sc_txq[i];
1188: rxq = &sc->sc_rxq[i];
1189:
1190: txq->txq_link_active = false;
1191:
1192: if (disable)
1193: vioif_rx_drain(rxq);
1194:
1195: vioif_tx_drain(txq);
1196: }
1.1 hannken 1197: }
1198:
1199: static void
1.48 msaitoh 1200: vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq,
1201: bool is_transmit)
1.1 hannken 1202: {
1203: struct vioif_softc *sc = ifp->if_softc;
1204: struct virtio_softc *vsc = sc->sc_virtio;
1.43 yamaguch 1205: struct virtqueue *vq = txq->txq_vq;
1.1 hannken 1206: struct mbuf *m;
1.36 jdolecek 1207: int queued = 0;
1.1 hannken 1208:
1.46 yamaguch 1209: KASSERT(mutex_owned(txq->txq_lock));
1210:
1211: if ((ifp->if_flags & IFF_RUNNING) == 0)
1212: return;
1.7 ozaki-r 1213:
1.46 yamaguch 1214: if (!txq->txq_link_active || txq->txq_stopping)
1215: return;
1.7 ozaki-r 1216:
1.46 yamaguch 1217: if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit)
1218: return;
1.1 hannken 1219:
1.2 jmcneill 1220: for (;;) {
1.1 hannken 1221: int slot, r;
1222:
1.46 yamaguch 1223: if (is_transmit)
1224: m = pcq_get(txq->txq_intrq);
1225: else
1226: IFQ_DEQUEUE(&ifp->if_snd, m);
1227:
1.2 jmcneill 1228: if (m == NULL)
1229: break;
1230:
1.1 hannken 1231: r = virtio_enqueue_prep(vsc, vq, &slot);
1232: if (r == EAGAIN) {
1233: ifp->if_flags |= IFF_OACTIVE;
1.36 jdolecek 1234: m_freem(m);
1235: break;
1.1 hannken 1236: }
1237: if (r != 0)
1238: panic("enqueue_prep for a tx buffer");
1.36 jdolecek 1239:
1.32 jdolecek 1240: r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1.50 christos 1241: txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1.1 hannken 1242: if (r != 0) {
1.36 jdolecek 1243: /* maybe just too fragmented */
1244: struct mbuf *newm;
1245:
1246: newm = m_defrag(m, M_NOWAIT);
1247: if (newm == NULL) {
1248: aprint_error_dev(sc->sc_dev,
1249: "m_defrag() failed\n");
1250: goto skip;
1251: }
1252:
1.37 jdolecek 1253: m = newm;
1.36 jdolecek 1254: r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1.50 christos 1255: txq->txq_dmamaps[slot], m,
1256: BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1.36 jdolecek 1257: if (r != 0) {
1258: aprint_error_dev(sc->sc_dev,
1.49 msaitoh 1259: "tx dmamap load failed, error code %d\n",
1.36 jdolecek 1260: r);
1261: skip:
1.37 jdolecek 1262: m_freem(m);
1.36 jdolecek 1263: virtio_enqueue_abort(vsc, vq, slot);
1264: continue;
1265: }
1.1 hannken 1266: }
1.36 jdolecek 1267:
1268: /* This should actually never fail */
1.1 hannken 1269: r = virtio_enqueue_reserve(vsc, vq, slot,
1.43 yamaguch 1270: txq->txq_dmamaps[slot]->dm_nsegs + 1);
1.1 hannken 1271: if (r != 0) {
1.36 jdolecek 1272: aprint_error_dev(sc->sc_dev,
1.49 msaitoh 1273: "virtio_enqueue_reserve failed, error code %d\n",
1.36 jdolecek 1274: r);
1.32 jdolecek 1275: bus_dmamap_unload(virtio_dmat(vsc),
1.43 yamaguch 1276: txq->txq_dmamaps[slot]);
1.36 jdolecek 1277: /* slot already freed by virtio_enqueue_reserve */
1.37 jdolecek 1278: m_freem(m);
1.36 jdolecek 1279: continue;
1.1 hannken 1280: }
1.7 ozaki-r 1281:
1.43 yamaguch 1282: txq->txq_mbufs[slot] = m;
1.1 hannken 1283:
1.43 yamaguch 1284: memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr));
1285: bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1.50 christos 1286: 0, txq->txq_dmamaps[slot]->dm_mapsize,
1287: BUS_DMASYNC_PREWRITE);
1.43 yamaguch 1288: bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1.50 christos 1289: 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize,
1290: BUS_DMASYNC_PREWRITE);
1.43 yamaguch 1291: virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true);
1292: virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true);
1.1 hannken 1293: virtio_enqueue_commit(vsc, vq, slot, false);
1.36 jdolecek 1294:
1.1 hannken 1295: queued++;
1.41 msaitoh 1296: bpf_mtap(ifp, m, BPF_D_OUT);
1.1 hannken 1297: }
1298:
1299: if (queued > 0) {
1300: virtio_enqueue_commit(vsc, vq, -1, true);
1301: ifp->if_timer = 5;
1302: }
1.46 yamaguch 1303: }
1304:
1305: static void
1306: vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1307: {
1308:
1309: /*
1310: * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
1311: */
1312: vioif_send_common_locked(ifp, txq, false);
1313:
1314: }
1315:
1316: static void
1317: vioif_start(struct ifnet *ifp)
1318: {
1319: struct vioif_softc *sc = ifp->if_softc;
1320: struct vioif_txqueue *txq = &sc->sc_txq[0];
1321:
1322: #ifdef VIOIF_MPSAFE
1323: KASSERT(if_is_mpsafe(ifp));
1324: #endif
1325:
1326: mutex_enter(txq->txq_lock);
1327: if (!txq->txq_stopping)
1328: vioif_start_locked(ifp, txq);
1329: mutex_exit(txq->txq_lock);
1330: }
1331:
1332: static inline int
1333: vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
1334: {
1335: struct vioif_softc *sc = ifp->if_softc;
1336: u_int cpuid = cpu_index(curcpu());
1337:
1338: return cpuid % sc->sc_act_nvq_pairs;
1339: }
1340:
1341: static void
1342: vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1343: {
1344:
1345: vioif_send_common_locked(ifp, txq, true);
1346: }
1347:
1348: static int
1349: vioif_transmit(struct ifnet *ifp, struct mbuf *m)
1350: {
1351: struct vioif_softc *sc = ifp->if_softc;
1352: struct vioif_txqueue *txq;
1353: int qid;
1354:
1355: qid = vioif_select_txqueue(ifp, m);
1356: txq = &sc->sc_txq[qid];
1357:
1358: if (__predict_false(!pcq_put(txq->txq_intrq, m))) {
1359: m_freem(m);
1360: return ENOBUFS;
1361: }
1362:
1.52 thorpej 1363: net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
1364: if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
1.46 yamaguch 1365: if (m->m_flags & M_MCAST)
1.52 thorpej 1366: if_statinc_ref(nsr, if_omcasts);
1367: IF_STAT_PUTREF(ifp);
1.46 yamaguch 1368:
1369: if (mutex_tryenter(txq->txq_lock)) {
1370: if (!txq->txq_stopping)
1371: vioif_transmit_locked(ifp, txq);
1372: mutex_exit(txq->txq_lock);
1373: }
1374:
1375: return 0;
1376: }
1377:
1378: static void
1379: vioif_deferred_transmit(void *arg)
1380: {
1381: struct vioif_txqueue *txq = arg;
1382: struct virtio_softc *vsc = txq->txq_vq->vq_owner;
1383: struct vioif_softc *sc = device_private(virtio_child(vsc));
1384: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.7 ozaki-r 1385:
1.46 yamaguch 1386: if (mutex_tryenter(txq->txq_lock)) {
1387: vioif_send_common_locked(ifp, txq, true);
1388: mutex_exit(txq->txq_lock);
1389: }
1.1 hannken 1390: }
1391:
1392: static int
1393: vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
1394: {
1395: int s, r;
1396:
1397: s = splnet();
1398:
1399: r = ether_ioctl(ifp, cmd, data);
1400: if ((r == 0 && cmd == SIOCSIFFLAGS) ||
1401: (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) {
1402: if (ifp->if_flags & IFF_RUNNING)
1403: r = vioif_rx_filter(ifp->if_softc);
1404: else
1405: r = 0;
1406: }
1407:
1408: splx(s);
1409:
1410: return r;
1411: }
1412:
1413: void
1414: vioif_watchdog(struct ifnet *ifp)
1415: {
1416: struct vioif_softc *sc = ifp->if_softc;
1.46 yamaguch 1417: int i;
1.1 hannken 1418:
1.46 yamaguch 1419: if (ifp->if_flags & IFF_RUNNING) {
1.55 yamaguch 1420: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1421: vioif_tx_queue_clear(&sc->sc_txq[i]);
1422: }
1.46 yamaguch 1423: }
1.1 hannken 1424: }
1425:
1426: /*
1.39 dholland 1427: * Receive implementation
1.1 hannken 1428: */
1.39 dholland 1429: /* allocate and initialize a mbuf for receive */
1.1 hannken 1430: static int
1.46 yamaguch 1431: vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1.1 hannken 1432: {
1.46 yamaguch 1433: struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1.1 hannken 1434: struct mbuf *m;
1435: int r;
1436:
1437: MGETHDR(m, M_DONTWAIT, MT_DATA);
1438: if (m == NULL)
1439: return ENOBUFS;
1440: MCLGET(m, M_DONTWAIT);
1441: if ((m->m_flags & M_EXT) == 0) {
1442: m_freem(m);
1443: return ENOBUFS;
1444: }
1.43 yamaguch 1445: rxq->rxq_mbufs[i] = m;
1.1 hannken 1446: m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
1.46 yamaguch 1447: r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1.50 christos 1448: rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT);
1.1 hannken 1449: if (r) {
1450: m_freem(m);
1.46 yamaguch 1451: rxq->rxq_mbufs[i] = NULL;
1.1 hannken 1452: return r;
1453: }
1454:
1455: return 0;
1456: }
1457:
1.39 dholland 1458: /* free a mbuf for receive */
1.1 hannken 1459: static void
1.46 yamaguch 1460: vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1.1 hannken 1461: {
1.46 yamaguch 1462: struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1.43 yamaguch 1463:
1.46 yamaguch 1464: bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]);
1.43 yamaguch 1465: m_freem(rxq->rxq_mbufs[i]);
1466: rxq->rxq_mbufs[i] = NULL;
1.1 hannken 1467: }
1468:
1.39 dholland 1469: /* add mbufs for all the empty receive slots */
1.1 hannken 1470: static void
1.46 yamaguch 1471: vioif_populate_rx_mbufs(struct vioif_rxqueue *rxq)
1.1 hannken 1472: {
1.43 yamaguch 1473:
1.46 yamaguch 1474: mutex_enter(rxq->rxq_lock);
1475: vioif_populate_rx_mbufs_locked(rxq);
1476: mutex_exit(rxq->rxq_lock);
1.12 ozaki-r 1477: }
1478:
1479: static void
1.46 yamaguch 1480: vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq)
1.12 ozaki-r 1481: {
1.46 yamaguch 1482: struct virtqueue *vq = rxq->rxq_vq;
1483: struct virtio_softc *vsc = vq->vq_owner;
1484: struct vioif_softc *sc = device_private(virtio_child(vsc));
1.1 hannken 1485: int i, r, ndone = 0;
1486:
1.46 yamaguch 1487: KASSERT(mutex_owned(rxq->rxq_lock));
1.7 ozaki-r 1488:
1.43 yamaguch 1489: if (rxq->rxq_stopping)
1.12 ozaki-r 1490: return;
1.7 ozaki-r 1491:
1.1 hannken 1492: for (i = 0; i < vq->vq_num; i++) {
1493: int slot;
1494: r = virtio_enqueue_prep(vsc, vq, &slot);
1495: if (r == EAGAIN)
1496: break;
1497: if (r != 0)
1498: panic("enqueue_prep for rx buffers");
1.43 yamaguch 1499: if (rxq->rxq_mbufs[slot] == NULL) {
1.46 yamaguch 1500: r = vioif_add_rx_mbuf(rxq, slot);
1.1 hannken 1501: if (r != 0) {
1.50 christos 1502: aprint_error_dev(sc->sc_dev,
1503: "rx mbuf allocation failed, "
1504: "error code %d\n", r);
1.1 hannken 1505: break;
1506: }
1507: }
1508: r = virtio_enqueue_reserve(vsc, vq, slot,
1.50 christos 1509: rxq->rxq_dmamaps[slot]->dm_nsegs + 1);
1.1 hannken 1510: if (r != 0) {
1.46 yamaguch 1511: vioif_free_rx_mbuf(rxq, slot);
1.1 hannken 1512: break;
1513: }
1.43 yamaguch 1514: bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1.50 christos 1515: 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD);
1.43 yamaguch 1516: bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1.50 christos 1517: 0, MCLBYTES, BUS_DMASYNC_PREREAD);
1518: virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot],
1519: false);
1.43 yamaguch 1520: virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false);
1.1 hannken 1521: virtio_enqueue_commit(vsc, vq, slot, false);
1522: ndone++;
1523: }
1524: if (ndone > 0)
1525: virtio_enqueue_commit(vsc, vq, -1, true);
1526: }
1527:
1.55 yamaguch 1528: static void
1529: vioif_rx_queue_clear(struct vioif_rxqueue *rxq)
1.1 hannken 1530: {
1.55 yamaguch 1531: struct virtqueue *vq = rxq->rxq_vq;
1532: struct virtio_softc *vsc = vq->vq_owner;
1533: struct vioif_softc *sc = device_private(virtio_child(vsc));
1534: u_int limit = UINT_MAX;
1535: bool more;
1.7 ozaki-r 1536:
1.43 yamaguch 1537: KASSERT(rxq->rxq_stopping);
1.7 ozaki-r 1538:
1.46 yamaguch 1539: mutex_enter(rxq->rxq_lock);
1.55 yamaguch 1540: for (;;) {
1541: more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1542: if (more == false)
1543: break;
1544: }
1.46 yamaguch 1545: mutex_exit(rxq->rxq_lock);
1.7 ozaki-r 1546: }
1547:
1.39 dholland 1548: /* dequeue received packets */
1.55 yamaguch 1549: static bool
1550: vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1551: struct vioif_rxqueue *rxq, u_int limit)
1.7 ozaki-r 1552: {
1.43 yamaguch 1553: struct virtqueue *vq = rxq->rxq_vq;
1.1 hannken 1554: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1555: struct mbuf *m;
1556: int slot, len;
1.55 yamaguch 1557: bool more = false, dequeued = false;
1.1 hannken 1558:
1.46 yamaguch 1559: KASSERT(mutex_owned(rxq->rxq_lock));
1.7 ozaki-r 1560:
1.54 yamaguch 1561: if (virtio_vq_is_enqueued(vsc, vq) == false)
1.55 yamaguch 1562: return false;
1563:
1564: for (;;) {
1565: if (limit-- == 0) {
1566: more = true;
1567: break;
1568: }
1569:
1570: if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1571: break;
1572:
1573: dequeued = true;
1.54 yamaguch 1574:
1.1 hannken 1575: len -= sizeof(struct virtio_net_hdr);
1.43 yamaguch 1576: bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1.50 christos 1577: 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD);
1.43 yamaguch 1578: bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1.50 christos 1579: 0, MCLBYTES, BUS_DMASYNC_POSTREAD);
1.43 yamaguch 1580: m = rxq->rxq_mbufs[slot];
1.1 hannken 1581: KASSERT(m != NULL);
1.43 yamaguch 1582: bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]);
1.46 yamaguch 1583: rxq->rxq_mbufs[slot] = NULL;
1.1 hannken 1584: virtio_dequeue_commit(vsc, vq, slot);
1.24 ozaki-r 1585: m_set_rcvif(m, ifp);
1.1 hannken 1586: m->m_len = m->m_pkthdr.len = len;
1.7 ozaki-r 1587:
1.46 yamaguch 1588: mutex_exit(rxq->rxq_lock);
1.22 ozaki-r 1589: if_percpuq_enqueue(ifp->if_percpuq, m);
1.46 yamaguch 1590: mutex_enter(rxq->rxq_lock);
1.7 ozaki-r 1591:
1.43 yamaguch 1592: if (rxq->rxq_stopping)
1.7 ozaki-r 1593: break;
1.1 hannken 1594: }
1.3 christos 1595:
1.55 yamaguch 1596: if (dequeued)
1597: softint_schedule(rxq->rxq_softint);
1598:
1599: return more;
1.1 hannken 1600: }
1601:
1602: /* rx interrupt; call _dequeue above and schedule a softint */
1603: static int
1.54 yamaguch 1604: vioif_rx_intr(void *arg)
1.1 hannken 1605: {
1.54 yamaguch 1606: struct vioif_rxqueue *rxq = arg;
1.55 yamaguch 1607: struct virtqueue *vq = rxq->rxq_vq;
1608: struct virtio_softc *vsc = vq->vq_owner;
1609: struct vioif_softc *sc = device_private(virtio_child(vsc));
1610: u_int limit;
1611: bool more;
1612:
1613: limit = sc->sc_rx_intr_process_limit;
1614:
1615: if (atomic_load_relaxed(&rxq->rxq_active) == true)
1616: return 1;
1.7 ozaki-r 1617:
1.46 yamaguch 1618: mutex_enter(rxq->rxq_lock);
1.7 ozaki-r 1619:
1.55 yamaguch 1620: if (!rxq->rxq_stopping) {
1621: rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl;
1622:
1623: virtio_stop_vq_intr(vsc, vq);
1624: atomic_store_relaxed(&rxq->rxq_active, true);
1625:
1626: more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1627: if (more) {
1628: vioif_rx_sched_handle(sc, rxq);
1629: } else {
1630: atomic_store_relaxed(&rxq->rxq_active, false);
1631: virtio_start_vq_intr(vsc, vq);
1632: }
1633: }
1634:
1635: mutex_exit(rxq->rxq_lock);
1636: return 1;
1637: }
1638:
1639: static void
1640: vioif_rx_handle(void *xrxq)
1641: {
1642: struct vioif_rxqueue *rxq = xrxq;
1643: struct virtqueue *vq = rxq->rxq_vq;
1644: struct virtio_softc *vsc = vq->vq_owner;
1645: struct vioif_softc *sc = device_private(virtio_child(vsc));
1646: u_int limit;
1647: bool more;
1648:
1649: limit = sc->sc_rx_process_limit;
1650:
1651: mutex_enter(rxq->rxq_lock);
1.1 hannken 1652:
1.55 yamaguch 1653: if (!rxq->rxq_stopping) {
1654: more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1655: if (more) {
1656: vioif_rx_sched_handle(sc, rxq);
1657: } else {
1658: atomic_store_relaxed(&rxq->rxq_active, false);
1659: virtio_start_vq_intr(vsc, rxq->rxq_vq);
1660: }
1661: }
1.1 hannken 1662:
1.46 yamaguch 1663: mutex_exit(rxq->rxq_lock);
1.55 yamaguch 1664: }
1665:
1666: static void
1667: vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
1668: {
1669:
1670: if (rxq->rxq_workqueue)
1671: vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work);
1672: else
1673: softint_schedule(rxq->rxq_handle_si);
1.1 hannken 1674: }
1675:
1.39 dholland 1676: /* softint: enqueue receive requests for new incoming packets */
1.1 hannken 1677: static void
1678: vioif_rx_softint(void *arg)
1679: {
1.46 yamaguch 1680: struct vioif_rxqueue *rxq = arg;
1.1 hannken 1681:
1.46 yamaguch 1682: vioif_populate_rx_mbufs(rxq);
1.1 hannken 1683: }
1684:
1685: /* free all the mbufs; called from if_stop(disable) */
1686: static void
1.46 yamaguch 1687: vioif_rx_drain(struct vioif_rxqueue *rxq)
1.1 hannken 1688: {
1.43 yamaguch 1689: struct virtqueue *vq = rxq->rxq_vq;
1.1 hannken 1690: int i;
1691:
1692: for (i = 0; i < vq->vq_num; i++) {
1.43 yamaguch 1693: if (rxq->rxq_mbufs[i] == NULL)
1.1 hannken 1694: continue;
1.46 yamaguch 1695: vioif_free_rx_mbuf(rxq, i);
1.1 hannken 1696: }
1697: }
1698:
1699: /*
1700: * Transmition implementation
1701: */
1702: /* actual transmission is done in if_start */
1703: /* tx interrupt; dequeue and free mbufs */
1704: /*
1705: * tx interrupt is actually disabled; this should be called upon
1706: * tx vq full and watchdog
1707: */
1.55 yamaguch 1708:
1.1 hannken 1709: static int
1.54 yamaguch 1710: vioif_tx_intr(void *arg)
1.1 hannken 1711: {
1.54 yamaguch 1712: struct vioif_txqueue *txq = arg;
1713: struct virtqueue *vq = txq->txq_vq;
1.1 hannken 1714: struct virtio_softc *vsc = vq->vq_owner;
1.32 jdolecek 1715: struct vioif_softc *sc = device_private(virtio_child(vsc));
1.23 pooka 1716: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.55 yamaguch 1717: bool more;
1718: u_int limit;
1719:
1720: limit = sc->sc_tx_intr_process_limit;
1721:
1722: if (atomic_load_relaxed(&txq->txq_active) == true)
1723: return 1;
1.7 ozaki-r 1724:
1.46 yamaguch 1725: mutex_enter(txq->txq_lock);
1.7 ozaki-r 1726:
1.55 yamaguch 1727: if (!txq->txq_stopping) {
1728: txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl;
1729:
1730: virtio_stop_vq_intr(vsc, vq);
1731: atomic_store_relaxed(&txq->txq_active, true);
1732:
1733: more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1734: if (more) {
1735: vioif_tx_sched_handle(sc, txq);
1736: } else {
1737: atomic_store_relaxed(&txq->txq_active, false);
1.7 ozaki-r 1738:
1.55 yamaguch 1739: /* for ALTQ */
1740: if (txq == &sc->sc_txq[0]) {
1741: if_schedule_deferred_start(ifp);
1742: ifp->if_flags &= ~IFF_OACTIVE;
1743: }
1744: softint_schedule(txq->txq_deferred_transmit);
1745:
1746: virtio_start_vq_intr(vsc, vq);
1747: }
1748: }
1.7 ozaki-r 1749:
1.46 yamaguch 1750: mutex_exit(txq->txq_lock);
1751:
1.55 yamaguch 1752: return 1;
1753: }
1754:
1755: static void
1756: vioif_tx_handle(void *xtxq)
1757: {
1758: struct vioif_txqueue *txq = xtxq;
1759: struct virtqueue *vq = txq->txq_vq;
1760: struct virtio_softc *vsc = vq->vq_owner;
1761: struct vioif_softc *sc = device_private(virtio_child(vsc));
1762: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1763: u_int limit;
1764: bool more;
1765:
1766: limit = sc->sc_tx_process_limit;
1767:
1768: mutex_enter(txq->txq_lock);
1769:
1770: if (!txq->txq_stopping) {
1771: more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1772: if (more) {
1773: vioif_tx_sched_handle(sc, txq);
1774: } else {
1775: atomic_store_relaxed(&txq->txq_active, false);
1776:
1777: /* for ALTQ */
1778: if (txq == &sc->sc_txq[0]) {
1779: if_schedule_deferred_start(ifp);
1780: ifp->if_flags &= ~IFF_OACTIVE;
1781: }
1782: softint_schedule(txq->txq_deferred_transmit);
1783:
1784: virtio_start_vq_intr(vsc, txq->txq_vq);
1785: }
1.46 yamaguch 1786: }
1.55 yamaguch 1787:
1788: mutex_exit(txq->txq_lock);
1789: }
1790:
1791: static void
1792: vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq)
1793: {
1794:
1795: if (txq->txq_workqueue)
1796: vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work);
1797: else
1798: softint_schedule(txq->txq_handle_si);
1.7 ozaki-r 1799: }
1800:
1.55 yamaguch 1801: static void
1802: vioif_tx_queue_clear(struct vioif_txqueue *txq)
1.7 ozaki-r 1803: {
1.55 yamaguch 1804: struct virtqueue *vq = txq->txq_vq;
1.7 ozaki-r 1805: struct virtio_softc *vsc = vq->vq_owner;
1.32 jdolecek 1806: struct vioif_softc *sc = device_private(virtio_child(vsc));
1.55 yamaguch 1807: u_int limit = UINT_MAX;
1808: bool more;
1809:
1810: mutex_enter(txq->txq_lock);
1811: for (;;) {
1812: more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1813: if (more == false)
1814: break;
1815: }
1816: mutex_exit(txq->txq_lock);
1817: }
1818:
1819: static bool
1820: vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1821: struct vioif_txqueue *txq, u_int limit)
1822: {
1823: struct virtqueue *vq = txq->txq_vq;
1.1 hannken 1824: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1825: struct mbuf *m;
1826: int slot, len;
1.55 yamaguch 1827: bool more = false;
1.1 hannken 1828:
1.46 yamaguch 1829: KASSERT(mutex_owned(txq->txq_lock));
1.7 ozaki-r 1830:
1.54 yamaguch 1831: if (virtio_vq_is_enqueued(vsc, vq) == false)
1.55 yamaguch 1832: return false;
1833:
1834: for (;;) {
1835: if (limit-- == 0) {
1836: more = true;
1837: break;
1838: }
1839:
1840: if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1841: break;
1.54 yamaguch 1842:
1.43 yamaguch 1843: bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1.50 christos 1844: 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE);
1.43 yamaguch 1845: bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1.50 christos 1846: 0, txq->txq_dmamaps[slot]->dm_mapsize,
1847: BUS_DMASYNC_POSTWRITE);
1.43 yamaguch 1848: m = txq->txq_mbufs[slot];
1849: bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]);
1.46 yamaguch 1850: txq->txq_mbufs[slot] = NULL;
1.1 hannken 1851: virtio_dequeue_commit(vsc, vq, slot);
1.52 thorpej 1852: if_statinc(ifp, if_opackets);
1.1 hannken 1853: m_freem(m);
1854: }
1855:
1.55 yamaguch 1856: return more;
1.1 hannken 1857: }
1858:
1859: /* free all the mbufs already put on vq; called from if_stop(disable) */
1860: static void
1.46 yamaguch 1861: vioif_tx_drain(struct vioif_txqueue *txq)
1.1 hannken 1862: {
1.43 yamaguch 1863: struct virtqueue *vq = txq->txq_vq;
1.46 yamaguch 1864: struct virtio_softc *vsc = vq->vq_owner;
1.1 hannken 1865: int i;
1866:
1.43 yamaguch 1867: KASSERT(txq->txq_stopping);
1.7 ozaki-r 1868:
1.1 hannken 1869: for (i = 0; i < vq->vq_num; i++) {
1.43 yamaguch 1870: if (txq->txq_mbufs[i] == NULL)
1.1 hannken 1871: continue;
1.43 yamaguch 1872: bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]);
1873: m_freem(txq->txq_mbufs[i]);
1874: txq->txq_mbufs[i] = NULL;
1.1 hannken 1875: }
1876: }
1877:
1878: /*
1879: * Control vq
1880: */
1881: /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
1.44 yamaguch 1882: static void
1883: vioif_ctrl_acquire(struct vioif_softc *sc)
1.1 hannken 1884: {
1.43 yamaguch 1885: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.1 hannken 1886:
1.43 yamaguch 1887: mutex_enter(&ctrlq->ctrlq_wait_lock);
1888: while (ctrlq->ctrlq_inuse != FREE)
1889: cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
1890: ctrlq->ctrlq_inuse = INUSE;
1.44 yamaguch 1891: ctrlq->ctrlq_owner = curlwp;
1892: mutex_exit(&ctrlq->ctrlq_wait_lock);
1893: }
1894:
1895: static void
1896: vioif_ctrl_release(struct vioif_softc *sc)
1897: {
1898: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1899:
1900: KASSERT(ctrlq->ctrlq_inuse != FREE);
1901: KASSERT(ctrlq->ctrlq_owner == curlwp);
1902:
1903: mutex_enter(&ctrlq->ctrlq_wait_lock);
1904: ctrlq->ctrlq_inuse = FREE;
1.46 yamaguch 1905: ctrlq->ctrlq_owner = NULL;
1.44 yamaguch 1906: cv_signal(&ctrlq->ctrlq_wait);
1.43 yamaguch 1907: mutex_exit(&ctrlq->ctrlq_wait_lock);
1.44 yamaguch 1908: }
1909:
1910: static int
1911: vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
1912: struct vioif_ctrl_cmdspec *specs, int nspecs)
1913: {
1914: struct virtio_softc *vsc = sc->sc_virtio;
1915: int i, r, loaded;
1916:
1917: loaded = 0;
1918: for (i = 0; i < nspecs; i++) {
1919: r = bus_dmamap_load(virtio_dmat(vsc),
1920: specs[i].dmamap, specs[i].buf, specs[i].bufsize,
1.48 msaitoh 1921: NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1.44 yamaguch 1922: if (r) {
1.50 christos 1923: aprint_error_dev(sc->sc_dev, "control command dmamap"
1924: " load failed, error code %d\n", r);
1.44 yamaguch 1925: goto err;
1926: }
1927: loaded++;
1928:
1929: }
1930:
1931: return r;
1932:
1933: err:
1934: for (i = 0; i < loaded; i++) {
1935: bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
1936: }
1937:
1938: return r;
1939: }
1940:
1941: static void
1942: vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
1943: struct vioif_ctrl_cmdspec *specs, int nspecs)
1944: {
1945: struct virtio_softc *vsc = sc->sc_virtio;
1946: int i;
1947:
1948: for (i = 0; i < nspecs; i++) {
1949: bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
1950: }
1951: }
1952:
1953: static int
1954: vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
1955: struct vioif_ctrl_cmdspec *specs, int nspecs)
1956: {
1957: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1958: struct virtqueue *vq = ctrlq->ctrlq_vq;
1959: struct virtio_softc *vsc = sc->sc_virtio;
1960: int i, r, slot;
1.43 yamaguch 1961:
1.44 yamaguch 1962: ctrlq->ctrlq_cmd->class = class;
1.43 yamaguch 1963: ctrlq->ctrlq_cmd->command = cmd;
1.1 hannken 1964:
1.43 yamaguch 1965: bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
1.50 christos 1966: 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
1.44 yamaguch 1967: for (i = 0; i < nspecs; i++) {
1968: bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
1.50 christos 1969: 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
1.44 yamaguch 1970: }
1.43 yamaguch 1971: bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
1.50 christos 1972: 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
1.1 hannken 1973:
1974: r = virtio_enqueue_prep(vsc, vq, &slot);
1975: if (r != 0)
1976: panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1.44 yamaguch 1977: r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
1.1 hannken 1978: if (r != 0)
1979: panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1.43 yamaguch 1980: virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
1.44 yamaguch 1981: for (i = 0; i < nspecs; i++) {
1982: virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
1983: }
1.43 yamaguch 1984: virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
1.1 hannken 1985: virtio_enqueue_commit(vsc, vq, slot, true);
1986:
1987: /* wait for done */
1.43 yamaguch 1988: mutex_enter(&ctrlq->ctrlq_wait_lock);
1989: while (ctrlq->ctrlq_inuse != DONE)
1990: cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
1991: mutex_exit(&ctrlq->ctrlq_wait_lock);
1.1 hannken 1992: /* already dequeueued */
1993:
1.43 yamaguch 1994: bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
1.50 christos 1995: sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
1.44 yamaguch 1996: for (i = 0; i < nspecs; i++) {
1997: bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
1.50 christos 1998: specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
1.44 yamaguch 1999: }
1.43 yamaguch 2000: bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
1.50 christos 2001: sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
1.1 hannken 2002:
1.43 yamaguch 2003: if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
1.1 hannken 2004: r = 0;
2005: else {
1.50 christos 2006: aprint_error_dev(sc->sc_dev, "failed setting rx mode\n");
1.1 hannken 2007: r = EIO;
2008: }
2009:
1.44 yamaguch 2010: return r;
2011: }
2012:
2013: static int
2014: vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
2015: {
2016: struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
2017: struct vioif_ctrl_cmdspec specs[1];
2018: int r;
2019:
2020: if (!sc->sc_has_ctrl)
2021: return ENOTSUP;
2022:
2023: vioif_ctrl_acquire(sc);
2024:
2025: rx->onoff = onoff;
2026: specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
2027: specs[0].buf = rx;
2028: specs[0].bufsize = sizeof(*rx);
2029:
2030: r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
2031: specs, __arraycount(specs));
1.3 christos 2032:
1.44 yamaguch 2033: vioif_ctrl_release(sc);
1.1 hannken 2034: return r;
2035: }
2036:
2037: static int
2038: vioif_set_promisc(struct vioif_softc *sc, bool onoff)
2039: {
1.50 christos 2040: return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
1.1 hannken 2041: }
2042:
2043: static int
2044: vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
2045: {
1.50 christos 2046: return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
1.1 hannken 2047: }
2048:
2049: /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
2050: static int
2051: vioif_set_rx_filter(struct vioif_softc *sc)
2052: {
1.43 yamaguch 2053: /* filter already set in ctrlq->ctrlq_mac_tbl */
1.44 yamaguch 2054: struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
2055: struct vioif_ctrl_cmdspec specs[2];
2056: int nspecs = __arraycount(specs);
2057: int r;
2058:
2059: mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
2060: mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
1.1 hannken 2061:
1.32 jdolecek 2062: if (!sc->sc_has_ctrl)
1.1 hannken 2063: return ENOTSUP;
2064:
1.44 yamaguch 2065: vioif_ctrl_acquire(sc);
1.1 hannken 2066:
1.44 yamaguch 2067: specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
2068: specs[0].buf = mac_tbl_uc;
2069: specs[0].bufsize = sizeof(*mac_tbl_uc)
2070: + (ETHER_ADDR_LEN * mac_tbl_uc->nentries);
2071:
2072: specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
2073: specs[1].buf = mac_tbl_mc;
2074: specs[1].bufsize = sizeof(*mac_tbl_mc)
2075: + (ETHER_ADDR_LEN * mac_tbl_mc->nentries);
1.1 hannken 2076:
1.44 yamaguch 2077: r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
2078: if (r != 0)
1.1 hannken 2079: goto out;
2080:
1.44 yamaguch 2081: r = vioif_ctrl_send_command(sc,
2082: VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
2083: specs, nspecs);
1.1 hannken 2084:
1.44 yamaguch 2085: vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
1.1 hannken 2086:
2087: out:
1.44 yamaguch 2088: vioif_ctrl_release(sc);
1.3 christos 2089:
1.1 hannken 2090: return r;
2091: }
2092:
1.46 yamaguch 2093: static int
2094: vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
2095: {
2096: struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
2097: struct vioif_ctrl_cmdspec specs[1];
2098: int r;
2099:
2100: if (!sc->sc_has_ctrl)
2101: return ENOTSUP;
2102:
2103: if (nvq_pairs <= 1)
2104: return EINVAL;
2105:
2106: vioif_ctrl_acquire(sc);
2107:
2108: mq->virtqueue_pairs = nvq_pairs;
2109: specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
2110: specs[0].buf = mq;
2111: specs[0].bufsize = sizeof(*mq);
2112:
2113: r = vioif_ctrl_send_command(sc,
2114: VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
2115: specs, __arraycount(specs));
2116:
2117: vioif_ctrl_release(sc);
2118:
2119: return r;
2120: }
2121:
1.1 hannken 2122: /* ctrl vq interrupt; wake up the command issuer */
2123: static int
1.54 yamaguch 2124: vioif_ctrl_intr(void *arg)
1.1 hannken 2125: {
1.54 yamaguch 2126: struct vioif_ctrlqueue *ctrlq = arg;
2127: struct virtqueue *vq = ctrlq->ctrlq_vq;
1.1 hannken 2128: struct virtio_softc *vsc = vq->vq_owner;
2129: int r, slot;
2130:
1.54 yamaguch 2131: if (virtio_vq_is_enqueued(vsc, vq) == false)
2132: return 0;
2133:
1.1 hannken 2134: r = virtio_dequeue(vsc, vq, &slot, NULL);
2135: if (r == ENOENT)
2136: return 0;
2137: virtio_dequeue_commit(vsc, vq, slot);
2138:
1.43 yamaguch 2139: mutex_enter(&ctrlq->ctrlq_wait_lock);
2140: ctrlq->ctrlq_inuse = DONE;
2141: cv_signal(&ctrlq->ctrlq_wait);
2142: mutex_exit(&ctrlq->ctrlq_wait_lock);
1.1 hannken 2143:
2144: return 1;
2145: }
2146:
2147: /*
2148: * If IFF_PROMISC requested, set promiscuous
2149: * If multicast filter small enough (<=MAXENTRIES) set rx filter
2150: * If large multicast filter exist use ALLMULTI
2151: */
2152: /*
2153: * If setting rx filter fails fall back to ALLMULTI
2154: * If ALLMULTI fails fall back to PROMISC
2155: */
2156: static int
2157: vioif_rx_filter(struct vioif_softc *sc)
2158: {
1.48 msaitoh 2159: struct ethercom *ec = &sc->sc_ethercom;
2160: struct ifnet *ifp = &ec->ec_if;
1.1 hannken 2161: struct ether_multi *enm;
2162: struct ether_multistep step;
1.43 yamaguch 2163: struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.1 hannken 2164: int nentries;
2165: int promisc = 0, allmulti = 0, rxfilter = 0;
2166: int r;
2167:
1.32 jdolecek 2168: if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */
1.1 hannken 2169: ifp->if_flags |= IFF_PROMISC;
2170: return 0;
2171: }
2172:
2173: if (ifp->if_flags & IFF_PROMISC) {
2174: promisc = 1;
2175: goto set;
2176: }
2177:
2178: nentries = -1;
1.48 msaitoh 2179: ETHER_LOCK(ec);
2180: ETHER_FIRST_MULTI(step, ec, enm);
1.1 hannken 2181: while (nentries++, enm != NULL) {
2182: if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
2183: allmulti = 1;
1.31 ozaki-r 2184: goto set_unlock;
1.1 hannken 2185: }
1.50 christos 2186: if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
1.1 hannken 2187: allmulti = 1;
1.31 ozaki-r 2188: goto set_unlock;
1.1 hannken 2189: }
1.43 yamaguch 2190: memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
1.50 christos 2191: enm->enm_addrlo, ETHER_ADDR_LEN);
1.1 hannken 2192: ETHER_NEXT_MULTI(step, enm);
2193: }
2194: rxfilter = 1;
2195:
1.31 ozaki-r 2196: set_unlock:
1.48 msaitoh 2197: ETHER_UNLOCK(ec);
1.30 ozaki-r 2198:
1.31 ozaki-r 2199: set:
1.1 hannken 2200: if (rxfilter) {
1.43 yamaguch 2201: ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
2202: ctrlq->ctrlq_mac_tbl_mc->nentries = nentries;
1.1 hannken 2203: r = vioif_set_rx_filter(sc);
2204: if (r != 0) {
2205: rxfilter = 0;
2206: allmulti = 1; /* fallback */
2207: }
2208: } else {
2209: /* remove rx filter */
1.43 yamaguch 2210: ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
2211: ctrlq->ctrlq_mac_tbl_mc->nentries = 0;
1.1 hannken 2212: r = vioif_set_rx_filter(sc);
2213: /* what to do on failure? */
2214: }
2215: if (allmulti) {
2216: r = vioif_set_allmulti(sc, true);
2217: if (r != 0) {
2218: allmulti = 0;
2219: promisc = 1; /* fallback */
2220: }
2221: } else {
2222: r = vioif_set_allmulti(sc, false);
2223: /* what to do on failure? */
2224: }
2225: if (promisc) {
2226: r = vioif_set_promisc(sc, true);
2227: } else {
2228: r = vioif_set_promisc(sc, false);
2229: }
2230:
2231: return r;
2232: }
2233:
1.34 ozaki-r 2234: static bool
2235: vioif_is_link_up(struct vioif_softc *sc)
2236: {
2237: struct virtio_softc *vsc = sc->sc_virtio;
2238: uint16_t status;
2239:
2240: if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
2241: status = virtio_read_device_config_2(vsc,
2242: VIRTIO_NET_CONFIG_STATUS);
2243: else
2244: status = VIRTIO_NET_S_LINK_UP;
2245:
2246: return ((status & VIRTIO_NET_S_LINK_UP) != 0);
2247: }
2248:
2249: /* change link status */
2250: static void
2251: vioif_update_link_status(struct vioif_softc *sc)
2252: {
2253: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.46 yamaguch 2254: struct vioif_txqueue *txq;
1.34 ozaki-r 2255: bool active, changed;
1.46 yamaguch 2256: int link, i;
1.34 ozaki-r 2257:
2258: active = vioif_is_link_up(sc);
2259: changed = false;
2260:
2261: if (active) {
2262: if (!sc->sc_link_active)
2263: changed = true;
2264:
2265: link = LINK_STATE_UP;
2266: sc->sc_link_active = true;
2267: } else {
2268: if (sc->sc_link_active)
2269: changed = true;
2270:
2271: link = LINK_STATE_DOWN;
2272: sc->sc_link_active = false;
2273: }
2274:
1.43 yamaguch 2275: if (changed) {
1.46 yamaguch 2276: for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
2277: txq = &sc->sc_txq[i];
2278:
2279: mutex_enter(txq->txq_lock);
2280: txq->txq_link_active = sc->sc_link_active;
2281: mutex_exit(txq->txq_lock);
2282: }
1.43 yamaguch 2283:
1.34 ozaki-r 2284: if_link_state_change(ifp, link);
1.43 yamaguch 2285: }
1.34 ozaki-r 2286: }
2287:
2288: static int
2289: vioif_config_change(struct virtio_softc *vsc)
2290: {
2291: struct vioif_softc *sc = device_private(virtio_child(vsc));
2292:
2293: softint_schedule(sc->sc_ctl_softint);
2294: return 0;
2295: }
2296:
2297: static void
2298: vioif_ctl_softint(void *arg)
2299: {
2300: struct vioif_softc *sc = arg;
2301: struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2302:
2303: vioif_update_link_status(sc);
2304: vioif_start(ifp);
2305: }
2306:
1.55 yamaguch 2307: static struct workqueue *
2308: vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
2309: {
2310: struct workqueue *wq;
2311: int error;
2312:
2313: error = workqueue_create(&wq, name, vioif_workq_work, NULL,
2314: prio, ipl, flags);
2315:
2316: if (error)
2317: return NULL;
2318:
2319: return wq;
2320: }
2321:
2322: static void
2323: vioif_workq_destroy(struct workqueue *wq)
2324: {
2325:
2326: workqueue_destroy(wq);
2327: }
2328:
2329: static void
2330: vioif_workq_work(struct work *wk, void *context)
2331: {
2332: struct vioif_work *work;
2333:
2334: work = container_of(wk, struct vioif_work, cookie);
2335:
2336: atomic_store_relaxed(&work->added, 0);
2337: work->func(work->arg);
2338: }
2339:
2340: static void
2341: vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
2342: {
2343:
2344: memset(work, 0, sizeof(*work));
2345: work->func = func;
2346: work->arg = arg;
2347: }
2348:
2349: static void
2350: vioif_work_add(struct workqueue *wq, struct vioif_work *work)
2351: {
2352:
2353: if (atomic_load_relaxed(&work->added) != 0)
2354: return;
2355:
2356: atomic_store_relaxed(&work->added, 1);
2357: kpreempt_disable();
2358: workqueue_enqueue(wq, &work->cookie, NULL);
2359: kpreempt_enable();
2360: }
2361:
2362: static void
2363: vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
2364: {
2365:
2366: workqueue_wait(wq, &work->cookie);
2367: }
2368:
2369: static int
2370: vioif_setup_sysctl(struct vioif_softc *sc)
2371: {
2372: const char *devname;
2373: struct sysctllog **log;
2374: const struct sysctlnode *rnode, *rxnode, *txnode;
2375: int error;
2376:
2377: log = &sc->sc_sysctllog;
2378: devname = device_xname(sc->sc_dev);
2379:
2380: error = sysctl_createv(log, 0, NULL, &rnode,
2381: 0, CTLTYPE_NODE, devname,
2382: SYSCTL_DESCR("virtio-net information and settings"),
2383: NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
2384: if (error)
2385: goto out;
2386:
2387: error = sysctl_createv(log, 0, &rnode, NULL,
2388: CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
2389: SYSCTL_DESCR("Use workqueue for packet processing"),
2390: NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
2391: if (error)
2392: goto out;
2393:
2394: error = sysctl_createv(log, 0, &rnode, &rxnode,
2395: 0, CTLTYPE_NODE, "rx",
2396: SYSCTL_DESCR("virtio-net information and settings for Rx"),
2397: NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2398: if (error)
2399: goto out;
2400:
2401: error = sysctl_createv(log, 0, &rxnode, NULL,
2402: CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2403: SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
2404: NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2405: if (error)
2406: goto out;
2407:
2408: error = sysctl_createv(log, 0, &rxnode, NULL,
2409: CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2410: SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
2411: NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
2412: if (error)
2413: goto out;
2414:
2415: error = sysctl_createv(log, 0, &rnode, &txnode,
2416: 0, CTLTYPE_NODE, "tx",
2417: SYSCTL_DESCR("virtio-net information and settings for Tx"),
2418: NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2419: if (error)
2420: goto out;
2421:
2422: error = sysctl_createv(log, 0, &txnode, NULL,
2423: CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2424: SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
2425: NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2426: if (error)
2427: goto out;
2428:
2429: error = sysctl_createv(log, 0, &txnode, NULL,
2430: CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2431: SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
2432: NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
2433:
2434: out:
2435: if (error)
2436: sysctl_teardown(log);
2437:
2438: return error;
2439: }
2440:
1.26 pgoyette 2441: MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
1.48 msaitoh 2442:
1.26 pgoyette 2443: #ifdef _MODULE
2444: #include "ioconf.c"
2445: #endif
1.48 msaitoh 2446:
2447: static int
1.26 pgoyette 2448: if_vioif_modcmd(modcmd_t cmd, void *opaque)
2449: {
2450: int error = 0;
1.48 msaitoh 2451:
1.26 pgoyette 2452: #ifdef _MODULE
2453: switch (cmd) {
2454: case MODULE_CMD_INIT:
1.48 msaitoh 2455: error = config_init_component(cfdriver_ioconf_if_vioif,
2456: cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
1.26 pgoyette 2457: break;
2458: case MODULE_CMD_FINI:
2459: error = config_fini_component(cfdriver_ioconf_if_vioif,
2460: cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2461: break;
2462: default:
2463: error = ENOTTY;
1.48 msaitoh 2464: break;
1.26 pgoyette 2465: }
2466: #endif
1.48 msaitoh 2467:
1.26 pgoyette 2468: return error;
2469: }
CVSweb <webmaster@jp.NetBSD.org>