[BACK]Return to if_vioif.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / dev / pci

Annotation of src/sys/dev/pci/if_vioif.c, Revision 1.56

1.56    ! yamaguch    1: /*     $NetBSD: if_vioif.c,v 1.55 2020/05/25 08:16:23 yamaguchi Exp $  */
1.1       hannken     2:
                      3: /*
                      4:  * Copyright (c) 2010 Minoura Makoto.
                      5:  * All rights reserved.
                      6:  *
                      7:  * Redistribution and use in source and binary forms, with or without
                      8:  * modification, are permitted provided that the following conditions
                      9:  * are met:
                     10:  * 1. Redistributions of source code must retain the above copyright
                     11:  *    notice, this list of conditions and the following disclaimer.
                     12:  * 2. Redistributions in binary form must reproduce the above copyright
                     13:  *    notice, this list of conditions and the following disclaimer in the
                     14:  *    documentation and/or other materials provided with the distribution.
                     15:  *
                     16:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     17:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
                     18:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
                     19:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
                     20:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
                     21:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
                     22:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
                     23:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
                     24:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
                     25:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
                     26:  */
                     27:
                     28: #include <sys/cdefs.h>
1.56    ! yamaguch   29: __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.55 2020/05/25 08:16:23 yamaguchi Exp $");
1.15      ozaki-r    30:
                     31: #ifdef _KERNEL_OPT
                     32: #include "opt_net_mpsafe.h"
                     33: #endif
1.1       hannken    34:
                     35: #include <sys/param.h>
                     36: #include <sys/systm.h>
                     37: #include <sys/kernel.h>
1.55      yamaguch   38: #include <sys/atomic.h>
1.1       hannken    39: #include <sys/bus.h>
                     40: #include <sys/condvar.h>
                     41: #include <sys/device.h>
                     42: #include <sys/intr.h>
                     43: #include <sys/kmem.h>
                     44: #include <sys/mbuf.h>
                     45: #include <sys/mutex.h>
                     46: #include <sys/sockio.h>
1.12      ozaki-r    47: #include <sys/cpu.h>
1.26      pgoyette   48: #include <sys/module.h>
1.46      yamaguch   49: #include <sys/pcq.h>
1.55      yamaguch   50: #include <sys/workqueue.h>
1.1       hannken    51:
                     52: #include <dev/pci/virtioreg.h>
                     53: #include <dev/pci/virtiovar.h>
                     54:
                     55: #include <net/if.h>
                     56: #include <net/if_media.h>
                     57: #include <net/if_ether.h>
                     58:
                     59: #include <net/bpf.h>
                     60:
1.26      pgoyette   61: #include "ioconf.h"
1.1       hannken    62:
1.7       ozaki-r    63: #ifdef NET_MPSAFE
                     64: #define VIOIF_MPSAFE   1
1.46      yamaguch   65: #define VIOIF_MULTIQ   1
1.7       ozaki-r    66: #endif
                     67:
1.1       hannken    68: /*
                     69:  * if_vioifreg.h:
                     70:  */
                     71: /* Configuration registers */
                     72: #define VIRTIO_NET_CONFIG_MAC          0 /* 8bit x 6byte */
                     73: #define VIRTIO_NET_CONFIG_STATUS       6 /* 16bit */
1.46      yamaguch   74: #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
1.1       hannken    75:
                     76: /* Feature bits */
1.46      yamaguch   77: #define VIRTIO_NET_F_CSUM              __BIT(0)
                     78: #define VIRTIO_NET_F_GUEST_CSUM                __BIT(1)
                     79: #define VIRTIO_NET_F_MAC               __BIT(5)
                     80: #define VIRTIO_NET_F_GSO               __BIT(6)
                     81: #define VIRTIO_NET_F_GUEST_TSO4                __BIT(7)
                     82: #define VIRTIO_NET_F_GUEST_TSO6                __BIT(8)
                     83: #define VIRTIO_NET_F_GUEST_ECN         __BIT(9)
                     84: #define VIRTIO_NET_F_GUEST_UFO         __BIT(10)
                     85: #define VIRTIO_NET_F_HOST_TSO4         __BIT(11)
                     86: #define VIRTIO_NET_F_HOST_TSO6         __BIT(12)
                     87: #define VIRTIO_NET_F_HOST_ECN          __BIT(13)
                     88: #define VIRTIO_NET_F_HOST_UFO          __BIT(14)
                     89: #define VIRTIO_NET_F_MRG_RXBUF         __BIT(15)
                     90: #define VIRTIO_NET_F_STATUS            __BIT(16)
                     91: #define VIRTIO_NET_F_CTRL_VQ           __BIT(17)
                     92: #define VIRTIO_NET_F_CTRL_RX           __BIT(18)
                     93: #define VIRTIO_NET_F_CTRL_VLAN         __BIT(19)
                     94: #define VIRTIO_NET_F_CTRL_RX_EXTRA     __BIT(20)
                     95: #define VIRTIO_NET_F_GUEST_ANNOUNCE    __BIT(21)
                     96: #define VIRTIO_NET_F_MQ                        __BIT(22)
1.1       hannken    97:
1.18      christos   98: #define VIRTIO_NET_FLAG_BITS \
                     99:        VIRTIO_COMMON_FLAG_BITS \
1.46      yamaguch  100:        "\x17""MQ" \
                    101:        "\x16""GUEST_ANNOUNCE" \
                    102:        "\x15""CTRL_RX_EXTRA" \
1.18      christos  103:        "\x14""CTRL_VLAN" \
                    104:        "\x13""CTRL_RX" \
                    105:        "\x12""CTRL_VQ" \
                    106:        "\x11""STATUS" \
                    107:        "\x10""MRG_RXBUF" \
                    108:        "\x0f""HOST_UFO" \
                    109:        "\x0e""HOST_ECN" \
                    110:        "\x0d""HOST_TSO6" \
                    111:        "\x0c""HOST_TSO4" \
                    112:        "\x0b""GUEST_UFO" \
                    113:        "\x0a""GUEST_ECN" \
                    114:        "\x09""GUEST_TSO6" \
                    115:        "\x08""GUEST_TSO4" \
                    116:        "\x07""GSO" \
                    117:        "\x06""MAC" \
                    118:        "\x02""GUEST_CSUM" \
                    119:        "\x01""CSUM"
                    120:
1.1       hannken   121: /* Status */
                    122: #define VIRTIO_NET_S_LINK_UP   1
                    123:
                    124: /* Packet header structure */
                    125: struct virtio_net_hdr {
                    126:        uint8_t         flags;
                    127:        uint8_t         gso_type;
                    128:        uint16_t        hdr_len;
                    129:        uint16_t        gso_size;
                    130:        uint16_t        csum_start;
                    131:        uint16_t        csum_offset;
                    132: #if 0
                    133:        uint16_t        num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */
                    134: #endif
                    135: } __packed;
                    136:
                    137: #define VIRTIO_NET_HDR_F_NEEDS_CSUM    1 /* flags */
                    138: #define VIRTIO_NET_HDR_GSO_NONE                0 /* gso_type */
                    139: #define VIRTIO_NET_HDR_GSO_TCPV4       1 /* gso_type */
                    140: #define VIRTIO_NET_HDR_GSO_UDP         3 /* gso_type */
                    141: #define VIRTIO_NET_HDR_GSO_TCPV6       4 /* gso_type */
                    142: #define VIRTIO_NET_HDR_GSO_ECN         0x80 /* gso_type, |'ed */
                    143:
                    144: #define VIRTIO_NET_MAX_GSO_LEN         (65536+ETHER_HDR_LEN)
                    145:
                    146: /* Control virtqueue */
                    147: struct virtio_net_ctrl_cmd {
                    148:        uint8_t class;
                    149:        uint8_t command;
                    150: } __packed;
                    151: #define VIRTIO_NET_CTRL_RX             0
                    152: # define VIRTIO_NET_CTRL_RX_PROMISC    0
                    153: # define VIRTIO_NET_CTRL_RX_ALLMULTI   1
                    154:
                    155: #define VIRTIO_NET_CTRL_MAC            1
                    156: # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
                    157:
                    158: #define VIRTIO_NET_CTRL_VLAN           2
                    159: # define VIRTIO_NET_CTRL_VLAN_ADD      0
                    160: # define VIRTIO_NET_CTRL_VLAN_DEL      1
                    161:
1.46      yamaguch  162: #define VIRTIO_NET_CTRL_MQ                     4
                    163: # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET       0
                    164: # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN       1
                    165: # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX       0x8000
                    166:
1.1       hannken   167: struct virtio_net_ctrl_status {
                    168:        uint8_t ack;
                    169: } __packed;
                    170: #define VIRTIO_NET_OK                  0
                    171: #define VIRTIO_NET_ERR                 1
                    172:
                    173: struct virtio_net_ctrl_rx {
                    174:        uint8_t onoff;
                    175: } __packed;
                    176:
                    177: struct virtio_net_ctrl_mac_tbl {
                    178:        uint32_t nentries;
                    179:        uint8_t macs[][ETHER_ADDR_LEN];
                    180: } __packed;
                    181:
                    182: struct virtio_net_ctrl_vlan {
                    183:        uint16_t id;
                    184: } __packed;
                    185:
1.46      yamaguch  186: struct virtio_net_ctrl_mq {
                    187:        uint16_t virtqueue_pairs;
                    188: } __packed;
                    189:
1.44      yamaguch  190: struct vioif_ctrl_cmdspec {
                    191:        bus_dmamap_t    dmamap;
                    192:        void            *buf;
                    193:        bus_size_t      bufsize;
                    194: };
1.1       hannken   195:
                    196: /*
                    197:  * if_vioifvar.h:
                    198:  */
1.43      yamaguch  199:
                    200: /*
                    201:  * Locking notes:
                    202:  * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and
                    203:  *   a filds in vioif_rxqueue is protected by rxq_lock (a spin mutex).
                    204:  *      - more than one lock cannot be held at onece
                    205:  * + ctrlq_inuse is protected by ctrlq_wait_lock.
                    206:  *      - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
                    207:  *      - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock
                    208:  */
                    209:
1.55      yamaguch  210: struct vioif_work {
                    211:        struct work      cookie;
                    212:        void            (*func)(void *);
                    213:        void            *arg;
                    214:        unsigned int     added;
                    215: };
                    216:
1.43      yamaguch  217: struct vioif_txqueue {
                    218:        kmutex_t                *txq_lock;      /* lock for tx operations */
                    219:
                    220:        struct virtqueue        *txq_vq;
                    221:        bool                    txq_stopping;
                    222:        bool                    txq_link_active;
1.46      yamaguch  223:        pcq_t                   *txq_intrq;
1.43      yamaguch  224:
                    225:        struct virtio_net_hdr   *txq_hdrs;
                    226:        bus_dmamap_t            *txq_hdr_dmamaps;
                    227:
                    228:        struct mbuf             **txq_mbufs;
                    229:        bus_dmamap_t            *txq_dmamaps;
1.46      yamaguch  230:
                    231:        void                    *txq_deferred_transmit;
1.55      yamaguch  232:        void                    *txq_handle_si;
                    233:        struct vioif_work        txq_work;
                    234:        bool                     txq_workqueue;
                    235:        bool                     txq_active;
1.43      yamaguch  236: };
                    237:
                    238: struct vioif_rxqueue {
                    239:        kmutex_t                *rxq_lock;      /* lock for rx operations */
                    240:
                    241:        struct virtqueue        *rxq_vq;
                    242:        bool                    rxq_stopping;
                    243:
                    244:        struct virtio_net_hdr   *rxq_hdrs;
                    245:        bus_dmamap_t            *rxq_hdr_dmamaps;
                    246:
                    247:        struct mbuf             **rxq_mbufs;
                    248:        bus_dmamap_t            *rxq_dmamaps;
                    249:
                    250:        void                    *rxq_softint;
1.55      yamaguch  251:        void                    *rxq_handle_si;
                    252:        struct vioif_work        rxq_work;
                    253:        bool                     rxq_workqueue;
                    254:        bool                     rxq_active;
1.43      yamaguch  255: };
                    256:
                    257: struct vioif_ctrlqueue {
                    258:        struct virtqueue                *ctrlq_vq;
                    259:        enum {
                    260:                FREE, INUSE, DONE
                    261:        }                               ctrlq_inuse;
                    262:        kcondvar_t                      ctrlq_wait;
                    263:        kmutex_t                        ctrlq_wait_lock;
1.44      yamaguch  264:        struct lwp                      *ctrlq_owner;
1.43      yamaguch  265:
                    266:        struct virtio_net_ctrl_cmd      *ctrlq_cmd;
                    267:        struct virtio_net_ctrl_status   *ctrlq_status;
                    268:        struct virtio_net_ctrl_rx       *ctrlq_rx;
                    269:        struct virtio_net_ctrl_mac_tbl  *ctrlq_mac_tbl_uc;
                    270:        struct virtio_net_ctrl_mac_tbl  *ctrlq_mac_tbl_mc;
1.46      yamaguch  271:        struct virtio_net_ctrl_mq       *ctrlq_mq;
1.43      yamaguch  272:
                    273:        bus_dmamap_t                    ctrlq_cmd_dmamap;
                    274:        bus_dmamap_t                    ctrlq_status_dmamap;
                    275:        bus_dmamap_t                    ctrlq_rx_dmamap;
                    276:        bus_dmamap_t                    ctrlq_tbl_uc_dmamap;
                    277:        bus_dmamap_t                    ctrlq_tbl_mc_dmamap;
1.46      yamaguch  278:        bus_dmamap_t                    ctrlq_mq_dmamap;
1.43      yamaguch  279: };
                    280:
1.1       hannken   281: struct vioif_softc {
                    282:        device_t                sc_dev;
1.55      yamaguch  283:        struct sysctllog        *sc_sysctllog;
1.1       hannken   284:
                    285:        struct virtio_softc     *sc_virtio;
1.46      yamaguch  286:        struct virtqueue        *sc_vqs;
                    287:
                    288:        int                     sc_max_nvq_pairs;
                    289:        int                     sc_req_nvq_pairs;
                    290:        int                     sc_act_nvq_pairs;
1.1       hannken   291:
                    292:        uint8_t                 sc_mac[ETHER_ADDR_LEN];
                    293:        struct ethercom         sc_ethercom;
1.8       pooka     294:        short                   sc_deferred_init_done;
1.34      ozaki-r   295:        bool                    sc_link_active;
1.1       hannken   296:
1.46      yamaguch  297:        struct vioif_txqueue    *sc_txq;
                    298:        struct vioif_rxqueue    *sc_rxq;
1.43      yamaguch  299:
                    300:        bool                    sc_has_ctrl;
                    301:        struct vioif_ctrlqueue  sc_ctrlq;
                    302:
1.1       hannken   303:        bus_dma_segment_t       sc_hdr_segs[1];
1.43      yamaguch  304:        void                    *sc_dmamem;
                    305:        void                    *sc_kmem;
1.1       hannken   306:
1.34      ozaki-r   307:        void                    *sc_ctl_softint;
1.55      yamaguch  308:
                    309:        struct workqueue        *sc_txrx_workqueue;
                    310:        bool                     sc_txrx_workqueue_sysctl;
                    311:        u_int                    sc_tx_intr_process_limit;
                    312:        u_int                    sc_tx_process_limit;
                    313:        u_int                    sc_rx_intr_process_limit;
                    314:        u_int                    sc_rx_process_limit;
1.1       hannken   315: };
                    316: #define VIRTIO_NET_TX_MAXNSEGS         (16) /* XXX */
                    317: #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
                    318:
1.55      yamaguch  319: #define VIOIF_TX_INTR_PROCESS_LIMIT    256
                    320: #define VIOIF_TX_PROCESS_LIMIT         256
                    321: #define VIOIF_RX_INTR_PROCESS_LIMIT    0U
                    322: #define VIOIF_RX_PROCESS_LIMIT         256
                    323:
                    324: #define VIOIF_WORKQUEUE_PRI            PRI_SOFTNET
                    325:
1.1       hannken   326: /* cfattach interface functions */
                    327: static int     vioif_match(device_t, cfdata_t, void *);
                    328: static void    vioif_attach(device_t, device_t, void *);
                    329: static void    vioif_deferred_init(device_t);
1.55      yamaguch  330: static int     vioif_finalize_teardown(device_t);
1.1       hannken   331:
                    332: /* ifnet interface functions */
                    333: static int     vioif_init(struct ifnet *);
                    334: static void    vioif_stop(struct ifnet *, int);
                    335: static void    vioif_start(struct ifnet *);
1.46      yamaguch  336: static void    vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
                    337: static int     vioif_transmit(struct ifnet *, struct mbuf *);
                    338: static void    vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
1.1       hannken   339: static int     vioif_ioctl(struct ifnet *, u_long, void *);
                    340: static void    vioif_watchdog(struct ifnet *);
                    341:
                    342: /* rx */
1.46      yamaguch  343: static int     vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
                    344: static void    vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
                    345: static void    vioif_populate_rx_mbufs(struct vioif_rxqueue *);
                    346: static void    vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *);
1.55      yamaguch  347: static void    vioif_rx_queue_clear(struct vioif_rxqueue *);
                    348: static bool    vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *,
                    349:                    struct vioif_rxqueue *, u_int);
1.54      yamaguch  350: static int     vioif_rx_intr(void *);
1.55      yamaguch  351: static void    vioif_rx_handle(void *);
                    352: static void    vioif_rx_sched_handle(struct vioif_softc *,
                    353:                    struct vioif_rxqueue *);
1.1       hannken   354: static void    vioif_rx_softint(void *);
1.46      yamaguch  355: static void    vioif_rx_drain(struct vioif_rxqueue *);
1.1       hannken   356:
                    357: /* tx */
1.54      yamaguch  358: static int     vioif_tx_intr(void *);
1.55      yamaguch  359: static void    vioif_tx_handle(void *);
                    360: static void    vioif_tx_sched_handle(struct vioif_softc *,
                    361:                    struct vioif_txqueue *);
                    362: static void    vioif_tx_queue_clear(struct vioif_txqueue *);
                    363: static bool    vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *,
                    364:                    struct vioif_txqueue *, u_int);
1.46      yamaguch  365: static void    vioif_tx_drain(struct vioif_txqueue *);
                    366: static void    vioif_deferred_transmit(void *);
1.1       hannken   367:
1.55      yamaguch  368: /* workqueue */
                    369: static struct workqueue*
                    370:                vioif_workq_create(const char *, pri_t, int, int);
                    371: static void    vioif_workq_destroy(struct workqueue *);
                    372: static void    vioif_workq_work(struct work *, void *);
                    373: static void    vioif_work_set(struct vioif_work *, void(*)(void *), void *);
                    374: static void    vioif_work_add(struct workqueue *, struct vioif_work *);
                    375: static void    vioif_work_wait(struct workqueue *, struct vioif_work *);
                    376:
1.1       hannken   377: /* other control */
1.34      ozaki-r   378: static bool    vioif_is_link_up(struct vioif_softc *);
                    379: static void    vioif_update_link_status(struct vioif_softc *);
1.1       hannken   380: static int     vioif_ctrl_rx(struct vioif_softc *, int, bool);
                    381: static int     vioif_set_promisc(struct vioif_softc *, bool);
                    382: static int     vioif_set_allmulti(struct vioif_softc *, bool);
                    383: static int     vioif_set_rx_filter(struct vioif_softc *);
                    384: static int     vioif_rx_filter(struct vioif_softc *);
1.54      yamaguch  385: static int     vioif_ctrl_intr(void *);
1.34      ozaki-r   386: static int     vioif_config_change(struct virtio_softc *);
                    387: static void    vioif_ctl_softint(void *);
1.46      yamaguch  388: static int     vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
                    389: static void    vioif_enable_interrupt_vqpairs(struct vioif_softc *);
                    390: static void    vioif_disable_interrupt_vqpairs(struct vioif_softc *);
1.55      yamaguch  391: static int     vioif_setup_sysctl(struct vioif_softc *);
1.1       hannken   392:
                    393: CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
                    394:                  vioif_match, vioif_attach, NULL, NULL);
                    395:
                    396: static int
                    397: vioif_match(device_t parent, cfdata_t match, void *aux)
                    398: {
1.32      jdolecek  399:        struct virtio_attach_args *va = aux;
1.1       hannken   400:
                    401:        if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK)
                    402:                return 1;
                    403:
                    404:        return 0;
                    405: }
                    406:
1.51      chs       407: static void
1.46      yamaguch  408: vioif_alloc_queues(struct vioif_softc *sc)
                    409: {
                    410:        int nvq_pairs = sc->sc_max_nvq_pairs;
                    411:        int nvqs = nvq_pairs * 2;
                    412:        int i;
                    413:
                    414:        KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
                    415:
                    416:        sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
1.51      chs       417:            KM_SLEEP);
1.46      yamaguch  418:        sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
1.51      chs       419:            KM_SLEEP);
1.46      yamaguch  420:
                    421:        if (sc->sc_has_ctrl)
                    422:                nvqs++;
                    423:
1.51      chs       424:        sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
1.46      yamaguch  425:        nvqs = 0;
                    426:        for (i = 0; i < nvq_pairs; i++) {
                    427:                sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
                    428:                sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
                    429:        }
                    430:
                    431:        if (sc->sc_has_ctrl)
                    432:                sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
                    433: }
                    434:
                    435: static void
                    436: vioif_free_queues(struct vioif_softc *sc)
                    437: {
                    438:        int nvq_pairs = sc->sc_max_nvq_pairs;
                    439:        int nvqs = nvq_pairs * 2;
                    440:
                    441:        if (sc->sc_ctrlq.ctrlq_vq)
                    442:                nvqs++;
                    443:
                    444:        if (sc->sc_txq) {
                    445:                kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
                    446:                sc->sc_txq = NULL;
                    447:        }
                    448:
                    449:        if (sc->sc_rxq) {
                    450:                kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
                    451:                sc->sc_rxq = NULL;
                    452:        }
                    453:
                    454:        if (sc->sc_vqs) {
                    455:                kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
                    456:                sc->sc_vqs = NULL;
                    457:        }
                    458: }
                    459:
1.1       hannken   460: /* allocate memory */
                    461: /*
                    462:  * dma memory is used for:
1.43      yamaguch  463:  *   rxq_hdrs[slot]:    metadata array for received frames (READ)
                    464:  *   txq_hdrs[slot]:    metadata array for frames to be sent (WRITE)
                    465:  *   ctrlq_cmd:                 command to be sent via ctrl vq (WRITE)
                    466:  *   ctrlq_status:      return value for a command via ctrl vq (READ)
                    467:  *   ctrlq_rx:          parameter for a VIRTIO_NET_CTRL_RX class command
1.1       hannken   468:  *                      (WRITE)
1.43      yamaguch  469:  *   ctrlq_mac_tbl_uc:  unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1.1       hannken   470:  *                      class command (WRITE)
1.43      yamaguch  471:  *   ctrlq_mac_tbl_mc:  multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1.1       hannken   472:  *                      class command (WRITE)
1.43      yamaguch  473:  * ctrlq_* structures are allocated only one each; they are protected by
                    474:  * ctrlq_inuse variable and ctrlq_wait condvar.
1.1       hannken   475:  */
                    476: /*
                    477:  * dynamically allocated memory is used for:
1.43      yamaguch  478:  *   rxq_hdr_dmamaps[slot]:    bus_dmamap_t array for sc_rx_hdrs[slot]
                    479:  *   txq_hdr_dmamaps[slot]:    bus_dmamap_t array for sc_tx_hdrs[slot]
                    480:  *   rxq_dmamaps[slot]:                bus_dmamap_t array for received payload
                    481:  *   txq_dmamaps[slot]:                bus_dmamap_t array for sent payload
                    482:  *   rxq_mbufs[slot]:          mbuf pointer array for received frames
                    483:  *   txq_mbufs[slot]:          mbuf pointer array for sent frames
1.1       hannken   484:  */
                    485: static int
                    486: vioif_alloc_mems(struct vioif_softc *sc)
                    487: {
                    488:        struct virtio_softc *vsc = sc->sc_virtio;
1.46      yamaguch  489:        struct vioif_txqueue *txq;
                    490:        struct vioif_rxqueue *rxq;
1.43      yamaguch  491:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46      yamaguch  492:        int allocsize, allocsize2, r, rsegs, i, qid;
1.1       hannken   493:        void *vaddr;
                    494:        intptr_t p;
                    495:
1.46      yamaguch  496:        allocsize = 0;
                    497:        for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
                    498:                rxq = &sc->sc_rxq[qid];
                    499:                txq = &sc->sc_txq[qid];
                    500:
                    501:                allocsize +=
                    502:                    sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num;
                    503:                allocsize +=
                    504:                    sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num;
                    505:        }
1.32      jdolecek  506:        if (sc->sc_has_ctrl) {
1.1       hannken   507:                allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1;
                    508:                allocsize += sizeof(struct virtio_net_ctrl_status) * 1;
                    509:                allocsize += sizeof(struct virtio_net_ctrl_rx) * 1;
                    510:                allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
1.50      christos  511:                    + sizeof(struct virtio_net_ctrl_mac_tbl)
                    512:                    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
1.46      yamaguch  513:                allocsize += sizeof(struct virtio_net_ctrl_mq) * 1;
1.1       hannken   514:        }
1.32      jdolecek  515:        r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0,
1.50      christos  516:            &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
1.1       hannken   517:        if (r != 0) {
                    518:                aprint_error_dev(sc->sc_dev,
1.50      christos  519:                    "DMA memory allocation failed, size %d, "
                    520:                    "error code %d\n", allocsize, r);
1.1       hannken   521:                goto err_none;
                    522:        }
1.32      jdolecek  523:        r = bus_dmamem_map(virtio_dmat(vsc),
1.50      christos  524:            &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT);
1.1       hannken   525:        if (r != 0) {
                    526:                aprint_error_dev(sc->sc_dev,
1.50      christos  527:                    "DMA memory map failed, error code %d\n", r);
1.1       hannken   528:                goto err_dmamem_alloc;
                    529:        }
1.42      yamaguch  530:
                    531: #define P(p, p0, p0size)       do { p0 = (void *) p;           \
                    532:                                     p += p0size; } while (0)
1.1       hannken   533:        memset(vaddr, 0, allocsize);
1.43      yamaguch  534:        sc->sc_dmamem = vaddr;
1.1       hannken   535:        p = (intptr_t) vaddr;
1.42      yamaguch  536:
1.46      yamaguch  537:        for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
                    538:                rxq = &sc->sc_rxq[qid];
                    539:                txq = &sc->sc_txq[qid];
                    540:
                    541:                P(p, rxq->rxq_hdrs,
                    542:                    sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num);
                    543:                P(p, txq->txq_hdrs,
                    544:                    sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num);
                    545:        }
1.32      jdolecek  546:        if (sc->sc_has_ctrl) {
1.43      yamaguch  547:                P(p, ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd));
                    548:                P(p, ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status));
                    549:                P(p, ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx));
1.50      christos  550:                P(p, ctrlq->ctrlq_mac_tbl_uc, sizeof(*ctrlq->ctrlq_mac_tbl_uc));
                    551:                P(p, ctrlq->ctrlq_mac_tbl_mc, sizeof(*ctrlq->ctrlq_mac_tbl_mc)
                    552:                    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
1.46      yamaguch  553:                P(p, ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq));
1.1       hannken   554:        }
                    555:
1.46      yamaguch  556:        allocsize2 = 0;
                    557:        for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
                    558:                int rxqsize, txqsize;
                    559:
                    560:                rxq = &sc->sc_rxq[qid];
                    561:                txq = &sc->sc_txq[qid];
                    562:                rxqsize = rxq->rxq_vq->vq_num;
                    563:                txqsize = txq->txq_vq->vq_num;
                    564:
                    565:                allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize;
                    566:                allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize;
                    567:                allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize;
                    568:
                    569:                allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize;
                    570:                allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize;
                    571:                allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize;
                    572:        }
1.42      yamaguch  573:        vaddr = kmem_zalloc(allocsize2, KM_SLEEP);
1.43      yamaguch  574:        sc->sc_kmem = vaddr;
1.42      yamaguch  575:        p = (intptr_t) vaddr;
                    576:
1.46      yamaguch  577:        for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
                    578:                int rxqsize, txqsize;
                    579:                rxq = &sc->sc_rxq[qid];
                    580:                txq = &sc->sc_txq[qid];
                    581:                rxqsize = rxq->rxq_vq->vq_num;
                    582:                txqsize = txq->txq_vq->vq_num;
                    583:
1.48      msaitoh   584:                P(p, rxq->rxq_hdr_dmamaps,
                    585:                    sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize);
                    586:                P(p, txq->txq_hdr_dmamaps,
                    587:                    sizeof(txq->txq_hdr_dmamaps[0]) * txqsize);
1.46      yamaguch  588:                P(p, rxq->rxq_dmamaps, sizeof(rxq->rxq_dmamaps[0]) * rxqsize);
                    589:                P(p, txq->txq_dmamaps, sizeof(txq->txq_dmamaps[0]) * txqsize);
                    590:                P(p, rxq->rxq_mbufs, sizeof(rxq->rxq_mbufs[0]) * rxqsize);
                    591:                P(p, txq->txq_mbufs, sizeof(txq->txq_mbufs[0]) * txqsize);
                    592:        }
1.42      yamaguch  593: #undef P
1.1       hannken   594:
1.48      msaitoh   595: #define C(map, size, nsegs, usage)                                           \
                    596:        do {                                                                  \
                    597:                r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \
                    598:                    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,                        \
                    599:                    &map);                                                    \
                    600:                if (r != 0) {                                                 \
                    601:                        aprint_error_dev(sc->sc_dev,                          \
1.50      christos  602:                            usage " dmamap creation failed, "                 \
                    603:                            "error code %d\n", r);                            \
1.48      msaitoh   604:                        goto err_reqs;                                        \
                    605:                }                                                             \
1.1       hannken   606:        } while (0)
1.42      yamaguch  607: #define C_L(map, buf, size, nsegs, rw, usage)                          \
                    608:        C(map, size, nsegs, usage);                                     \
1.1       hannken   609:        do {                                                            \
1.42      yamaguch  610:                r = bus_dmamap_load(virtio_dmat(vsc), map,              \
                    611:                                    buf, size, NULL,                    \
                    612:                                    rw | BUS_DMA_NOWAIT);               \
1.1       hannken   613:                if (r != 0) {                                           \
                    614:                        aprint_error_dev(sc->sc_dev,                    \
1.50      christos  615:                            usage " dmamap load failed, "               \
                    616:                            "error code %d\n", r);                      \
1.1       hannken   617:                        goto err_reqs;                                  \
                    618:                }                                                       \
                    619:        } while (0)
                    620:
1.46      yamaguch  621:        for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
                    622:                rxq = &sc->sc_rxq[qid];
                    623:                txq = &sc->sc_txq[qid];
                    624:
                    625:                for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
                    626:                        C_L(rxq->rxq_hdr_dmamaps[i], &rxq->rxq_hdrs[i],
                    627:                            sizeof(rxq->rxq_hdrs[0]), 1,
                    628:                            BUS_DMA_READ, "rx header");
                    629:                        C(rxq->rxq_dmamaps[i], MCLBYTES, 1, "rx payload");
                    630:                }
                    631:
                    632:                for (i = 0; i < txq->txq_vq->vq_num; i++) {
                    633:                        C_L(txq->txq_hdr_dmamaps[i], &txq->txq_hdrs[i],
                    634:                            sizeof(txq->txq_hdrs[0]), 1,
                    635:                            BUS_DMA_READ, "tx header");
                    636:                        C(txq->txq_dmamaps[i], ETHER_MAX_LEN,
                    637:                            VIRTIO_NET_TX_MAXNSEGS, "tx payload");
                    638:                }
1.1       hannken   639:        }
                    640:
1.32      jdolecek  641:        if (sc->sc_has_ctrl) {
1.1       hannken   642:                /* control vq class & command */
1.43      yamaguch  643:                C_L(ctrlq->ctrlq_cmd_dmamap,
                    644:                    ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
1.42      yamaguch  645:                    BUS_DMA_WRITE, "control command");
1.43      yamaguch  646:                C_L(ctrlq->ctrlq_status_dmamap,
                    647:                    ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
1.42      yamaguch  648:                    BUS_DMA_READ, "control status");
1.1       hannken   649:
                    650:                /* control vq rx mode command parameter */
1.43      yamaguch  651:                C_L(ctrlq->ctrlq_rx_dmamap,
                    652:                    ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
1.42      yamaguch  653:                    BUS_DMA_WRITE, "rx mode control command");
1.1       hannken   654:
1.46      yamaguch  655:                /* multiqueue set command */
                    656:                C_L(ctrlq->ctrlq_mq_dmamap,
                    657:                    ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
                    658:                    BUS_DMA_WRITE, "multiqueue set command");
                    659:
1.1       hannken   660:                /* control vq MAC filter table for unicast */
                    661:                /* do not load now since its length is variable */
1.43      yamaguch  662:                C(ctrlq->ctrlq_tbl_uc_dmamap,
                    663:                    sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1,
1.42      yamaguch  664:                    "unicast MAC address filter command");
1.1       hannken   665:
                    666:                /* control vq MAC filter table for multicast */
1.43      yamaguch  667:                C(ctrlq->ctrlq_tbl_mc_dmamap,
                    668:                    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
1.42      yamaguch  669:                    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
                    670:                    "multicast MAC address filter command");
1.1       hannken   671:        }
1.42      yamaguch  672: #undef C_L
1.1       hannken   673: #undef C
                    674:
                    675:        return 0;
                    676:
                    677: err_reqs:
                    678: #define D(map)                                                         \
                    679:        do {                                                            \
1.42      yamaguch  680:                if (map) {                                              \
                    681:                        bus_dmamap_destroy(virtio_dmat(vsc), map);      \
                    682:                        map = NULL;                                     \
1.1       hannken   683:                }                                                       \
                    684:        } while (0)
1.43      yamaguch  685:        D(ctrlq->ctrlq_tbl_mc_dmamap);
                    686:        D(ctrlq->ctrlq_tbl_uc_dmamap);
                    687:        D(ctrlq->ctrlq_rx_dmamap);
                    688:        D(ctrlq->ctrlq_status_dmamap);
                    689:        D(ctrlq->ctrlq_cmd_dmamap);
1.46      yamaguch  690:        for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
                    691:                rxq = &sc->sc_rxq[qid];
                    692:                txq = &sc->sc_txq[qid];
                    693:
                    694:                for (i = 0; i < txq->txq_vq->vq_num; i++) {
                    695:                        D(txq->txq_dmamaps[i]);
                    696:                        D(txq->txq_hdr_dmamaps[i]);
                    697:                }
                    698:                for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
                    699:                        D(rxq->rxq_dmamaps[i]);
                    700:                        D(rxq->rxq_hdr_dmamaps[i]);
                    701:                }
1.1       hannken   702:        }
                    703: #undef D
1.43      yamaguch  704:        if (sc->sc_kmem) {
                    705:                kmem_free(sc->sc_kmem, allocsize2);
                    706:                sc->sc_kmem = NULL;
1.1       hannken   707:        }
1.43      yamaguch  708:        bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize);
1.1       hannken   709: err_dmamem_alloc:
1.32      jdolecek  710:        bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
1.1       hannken   711: err_none:
                    712:        return -1;
                    713: }
                    714:
                    715: static void
                    716: vioif_attach(device_t parent, device_t self, void *aux)
                    717: {
                    718:        struct vioif_softc *sc = device_private(self);
                    719:        struct virtio_softc *vsc = device_private(parent);
1.43      yamaguch  720:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46      yamaguch  721:        struct vioif_txqueue *txq;
                    722:        struct vioif_rxqueue *rxq;
                    723:        uint32_t features, req_features;
1.1       hannken   724:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.43      yamaguch  725:        u_int softint_flags;
1.46      yamaguch  726:        int r, i, nvqs=0, req_flags;
1.55      yamaguch  727:        char xnamebuf[MAXCOMLEN];
1.1       hannken   728:
1.32      jdolecek  729:        if (virtio_child(vsc) != NULL) {
1.1       hannken   730:                aprint_normal(": child already attached for %s; "
1.50      christos  731:                    "something wrong...\n", device_xname(parent));
1.1       hannken   732:                return;
                    733:        }
                    734:
                    735:        sc->sc_dev = self;
                    736:        sc->sc_virtio = vsc;
1.34      ozaki-r   737:        sc->sc_link_active = false;
1.1       hannken   738:
1.46      yamaguch  739:        sc->sc_max_nvq_pairs = 1;
                    740:        sc->sc_req_nvq_pairs = 1;
                    741:        sc->sc_act_nvq_pairs = 1;
1.55      yamaguch  742:        sc->sc_txrx_workqueue_sysctl = true;
                    743:        sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
                    744:        sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
                    745:        sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
                    746:        sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
                    747:
                    748:        snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
                    749:        sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
                    750:            IPL_NET, WQ_PERCPU | WQ_MPSAFE);
                    751:        if (sc->sc_txrx_workqueue == NULL)
                    752:                goto err;
1.46      yamaguch  753:
1.32      jdolecek  754:        req_flags = 0;
1.1       hannken   755:
1.7       ozaki-r   756: #ifdef VIOIF_MPSAFE
1.32      jdolecek  757:        req_flags |= VIRTIO_F_PCI_INTR_MPSAFE;
1.7       ozaki-r   758: #endif
1.32      jdolecek  759:        req_flags |= VIRTIO_F_PCI_INTR_MSIX;
                    760:
1.46      yamaguch  761:        req_features =
                    762:            VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
                    763:            VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
                    764: #ifdef VIOIF_MULTIQ
                    765:        req_features |= VIRTIO_NET_F_MQ;
                    766: #endif
                    767:        virtio_child_attach_start(vsc, self, IPL_NET, NULL,
1.54      yamaguch  768:            vioif_config_change, virtio_vq_intrhand, req_flags,
1.46      yamaguch  769:            req_features, VIRTIO_NET_FLAG_BITS);
1.32      jdolecek  770:
                    771:        features = virtio_features(vsc);
1.7       ozaki-r   772:
1.1       hannken   773:        if (features & VIRTIO_NET_F_MAC) {
1.50      christos  774:                for (i = 0; i < __arraycount(sc->sc_mac); i++) {
                    775:                        sc->sc_mac[i] = virtio_read_device_config_1(vsc,
                    776:                            VIRTIO_NET_CONFIG_MAC + i);
                    777:                }
1.1       hannken   778:        } else {
                    779:                /* code stolen from sys/net/if_tap.c */
                    780:                struct timeval tv;
                    781:                uint32_t ui;
                    782:                getmicrouptime(&tv);
                    783:                ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
                    784:                memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
1.50      christos  785:                for (i = 0; i < __arraycount(sc->sc_mac); i++) {
                    786:                        virtio_write_device_config_1(vsc,
                    787:                            VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
                    788:                }
1.1       hannken   789:        }
1.32      jdolecek  790:
1.50      christos  791:        aprint_normal_dev(self, "Ethernet address %s\n",
                    792:            ether_sprintf(sc->sc_mac));
1.1       hannken   793:
1.46      yamaguch  794:        if ((features & VIRTIO_NET_F_CTRL_VQ) &&
                    795:            (features & VIRTIO_NET_F_CTRL_RX)) {
                    796:                sc->sc_has_ctrl = true;
                    797:
                    798:                cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
                    799:                mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
                    800:                ctrlq->ctrlq_inuse = FREE;
                    801:        } else {
                    802:                sc->sc_has_ctrl = false;
                    803:        }
                    804:
                    805:        if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
                    806:                sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
                    807:                    VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
                    808:
                    809:                if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
                    810:                        goto err;
                    811:
                    812:                /* Limit the number of queue pairs to use */
                    813:                sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
                    814:        }
                    815:
1.51      chs       816:        vioif_alloc_queues(sc);
1.46      yamaguch  817:        virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
                    818:
1.43      yamaguch  819: #ifdef VIOIF_MPSAFE
                    820:        softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
                    821: #else
                    822:        softint_flags = SOFTINT_NET;
                    823: #endif
1.7       ozaki-r   824:
1.17      ozaki-r   825:        /*
1.46      yamaguch  826:         * Allocating a virtqueues
1.17      ozaki-r   827:         */
1.46      yamaguch  828:        for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
                    829:                rxq = &sc->sc_rxq[i];
                    830:                txq = &sc->sc_txq[i];
                    831:                char qname[32];
                    832:
                    833:                rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
                    834:
1.50      christos  835:                rxq->rxq_softint = softint_establish(softint_flags,
                    836:                    vioif_rx_softint, rxq);
1.46      yamaguch  837:                if (rxq->rxq_softint == NULL) {
                    838:                        aprint_error_dev(self, "cannot establish rx softint\n");
                    839:                        goto err;
                    840:                }
1.55      yamaguch  841:                rxq->rxq_handle_si = softint_establish(softint_flags,
                    842:                    vioif_rx_handle, rxq);
                    843:                if (rxq->rxq_handle_si == NULL) {
                    844:                        aprint_error_dev(self, "cannot establish rx softint\n");
                    845:                        goto err;
                    846:                }
                    847:
1.46      yamaguch  848:                snprintf(qname, sizeof(qname), "rx%d", i);
                    849:                r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs,
1.56    ! yamaguch  850:                    MCLBYTES+sizeof(struct virtio_net_hdr), 2, qname);
1.46      yamaguch  851:                if (r != 0)
                    852:                        goto err;
                    853:                nvqs++;
1.54      yamaguch  854:                rxq->rxq_vq->vq_intrhand = vioif_rx_intr;
                    855:                rxq->rxq_vq->vq_intrhand_arg = (void *)rxq;
1.46      yamaguch  856:                rxq->rxq_stopping = true;
1.55      yamaguch  857:                vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq);
1.46      yamaguch  858:
                    859:                txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
1.55      yamaguch  860:
1.46      yamaguch  861:                txq->txq_deferred_transmit = softint_establish(softint_flags,
                    862:                    vioif_deferred_transmit, txq);
                    863:                if (txq->txq_deferred_transmit == NULL) {
                    864:                        aprint_error_dev(self, "cannot establish tx softint\n");
                    865:                        goto err;
                    866:                }
1.55      yamaguch  867:                txq->txq_handle_si = softint_establish(softint_flags,
                    868:                    vioif_tx_handle, txq);
                    869:                if (txq->txq_handle_si == NULL) {
                    870:                        aprint_error_dev(self, "cannot establish tx softint\n");
                    871:                        goto err;
                    872:                }
                    873:
1.46      yamaguch  874:                snprintf(qname, sizeof(qname), "tx%d", i);
                    875:                r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs,
1.50      christos  876:                    sizeof(struct virtio_net_hdr)
                    877:                    + (ETHER_MAX_LEN - ETHER_HDR_LEN),
1.46      yamaguch  878:                    VIRTIO_NET_TX_MAXNSEGS + 1, qname);
                    879:                if (r != 0)
                    880:                        goto err;
                    881:                nvqs++;
1.54      yamaguch  882:                txq->txq_vq->vq_intrhand = vioif_tx_intr;
                    883:                txq->txq_vq->vq_intrhand_arg = (void *)txq;
1.46      yamaguch  884:                txq->txq_link_active = sc->sc_link_active;
                    885:                txq->txq_stopping = false;
1.51      chs       886:                txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP);
1.55      yamaguch  887:                vioif_work_set(&txq->txq_work, vioif_tx_handle, txq);
1.43      yamaguch  888:        }
1.17      ozaki-r   889:
1.46      yamaguch  890:        if (sc->sc_has_ctrl) {
1.17      ozaki-r   891:                /*
                    892:                 * Allocating a virtqueue for control channel
                    893:                 */
1.46      yamaguch  894:                r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs,
1.17      ozaki-r   895:                    NBPG, 1, "control");
                    896:                if (r != 0) {
                    897:                        aprint_error_dev(self, "failed to allocate "
1.50      christos  898:                            "a virtqueue for control channel, error code %d\n",
                    899:                            r);
1.46      yamaguch  900:
                    901:                        sc->sc_has_ctrl = false;
                    902:                        cv_destroy(&ctrlq->ctrlq_wait);
                    903:                        mutex_destroy(&ctrlq->ctrlq_wait_lock);
                    904:                } else {
                    905:                        nvqs++;
1.54      yamaguch  906:                        ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
                    907:                        ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
1.1       hannken   908:                }
                    909:        }
1.34      ozaki-r   910:
1.48      msaitoh   911:        sc->sc_ctl_softint = softint_establish(softint_flags,
                    912:            vioif_ctl_softint, sc);
1.34      ozaki-r   913:        if (sc->sc_ctl_softint == NULL) {
                    914:                aprint_error_dev(self, "cannot establish ctl softint\n");
1.1       hannken   915:                goto err;
                    916:        }
                    917:
                    918:        if (vioif_alloc_mems(sc) < 0)
                    919:                goto err;
                    920:
1.32      jdolecek  921:        if (virtio_child_attach_finish(vsc) != 0)
                    922:                goto err;
                    923:
1.55      yamaguch  924:        if (vioif_setup_sysctl(sc) != 0) {
                    925:                aprint_error_dev(self, "unable to create sysctl node\n");
                    926:                /* continue */
                    927:        }
                    928:
1.1       hannken   929:        strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
                    930:        ifp->if_softc = sc;
                    931:        ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1.45      yamaguch  932: #ifdef VIOIF_MPSAFE
                    933:        ifp->if_extflags = IFEF_MPSAFE;
                    934: #endif
1.1       hannken   935:        ifp->if_start = vioif_start;
1.46      yamaguch  936:        if (sc->sc_req_nvq_pairs > 1)
                    937:                ifp->if_transmit = vioif_transmit;
1.1       hannken   938:        ifp->if_ioctl = vioif_ioctl;
                    939:        ifp->if_init = vioif_init;
                    940:        ifp->if_stop = vioif_stop;
                    941:        ifp->if_capabilities = 0;
                    942:        ifp->if_watchdog = vioif_watchdog;
1.46      yamaguch  943:        txq = &sc->sc_txq[0];
1.43      yamaguch  944:        IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN));
1.36      jdolecek  945:        IFQ_SET_READY(&ifp->if_snd);
1.1       hannken   946:
1.11      ozaki-r   947:        sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
                    948:
1.1       hannken   949:        if_attach(ifp);
1.28      ozaki-r   950:        if_deferred_start_init(ifp, NULL);
1.1       hannken   951:        ether_ifattach(ifp, sc->sc_mac);
                    952:
                    953:        return;
                    954:
                    955: err:
1.46      yamaguch  956:        for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
                    957:                rxq = &sc->sc_rxq[i];
                    958:                txq = &sc->sc_txq[i];
                    959:
                    960:                if (rxq->rxq_lock) {
                    961:                        mutex_obj_free(rxq->rxq_lock);
                    962:                        rxq->rxq_lock = NULL;
                    963:                }
                    964:
                    965:                if (rxq->rxq_softint) {
                    966:                        softint_disestablish(rxq->rxq_softint);
                    967:                        rxq->rxq_softint = NULL;
                    968:                }
                    969:
1.55      yamaguch  970:                if (rxq->rxq_handle_si) {
                    971:                        softint_disestablish(rxq->rxq_handle_si);
                    972:                        rxq->rxq_handle_si = NULL;
                    973:                }
                    974:
1.46      yamaguch  975:                if (txq->txq_lock) {
                    976:                        mutex_obj_free(txq->txq_lock);
                    977:                        txq->txq_lock = NULL;
                    978:                }
1.43      yamaguch  979:
1.55      yamaguch  980:                if (txq->txq_handle_si) {
                    981:                        softint_disestablish(txq->txq_handle_si);
                    982:                        txq->txq_handle_si = NULL;
                    983:                }
                    984:
1.46      yamaguch  985:                if (txq->txq_deferred_transmit) {
                    986:                        softint_disestablish(txq->txq_deferred_transmit);
                    987:                        txq->txq_deferred_transmit = NULL;
                    988:                }
1.43      yamaguch  989:
1.46      yamaguch  990:                if (txq->txq_intrq) {
                    991:                        pcq_destroy(txq->txq_intrq);
                    992:                        txq->txq_intrq = NULL;
                    993:                }
1.43      yamaguch  994:        }
1.7       ozaki-r   995:
1.32      jdolecek  996:        if (sc->sc_has_ctrl) {
1.43      yamaguch  997:                cv_destroy(&ctrlq->ctrlq_wait);
                    998:                mutex_destroy(&ctrlq->ctrlq_wait_lock);
1.1       hannken   999:        }
1.20      christos 1000:
1.32      jdolecek 1001:        while (nvqs > 0)
1.46      yamaguch 1002:                virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]);
                   1003:
                   1004:        vioif_free_queues(sc);
1.55      yamaguch 1005:        virtio_child_attach_failed(vsc);
                   1006:        config_finalize_register(self, vioif_finalize_teardown);
1.20      christos 1007:
1.1       hannken  1008:        return;
                   1009: }
                   1010:
1.55      yamaguch 1011: static int
                   1012: vioif_finalize_teardown(device_t self)
                   1013: {
                   1014:        struct vioif_softc *sc = device_private(self);
                   1015:
                   1016:        if (sc->sc_txrx_workqueue != NULL) {
                   1017:                vioif_workq_destroy(sc->sc_txrx_workqueue);
                   1018:                sc->sc_txrx_workqueue = NULL;
                   1019:        }
                   1020:
                   1021:        return 0;
                   1022: }
                   1023:
1.1       hannken  1024: /* we need interrupts to make promiscuous mode off */
                   1025: static void
                   1026: vioif_deferred_init(device_t self)
                   1027: {
                   1028:        struct vioif_softc *sc = device_private(self);
                   1029:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
                   1030:        int r;
                   1031:
1.9       ozaki-r  1032:        if (ifp->if_flags & IFF_PROMISC)
1.10      ozaki-r  1033:                return;
1.9       ozaki-r  1034:
1.1       hannken  1035:        r =  vioif_set_promisc(sc, false);
                   1036:        if (r != 0)
                   1037:                aprint_error_dev(self, "resetting promisc mode failed, "
1.50      christos 1038:                    "error code %d\n", r);
1.1       hannken  1039: }
                   1040:
1.46      yamaguch 1041: static void
                   1042: vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
                   1043: {
                   1044:        struct virtio_softc *vsc = sc->sc_virtio;
                   1045:        struct vioif_txqueue *txq;
                   1046:        struct vioif_rxqueue *rxq;
                   1047:        int i;
                   1048:
                   1049:        for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   1050:                txq = &sc->sc_txq[i];
                   1051:                rxq = &sc->sc_rxq[i];
                   1052:
                   1053:                virtio_start_vq_intr(vsc, txq->txq_vq);
                   1054:                virtio_start_vq_intr(vsc, rxq->rxq_vq);
                   1055:        }
                   1056: }
                   1057:
                   1058: static void
                   1059: vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
                   1060: {
                   1061:        struct virtio_softc *vsc = sc->sc_virtio;
                   1062:        struct vioif_txqueue *txq;
                   1063:        struct vioif_rxqueue *rxq;
                   1064:        int i;
                   1065:
                   1066:        for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   1067:                txq = &sc->sc_txq[i];
                   1068:                rxq = &sc->sc_rxq[i];
                   1069:
                   1070:                virtio_stop_vq_intr(vsc, txq->txq_vq);
                   1071:                virtio_stop_vq_intr(vsc, rxq->rxq_vq);
                   1072:        }
                   1073: }
                   1074:
1.1       hannken  1075: /*
                   1076:  * Interface functions for ifnet
                   1077:  */
                   1078: static int
                   1079: vioif_init(struct ifnet *ifp)
                   1080: {
                   1081:        struct vioif_softc *sc = ifp->if_softc;
1.33      ozaki-r  1082:        struct virtio_softc *vsc = sc->sc_virtio;
1.46      yamaguch 1083:        struct vioif_rxqueue *rxq;
1.43      yamaguch 1084:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46      yamaguch 1085:        int r, i;
1.1       hannken  1086:
                   1087:        vioif_stop(ifp, 0);
1.7       ozaki-r  1088:
1.33      ozaki-r  1089:        virtio_reinit_start(vsc);
                   1090:        virtio_negotiate_features(vsc, virtio_features(vsc));
1.46      yamaguch 1091:
                   1092:        for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
                   1093:                rxq = &sc->sc_rxq[i];
                   1094:
                   1095:                /* Have to set false before vioif_populate_rx_mbufs */
                   1096:                rxq->rxq_stopping = false;
                   1097:                vioif_populate_rx_mbufs(rxq);
                   1098:        }
                   1099:
                   1100:        virtio_reinit_end(vsc);
1.47      yamaguch 1101:
                   1102:        if (sc->sc_has_ctrl)
1.43      yamaguch 1103:                virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
1.46      yamaguch 1104:
                   1105:        r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
                   1106:        if (r == 0)
                   1107:                sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
                   1108:        else
                   1109:                sc->sc_act_nvq_pairs = 1;
                   1110:
                   1111:        for (i = 0; i < sc->sc_act_nvq_pairs; i++)
                   1112:                sc->sc_txq[i].txq_stopping = false;
                   1113:
                   1114:        vioif_enable_interrupt_vqpairs(sc);
1.33      ozaki-r  1115:
1.8       pooka    1116:        if (!sc->sc_deferred_init_done) {
                   1117:                sc->sc_deferred_init_done = 1;
1.32      jdolecek 1118:                if (sc->sc_has_ctrl)
1.8       pooka    1119:                        vioif_deferred_init(sc->sc_dev);
                   1120:        }
                   1121:
1.34      ozaki-r  1122:        vioif_update_link_status(sc);
1.1       hannken  1123:        ifp->if_flags |= IFF_RUNNING;
                   1124:        ifp->if_flags &= ~IFF_OACTIVE;
                   1125:        vioif_rx_filter(sc);
                   1126:
                   1127:        return 0;
                   1128: }
                   1129:
                   1130: static void
                   1131: vioif_stop(struct ifnet *ifp, int disable)
                   1132: {
                   1133:        struct vioif_softc *sc = ifp->if_softc;
                   1134:        struct virtio_softc *vsc = sc->sc_virtio;
1.46      yamaguch 1135:        struct vioif_txqueue *txq;
                   1136:        struct vioif_rxqueue *rxq;
1.43      yamaguch 1137:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.46      yamaguch 1138:        int i;
1.1       hannken  1139:
1.13      ozaki-r  1140:        /* Take the locks to ensure that ongoing TX/RX finish */
1.46      yamaguch 1141:        for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   1142:                txq = &sc->sc_txq[i];
                   1143:                rxq = &sc->sc_rxq[i];
                   1144:
                   1145:                mutex_enter(txq->txq_lock);
                   1146:                txq->txq_stopping = true;
                   1147:                mutex_exit(txq->txq_lock);
                   1148:
                   1149:                mutex_enter(rxq->rxq_lock);
                   1150:                rxq->rxq_stopping = true;
                   1151:                mutex_exit(rxq->rxq_lock);
                   1152:        }
1.7       ozaki-r  1153:
1.33      ozaki-r  1154:        /* disable interrupts */
1.46      yamaguch 1155:        vioif_disable_interrupt_vqpairs(sc);
                   1156:
1.33      ozaki-r  1157:        if (sc->sc_has_ctrl)
1.43      yamaguch 1158:                virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
1.33      ozaki-r  1159:
1.1       hannken  1160:        /* only way to stop I/O and DMA is resetting... */
                   1161:        virtio_reset(vsc);
1.55      yamaguch 1162:
                   1163:        /* rendezvous for finish of handlers */
                   1164:        for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   1165:                txq = &sc->sc_txq[i];
                   1166:                rxq = &sc->sc_rxq[i];
                   1167:
                   1168:                mutex_enter(txq->txq_lock);
                   1169:                mutex_exit(txq->txq_lock);
                   1170:
                   1171:                mutex_enter(rxq->rxq_lock);
                   1172:                mutex_exit(rxq->rxq_lock);
                   1173:
                   1174:                vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work);
                   1175:                vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work);
                   1176:        }
                   1177:
                   1178:        for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   1179:                vioif_rx_queue_clear(&sc->sc_rxq[i]);
                   1180:                vioif_tx_queue_clear(&sc->sc_txq[i]);
                   1181:        }
1.46      yamaguch 1182:
1.1       hannken  1183:        ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1.34      ozaki-r  1184:        sc->sc_link_active = false;
1.1       hannken  1185:
1.46      yamaguch 1186:        for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   1187:                txq = &sc->sc_txq[i];
                   1188:                rxq = &sc->sc_rxq[i];
                   1189:
                   1190:                txq->txq_link_active = false;
                   1191:
                   1192:                if (disable)
                   1193:                        vioif_rx_drain(rxq);
                   1194:
                   1195:                vioif_tx_drain(txq);
                   1196:        }
1.1       hannken  1197: }
                   1198:
                   1199: static void
1.48      msaitoh  1200: vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq,
                   1201:     bool is_transmit)
1.1       hannken  1202: {
                   1203:        struct vioif_softc *sc = ifp->if_softc;
                   1204:        struct virtio_softc *vsc = sc->sc_virtio;
1.43      yamaguch 1205:        struct virtqueue *vq = txq->txq_vq;
1.1       hannken  1206:        struct mbuf *m;
1.36      jdolecek 1207:        int queued = 0;
1.1       hannken  1208:
1.46      yamaguch 1209:        KASSERT(mutex_owned(txq->txq_lock));
                   1210:
                   1211:        if ((ifp->if_flags & IFF_RUNNING) == 0)
                   1212:                return;
1.7       ozaki-r  1213:
1.46      yamaguch 1214:        if (!txq->txq_link_active || txq->txq_stopping)
                   1215:                return;
1.7       ozaki-r  1216:
1.46      yamaguch 1217:        if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit)
                   1218:                return;
1.1       hannken  1219:
1.2       jmcneill 1220:        for (;;) {
1.1       hannken  1221:                int slot, r;
                   1222:
1.46      yamaguch 1223:                if (is_transmit)
                   1224:                        m = pcq_get(txq->txq_intrq);
                   1225:                else
                   1226:                        IFQ_DEQUEUE(&ifp->if_snd, m);
                   1227:
1.2       jmcneill 1228:                if (m == NULL)
                   1229:                        break;
                   1230:
1.1       hannken  1231:                r = virtio_enqueue_prep(vsc, vq, &slot);
                   1232:                if (r == EAGAIN) {
                   1233:                        ifp->if_flags |= IFF_OACTIVE;
1.36      jdolecek 1234:                        m_freem(m);
                   1235:                        break;
1.1       hannken  1236:                }
                   1237:                if (r != 0)
                   1238:                        panic("enqueue_prep for a tx buffer");
1.36      jdolecek 1239:
1.32      jdolecek 1240:                r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1.50      christos 1241:                    txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1.1       hannken  1242:                if (r != 0) {
1.36      jdolecek 1243:                        /* maybe just too fragmented */
                   1244:                        struct mbuf *newm;
                   1245:
                   1246:                        newm = m_defrag(m, M_NOWAIT);
                   1247:                        if (newm == NULL) {
                   1248:                                aprint_error_dev(sc->sc_dev,
                   1249:                                    "m_defrag() failed\n");
                   1250:                                goto skip;
                   1251:                        }
                   1252:
1.37      jdolecek 1253:                        m = newm;
1.36      jdolecek 1254:                        r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1.50      christos 1255:                            txq->txq_dmamaps[slot], m,
                   1256:                            BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1.36      jdolecek 1257:                        if (r != 0) {
                   1258:                                aprint_error_dev(sc->sc_dev,
1.49      msaitoh  1259:                                    "tx dmamap load failed, error code %d\n",
1.36      jdolecek 1260:                                    r);
                   1261: skip:
1.37      jdolecek 1262:                                m_freem(m);
1.36      jdolecek 1263:                                virtio_enqueue_abort(vsc, vq, slot);
                   1264:                                continue;
                   1265:                        }
1.1       hannken  1266:                }
1.36      jdolecek 1267:
                   1268:                /* This should actually never fail */
1.1       hannken  1269:                r = virtio_enqueue_reserve(vsc, vq, slot,
1.43      yamaguch 1270:                                        txq->txq_dmamaps[slot]->dm_nsegs + 1);
1.1       hannken  1271:                if (r != 0) {
1.36      jdolecek 1272:                        aprint_error_dev(sc->sc_dev,
1.49      msaitoh  1273:                            "virtio_enqueue_reserve failed, error code %d\n",
1.36      jdolecek 1274:                            r);
1.32      jdolecek 1275:                        bus_dmamap_unload(virtio_dmat(vsc),
1.43      yamaguch 1276:                                          txq->txq_dmamaps[slot]);
1.36      jdolecek 1277:                        /* slot already freed by virtio_enqueue_reserve */
1.37      jdolecek 1278:                        m_freem(m);
1.36      jdolecek 1279:                        continue;
1.1       hannken  1280:                }
1.7       ozaki-r  1281:
1.43      yamaguch 1282:                txq->txq_mbufs[slot] = m;
1.1       hannken  1283:
1.43      yamaguch 1284:                memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr));
                   1285:                bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1.50      christos 1286:                    0, txq->txq_dmamaps[slot]->dm_mapsize,
                   1287:                    BUS_DMASYNC_PREWRITE);
1.43      yamaguch 1288:                bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1.50      christos 1289:                    0, txq->txq_hdr_dmamaps[slot]->dm_mapsize,
                   1290:                    BUS_DMASYNC_PREWRITE);
1.43      yamaguch 1291:                virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true);
                   1292:                virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true);
1.1       hannken  1293:                virtio_enqueue_commit(vsc, vq, slot, false);
1.36      jdolecek 1294:
1.1       hannken  1295:                queued++;
1.41      msaitoh  1296:                bpf_mtap(ifp, m, BPF_D_OUT);
1.1       hannken  1297:        }
                   1298:
                   1299:        if (queued > 0) {
                   1300:                virtio_enqueue_commit(vsc, vq, -1, true);
                   1301:                ifp->if_timer = 5;
                   1302:        }
1.46      yamaguch 1303: }
                   1304:
                   1305: static void
                   1306: vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
                   1307: {
                   1308:
                   1309:        /*
                   1310:         * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
                   1311:         */
                   1312:        vioif_send_common_locked(ifp, txq, false);
                   1313:
                   1314: }
                   1315:
                   1316: static void
                   1317: vioif_start(struct ifnet *ifp)
                   1318: {
                   1319:        struct vioif_softc *sc = ifp->if_softc;
                   1320:        struct vioif_txqueue *txq = &sc->sc_txq[0];
                   1321:
                   1322: #ifdef VIOIF_MPSAFE
                   1323:        KASSERT(if_is_mpsafe(ifp));
                   1324: #endif
                   1325:
                   1326:        mutex_enter(txq->txq_lock);
                   1327:        if (!txq->txq_stopping)
                   1328:                vioif_start_locked(ifp, txq);
                   1329:        mutex_exit(txq->txq_lock);
                   1330: }
                   1331:
                   1332: static inline int
                   1333: vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
                   1334: {
                   1335:        struct vioif_softc *sc = ifp->if_softc;
                   1336:        u_int cpuid = cpu_index(curcpu());
                   1337:
                   1338:        return cpuid % sc->sc_act_nvq_pairs;
                   1339: }
                   1340:
                   1341: static void
                   1342: vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
                   1343: {
                   1344:
                   1345:        vioif_send_common_locked(ifp, txq, true);
                   1346: }
                   1347:
                   1348: static int
                   1349: vioif_transmit(struct ifnet *ifp, struct mbuf *m)
                   1350: {
                   1351:        struct vioif_softc *sc = ifp->if_softc;
                   1352:        struct vioif_txqueue *txq;
                   1353:        int qid;
                   1354:
                   1355:        qid = vioif_select_txqueue(ifp, m);
                   1356:        txq = &sc->sc_txq[qid];
                   1357:
                   1358:        if (__predict_false(!pcq_put(txq->txq_intrq, m))) {
                   1359:                m_freem(m);
                   1360:                return ENOBUFS;
                   1361:        }
                   1362:
1.52      thorpej  1363:        net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
                   1364:        if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
1.46      yamaguch 1365:        if (m->m_flags & M_MCAST)
1.52      thorpej  1366:                if_statinc_ref(nsr, if_omcasts);
                   1367:        IF_STAT_PUTREF(ifp);
1.46      yamaguch 1368:
                   1369:        if (mutex_tryenter(txq->txq_lock)) {
                   1370:                if (!txq->txq_stopping)
                   1371:                        vioif_transmit_locked(ifp, txq);
                   1372:                mutex_exit(txq->txq_lock);
                   1373:        }
                   1374:
                   1375:        return 0;
                   1376: }
                   1377:
                   1378: static void
                   1379: vioif_deferred_transmit(void *arg)
                   1380: {
                   1381:        struct vioif_txqueue *txq = arg;
                   1382:        struct virtio_softc *vsc = txq->txq_vq->vq_owner;
                   1383:        struct vioif_softc *sc = device_private(virtio_child(vsc));
                   1384:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.7       ozaki-r  1385:
1.46      yamaguch 1386:        if (mutex_tryenter(txq->txq_lock)) {
                   1387:                vioif_send_common_locked(ifp, txq, true);
                   1388:                mutex_exit(txq->txq_lock);
                   1389:        }
1.1       hannken  1390: }
                   1391:
                   1392: static int
                   1393: vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
                   1394: {
                   1395:        int s, r;
                   1396:
                   1397:        s = splnet();
                   1398:
                   1399:        r = ether_ioctl(ifp, cmd, data);
                   1400:        if ((r == 0 && cmd == SIOCSIFFLAGS) ||
                   1401:            (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) {
                   1402:                if (ifp->if_flags & IFF_RUNNING)
                   1403:                        r = vioif_rx_filter(ifp->if_softc);
                   1404:                else
                   1405:                        r = 0;
                   1406:        }
                   1407:
                   1408:        splx(s);
                   1409:
                   1410:        return r;
                   1411: }
                   1412:
                   1413: void
                   1414: vioif_watchdog(struct ifnet *ifp)
                   1415: {
                   1416:        struct vioif_softc *sc = ifp->if_softc;
1.46      yamaguch 1417:        int i;
1.1       hannken  1418:
1.46      yamaguch 1419:        if (ifp->if_flags & IFF_RUNNING) {
1.55      yamaguch 1420:                for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   1421:                        vioif_tx_queue_clear(&sc->sc_txq[i]);
                   1422:                }
1.46      yamaguch 1423:        }
1.1       hannken  1424: }
                   1425:
                   1426: /*
1.39      dholland 1427:  * Receive implementation
1.1       hannken  1428:  */
1.39      dholland 1429: /* allocate and initialize a mbuf for receive */
1.1       hannken  1430: static int
1.46      yamaguch 1431: vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1.1       hannken  1432: {
1.46      yamaguch 1433:        struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1.1       hannken  1434:        struct mbuf *m;
                   1435:        int r;
                   1436:
                   1437:        MGETHDR(m, M_DONTWAIT, MT_DATA);
                   1438:        if (m == NULL)
                   1439:                return ENOBUFS;
                   1440:        MCLGET(m, M_DONTWAIT);
                   1441:        if ((m->m_flags & M_EXT) == 0) {
                   1442:                m_freem(m);
                   1443:                return ENOBUFS;
                   1444:        }
1.43      yamaguch 1445:        rxq->rxq_mbufs[i] = m;
1.1       hannken  1446:        m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
1.46      yamaguch 1447:        r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1.50      christos 1448:            rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT);
1.1       hannken  1449:        if (r) {
                   1450:                m_freem(m);
1.46      yamaguch 1451:                rxq->rxq_mbufs[i] = NULL;
1.1       hannken  1452:                return r;
                   1453:        }
                   1454:
                   1455:        return 0;
                   1456: }
                   1457:
1.39      dholland 1458: /* free a mbuf for receive */
1.1       hannken  1459: static void
1.46      yamaguch 1460: vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1.1       hannken  1461: {
1.46      yamaguch 1462:        struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1.43      yamaguch 1463:
1.46      yamaguch 1464:        bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]);
1.43      yamaguch 1465:        m_freem(rxq->rxq_mbufs[i]);
                   1466:        rxq->rxq_mbufs[i] = NULL;
1.1       hannken  1467: }
                   1468:
1.39      dholland 1469: /* add mbufs for all the empty receive slots */
1.1       hannken  1470: static void
1.46      yamaguch 1471: vioif_populate_rx_mbufs(struct vioif_rxqueue *rxq)
1.1       hannken  1472: {
1.43      yamaguch 1473:
1.46      yamaguch 1474:        mutex_enter(rxq->rxq_lock);
                   1475:        vioif_populate_rx_mbufs_locked(rxq);
                   1476:        mutex_exit(rxq->rxq_lock);
1.12      ozaki-r  1477: }
                   1478:
                   1479: static void
1.46      yamaguch 1480: vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq)
1.12      ozaki-r  1481: {
1.46      yamaguch 1482:        struct virtqueue *vq = rxq->rxq_vq;
                   1483:        struct virtio_softc *vsc = vq->vq_owner;
                   1484:        struct vioif_softc *sc = device_private(virtio_child(vsc));
1.1       hannken  1485:        int i, r, ndone = 0;
                   1486:
1.46      yamaguch 1487:        KASSERT(mutex_owned(rxq->rxq_lock));
1.7       ozaki-r  1488:
1.43      yamaguch 1489:        if (rxq->rxq_stopping)
1.12      ozaki-r  1490:                return;
1.7       ozaki-r  1491:
1.1       hannken  1492:        for (i = 0; i < vq->vq_num; i++) {
                   1493:                int slot;
                   1494:                r = virtio_enqueue_prep(vsc, vq, &slot);
                   1495:                if (r == EAGAIN)
                   1496:                        break;
                   1497:                if (r != 0)
                   1498:                        panic("enqueue_prep for rx buffers");
1.43      yamaguch 1499:                if (rxq->rxq_mbufs[slot] == NULL) {
1.46      yamaguch 1500:                        r = vioif_add_rx_mbuf(rxq, slot);
1.1       hannken  1501:                        if (r != 0) {
1.50      christos 1502:                                aprint_error_dev(sc->sc_dev,
                   1503:                                    "rx mbuf allocation failed, "
                   1504:                                    "error code %d\n", r);
1.1       hannken  1505:                                break;
                   1506:                        }
                   1507:                }
                   1508:                r = virtio_enqueue_reserve(vsc, vq, slot,
1.50      christos 1509:                    rxq->rxq_dmamaps[slot]->dm_nsegs + 1);
1.1       hannken  1510:                if (r != 0) {
1.46      yamaguch 1511:                        vioif_free_rx_mbuf(rxq, slot);
1.1       hannken  1512:                        break;
                   1513:                }
1.43      yamaguch 1514:                bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1.50      christos 1515:                    0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD);
1.43      yamaguch 1516:                bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1.50      christos 1517:                    0, MCLBYTES, BUS_DMASYNC_PREREAD);
                   1518:                virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot],
                   1519:                    false);
1.43      yamaguch 1520:                virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false);
1.1       hannken  1521:                virtio_enqueue_commit(vsc, vq, slot, false);
                   1522:                ndone++;
                   1523:        }
                   1524:        if (ndone > 0)
                   1525:                virtio_enqueue_commit(vsc, vq, -1, true);
                   1526: }
                   1527:
1.55      yamaguch 1528: static void
                   1529: vioif_rx_queue_clear(struct vioif_rxqueue *rxq)
1.1       hannken  1530: {
1.55      yamaguch 1531:        struct virtqueue *vq = rxq->rxq_vq;
                   1532:        struct virtio_softc *vsc = vq->vq_owner;
                   1533:        struct vioif_softc *sc = device_private(virtio_child(vsc));
                   1534:        u_int limit = UINT_MAX;
                   1535:        bool more;
1.7       ozaki-r  1536:
1.43      yamaguch 1537:        KASSERT(rxq->rxq_stopping);
1.7       ozaki-r  1538:
1.46      yamaguch 1539:        mutex_enter(rxq->rxq_lock);
1.55      yamaguch 1540:        for (;;) {
                   1541:                more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
                   1542:                if (more == false)
                   1543:                        break;
                   1544:        }
1.46      yamaguch 1545:        mutex_exit(rxq->rxq_lock);
1.7       ozaki-r  1546: }
                   1547:
1.39      dholland 1548: /* dequeue received packets */
1.55      yamaguch 1549: static bool
                   1550: vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
                   1551:     struct vioif_rxqueue *rxq, u_int limit)
1.7       ozaki-r  1552: {
1.43      yamaguch 1553:        struct virtqueue *vq = rxq->rxq_vq;
1.1       hannken  1554:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
                   1555:        struct mbuf *m;
                   1556:        int slot, len;
1.55      yamaguch 1557:        bool more = false, dequeued = false;
1.1       hannken  1558:
1.46      yamaguch 1559:        KASSERT(mutex_owned(rxq->rxq_lock));
1.7       ozaki-r  1560:
1.54      yamaguch 1561:        if (virtio_vq_is_enqueued(vsc, vq) == false)
1.55      yamaguch 1562:                return false;
                   1563:
                   1564:        for (;;) {
                   1565:                if (limit-- == 0) {
                   1566:                        more = true;
                   1567:                        break;
                   1568:                }
                   1569:
                   1570:                if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
                   1571:                        break;
                   1572:
                   1573:                dequeued = true;
1.54      yamaguch 1574:
1.1       hannken  1575:                len -= sizeof(struct virtio_net_hdr);
1.43      yamaguch 1576:                bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1.50      christos 1577:                    0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD);
1.43      yamaguch 1578:                bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1.50      christos 1579:                    0, MCLBYTES, BUS_DMASYNC_POSTREAD);
1.43      yamaguch 1580:                m = rxq->rxq_mbufs[slot];
1.1       hannken  1581:                KASSERT(m != NULL);
1.43      yamaguch 1582:                bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]);
1.46      yamaguch 1583:                rxq->rxq_mbufs[slot] = NULL;
1.1       hannken  1584:                virtio_dequeue_commit(vsc, vq, slot);
1.24      ozaki-r  1585:                m_set_rcvif(m, ifp);
1.1       hannken  1586:                m->m_len = m->m_pkthdr.len = len;
1.7       ozaki-r  1587:
1.46      yamaguch 1588:                mutex_exit(rxq->rxq_lock);
1.22      ozaki-r  1589:                if_percpuq_enqueue(ifp->if_percpuq, m);
1.46      yamaguch 1590:                mutex_enter(rxq->rxq_lock);
1.7       ozaki-r  1591:
1.43      yamaguch 1592:                if (rxq->rxq_stopping)
1.7       ozaki-r  1593:                        break;
1.1       hannken  1594:        }
1.3       christos 1595:
1.55      yamaguch 1596:        if (dequeued)
                   1597:                softint_schedule(rxq->rxq_softint);
                   1598:
                   1599:        return more;
1.1       hannken  1600: }
                   1601:
                   1602: /* rx interrupt; call _dequeue above and schedule a softint */
                   1603: static int
1.54      yamaguch 1604: vioif_rx_intr(void *arg)
1.1       hannken  1605: {
1.54      yamaguch 1606:        struct vioif_rxqueue *rxq = arg;
1.55      yamaguch 1607:        struct virtqueue *vq = rxq->rxq_vq;
                   1608:        struct virtio_softc *vsc = vq->vq_owner;
                   1609:        struct vioif_softc *sc = device_private(virtio_child(vsc));
                   1610:        u_int limit;
                   1611:        bool more;
                   1612:
                   1613:        limit = sc->sc_rx_intr_process_limit;
                   1614:
                   1615:        if (atomic_load_relaxed(&rxq->rxq_active) == true)
                   1616:                return 1;
1.7       ozaki-r  1617:
1.46      yamaguch 1618:        mutex_enter(rxq->rxq_lock);
1.7       ozaki-r  1619:
1.55      yamaguch 1620:        if (!rxq->rxq_stopping) {
                   1621:                rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl;
                   1622:
                   1623:                virtio_stop_vq_intr(vsc, vq);
                   1624:                atomic_store_relaxed(&rxq->rxq_active, true);
                   1625:
                   1626:                more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
                   1627:                if (more) {
                   1628:                        vioif_rx_sched_handle(sc, rxq);
                   1629:                } else {
                   1630:                        atomic_store_relaxed(&rxq->rxq_active, false);
                   1631:                        virtio_start_vq_intr(vsc, vq);
                   1632:                }
                   1633:        }
                   1634:
                   1635:        mutex_exit(rxq->rxq_lock);
                   1636:        return 1;
                   1637: }
                   1638:
                   1639: static void
                   1640: vioif_rx_handle(void *xrxq)
                   1641: {
                   1642:        struct vioif_rxqueue *rxq = xrxq;
                   1643:        struct virtqueue *vq = rxq->rxq_vq;
                   1644:        struct virtio_softc *vsc = vq->vq_owner;
                   1645:        struct vioif_softc *sc = device_private(virtio_child(vsc));
                   1646:        u_int limit;
                   1647:        bool more;
                   1648:
                   1649:        limit = sc->sc_rx_process_limit;
                   1650:
                   1651:        mutex_enter(rxq->rxq_lock);
1.1       hannken  1652:
1.55      yamaguch 1653:        if (!rxq->rxq_stopping) {
                   1654:                more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
                   1655:                if (more) {
                   1656:                        vioif_rx_sched_handle(sc, rxq);
                   1657:                } else {
                   1658:                        atomic_store_relaxed(&rxq->rxq_active, false);
                   1659:                        virtio_start_vq_intr(vsc, rxq->rxq_vq);
                   1660:                }
                   1661:        }
1.1       hannken  1662:
1.46      yamaguch 1663:        mutex_exit(rxq->rxq_lock);
1.55      yamaguch 1664: }
                   1665:
                   1666: static void
                   1667: vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
                   1668: {
                   1669:
                   1670:        if (rxq->rxq_workqueue)
                   1671:                vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work);
                   1672:        else
                   1673:                softint_schedule(rxq->rxq_handle_si);
1.1       hannken  1674: }
                   1675:
1.39      dholland 1676: /* softint: enqueue receive requests for new incoming packets */
1.1       hannken  1677: static void
                   1678: vioif_rx_softint(void *arg)
                   1679: {
1.46      yamaguch 1680:        struct vioif_rxqueue *rxq = arg;
1.1       hannken  1681:
1.46      yamaguch 1682:        vioif_populate_rx_mbufs(rxq);
1.1       hannken  1683: }
                   1684:
                   1685: /* free all the mbufs; called from if_stop(disable) */
                   1686: static void
1.46      yamaguch 1687: vioif_rx_drain(struct vioif_rxqueue *rxq)
1.1       hannken  1688: {
1.43      yamaguch 1689:        struct virtqueue *vq = rxq->rxq_vq;
1.1       hannken  1690:        int i;
                   1691:
                   1692:        for (i = 0; i < vq->vq_num; i++) {
1.43      yamaguch 1693:                if (rxq->rxq_mbufs[i] == NULL)
1.1       hannken  1694:                        continue;
1.46      yamaguch 1695:                vioif_free_rx_mbuf(rxq, i);
1.1       hannken  1696:        }
                   1697: }
                   1698:
                   1699: /*
                   1700:  * Transmition implementation
                   1701:  */
                   1702: /* actual transmission is done in if_start */
                   1703: /* tx interrupt; dequeue and free mbufs */
                   1704: /*
                   1705:  * tx interrupt is actually disabled; this should be called upon
                   1706:  * tx vq full and watchdog
                   1707:  */
1.55      yamaguch 1708:
1.1       hannken  1709: static int
1.54      yamaguch 1710: vioif_tx_intr(void *arg)
1.1       hannken  1711: {
1.54      yamaguch 1712:        struct vioif_txqueue *txq = arg;
                   1713:        struct virtqueue *vq = txq->txq_vq;
1.1       hannken  1714:        struct virtio_softc *vsc = vq->vq_owner;
1.32      jdolecek 1715:        struct vioif_softc *sc = device_private(virtio_child(vsc));
1.23      pooka    1716:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.55      yamaguch 1717:        bool more;
                   1718:        u_int limit;
                   1719:
                   1720:        limit = sc->sc_tx_intr_process_limit;
                   1721:
                   1722:        if (atomic_load_relaxed(&txq->txq_active) == true)
                   1723:                return 1;
1.7       ozaki-r  1724:
1.46      yamaguch 1725:        mutex_enter(txq->txq_lock);
1.7       ozaki-r  1726:
1.55      yamaguch 1727:        if (!txq->txq_stopping) {
                   1728:                txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl;
                   1729:
                   1730:                virtio_stop_vq_intr(vsc, vq);
                   1731:                atomic_store_relaxed(&txq->txq_active, true);
                   1732:
                   1733:                more = vioif_tx_deq_locked(sc, vsc, txq, limit);
                   1734:                if (more) {
                   1735:                        vioif_tx_sched_handle(sc, txq);
                   1736:                } else {
                   1737:                        atomic_store_relaxed(&txq->txq_active, false);
1.7       ozaki-r  1738:
1.55      yamaguch 1739:                        /* for ALTQ */
                   1740:                        if (txq == &sc->sc_txq[0]) {
                   1741:                                if_schedule_deferred_start(ifp);
                   1742:                                ifp->if_flags &= ~IFF_OACTIVE;
                   1743:                        }
                   1744:                        softint_schedule(txq->txq_deferred_transmit);
                   1745:
                   1746:                        virtio_start_vq_intr(vsc, vq);
                   1747:                }
                   1748:        }
1.7       ozaki-r  1749:
1.46      yamaguch 1750:        mutex_exit(txq->txq_lock);
                   1751:
1.55      yamaguch 1752:        return 1;
                   1753: }
                   1754:
                   1755: static void
                   1756: vioif_tx_handle(void *xtxq)
                   1757: {
                   1758:        struct vioif_txqueue *txq = xtxq;
                   1759:        struct virtqueue *vq = txq->txq_vq;
                   1760:        struct virtio_softc *vsc = vq->vq_owner;
                   1761:        struct vioif_softc *sc = device_private(virtio_child(vsc));
                   1762:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
                   1763:        u_int limit;
                   1764:        bool more;
                   1765:
                   1766:        limit = sc->sc_tx_process_limit;
                   1767:
                   1768:        mutex_enter(txq->txq_lock);
                   1769:
                   1770:        if (!txq->txq_stopping) {
                   1771:                more = vioif_tx_deq_locked(sc, vsc, txq, limit);
                   1772:                if (more) {
                   1773:                        vioif_tx_sched_handle(sc, txq);
                   1774:                } else {
                   1775:                        atomic_store_relaxed(&txq->txq_active, false);
                   1776:
                   1777:                        /* for ALTQ */
                   1778:                        if (txq == &sc->sc_txq[0]) {
                   1779:                                if_schedule_deferred_start(ifp);
                   1780:                                ifp->if_flags &= ~IFF_OACTIVE;
                   1781:                        }
                   1782:                        softint_schedule(txq->txq_deferred_transmit);
                   1783:
                   1784:                        virtio_start_vq_intr(vsc, txq->txq_vq);
                   1785:                }
1.46      yamaguch 1786:        }
1.55      yamaguch 1787:
                   1788:        mutex_exit(txq->txq_lock);
                   1789: }
                   1790:
                   1791: static void
                   1792: vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq)
                   1793: {
                   1794:
                   1795:        if (txq->txq_workqueue)
                   1796:                vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work);
                   1797:        else
                   1798:                softint_schedule(txq->txq_handle_si);
1.7       ozaki-r  1799: }
                   1800:
1.55      yamaguch 1801: static void
                   1802: vioif_tx_queue_clear(struct vioif_txqueue *txq)
1.7       ozaki-r  1803: {
1.55      yamaguch 1804:        struct virtqueue *vq = txq->txq_vq;
1.7       ozaki-r  1805:        struct virtio_softc *vsc = vq->vq_owner;
1.32      jdolecek 1806:        struct vioif_softc *sc = device_private(virtio_child(vsc));
1.55      yamaguch 1807:        u_int limit = UINT_MAX;
                   1808:        bool more;
                   1809:
                   1810:        mutex_enter(txq->txq_lock);
                   1811:        for (;;) {
                   1812:                more = vioif_tx_deq_locked(sc, vsc, txq, limit);
                   1813:                if (more == false)
                   1814:                        break;
                   1815:        }
                   1816:        mutex_exit(txq->txq_lock);
                   1817: }
                   1818:
                   1819: static bool
                   1820: vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
                   1821:     struct vioif_txqueue *txq, u_int limit)
                   1822: {
                   1823:        struct virtqueue *vq = txq->txq_vq;
1.1       hannken  1824:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
                   1825:        struct mbuf *m;
                   1826:        int slot, len;
1.55      yamaguch 1827:        bool more = false;
1.1       hannken  1828:
1.46      yamaguch 1829:        KASSERT(mutex_owned(txq->txq_lock));
1.7       ozaki-r  1830:
1.54      yamaguch 1831:        if (virtio_vq_is_enqueued(vsc, vq) == false)
1.55      yamaguch 1832:                return false;
                   1833:
                   1834:        for (;;) {
                   1835:                if (limit-- == 0) {
                   1836:                        more = true;
                   1837:                        break;
                   1838:                }
                   1839:
                   1840:                if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
                   1841:                        break;
1.54      yamaguch 1842:
1.43      yamaguch 1843:                bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1.50      christos 1844:                    0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE);
1.43      yamaguch 1845:                bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1.50      christos 1846:                    0, txq->txq_dmamaps[slot]->dm_mapsize,
                   1847:                    BUS_DMASYNC_POSTWRITE);
1.43      yamaguch 1848:                m = txq->txq_mbufs[slot];
                   1849:                bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]);
1.46      yamaguch 1850:                txq->txq_mbufs[slot] = NULL;
1.1       hannken  1851:                virtio_dequeue_commit(vsc, vq, slot);
1.52      thorpej  1852:                if_statinc(ifp, if_opackets);
1.1       hannken  1853:                m_freem(m);
                   1854:        }
                   1855:
1.55      yamaguch 1856:        return more;
1.1       hannken  1857: }
                   1858:
                   1859: /* free all the mbufs already put on vq; called from if_stop(disable) */
                   1860: static void
1.46      yamaguch 1861: vioif_tx_drain(struct vioif_txqueue *txq)
1.1       hannken  1862: {
1.43      yamaguch 1863:        struct virtqueue *vq = txq->txq_vq;
1.46      yamaguch 1864:        struct virtio_softc *vsc = vq->vq_owner;
1.1       hannken  1865:        int i;
                   1866:
1.43      yamaguch 1867:        KASSERT(txq->txq_stopping);
1.7       ozaki-r  1868:
1.1       hannken  1869:        for (i = 0; i < vq->vq_num; i++) {
1.43      yamaguch 1870:                if (txq->txq_mbufs[i] == NULL)
1.1       hannken  1871:                        continue;
1.43      yamaguch 1872:                bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]);
                   1873:                m_freem(txq->txq_mbufs[i]);
                   1874:                txq->txq_mbufs[i] = NULL;
1.1       hannken  1875:        }
                   1876: }
                   1877:
                   1878: /*
                   1879:  * Control vq
                   1880:  */
                   1881: /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
1.44      yamaguch 1882: static void
                   1883: vioif_ctrl_acquire(struct vioif_softc *sc)
1.1       hannken  1884: {
1.43      yamaguch 1885:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.1       hannken  1886:
1.43      yamaguch 1887:        mutex_enter(&ctrlq->ctrlq_wait_lock);
                   1888:        while (ctrlq->ctrlq_inuse != FREE)
                   1889:                cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
                   1890:        ctrlq->ctrlq_inuse = INUSE;
1.44      yamaguch 1891:        ctrlq->ctrlq_owner = curlwp;
                   1892:        mutex_exit(&ctrlq->ctrlq_wait_lock);
                   1893: }
                   1894:
                   1895: static void
                   1896: vioif_ctrl_release(struct vioif_softc *sc)
                   1897: {
                   1898:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
                   1899:
                   1900:        KASSERT(ctrlq->ctrlq_inuse != FREE);
                   1901:        KASSERT(ctrlq->ctrlq_owner == curlwp);
                   1902:
                   1903:        mutex_enter(&ctrlq->ctrlq_wait_lock);
                   1904:        ctrlq->ctrlq_inuse = FREE;
1.46      yamaguch 1905:        ctrlq->ctrlq_owner = NULL;
1.44      yamaguch 1906:        cv_signal(&ctrlq->ctrlq_wait);
1.43      yamaguch 1907:        mutex_exit(&ctrlq->ctrlq_wait_lock);
1.44      yamaguch 1908: }
                   1909:
                   1910: static int
                   1911: vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
                   1912:     struct vioif_ctrl_cmdspec *specs, int nspecs)
                   1913: {
                   1914:        struct virtio_softc *vsc = sc->sc_virtio;
                   1915:        int i, r, loaded;
                   1916:
                   1917:        loaded = 0;
                   1918:        for (i = 0; i < nspecs; i++) {
                   1919:                r = bus_dmamap_load(virtio_dmat(vsc),
                   1920:                    specs[i].dmamap, specs[i].buf, specs[i].bufsize,
1.48      msaitoh  1921:                    NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1.44      yamaguch 1922:                if (r) {
1.50      christos 1923:                        aprint_error_dev(sc->sc_dev, "control command dmamap"
                   1924:                            " load failed, error code %d\n", r);
1.44      yamaguch 1925:                        goto err;
                   1926:                }
                   1927:                loaded++;
                   1928:
                   1929:        }
                   1930:
                   1931:        return r;
                   1932:
                   1933: err:
                   1934:        for (i = 0; i < loaded; i++) {
                   1935:                bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
                   1936:        }
                   1937:
                   1938:        return r;
                   1939: }
                   1940:
                   1941: static void
                   1942: vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
                   1943:     struct vioif_ctrl_cmdspec *specs, int nspecs)
                   1944: {
                   1945:        struct virtio_softc *vsc = sc->sc_virtio;
                   1946:        int i;
                   1947:
                   1948:        for (i = 0; i < nspecs; i++) {
                   1949:                bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
                   1950:        }
                   1951: }
                   1952:
                   1953: static int
                   1954: vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
                   1955:     struct vioif_ctrl_cmdspec *specs, int nspecs)
                   1956: {
                   1957:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
                   1958:        struct virtqueue *vq = ctrlq->ctrlq_vq;
                   1959:        struct virtio_softc *vsc = sc->sc_virtio;
                   1960:        int i, r, slot;
1.43      yamaguch 1961:
1.44      yamaguch 1962:        ctrlq->ctrlq_cmd->class = class;
1.43      yamaguch 1963:        ctrlq->ctrlq_cmd->command = cmd;
1.1       hannken  1964:
1.43      yamaguch 1965:        bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
1.50      christos 1966:            0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
1.44      yamaguch 1967:        for (i = 0; i < nspecs; i++) {
                   1968:                bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
1.50      christos 1969:                    0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
1.44      yamaguch 1970:        }
1.43      yamaguch 1971:        bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
1.50      christos 1972:            0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
1.1       hannken  1973:
                   1974:        r = virtio_enqueue_prep(vsc, vq, &slot);
                   1975:        if (r != 0)
                   1976:                panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1.44      yamaguch 1977:        r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
1.1       hannken  1978:        if (r != 0)
                   1979:                panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1.43      yamaguch 1980:        virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
1.44      yamaguch 1981:        for (i = 0; i < nspecs; i++) {
                   1982:                virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
                   1983:        }
1.43      yamaguch 1984:        virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
1.1       hannken  1985:        virtio_enqueue_commit(vsc, vq, slot, true);
                   1986:
                   1987:        /* wait for done */
1.43      yamaguch 1988:        mutex_enter(&ctrlq->ctrlq_wait_lock);
                   1989:        while (ctrlq->ctrlq_inuse != DONE)
                   1990:                cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
                   1991:        mutex_exit(&ctrlq->ctrlq_wait_lock);
1.1       hannken  1992:        /* already dequeueued */
                   1993:
1.43      yamaguch 1994:        bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
1.50      christos 1995:            sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
1.44      yamaguch 1996:        for (i = 0; i < nspecs; i++) {
                   1997:                bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
1.50      christos 1998:                    specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
1.44      yamaguch 1999:        }
1.43      yamaguch 2000:        bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
1.50      christos 2001:            sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
1.1       hannken  2002:
1.43      yamaguch 2003:        if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
1.1       hannken  2004:                r = 0;
                   2005:        else {
1.50      christos 2006:                aprint_error_dev(sc->sc_dev, "failed setting rx mode\n");
1.1       hannken  2007:                r = EIO;
                   2008:        }
                   2009:
1.44      yamaguch 2010:        return r;
                   2011: }
                   2012:
                   2013: static int
                   2014: vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
                   2015: {
                   2016:        struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
                   2017:        struct vioif_ctrl_cmdspec specs[1];
                   2018:        int r;
                   2019:
                   2020:        if (!sc->sc_has_ctrl)
                   2021:                return ENOTSUP;
                   2022:
                   2023:        vioif_ctrl_acquire(sc);
                   2024:
                   2025:        rx->onoff = onoff;
                   2026:        specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
                   2027:        specs[0].buf = rx;
                   2028:        specs[0].bufsize = sizeof(*rx);
                   2029:
                   2030:        r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
                   2031:            specs, __arraycount(specs));
1.3       christos 2032:
1.44      yamaguch 2033:        vioif_ctrl_release(sc);
1.1       hannken  2034:        return r;
                   2035: }
                   2036:
                   2037: static int
                   2038: vioif_set_promisc(struct vioif_softc *sc, bool onoff)
                   2039: {
1.50      christos 2040:        return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
1.1       hannken  2041: }
                   2042:
                   2043: static int
                   2044: vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
                   2045: {
1.50      christos 2046:        return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
1.1       hannken  2047: }
                   2048:
                   2049: /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
                   2050: static int
                   2051: vioif_set_rx_filter(struct vioif_softc *sc)
                   2052: {
1.43      yamaguch 2053:        /* filter already set in ctrlq->ctrlq_mac_tbl */
1.44      yamaguch 2054:        struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
                   2055:        struct vioif_ctrl_cmdspec specs[2];
                   2056:        int nspecs = __arraycount(specs);
                   2057:        int r;
                   2058:
                   2059:        mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
                   2060:        mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
1.1       hannken  2061:
1.32      jdolecek 2062:        if (!sc->sc_has_ctrl)
1.1       hannken  2063:                return ENOTSUP;
                   2064:
1.44      yamaguch 2065:        vioif_ctrl_acquire(sc);
1.1       hannken  2066:
1.44      yamaguch 2067:        specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
                   2068:        specs[0].buf = mac_tbl_uc;
                   2069:        specs[0].bufsize = sizeof(*mac_tbl_uc)
                   2070:            + (ETHER_ADDR_LEN * mac_tbl_uc->nentries);
                   2071:
                   2072:        specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
                   2073:        specs[1].buf = mac_tbl_mc;
                   2074:        specs[1].bufsize = sizeof(*mac_tbl_mc)
                   2075:            + (ETHER_ADDR_LEN * mac_tbl_mc->nentries);
1.1       hannken  2076:
1.44      yamaguch 2077:        r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
                   2078:        if (r != 0)
1.1       hannken  2079:                goto out;
                   2080:
1.44      yamaguch 2081:        r = vioif_ctrl_send_command(sc,
                   2082:            VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
                   2083:            specs, nspecs);
1.1       hannken  2084:
1.44      yamaguch 2085:        vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
1.1       hannken  2086:
                   2087: out:
1.44      yamaguch 2088:        vioif_ctrl_release(sc);
1.3       christos 2089:
1.1       hannken  2090:        return r;
                   2091: }
                   2092:
1.46      yamaguch 2093: static int
                   2094: vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
                   2095: {
                   2096:        struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
                   2097:        struct vioif_ctrl_cmdspec specs[1];
                   2098:        int r;
                   2099:
                   2100:        if (!sc->sc_has_ctrl)
                   2101:                return ENOTSUP;
                   2102:
                   2103:        if (nvq_pairs <= 1)
                   2104:                return EINVAL;
                   2105:
                   2106:        vioif_ctrl_acquire(sc);
                   2107:
                   2108:        mq->virtqueue_pairs = nvq_pairs;
                   2109:        specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
                   2110:        specs[0].buf = mq;
                   2111:        specs[0].bufsize = sizeof(*mq);
                   2112:
                   2113:        r = vioif_ctrl_send_command(sc,
                   2114:            VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
                   2115:            specs, __arraycount(specs));
                   2116:
                   2117:        vioif_ctrl_release(sc);
                   2118:
                   2119:        return r;
                   2120: }
                   2121:
1.1       hannken  2122: /* ctrl vq interrupt; wake up the command issuer */
                   2123: static int
1.54      yamaguch 2124: vioif_ctrl_intr(void *arg)
1.1       hannken  2125: {
1.54      yamaguch 2126:        struct vioif_ctrlqueue *ctrlq = arg;
                   2127:        struct virtqueue *vq = ctrlq->ctrlq_vq;
1.1       hannken  2128:        struct virtio_softc *vsc = vq->vq_owner;
                   2129:        int r, slot;
                   2130:
1.54      yamaguch 2131:        if (virtio_vq_is_enqueued(vsc, vq) == false)
                   2132:                return 0;
                   2133:
1.1       hannken  2134:        r = virtio_dequeue(vsc, vq, &slot, NULL);
                   2135:        if (r == ENOENT)
                   2136:                return 0;
                   2137:        virtio_dequeue_commit(vsc, vq, slot);
                   2138:
1.43      yamaguch 2139:        mutex_enter(&ctrlq->ctrlq_wait_lock);
                   2140:        ctrlq->ctrlq_inuse = DONE;
                   2141:        cv_signal(&ctrlq->ctrlq_wait);
                   2142:        mutex_exit(&ctrlq->ctrlq_wait_lock);
1.1       hannken  2143:
                   2144:        return 1;
                   2145: }
                   2146:
                   2147: /*
                   2148:  * If IFF_PROMISC requested,  set promiscuous
                   2149:  * If multicast filter small enough (<=MAXENTRIES) set rx filter
                   2150:  * If large multicast filter exist use ALLMULTI
                   2151:  */
                   2152: /*
                   2153:  * If setting rx filter fails fall back to ALLMULTI
                   2154:  * If ALLMULTI fails fall back to PROMISC
                   2155:  */
                   2156: static int
                   2157: vioif_rx_filter(struct vioif_softc *sc)
                   2158: {
1.48      msaitoh  2159:        struct ethercom *ec = &sc->sc_ethercom;
                   2160:        struct ifnet *ifp = &ec->ec_if;
1.1       hannken  2161:        struct ether_multi *enm;
                   2162:        struct ether_multistep step;
1.43      yamaguch 2163:        struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1.1       hannken  2164:        int nentries;
                   2165:        int promisc = 0, allmulti = 0, rxfilter = 0;
                   2166:        int r;
                   2167:
1.32      jdolecek 2168:        if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */
1.1       hannken  2169:                ifp->if_flags |= IFF_PROMISC;
                   2170:                return 0;
                   2171:        }
                   2172:
                   2173:        if (ifp->if_flags & IFF_PROMISC) {
                   2174:                promisc = 1;
                   2175:                goto set;
                   2176:        }
                   2177:
                   2178:        nentries = -1;
1.48      msaitoh  2179:        ETHER_LOCK(ec);
                   2180:        ETHER_FIRST_MULTI(step, ec, enm);
1.1       hannken  2181:        while (nentries++, enm != NULL) {
                   2182:                if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
                   2183:                        allmulti = 1;
1.31      ozaki-r  2184:                        goto set_unlock;
1.1       hannken  2185:                }
1.50      christos 2186:                if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
1.1       hannken  2187:                        allmulti = 1;
1.31      ozaki-r  2188:                        goto set_unlock;
1.1       hannken  2189:                }
1.43      yamaguch 2190:                memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
1.50      christos 2191:                    enm->enm_addrlo, ETHER_ADDR_LEN);
1.1       hannken  2192:                ETHER_NEXT_MULTI(step, enm);
                   2193:        }
                   2194:        rxfilter = 1;
                   2195:
1.31      ozaki-r  2196: set_unlock:
1.48      msaitoh  2197:        ETHER_UNLOCK(ec);
1.30      ozaki-r  2198:
1.31      ozaki-r  2199: set:
1.1       hannken  2200:        if (rxfilter) {
1.43      yamaguch 2201:                ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
                   2202:                ctrlq->ctrlq_mac_tbl_mc->nentries = nentries;
1.1       hannken  2203:                r = vioif_set_rx_filter(sc);
                   2204:                if (r != 0) {
                   2205:                        rxfilter = 0;
                   2206:                        allmulti = 1; /* fallback */
                   2207:                }
                   2208:        } else {
                   2209:                /* remove rx filter */
1.43      yamaguch 2210:                ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
                   2211:                ctrlq->ctrlq_mac_tbl_mc->nentries = 0;
1.1       hannken  2212:                r = vioif_set_rx_filter(sc);
                   2213:                /* what to do on failure? */
                   2214:        }
                   2215:        if (allmulti) {
                   2216:                r = vioif_set_allmulti(sc, true);
                   2217:                if (r != 0) {
                   2218:                        allmulti = 0;
                   2219:                        promisc = 1; /* fallback */
                   2220:                }
                   2221:        } else {
                   2222:                r = vioif_set_allmulti(sc, false);
                   2223:                /* what to do on failure? */
                   2224:        }
                   2225:        if (promisc) {
                   2226:                r = vioif_set_promisc(sc, true);
                   2227:        } else {
                   2228:                r = vioif_set_promisc(sc, false);
                   2229:        }
                   2230:
                   2231:        return r;
                   2232: }
                   2233:
1.34      ozaki-r  2234: static bool
                   2235: vioif_is_link_up(struct vioif_softc *sc)
                   2236: {
                   2237:        struct virtio_softc *vsc = sc->sc_virtio;
                   2238:        uint16_t status;
                   2239:
                   2240:        if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
                   2241:                status = virtio_read_device_config_2(vsc,
                   2242:                    VIRTIO_NET_CONFIG_STATUS);
                   2243:        else
                   2244:                status = VIRTIO_NET_S_LINK_UP;
                   2245:
                   2246:        return ((status & VIRTIO_NET_S_LINK_UP) != 0);
                   2247: }
                   2248:
                   2249: /* change link status */
                   2250: static void
                   2251: vioif_update_link_status(struct vioif_softc *sc)
                   2252: {
                   2253:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1.46      yamaguch 2254:        struct vioif_txqueue *txq;
1.34      ozaki-r  2255:        bool active, changed;
1.46      yamaguch 2256:        int link, i;
1.34      ozaki-r  2257:
                   2258:        active = vioif_is_link_up(sc);
                   2259:        changed = false;
                   2260:
                   2261:        if (active) {
                   2262:                if (!sc->sc_link_active)
                   2263:                        changed = true;
                   2264:
                   2265:                link = LINK_STATE_UP;
                   2266:                sc->sc_link_active = true;
                   2267:        } else {
                   2268:                if (sc->sc_link_active)
                   2269:                        changed = true;
                   2270:
                   2271:                link = LINK_STATE_DOWN;
                   2272:                sc->sc_link_active = false;
                   2273:        }
                   2274:
1.43      yamaguch 2275:        if (changed) {
1.46      yamaguch 2276:                for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
                   2277:                        txq = &sc->sc_txq[i];
                   2278:
                   2279:                        mutex_enter(txq->txq_lock);
                   2280:                        txq->txq_link_active = sc->sc_link_active;
                   2281:                        mutex_exit(txq->txq_lock);
                   2282:                }
1.43      yamaguch 2283:
1.34      ozaki-r  2284:                if_link_state_change(ifp, link);
1.43      yamaguch 2285:        }
1.34      ozaki-r  2286: }
                   2287:
                   2288: static int
                   2289: vioif_config_change(struct virtio_softc *vsc)
                   2290: {
                   2291:        struct vioif_softc *sc = device_private(virtio_child(vsc));
                   2292:
                   2293:        softint_schedule(sc->sc_ctl_softint);
                   2294:        return 0;
                   2295: }
                   2296:
                   2297: static void
                   2298: vioif_ctl_softint(void *arg)
                   2299: {
                   2300:        struct vioif_softc *sc = arg;
                   2301:        struct ifnet *ifp = &sc->sc_ethercom.ec_if;
                   2302:
                   2303:        vioif_update_link_status(sc);
                   2304:        vioif_start(ifp);
                   2305: }
                   2306:
1.55      yamaguch 2307: static struct workqueue *
                   2308: vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
                   2309: {
                   2310:        struct workqueue *wq;
                   2311:        int error;
                   2312:
                   2313:        error = workqueue_create(&wq, name, vioif_workq_work, NULL,
                   2314:            prio, ipl, flags);
                   2315:
                   2316:        if (error)
                   2317:                return NULL;
                   2318:
                   2319:        return wq;
                   2320: }
                   2321:
                   2322: static void
                   2323: vioif_workq_destroy(struct workqueue *wq)
                   2324: {
                   2325:
                   2326:        workqueue_destroy(wq);
                   2327: }
                   2328:
                   2329: static void
                   2330: vioif_workq_work(struct work *wk, void *context)
                   2331: {
                   2332:        struct vioif_work *work;
                   2333:
                   2334:        work = container_of(wk, struct vioif_work, cookie);
                   2335:
                   2336:        atomic_store_relaxed(&work->added, 0);
                   2337:        work->func(work->arg);
                   2338: }
                   2339:
                   2340: static void
                   2341: vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
                   2342: {
                   2343:
                   2344:        memset(work, 0, sizeof(*work));
                   2345:        work->func = func;
                   2346:        work->arg = arg;
                   2347: }
                   2348:
                   2349: static void
                   2350: vioif_work_add(struct workqueue *wq, struct vioif_work *work)
                   2351: {
                   2352:
                   2353:        if (atomic_load_relaxed(&work->added) != 0)
                   2354:                return;
                   2355:
                   2356:        atomic_store_relaxed(&work->added, 1);
                   2357:        kpreempt_disable();
                   2358:        workqueue_enqueue(wq, &work->cookie, NULL);
                   2359:        kpreempt_enable();
                   2360: }
                   2361:
                   2362: static void
                   2363: vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
                   2364: {
                   2365:
                   2366:        workqueue_wait(wq, &work->cookie);
                   2367: }
                   2368:
                   2369: static int
                   2370: vioif_setup_sysctl(struct vioif_softc *sc)
                   2371: {
                   2372:        const char *devname;
                   2373:        struct sysctllog **log;
                   2374:        const struct sysctlnode *rnode, *rxnode, *txnode;
                   2375:        int error;
                   2376:
                   2377:        log = &sc->sc_sysctllog;
                   2378:        devname = device_xname(sc->sc_dev);
                   2379:
                   2380:        error = sysctl_createv(log, 0, NULL, &rnode,
                   2381:            0, CTLTYPE_NODE, devname,
                   2382:            SYSCTL_DESCR("virtio-net information and settings"),
                   2383:            NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
                   2384:        if (error)
                   2385:                goto out;
                   2386:
                   2387:        error = sysctl_createv(log, 0, &rnode, NULL,
                   2388:            CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
                   2389:            SYSCTL_DESCR("Use workqueue for packet processing"),
                   2390:            NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
                   2391:        if (error)
                   2392:                goto out;
                   2393:
                   2394:        error = sysctl_createv(log, 0, &rnode, &rxnode,
                   2395:            0, CTLTYPE_NODE, "rx",
                   2396:            SYSCTL_DESCR("virtio-net information and settings for Rx"),
                   2397:            NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
                   2398:        if (error)
                   2399:                goto out;
                   2400:
                   2401:        error = sysctl_createv(log, 0, &rxnode, NULL,
                   2402:            CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
                   2403:            SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
                   2404:            NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
                   2405:        if (error)
                   2406:                goto out;
                   2407:
                   2408:        error = sysctl_createv(log, 0, &rxnode, NULL,
                   2409:            CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
                   2410:            SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
                   2411:            NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
                   2412:        if (error)
                   2413:                goto out;
                   2414:
                   2415:        error = sysctl_createv(log, 0, &rnode, &txnode,
                   2416:            0, CTLTYPE_NODE, "tx",
                   2417:            SYSCTL_DESCR("virtio-net information and settings for Tx"),
                   2418:            NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
                   2419:        if (error)
                   2420:                goto out;
                   2421:
                   2422:        error = sysctl_createv(log, 0, &txnode, NULL,
                   2423:            CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
                   2424:            SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
                   2425:            NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
                   2426:        if (error)
                   2427:                goto out;
                   2428:
                   2429:        error = sysctl_createv(log, 0, &txnode, NULL,
                   2430:            CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
                   2431:            SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
                   2432:            NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
                   2433:
                   2434: out:
                   2435:        if (error)
                   2436:                sysctl_teardown(log);
                   2437:
                   2438:        return error;
                   2439: }
                   2440:
1.26      pgoyette 2441: MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
1.48      msaitoh  2442:
1.26      pgoyette 2443: #ifdef _MODULE
                   2444: #include "ioconf.c"
                   2445: #endif
1.48      msaitoh  2446:
                   2447: static int
1.26      pgoyette 2448: if_vioif_modcmd(modcmd_t cmd, void *opaque)
                   2449: {
                   2450:        int error = 0;
1.48      msaitoh  2451:
1.26      pgoyette 2452: #ifdef _MODULE
                   2453:        switch (cmd) {
                   2454:        case MODULE_CMD_INIT:
1.48      msaitoh  2455:                error = config_init_component(cfdriver_ioconf_if_vioif,
                   2456:                    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
1.26      pgoyette 2457:                break;
                   2458:        case MODULE_CMD_FINI:
                   2459:                error = config_fini_component(cfdriver_ioconf_if_vioif,
                   2460:                    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
                   2461:                break;
                   2462:        default:
                   2463:                error = ENOTTY;
1.48      msaitoh  2464:                break;
1.26      pgoyette 2465:        }
                   2466: #endif
1.48      msaitoh  2467:
1.26      pgoyette 2468:        return error;
                   2469: }

CVSweb <webmaster@jp.NetBSD.org>