Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/dev/usb/xhci.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/dev/usb/xhci.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.16.4.1 retrieving revision 1.16.4.2 diff -u -p -r1.16.4.1 -r1.16.4.2 --- src/sys/dev/usb/xhci.c 2014/03/10 13:21:22 1.16.4.1 +++ src/sys/dev/usb/xhci.c 2014/05/18 17:45:48 1.16.4.2 @@ -0,0 +1,2911 @@ +/* $NetBSD: xhci.c,v 1.16.4.2 2014/05/18 17:45:48 rmind Exp $ */ + +/* + * Copyright (c) 2013 Jonathan A. Kollasch + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.16.4.2 2014/05/18 17:45:48 rmind Exp $"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef XHCI_DEBUG +int xhcidebug = 0; +#define DPRINTF(x) do { if (xhcidebug) printf x; } while(0) +#define DPRINTFN(n,x) do { if (xhcidebug>(n)) printf x; } while (0) +#else +#define DPRINTF(x) +#define DPRINTFN(n,x) +#endif + +#define XHCI_DCI_SLOT 0 +#define XHCI_DCI_EP_CONTROL 1 + +#define XHCI_ICI_INPUT_CONTROL 0 + +struct xhci_pipe { + struct usbd_pipe xp_pipe; +}; + +#define XHCI_INTR_ENDPT 1 +#define XHCI_COMMAND_RING_TRBS 256 +#define XHCI_EVENT_RING_TRBS 256 +#define XHCI_EVENT_RING_SEGMENTS 1 +#define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT + +static usbd_status xhci_open(usbd_pipe_handle); +static int xhci_intr1(struct xhci_softc * const); +static void xhci_softintr(void *); +static void xhci_poll(struct usbd_bus *); +static usbd_status xhci_allocm(struct usbd_bus *, usb_dma_t *, uint32_t); +static void xhci_freem(struct usbd_bus *, usb_dma_t *); +static usbd_xfer_handle xhci_allocx(struct usbd_bus *); +static void xhci_freex(struct usbd_bus *, usbd_xfer_handle); +static void xhci_get_lock(struct usbd_bus *, kmutex_t **); +static usbd_status xhci_new_device(device_t, usbd_bus_handle, int, int, int, + struct usbd_port *); + +static usbd_status xhci_configure_endpoint(usbd_pipe_handle); +static usbd_status xhci_unconfigure_endpoint(usbd_pipe_handle); +static usbd_status xhci_reset_endpoint(usbd_pipe_handle); +//static usbd_status xhci_stop_endpoint(usbd_pipe_handle); + +static usbd_status xhci_set_dequeue(usbd_pipe_handle); + +static usbd_status xhci_do_command(struct xhci_softc * const, + struct xhci_trb * const, int); +static usbd_status xhci_init_slot(struct xhci_softc * const, uint32_t, + int, int, int, int); +static usbd_status xhci_enable_slot(struct xhci_softc * const, + uint8_t * const); +static usbd_status xhci_address_device(struct xhci_softc * const, + uint64_t, uint8_t, bool); +static usbd_status xhci_update_ep0_mps(struct xhci_softc * const, + struct xhci_slot * const, u_int); +static usbd_status xhci_ring_init(struct xhci_softc * const, + struct xhci_ring * const, size_t, size_t); +static void xhci_ring_free(struct xhci_softc * const, struct xhci_ring * const); + +static void xhci_noop(usbd_pipe_handle); + +static usbd_status xhci_root_ctrl_transfer(usbd_xfer_handle); +static usbd_status xhci_root_ctrl_start(usbd_xfer_handle); +static void xhci_root_ctrl_abort(usbd_xfer_handle); +static void xhci_root_ctrl_close(usbd_pipe_handle); +static void xhci_root_ctrl_done(usbd_xfer_handle); + +static usbd_status xhci_root_intr_transfer(usbd_xfer_handle); +static usbd_status xhci_root_intr_start(usbd_xfer_handle); +static void xhci_root_intr_abort(usbd_xfer_handle); +static void xhci_root_intr_close(usbd_pipe_handle); +static void xhci_root_intr_done(usbd_xfer_handle); + +static usbd_status xhci_device_ctrl_transfer(usbd_xfer_handle); +static usbd_status xhci_device_ctrl_start(usbd_xfer_handle); +static void xhci_device_ctrl_abort(usbd_xfer_handle); +static void xhci_device_ctrl_close(usbd_pipe_handle); +static void xhci_device_ctrl_done(usbd_xfer_handle); + +static usbd_status xhci_device_intr_transfer(usbd_xfer_handle); +static usbd_status xhci_device_intr_start(usbd_xfer_handle); +static void xhci_device_intr_abort(usbd_xfer_handle); +static void xhci_device_intr_close(usbd_pipe_handle); +static void xhci_device_intr_done(usbd_xfer_handle); + +static usbd_status xhci_device_bulk_transfer(usbd_xfer_handle); +static usbd_status xhci_device_bulk_start(usbd_xfer_handle); +static void xhci_device_bulk_abort(usbd_xfer_handle); +static void xhci_device_bulk_close(usbd_pipe_handle); +static void xhci_device_bulk_done(usbd_xfer_handle); + +static void xhci_timeout(void *); +static void xhci_timeout_task(void *); + +static const struct usbd_bus_methods xhci_bus_methods = { + .open_pipe = xhci_open, + .soft_intr = xhci_softintr, + .do_poll = xhci_poll, + .allocm = xhci_allocm, + .freem = xhci_freem, + .allocx = xhci_allocx, + .freex = xhci_freex, + .get_lock = xhci_get_lock, + .new_device = xhci_new_device, +}; + +static const struct usbd_pipe_methods xhci_root_ctrl_methods = { + .transfer = xhci_root_ctrl_transfer, + .start = xhci_root_ctrl_start, + .abort = xhci_root_ctrl_abort, + .close = xhci_root_ctrl_close, + .cleartoggle = xhci_noop, + .done = xhci_root_ctrl_done, +}; + +static const struct usbd_pipe_methods xhci_root_intr_methods = { + .transfer = xhci_root_intr_transfer, + .start = xhci_root_intr_start, + .abort = xhci_root_intr_abort, + .close = xhci_root_intr_close, + .cleartoggle = xhci_noop, + .done = xhci_root_intr_done, +}; + + +static const struct usbd_pipe_methods xhci_device_ctrl_methods = { + .transfer = xhci_device_ctrl_transfer, + .start = xhci_device_ctrl_start, + .abort = xhci_device_ctrl_abort, + .close = xhci_device_ctrl_close, + .cleartoggle = xhci_noop, + .done = xhci_device_ctrl_done, +}; + +static const struct usbd_pipe_methods xhci_device_isoc_methods = { + .cleartoggle = xhci_noop, +}; + +static const struct usbd_pipe_methods xhci_device_bulk_methods = { + .transfer = xhci_device_bulk_transfer, + .start = xhci_device_bulk_start, + .abort = xhci_device_bulk_abort, + .close = xhci_device_bulk_close, + .cleartoggle = xhci_noop, + .done = xhci_device_bulk_done, +}; + +static const struct usbd_pipe_methods xhci_device_intr_methods = { + .transfer = xhci_device_intr_transfer, + .start = xhci_device_intr_start, + .abort = xhci_device_intr_abort, + .close = xhci_device_intr_close, + .cleartoggle = xhci_noop, + .done = xhci_device_intr_done, +}; + +static inline uint32_t +xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset) +{ + return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset); +} + +#if 0 /* unused */ +static inline void +xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset, + uint32_t value) +{ + bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value); +} +#endif /* unused */ + +static inline uint32_t +xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset) +{ + return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset); +} + +static inline uint32_t +xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset) +{ + return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); +} + +static inline void +xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset, + uint32_t value) +{ + bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); +} + +#if 0 /* unused */ +static inline uint64_t +xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset) +{ + uint64_t value; + + if (sc->sc_ac64) { +#ifdef XHCI_USE_BUS_SPACE_8 + value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset); +#else + value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); + value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh, + offset + 4) << 32; +#endif + } else { + value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); + } + + return value; +} +#endif /* unused */ + +static inline void +xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset, + uint64_t value) +{ + if (sc->sc_ac64) { +#ifdef XHCI_USE_BUS_SPACE_8 + bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value); +#else + bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0, + (value >> 0) & 0xffffffff); + bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4, + (value >> 32) & 0xffffffff); +#endif + } else { + bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); + } +} + +static inline uint32_t +xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset) +{ + return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); +} + +static inline void +xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset, + uint32_t value) +{ + bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); +} + +#if 0 /* unused */ +static inline uint64_t +xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset) +{ + uint64_t value; + + if (sc->sc_ac64) { +#ifdef XHCI_USE_BUS_SPACE_8 + value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset); +#else + value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); + value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh, + offset + 4) << 32; +#endif + } else { + value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); + } + + return value; +} +#endif /* unused */ + +static inline void +xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset, + uint64_t value) +{ + if (sc->sc_ac64) { +#ifdef XHCI_USE_BUS_SPACE_8 + bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value); +#else + bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0, + (value >> 0) & 0xffffffff); + bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4, + (value >> 32) & 0xffffffff); +#endif + } else { + bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); + } +} + +#if 0 /* unused */ +static inline uint32_t +xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset) +{ + return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset); +} +#endif /* unused */ + +static inline void +xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset, + uint32_t value) +{ + bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value); +} + +/* --- */ + +static inline uint8_t +xhci_ep_get_type(usb_endpoint_descriptor_t * const ed) +{ + u_int eptype; + + switch (UE_GET_XFERTYPE(ed->bmAttributes)) { + case UE_CONTROL: + eptype = 0x0; + break; + case UE_ISOCHRONOUS: + eptype = 0x1; + break; + case UE_BULK: + eptype = 0x2; + break; + case UE_INTERRUPT: + eptype = 0x3; + break; + } + + if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || + (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) + return eptype | 0x4; + else + return eptype; +} + +static u_int +xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed) +{ + /* xHCI 1.0 section 4.5.1 */ + u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress); + u_int in = 0; + + if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || + (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) + in = 1; + + return epaddr * 2 + in; +} + +static inline u_int +xhci_dci_to_ici(const u_int i) +{ + return i + 1; +} + +static inline void * +xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs, + const u_int dci) +{ + return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); +} + +#if 0 /* unused */ +static inline bus_addr_t +xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs, + const u_int dci) +{ + return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); +} +#endif /* unused */ + +static inline void * +xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs, + const u_int ici) +{ + return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); +} + +static inline bus_addr_t +xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs, + const u_int ici) +{ + return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); +} + +static inline struct xhci_trb * +xhci_ring_trbv(struct xhci_ring * const xr, u_int idx) +{ + return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); +} + +static inline bus_addr_t +xhci_ring_trbp(struct xhci_ring * const xr, u_int idx) +{ + return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); +} + +static inline void +xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status, + uint32_t control) +{ + trb->trb_0 = parameter; + trb->trb_2 = status; + trb->trb_3 = control; +} + +/* --- */ + +void +xhci_childdet(device_t self, device_t child) +{ + struct xhci_softc * const sc = device_private(self); + + KASSERT(sc->sc_child == child); + if (child == sc->sc_child) + sc->sc_child = NULL; +} + +int +xhci_detach(struct xhci_softc *sc, int flags) +{ + int rv = 0; + + if (sc->sc_child != NULL) + rv = config_detach(sc->sc_child, flags); + + if (rv != 0) + return (rv); + + /* XXX unconfigure/free slots */ + + /* verify: */ + xhci_rt_write_4(sc, XHCI_IMAN(0), 0); + xhci_op_write_4(sc, XHCI_USBCMD, 0); + /* do we need to wait for stop? */ + + xhci_op_write_8(sc, XHCI_CRCR, 0); + xhci_ring_free(sc, &sc->sc_cr); + cv_destroy(&sc->sc_command_cv); + + xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0); + xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0); + xhci_rt_write_8(sc, XHCI_ERDP(0), 0|XHCI_ERDP_LO_BUSY); + xhci_ring_free(sc, &sc->sc_er); + + usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); + + xhci_op_write_8(sc, XHCI_DCBAAP, 0); + usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); + + kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots); + + mutex_destroy(&sc->sc_lock); + mutex_destroy(&sc->sc_intr_lock); + + pool_cache_destroy(sc->sc_xferpool); + + return rv; +} + +int +xhci_activate(device_t self, enum devact act) +{ + struct xhci_softc * const sc = device_private(self); + + switch (act) { + case DVACT_DEACTIVATE: + sc->sc_dying = true; + return 0; + default: + return EOPNOTSUPP; + } +} + +bool +xhci_suspend(device_t dv, const pmf_qual_t *qual) +{ + return false; +} + +bool +xhci_resume(device_t dv, const pmf_qual_t *qual) +{ + return false; +} + +bool +xhci_shutdown(device_t self, int flags) +{ + return false; +} + + +static void +hexdump(const char *msg, const void *base, size_t len) +{ +#if 0 + size_t cnt; + const uint32_t *p; + extern paddr_t vtophys(vaddr_t); + + p = base; + cnt = 0; + + printf("*** %s (%zu bytes @ %p %p)\n", msg, len, base, + (void *)vtophys((vaddr_t)base)); + + while (cnt < len) { + if (cnt % 16 == 0) + printf("%p: ", p); + else if (cnt % 8 == 0) + printf(" |"); + printf(" %08x", *p++); + cnt += 4; + if (cnt % 16 == 0) + printf("\n"); + } +#endif +} + + +int +xhci_init(struct xhci_softc *sc) +{ + bus_size_t bsz; + uint32_t cap, hcs1, hcs2, hcc, dboff, rtsoff; + uint32_t ecp, ecr; + uint32_t usbcmd, usbsts, pagesize, config; + int i; + uint16_t hciversion; + uint8_t caplength; + + DPRINTF(("%s\n", __func__)); + + /* XXX Low/Full/High speeds for now */ + sc->sc_bus.usbrev = USBREV_2_0; + + cap = xhci_read_4(sc, XHCI_CAPLENGTH); + caplength = XHCI_CAP_CAPLENGTH(cap); + hciversion = XHCI_CAP_HCIVERSION(cap); + + if ((hciversion < 0x0096) || (hciversion > 0x0100)) { + aprint_normal_dev(sc->sc_dev, + "xHCI version %x.%x not known to be supported\n", + (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); + } else { + aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n", + (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); + } + + if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength, + &sc->sc_cbh) != 0) { + aprint_error_dev(sc->sc_dev, "capability subregion failure\n"); + return ENOMEM; + } + + hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1); + sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1); + sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1); + sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1); + hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2); + (void)xhci_cap_read_4(sc, XHCI_HCSPARAMS3); + hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS); + + sc->sc_ac64 = XHCI_HCC_AC64(hcc); + sc->sc_ctxsz = XHCI_HCC_CSZ(hcc) ? 64 : 32; + aprint_debug_dev(sc->sc_dev, "ac64 %d ctxsz %d\n", sc->sc_ac64, + sc->sc_ctxsz); + + aprint_debug_dev(sc->sc_dev, "xECP %x\n", XHCI_HCC_XECP(hcc) * 4); + ecp = XHCI_HCC_XECP(hcc) * 4; + while (ecp != 0) { + ecr = xhci_read_4(sc, ecp); + aprint_debug_dev(sc->sc_dev, "ECR %x: %08x\n", ecp, ecr); + switch (XHCI_XECP_ID(ecr)) { + case XHCI_ID_PROTOCOLS: { + uint32_t w0, w4, w8; + uint16_t w2; + w0 = xhci_read_4(sc, ecp + 0); + w2 = (w0 >> 16) & 0xffff; + w4 = xhci_read_4(sc, ecp + 4); + w8 = xhci_read_4(sc, ecp + 8); + aprint_debug_dev(sc->sc_dev, "SP: %08x %08x %08x\n", + w0, w4, w8); + if (w4 == 0x20425355 && w2 == 0x0300) { + sc->sc_ss_port_start = (w8 >> 0) & 0xff;; + sc->sc_ss_port_count = (w8 >> 8) & 0xff;; + } + if (w4 == 0x20425355 && w2 == 0x0200) { + sc->sc_hs_port_start = (w8 >> 0) & 0xff; + sc->sc_hs_port_count = (w8 >> 8) & 0xff; + } + break; + } + default: + break; + } + ecr = xhci_read_4(sc, ecp); + if (XHCI_XECP_NEXT(ecr) == 0) { + ecp = 0; + } else { + ecp += XHCI_XECP_NEXT(ecr) * 4; + } + } + + bsz = XHCI_PORTSC(sc->sc_maxports + 1); + if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz, + &sc->sc_obh) != 0) { + aprint_error_dev(sc->sc_dev, "operational subregion failure\n"); + return ENOMEM; + } + + dboff = xhci_cap_read_4(sc, XHCI_DBOFF); + if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff, + sc->sc_maxslots * 4, &sc->sc_dbh) != 0) { + aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n"); + return ENOMEM; + } + + rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF); + if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff, + sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) { + aprint_error_dev(sc->sc_dev, "runtime subregion failure\n"); + return ENOMEM; + } + + for (i = 0; i < 100; i++) { + usbsts = xhci_op_read_4(sc, XHCI_USBSTS); + if ((usbsts & XHCI_STS_CNR) == 0) + break; + usb_delay_ms(&sc->sc_bus, 1); + } + if (i >= 100) + return EIO; + + usbcmd = 0; + xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); + usb_delay_ms(&sc->sc_bus, 1); + + usbcmd = XHCI_CMD_HCRST; + xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); + for (i = 0; i < 100; i++) { + usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); + if ((usbcmd & XHCI_CMD_HCRST) == 0) + break; + usb_delay_ms(&sc->sc_bus, 1); + } + if (i >= 100) + return EIO; + + for (i = 0; i < 100; i++) { + usbsts = xhci_op_read_4(sc, XHCI_USBSTS); + if ((usbsts & XHCI_STS_CNR) == 0) + break; + usb_delay_ms(&sc->sc_bus, 1); + } + if (i >= 100) + return EIO; + + pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE); + aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize); + pagesize = ffs(pagesize); + if (pagesize == 0) + return EIO; + sc->sc_pgsz = 1 << (12 + (pagesize - 1)); + aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz); + aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n", + (uint32_t)sc->sc_maxslots); + + usbd_status err; + + sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2); + aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf); + if (sc->sc_maxspbuf != 0) { + err = usb_allocmem(&sc->sc_bus, + sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t), + &sc->sc_spbufarray_dma); + if (err) + return err; + + sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) * sc->sc_maxspbuf, KM_SLEEP); + uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0); + for (i = 0; i < sc->sc_maxspbuf; i++) { + usb_dma_t * const dma = &sc->sc_spbuf_dma[i]; + /* allocate contexts */ + err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, + sc->sc_pgsz, dma); + if (err) + return err; + spbufarray[i] = htole64(DMAADDR(dma, 0)); + usb_syncmem(dma, 0, sc->sc_pgsz, + BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); + } + + usb_syncmem(&sc->sc_spbufarray_dma, 0, + sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE); + } + + config = xhci_op_read_4(sc, XHCI_CONFIG); + config &= ~0xFF; + config |= sc->sc_maxslots & 0xFF; + xhci_op_write_4(sc, XHCI_CONFIG, config); + + err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS, + XHCI_COMMAND_RING_SEGMENTS_ALIGN); + if (err) { + aprint_error_dev(sc->sc_dev, "command ring init fail\n"); + return err; + } + + err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS, + XHCI_EVENT_RING_SEGMENTS_ALIGN); + if (err) { + aprint_error_dev(sc->sc_dev, "event ring init fail\n"); + return err; + } + + usb_dma_t *dma; + size_t size; + size_t align; + + dma = &sc->sc_eventst_dma; + size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE, + XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN); + KASSERT(size <= (512 * 1024)); + align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN; + err = usb_allocmem(&sc->sc_bus, size, align, dma); + + memset(KERNADDR(dma, 0), 0, size); + usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); + aprint_debug_dev(sc->sc_dev, "eventst: %s %016jx %p %zx\n", + usbd_errstr(err), + (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0), + KERNADDR(&sc->sc_eventst_dma, 0), + sc->sc_eventst_dma.block->size); + + dma = &sc->sc_dcbaa_dma; + size = (1 + sc->sc_maxslots) * sizeof(uint64_t); + KASSERT(size <= 2048); + align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN; + err = usb_allocmem(&sc->sc_bus, size, align, dma); + + memset(KERNADDR(dma, 0), 0, size); + if (sc->sc_maxspbuf != 0) { + /* + * DCBA entry 0 hold the scratchbuf array pointer. + */ + *(uint64_t *)KERNADDR(dma, 0) = + htole64(DMAADDR(&sc->sc_spbufarray_dma, 0)); + } + usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); + aprint_debug_dev(sc->sc_dev, "dcbaa: %s %016jx %p %zx\n", + usbd_errstr(err), + (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0), + KERNADDR(&sc->sc_dcbaa_dma, 0), + sc->sc_dcbaa_dma.block->size); + + sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots, + KM_SLEEP); + + cv_init(&sc->sc_command_cv, "xhcicmd"); + + struct xhci_erste *erst; + erst = KERNADDR(&sc->sc_eventst_dma, 0); + erst[0].erste_0 = htole64(xhci_ring_trbp(&sc->sc_er, 0)); + erst[0].erste_2 = htole32(XHCI_EVENT_RING_TRBS); + erst[0].erste_3 = htole32(0); + usb_syncmem(&sc->sc_eventst_dma, 0, + XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE); + + xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS); + xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0)); + xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(&sc->sc_er, 0) | + XHCI_ERDP_LO_BUSY); + xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0)); + xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) | + sc->sc_cr.xr_cs); + +#if 0 + hexdump("eventst", KERNADDR(&sc->sc_eventst_dma, 0), + XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS); +#endif + + xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA); + xhci_rt_write_4(sc, XHCI_IMOD(0), 0); + + xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); /* Go! */ + aprint_debug_dev(sc->sc_dev, "USBCMD %08"PRIx32"\n", + xhci_op_read_4(sc, XHCI_USBCMD)); + + mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB); + mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_SCHED); + cv_init(&sc->sc_softwake_cv, "xhciab"); + + sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0, + "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL); + + /* Set up the bus struct. */ + sc->sc_bus.methods = &xhci_bus_methods; + sc->sc_bus.pipe_size = sizeof(struct xhci_pipe); + + return USBD_NORMAL_COMPLETION; +} + +int +xhci_intr(void *v) +{ + struct xhci_softc * const sc = v; + + if (sc == NULL || sc->sc_dying || !device_has_power(sc->sc_dev)) + return 0; + + DPRINTF(("%s: %s\n", __func__, device_xname(sc->sc_dev))); + + /* If we get an interrupt while polling, then just ignore it. */ + if (sc->sc_bus.use_polling) { +#ifdef DIAGNOSTIC + DPRINTFN(16, ("xhci_intr: ignored interrupt while polling\n")); +#endif + return 0; + } + + return xhci_intr1(sc); +} + +int +xhci_intr1(struct xhci_softc * const sc) +{ + uint32_t usbsts; + uint32_t iman; + + usbsts = xhci_op_read_4(sc, XHCI_USBSTS); + //device_printf(sc->sc_dev, "%s USBSTS %08x\n", __func__, usbsts); +#if 0 + if ((usbsts & (XHCI_STS_EINT|XHCI_STS_PCD)) == 0) { + return 0; + } +#endif + xhci_op_write_4(sc, XHCI_USBSTS, + usbsts & (2|XHCI_STS_EINT|XHCI_STS_PCD)); /* XXX */ + usbsts = xhci_op_read_4(sc, XHCI_USBSTS); + //device_printf(sc->sc_dev, "%s USBSTS %08x\n", __func__, usbsts); + + iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); + //device_printf(sc->sc_dev, "%s IMAN0 %08x\n", __func__, iman); + if ((iman & XHCI_IMAN_INTR_PEND) == 0) { + return 0; + } + xhci_rt_write_4(sc, XHCI_IMAN(0), iman); + iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); + //device_printf(sc->sc_dev, "%s IMAN0 %08x\n", __func__, iman); + usbsts = xhci_op_read_4(sc, XHCI_USBSTS); + //device_printf(sc->sc_dev, "%s USBSTS %08x\n", __func__, usbsts); + + sc->sc_bus.no_intrs++; + usb_schedsoftintr(&sc->sc_bus); + + return 1; +} + +static usbd_status +xhci_configure_endpoint(usbd_pipe_handle pipe) +{ + struct xhci_softc * const sc = pipe->device->bus->hci_private; + struct xhci_slot * const xs = pipe->device->hci_private; + const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); + usb_endpoint_descriptor_t * const ed = pipe->endpoint->edesc; + const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); + struct xhci_trb trb; + usbd_status err; + uint32_t *cp; + + device_printf(sc->sc_dev, "%s dci %u (0x%x)\n", __func__, dci, + pipe->endpoint->edesc->bEndpointAddress); + + /* XXX ensure input context is available? */ + + memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz); + + cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); + cp[0] = htole32(0); + cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci)); + + /* set up input slot context */ + cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); + cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); + cp[1] = htole32(0); + cp[2] = htole32(0); + cp[3] = htole32(0); + + cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci)); + if (xfertype == UE_INTERRUPT) { + cp[0] = htole32( + XHCI_EPCTX_0_IVAL_SET(3) /* XXX */ + ); + cp[1] = htole32( + XHCI_EPCTX_1_CERR_SET(3) | + XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(pipe->endpoint->edesc)) | + XHCI_EPCTX_1_MAXB_SET(0) | + XHCI_EPCTX_1_MAXP_SIZE_SET(8) /* XXX */ + ); + cp[4] = htole32( + XHCI_EPCTX_4_AVG_TRB_LEN_SET(8) + ); + } else { + cp[0] = htole32(0); + cp[1] = htole32( + XHCI_EPCTX_1_CERR_SET(3) | + XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(pipe->endpoint->edesc)) | + XHCI_EPCTX_1_MAXB_SET(0) | + XHCI_EPCTX_1_MAXP_SIZE_SET(512) /* XXX */ + ); + } + *(uint64_t *)(&cp[2]) = htole64( + xhci_ring_trbp(&xs->xs_ep[dci].xe_tr, 0) | + XHCI_EPCTX_2_DCS_SET(1)); + + /* sync input contexts before they are read from memory */ + usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); + hexdump("input control context", xhci_slot_get_icv(sc, xs, 0), + sc->sc_ctxsz * 1); + hexdump("input endpoint context", xhci_slot_get_icv(sc, xs, + xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1); + + trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); + trb.trb_2 = 0; + trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); + + err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); + + usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); + hexdump("output context", xhci_slot_get_dcv(sc, xs, dci), + sc->sc_ctxsz * 1); + + return err; +} + +static usbd_status +xhci_unconfigure_endpoint(usbd_pipe_handle pipe) +{ + return USBD_NORMAL_COMPLETION; +} + +static usbd_status +xhci_reset_endpoint(usbd_pipe_handle pipe) +{ + struct xhci_softc * const sc = pipe->device->bus->hci_private; + struct xhci_slot * const xs = pipe->device->hci_private; + const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); + struct xhci_trb trb; + usbd_status err; + + device_printf(sc->sc_dev, "%s\n", __func__); + + trb.trb_0 = 0; + trb.trb_2 = 0; + trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | + XHCI_TRB_3_EP_SET(dci) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP); + + err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); + + return err; +} + +#if 0 +static usbd_status +xhci_stop_endpoint(usbd_pipe_handle pipe) +{ + struct xhci_softc * const sc = pipe->device->bus->hci_private; + struct xhci_slot * const xs = pipe->device->hci_private; + struct xhci_trb trb; + usbd_status err; + const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); + + device_printf(sc->sc_dev, "%s\n", __func__); + + trb.trb_0 = 0; + trb.trb_2 = 0; + trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | + XHCI_TRB_3_EP_SET(dci) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP); + + err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); + + return err; +} +#endif + +static usbd_status +xhci_set_dequeue(usbd_pipe_handle pipe) +{ + struct xhci_softc * const sc = pipe->device->bus->hci_private; + struct xhci_slot * const xs = pipe->device->hci_private; + const u_int dci = xhci_ep_get_dci(pipe->endpoint->edesc); + struct xhci_ring * const xr = &xs->xs_ep[dci].xe_tr; + struct xhci_trb trb; + usbd_status err; + + device_printf(sc->sc_dev, "%s\n", __func__); + + memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE); + usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE, + BUS_DMASYNC_PREWRITE); + + xr->xr_ep = 0; + xr->xr_cs = 1; + + trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */ + trb.trb_2 = 0; + trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | + XHCI_TRB_3_EP_SET(dci) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE); + + err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); + + return err; +} + +static usbd_status +xhci_open(usbd_pipe_handle pipe) +{ + usbd_device_handle const dev = pipe->device; + struct xhci_softc * const sc = dev->bus->hci_private; + usb_endpoint_descriptor_t * const ed = pipe->endpoint->edesc; + const int8_t addr = dev->address; + const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); + + DPRINTF(("%s\n", __func__)); + DPRINTF(("addr %d\n", addr)); + device_printf(sc->sc_dev, "%s addr %d depth %d port %d speed %d\n", + __func__, addr, dev->depth, dev->powersrc->portno, dev->speed); + + if (sc->sc_dying) + return USBD_IOERROR; + + /* Root Hub */ + if (dev->depth == 0 && dev->powersrc->portno == 0 && + dev->speed != USB_SPEED_SUPER) { + switch (ed->bEndpointAddress) { + case USB_CONTROL_ENDPOINT: + pipe->methods = &xhci_root_ctrl_methods; + break; + case UE_DIR_IN | XHCI_INTR_ENDPT: + pipe->methods = &xhci_root_intr_methods; + break; + default: + pipe->methods = NULL; + DPRINTF(("xhci_open: bad bEndpointAddress 0x%02x\n", + ed->bEndpointAddress)); + return USBD_INVAL; + } + return USBD_NORMAL_COMPLETION; + } + + switch (xfertype) { + case UE_CONTROL: + pipe->methods = &xhci_device_ctrl_methods; + break; + case UE_ISOCHRONOUS: + pipe->methods = &xhci_device_isoc_methods; + return USBD_INVAL; + break; + case UE_BULK: + pipe->methods = &xhci_device_bulk_methods; + break; + case UE_INTERRUPT: + pipe->methods = &xhci_device_intr_methods; + break; + default: + return USBD_IOERROR; + break; + } + + if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT) + xhci_configure_endpoint(pipe); + + return USBD_NORMAL_COMPLETION; +} + +static void +xhci_rhpsc(struct xhci_softc * const sc, u_int port) +{ + usbd_xfer_handle const xfer = sc->sc_intrxfer; + uint8_t *p; + + device_printf(sc->sc_dev, "port %u status change\n", port); + + if (xfer == NULL) + return; + + if (!(port >= sc->sc_hs_port_start && + port < sc->sc_hs_port_start + sc->sc_hs_port_count)) + return; + + port -= sc->sc_hs_port_start; + port += 1; + device_printf(sc->sc_dev, "hs port %u status change\n", port); + + p = KERNADDR(&xfer->dmabuf, 0); + memset(p, 0, xfer->length); + p[port/NBBY] |= 1 << (port%NBBY); + xfer->actlen = xfer->length; + xfer->status = USBD_NORMAL_COMPLETION; + usb_transfer_complete(xfer); +} + +static void +xhci_handle_event(struct xhci_softc * const sc, const struct xhci_trb * const trb) +{ + uint64_t trb_0; + uint32_t trb_2, trb_3; + + DPRINTF(("%s: %s\n", __func__, device_xname(sc->sc_dev))); + + trb_0 = le64toh(trb->trb_0); + trb_2 = le32toh(trb->trb_2); + trb_3 = le32toh(trb->trb_3); + +#if 0 + device_printf(sc->sc_dev, + "event: %p 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32"\n", trb, + trb_0, trb_2, trb_3); +#endif + + switch (XHCI_TRB_3_TYPE_GET(trb_3)){ + case XHCI_TRB_EVENT_TRANSFER: { + u_int slot, dci; + struct xhci_slot *xs; + struct xhci_ring *xr; + struct xhci_xfer *xx; + usbd_xfer_handle xfer; + usbd_status err; + + slot = XHCI_TRB_3_SLOT_GET(trb_3); + dci = XHCI_TRB_3_EP_GET(trb_3); + + xs = &sc->sc_slots[slot]; + xr = &xs->xs_ep[dci].xe_tr; + + if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { + xx = xr->xr_cookies[(trb_0 - xhci_ring_trbp(xr, 0))/ + sizeof(struct xhci_trb)]; + } else { + xx = (void *)(uintptr_t)(trb_0 & ~0x3); + } + xfer = &xx->xx_xfer; +#if 0 + device_printf(sc->sc_dev, "%s xfer %p\n", __func__, xfer); +#endif + + if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { +#if 0 + device_printf(sc->sc_dev, "transfer event data: " + "0x%016"PRIx64" 0x%08"PRIx32" %02x\n", + trb_0, XHCI_TRB_2_REM_GET(trb_2), + XHCI_TRB_2_ERROR_GET(trb_2)); +#endif + if ((trb_0 & 0x3) == 0x3) { + xfer->actlen = XHCI_TRB_2_REM_GET(trb_2); + } + } + + if (XHCI_TRB_2_ERROR_GET(trb_2) == + XHCI_TRB_ERROR_SUCCESS) { + xfer->actlen = xfer->length - XHCI_TRB_2_REM_GET(trb_2); + err = USBD_NORMAL_COMPLETION; + } else if (XHCI_TRB_2_ERROR_GET(trb_2) == + XHCI_TRB_ERROR_SHORT_PKT) { + xfer->actlen = xfer->length - XHCI_TRB_2_REM_GET(trb_2); + err = USBD_NORMAL_COMPLETION; + } else if (XHCI_TRB_2_ERROR_GET(trb_2) == + XHCI_TRB_ERROR_STALL) { + err = USBD_STALLED; + xr->is_halted = true; + } else { + err = USBD_IOERROR; + } + xfer->status = err; + + //mutex_enter(&sc->sc_lock); /* XXX ??? */ + if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { + if ((trb_0 & 0x3) == 0x0) { + usb_transfer_complete(xfer); + } + } else { + usb_transfer_complete(xfer); + } + //mutex_exit(&sc->sc_lock); /* XXX ??? */ + + } + break; + case XHCI_TRB_EVENT_CMD_COMPLETE: + if (trb_0 == sc->sc_command_addr) { + sc->sc_result_trb.trb_0 = trb_0; + sc->sc_result_trb.trb_2 = trb_2; + sc->sc_result_trb.trb_3 = trb_3; + if (XHCI_TRB_2_ERROR_GET(trb_2) != + XHCI_TRB_ERROR_SUCCESS) { + device_printf(sc->sc_dev, "command completion " + "failure: 0x%016"PRIx64" 0x%08"PRIx32" " + "0x%08"PRIx32"\n", trb_0, trb_2, trb_3); + } + cv_signal(&sc->sc_command_cv); + } else { + device_printf(sc->sc_dev, "event: %p 0x%016"PRIx64" " + "0x%08"PRIx32" 0x%08"PRIx32"\n", trb, trb_0, + trb_2, trb_3); + } + break; + case XHCI_TRB_EVENT_PORT_STS_CHANGE: + xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff)); + break; + default: + break; + } +} + +static void +xhci_softintr(void *v) +{ + struct usbd_bus * const bus = v; + struct xhci_softc * const sc = bus->hci_private; + struct xhci_ring * const er = &sc->sc_er; + struct xhci_trb *trb; + int i, j, k; + + DPRINTF(("%s: %s\n", __func__, device_xname(sc->sc_dev))); + + i = er->xr_ep; + j = er->xr_cs; + + while (1) { + usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE, + BUS_DMASYNC_POSTREAD); + trb = &er->xr_trb[i]; + k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; + + if (j != k) + break; + + xhci_handle_event(sc, trb); + + i++; + if (i == XHCI_EVENT_RING_TRBS) { + i = 0; + j ^= 1; + } + } + + er->xr_ep = i; + er->xr_cs = j; + + xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) | + XHCI_ERDP_LO_BUSY); + + DPRINTF(("%s: %s ends\n", __func__, device_xname(sc->sc_dev))); + + return; +} + +static void +xhci_poll(struct usbd_bus *bus) +{ + struct xhci_softc * const sc = bus->hci_private; + + DPRINTF(("%s: %s\n", __func__, device_xname(sc->sc_dev))); + + xhci_intr1(sc); + + return; +} + +static usbd_status +xhci_allocm(struct usbd_bus *bus, usb_dma_t *dma, uint32_t size) +{ + struct xhci_softc * const sc = bus->hci_private; + usbd_status err; + + DPRINTF(("%s\n", __func__)); + + err = usb_allocmem_flags(&sc->sc_bus, size, 0, dma, 0); +#if 0 + if (err == USBD_NOMEM) + err = usb_reserve_allocm(&sc->sc_dma_reserve, dma, size); +#endif +#ifdef XHCI_DEBUG + if (err) + device_printf(sc->sc_dev, "xhci_allocm: usb_allocmem()=%d\n", + err); +#endif + + return err; +} + +static void +xhci_freem(struct usbd_bus *bus, usb_dma_t *dma) +{ + struct xhci_softc * const sc = bus->hci_private; + +// DPRINTF(("%s\n", __func__)); + +#if 0 + if (dma->block->flags & USB_DMA_RESERVE) { + usb_reserve_freem(&sc->sc_dma_reserve, dma); + return; + } +#endif + usb_freemem(&sc->sc_bus, dma); +} + +static usbd_xfer_handle +xhci_allocx(struct usbd_bus *bus) +{ + struct xhci_softc * const sc = bus->hci_private; + usbd_xfer_handle xfer; + +// DPRINTF(("%s\n", __func__)); + + xfer = pool_cache_get(sc->sc_xferpool, PR_NOWAIT); + if (xfer != NULL) { + memset(xfer, 0, sizeof(struct xhci_xfer)); +#ifdef DIAGNOSTIC + xfer->busy_free = XFER_BUSY; +#endif + } + + return xfer; +} + +static void +xhci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = bus->hci_private; + +// DPRINTF(("%s\n", __func__)); + +#ifdef DIAGNOSTIC + if (xfer->busy_free != XFER_BUSY) { + device_printf(sc->sc_dev, "xhci_freex: xfer=%p " + "not busy, 0x%08x\n", xfer, xfer->busy_free); + } + xfer->busy_free = XFER_FREE; +#endif + pool_cache_put(sc->sc_xferpool, xfer); +} + +static void +xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock) +{ + struct xhci_softc * const sc = bus->hci_private; + + *lock = &sc->sc_lock; +} + +extern u_int32_t usb_cookie_no; + +static usbd_status +xhci_new_device(device_t parent, usbd_bus_handle bus, int depth, + int speed, int port, struct usbd_port *up) +{ + struct xhci_softc * const sc = bus->hci_private; + usbd_device_handle dev; + usbd_status err; + usb_device_descriptor_t *dd; + struct usbd_device *hub; + struct usbd_device *adev; + int rhport = 0; + struct xhci_slot *xs; + uint32_t *cp; + uint8_t slot; + uint8_t addr; + + dev = malloc(sizeof *dev, M_USB, M_NOWAIT|M_ZERO); + if (dev == NULL) + return USBD_NOMEM; + + dev->bus = bus; + + /* Set up default endpoint handle. */ + dev->def_ep.edesc = &dev->def_ep_desc; + + /* Set up default endpoint descriptor. */ + dev->def_ep_desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; + dev->def_ep_desc.bDescriptorType = UDESC_ENDPOINT; + dev->def_ep_desc.bEndpointAddress = USB_CONTROL_ENDPOINT; + dev->def_ep_desc.bmAttributes = UE_CONTROL; + /* XXX */ + USETW(dev->def_ep_desc.wMaxPacketSize, 64); + dev->def_ep_desc.bInterval = 0; + + /* doesn't matter, just don't let it uninitialized */ + dev->def_ep.datatoggle = 0; + + device_printf(sc->sc_dev, "%s up %p portno %d\n", __func__, up, + up->portno); + + dev->quirks = &usbd_no_quirk; + dev->address = 0; + dev->ddesc.bMaxPacketSize = 0; + dev->depth = depth; + dev->powersrc = up; + dev->myhub = up->parent; + + up->device = dev; + + /* Locate root hub port */ + for (adev = dev, hub = dev; + hub != NULL; + adev = hub, hub = hub->myhub) { + device_printf(sc->sc_dev, "%s hub %p\n", __func__, hub); + } + device_printf(sc->sc_dev, "%s hub %p\n", __func__, hub); + + if (hub != NULL) { + for (int p = 0; p < hub->hub->hubdesc.bNbrPorts; p++) { + if (hub->hub->ports[p].device == adev) { + rhport = p; + } + } + } else { + rhport = port; + } + if (speed == USB_SPEED_SUPER) { + rhport += sc->sc_ss_port_start - 1; + } else { + rhport += sc->sc_hs_port_start - 1; + } + device_printf(sc->sc_dev, "%s rhport %d\n", __func__, rhport); + + dev->speed = speed; + dev->langid = USBD_NOLANG; + dev->cookie.cookie = ++usb_cookie_no; + + /* Establish the default pipe. */ + err = usbd_setup_pipe(dev, 0, &dev->def_ep, USBD_DEFAULT_INTERVAL, + &dev->default_pipe); + if (err) { + usbd_remove_device(dev, up); + return (err); + } + + dd = &dev->ddesc; + + if ((depth == 0) && (port == 0)) { + KASSERT(bus->devices[dev->address] == NULL); + bus->devices[dev->address] = dev; + err = usbd_get_initial_ddesc(dev, dd); + if (err) + return err; + err = usbd_reload_device_desc(dev); + if (err) + return err; + } else { + err = xhci_enable_slot(sc, &slot); + if (err) + return err; + err = xhci_init_slot(sc, slot, depth, speed, port, rhport); + if (err) + return err; + xs = &sc->sc_slots[slot]; + dev->hci_private = xs; + cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT); + //hexdump("slot context", cp, sc->sc_ctxsz); + addr = XHCI_SCTX_3_DEV_ADDR_GET(cp[3]); + device_printf(sc->sc_dev, "%s device address %u\n", + __func__, addr); + /* XXX ensure we know when the hardware does something + we can't yet cope with */ + KASSERT(addr >= 1 && addr <= 127); + dev->address = addr; + /* XXX dev->address not necessarily unique on bus */ + KASSERT(bus->devices[dev->address] == NULL); + bus->devices[dev->address] = dev; + + err = usbd_get_initial_ddesc(dev, dd); + if (err) + return err; + USETW(dev->def_ep_desc.wMaxPacketSize, dd->bMaxPacketSize); + device_printf(sc->sc_dev, "%s bMaxPacketSize %u\n", __func__, + dd->bMaxPacketSize); + xhci_update_ep0_mps(sc, xs, dd->bMaxPacketSize); + err = usbd_reload_device_desc(dev); + if (err) + return err; + + usbd_kill_pipe(dev->default_pipe); + err = usbd_setup_pipe(dev, 0, &dev->def_ep, + USBD_DEFAULT_INTERVAL, &dev->default_pipe); + } + + DPRINTF(("usbd_new_device: adding unit addr=%d, rev=%02x, class=%d, " + "subclass=%d, protocol=%d, maxpacket=%d, len=%d, noconf=%d, " + "speed=%d\n", dev->address,UGETW(dd->bcdUSB), + dd->bDeviceClass, dd->bDeviceSubClass, dd->bDeviceProtocol, + dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations, + dev->speed)); + + usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); + + if ((depth == 0) && (port == 0)) { + usbd_attach_roothub(parent, dev); + device_printf(sc->sc_dev, "root_hub %p\n", bus->root_hub); + return USBD_NORMAL_COMPLETION; + } + + + err = usbd_probe_and_attach(parent, dev, port, dev->address); + if (err) { + usbd_remove_device(dev, up); + return (err); + } + + return USBD_NORMAL_COMPLETION; +} + +static usbd_status +xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring * const xr, + size_t ntrb, size_t align) +{ + usbd_status err; + size_t size = ntrb * XHCI_TRB_SIZE; + + err = usb_allocmem(&sc->sc_bus, size, align, &xr->xr_dma); + if (err) + return err; + mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB); + xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP); + xr->xr_trb = xhci_ring_trbv(xr, 0); + xr->xr_ntrb = ntrb; + xr->xr_ep = 0; + xr->xr_cs = 1; + memset(xr->xr_trb, 0, size); + usb_syncmem(&xr->xr_dma, 0, size, BUS_DMASYNC_PREWRITE); + xr->is_halted = false; + + return USBD_NORMAL_COMPLETION; +} + +static void +xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring * const xr) +{ + usb_freemem(&sc->sc_bus, &xr->xr_dma); + mutex_destroy(&xr->xr_lock); + kmem_free(xr->xr_cookies, sizeof(*xr->xr_cookies) * xr->xr_ntrb); +} + +static void +xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr, + void *cookie, struct xhci_trb * const trbs, size_t ntrbs) +{ + size_t i; + u_int ri; + u_int cs; + uint64_t parameter; + uint32_t status; + uint32_t control; + + for (i = 0; i < ntrbs; i++) { +#if 0 + device_printf(sc->sc_dev, "%s %p %p %zu " + "%016"PRIx64" %08"PRIx32" %08"PRIx32"\n", __func__, xr, + trbs, i, trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3); +#endif + KASSERT(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) != + XHCI_TRB_TYPE_LINK); + } + +#if 0 + device_printf(sc->sc_dev, "%s %p xr_ep 0x%x xr_cs %u\n", __func__, + xr, xr->xr_ep, xr->xr_cs); +#endif + + ri = xr->xr_ep; + cs = xr->xr_cs; + + /* + * Although the xhci hardware can do scatter/gather dma from + * arbitrary sized buffers, there is a non-obvious restriction + * that a LINK trb is only allowed at the end of a burst of + * transfers - which might be 16kB. + * Arbitrary aligned LINK trb definitely fail on Ivy bridge. + * The simple solution is not to allow a LINK trb in the middle + * of anything - as here. + * XXX: (dsl) There are xhci controllers out there (eg some made by + * ASMedia) that seem to lock up if they process a LINK trb but + * cannot process the linked-to trb yet. + * The code should write the 'cycle' bit on the link trb AFTER + * adding the other trb. + */ + if (ri + ntrbs >= (xr->xr_ntrb - 1)) { + parameter = xhci_ring_trbp(xr, 0); + status = 0; + control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | + XHCI_TRB_3_TC_BIT | (cs ? XHCI_TRB_3_CYCLE_BIT : 0); + xhci_trb_put(&xr->xr_trb[ri], htole64(parameter), + htole32(status), htole32(control)); + usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, + BUS_DMASYNC_PREWRITE); + xr->xr_cookies[ri] = NULL; + xr->xr_ep = 0; + xr->xr_cs ^= 1; + ri = xr->xr_ep; + cs = xr->xr_cs; + } + + ri++; + + /* Write any subsequent TRB first */ + for (i = 1; i < ntrbs; i++) { + parameter = trbs[i].trb_0; + status = trbs[i].trb_2; + control = trbs[i].trb_3; + + if (cs) { + control |= XHCI_TRB_3_CYCLE_BIT; + } else { + control &= ~XHCI_TRB_3_CYCLE_BIT; + } + + xhci_trb_put(&xr->xr_trb[ri], htole64(parameter), + htole32(status), htole32(control)); + usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, + BUS_DMASYNC_PREWRITE); + xr->xr_cookies[ri] = cookie; + ri++; + } + + /* Write the first TRB last */ + i = 0; + { + parameter = trbs[i].trb_0; + status = trbs[i].trb_2; + control = trbs[i].trb_3; + + if (xr->xr_cs) { + control |= XHCI_TRB_3_CYCLE_BIT; + } else { + control &= ~XHCI_TRB_3_CYCLE_BIT; + } + + xhci_trb_put(&xr->xr_trb[xr->xr_ep], htole64(parameter), + htole32(status), htole32(control)); + usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, + BUS_DMASYNC_PREWRITE); + xr->xr_cookies[xr->xr_ep] = cookie; + } + + xr->xr_ep = ri; + xr->xr_cs = cs; + +#if 0 + device_printf(sc->sc_dev, "%s %p xr_ep 0x%x xr_cs %u\n", __func__, + xr, xr->xr_ep, xr->xr_cs); +#endif +} + +static usbd_status +xhci_do_command(struct xhci_softc * const sc, struct xhci_trb * const trb, + int timeout) +{ + struct xhci_ring * const cr = &sc->sc_cr; + usbd_status err; + + device_printf(sc->sc_dev, "%s input: " + "0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32"\n", __func__, + trb->trb_0, trb->trb_2, trb->trb_3); + + mutex_enter(&sc->sc_lock); + + KASSERT(sc->sc_command_addr == 0); + sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep); + + mutex_enter(&cr->xr_lock); + xhci_ring_put(sc, cr, NULL, trb, 1); + mutex_exit(&cr->xr_lock); + + xhci_db_write_4(sc, XHCI_DOORBELL(0), 0); + + if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock, + MAX(1, mstohz(timeout))) == EWOULDBLOCK) { + err = USBD_TIMEOUT; + goto timedout; + } + + trb->trb_0 = sc->sc_result_trb.trb_0; + trb->trb_2 = sc->sc_result_trb.trb_2; + trb->trb_3 = sc->sc_result_trb.trb_3; + + device_printf(sc->sc_dev, "%s output: " + "0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32"\n", __func__, + trb->trb_0, trb->trb_2, trb->trb_3); + + switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) { + case XHCI_TRB_ERROR_SUCCESS: + err = USBD_NORMAL_COMPLETION; + break; + default: + case 192 ... 223: + err = USBD_IOERROR; + break; + case 224 ... 255: + err = USBD_NORMAL_COMPLETION; + break; + } + +timedout: + sc->sc_command_addr = 0; + mutex_exit(&sc->sc_lock); + return err; +} + +static usbd_status +xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp) +{ + struct xhci_trb trb; + usbd_status err; + + trb.trb_0 = 0; + trb.trb_2 = 0; + trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT); + + err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); + if (err != USBD_NORMAL_COMPLETION) { + return err; + } + + *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3); + + return err; +} + +static usbd_status +xhci_address_device(struct xhci_softc * const sc, + uint64_t icp, uint8_t slot_id, bool bsr) +{ + struct xhci_trb trb; + usbd_status err; + + trb.trb_0 = icp; + trb.trb_2 = 0; + trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | + (bsr ? XHCI_TRB_3_BSR_BIT : 0); + + err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); + return err; +} + +static usbd_status +xhci_update_ep0_mps(struct xhci_softc * const sc, + struct xhci_slot * const xs, u_int mps) +{ + struct xhci_trb trb; + usbd_status err; + uint32_t * cp; + + device_printf(sc->sc_dev, "%s\n", __func__); + + cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); + cp[0] = htole32(0); + cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)); + + cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); + cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); + + /* sync input contexts before they are read from memory */ + usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); + hexdump("input context", xhci_slot_get_icv(sc, xs, 0), + sc->sc_ctxsz * 4); + + trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); + trb.trb_2 = 0; + trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX); + + err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); + KASSERT(err == USBD_NORMAL_COMPLETION); /* XXX */ + return err; +} + +static void +xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si) +{ + uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0); + + device_printf(sc->sc_dev, "dcbaa %p dc %016"PRIx64" slot %d\n", + &dcbaa[si], dcba, si); + + dcbaa[si] = htole64(dcba); + usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t), + BUS_DMASYNC_PREWRITE); +} + +static usbd_status +xhci_init_slot(struct xhci_softc * const sc, uint32_t slot, int depth, + int speed, int port, int rhport) +{ + struct xhci_slot *xs; + usbd_status err; + u_int dci; + uint32_t *cp; + uint32_t mps; + uint32_t xspeed; + + switch (speed) { + case USB_SPEED_LOW: + xspeed = 2; + mps = USB_MAX_IPACKET; + break; + case USB_SPEED_FULL: + xspeed = 1; + mps = 64; + break; + case USB_SPEED_HIGH: + xspeed = 3; + mps = USB_2_MAX_CTRL_PACKET; + break; + case USB_SPEED_SUPER: + xspeed = 4; + mps = USB_3_MAX_CTRL_PACKET; + break; + default: + device_printf(sc->sc_dev, "%s: impossible speed: %x", + __func__, speed); + return USBD_INVAL; + } + + xs = &sc->sc_slots[slot]; + xs->xs_idx = slot; + + /* allocate contexts */ + err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, + &xs->xs_dc_dma); + if (err) + return err; + memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz); + + err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, + &xs->xs_ic_dma); + if (err) + return err; + memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz); + + for (dci = 0; dci < 32; dci++) { + //CTASSERT(sizeof(xs->xs_ep[dci]) == sizeof(struct xhci_endpoint)); + memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); + if (dci == XHCI_DCI_SLOT) + continue; + err = xhci_ring_init(sc, &xs->xs_ep[dci].xe_tr, + XHCI_TRANSFER_RING_TRBS, XHCI_TRB_ALIGN); + if (err) { + device_printf(sc->sc_dev, "ring init failure\n"); + return err; + } + } + + /* set up initial input control context */ + cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); + cp[0] = htole32(0); + cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)| + XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT)); + + /* set up input slot context */ + cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); + cp[0] = htole32( + XHCI_SCTX_0_CTX_NUM_SET(1) | + XHCI_SCTX_0_SPEED_SET(xspeed) + ); + cp[1] = htole32( + XHCI_SCTX_1_RH_PORT_SET(rhport) + ); + cp[2] = htole32( + XHCI_SCTX_2_IRQ_TARGET_SET(0) + ); + cp[3] = htole32(0); + + /* set up input EP0 context */ + cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); + cp[0] = htole32(0); + cp[1] = htole32( + XHCI_EPCTX_1_MAXP_SIZE_SET(mps) | + XHCI_EPCTX_1_EPTYPE_SET(4) | + XHCI_EPCTX_1_CERR_SET(3) + ); + /* can't use xhci_ep_get_dci() yet? */ + *(uint64_t *)(&cp[2]) = htole64( + xhci_ring_trbp(&xs->xs_ep[XHCI_DCI_EP_CONTROL].xe_tr, 0) | + XHCI_EPCTX_2_DCS_SET(1)); + cp[4] = htole32( + XHCI_EPCTX_4_AVG_TRB_LEN_SET(8) + ); + + /* sync input contexts before they are read from memory */ + usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); + hexdump("input context", xhci_slot_get_icv(sc, xs, 0), + sc->sc_ctxsz * 3); + + xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot); + + err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, + false); + + usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); + hexdump("output context", xhci_slot_get_dcv(sc, xs, 0), + sc->sc_ctxsz * 2); + + return err; +} + +/* ----- */ + +static void +xhci_noop(usbd_pipe_handle pipe) +{ + DPRINTF(("%s\n", __func__)); +} + +/* root hub descriptors */ + +static const usb_device_descriptor_t xhci_devd = { + USB_DEVICE_DESCRIPTOR_SIZE, + UDESC_DEVICE, /* type */ + {0x00, 0x02}, /* USB version */ + UDCLASS_HUB, /* class */ + UDSUBCLASS_HUB, /* subclass */ + UDPROTO_HSHUBSTT, /* protocol */ + 64, /* max packet */ + {0},{0},{0x00,0x01}, /* device id */ + 1,2,0, /* string indexes */ + 1 /* # of configurations */ +}; + +static const usb_device_qualifier_t xhci_odevd = { + USB_DEVICE_DESCRIPTOR_SIZE, + UDESC_DEVICE_QUALIFIER, /* type */ + {0x00, 0x02}, /* USB version */ + UDCLASS_HUB, /* class */ + UDSUBCLASS_HUB, /* subclass */ + UDPROTO_FSHUB, /* protocol */ + 64, /* max packet */ + 1, /* # of configurations */ + 0 +}; + +static const usb_config_descriptor_t xhci_confd = { + USB_CONFIG_DESCRIPTOR_SIZE, + UDESC_CONFIG, + {USB_CONFIG_DESCRIPTOR_SIZE + + USB_INTERFACE_DESCRIPTOR_SIZE + + USB_ENDPOINT_DESCRIPTOR_SIZE}, + 1, + 1, + 0, + UC_ATTR_MBO | UC_SELF_POWERED, + 0 /* max power */ +}; + +static const usb_interface_descriptor_t xhci_ifcd = { + USB_INTERFACE_DESCRIPTOR_SIZE, + UDESC_INTERFACE, + 0, + 0, + 1, + UICLASS_HUB, + UISUBCLASS_HUB, + UIPROTO_HSHUBSTT, + 0 +}; + +static const usb_endpoint_descriptor_t xhci_endpd = { + USB_ENDPOINT_DESCRIPTOR_SIZE, + UDESC_ENDPOINT, + UE_DIR_IN | XHCI_INTR_ENDPT, + UE_INTERRUPT, + {8, 0}, /* max packet */ + 12 +}; + +static const usb_hub_descriptor_t xhci_hubd = { + USB_HUB_DESCRIPTOR_SIZE, + UDESC_HUB, + 0, + {0,0}, + 0, + 0, + {""}, + {""}, +}; + +/* root hub control */ + +static usbd_status +xhci_root_ctrl_transfer(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + usbd_status err; + + DPRINTF(("%s\n", __func__)); + + /* Insert last in queue. */ + mutex_enter(&sc->sc_lock); + err = usb_insert_transfer(xfer); + mutex_exit(&sc->sc_lock); + if (err) + return err; + + /* Pipe isn't running, start first */ + return (xhci_root_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); +} + +static usbd_status +xhci_root_ctrl_start(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + usb_port_status_t ps; + usb_device_request_t *req; + void *buf = NULL; + usb_hub_descriptor_t hubd; + usbd_status err; + int len, value, index; + int l, totlen = 0; + int port, i; + uint32_t v; + + DPRINTF(("%s\n", __func__)); + + if (sc->sc_dying) + return USBD_IOERROR; + + req = &xfer->request; + + value = UGETW(req->wValue); + index = UGETW(req->wIndex); + len = UGETW(req->wLength); + + if (len != 0) + buf = KERNADDR(&xfer->dmabuf, 0); + + DPRINTF(("root req: %02x %02x %04x %04x %04x\n", req->bmRequestType, + req->bRequest, value, index, len)); + +#define C(x,y) ((x) | ((y) << 8)) + switch(C(req->bRequest, req->bmRequestType)) { + case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): + case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): + case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): + /* + * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops + * for the integrated root hub. + */ + break; + case C(UR_GET_CONFIG, UT_READ_DEVICE): + if (len > 0) { + *(uint8_t *)buf = sc->sc_conf; + totlen = 1; + } + break; + case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): + DPRINTFN(8,("xhci_root_ctrl_start: wValue=0x%04x\n", value)); + if (len == 0) + break; + switch(value >> 8) { + case UDESC_DEVICE: + if ((value & 0xff) != 0) { + err = USBD_IOERROR; + goto ret; + } + totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE); + memcpy(buf, &xhci_devd, min(l, sizeof(xhci_devd))); + break; + case UDESC_DEVICE_QUALIFIER: + if ((value & 0xff) != 0) { + } + totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE); + memcpy(buf, &xhci_odevd, min(l, sizeof(xhci_odevd))); + break; + case UDESC_OTHER_SPEED_CONFIGURATION: + case UDESC_CONFIG: + if ((value & 0xff) != 0) { + err = USBD_IOERROR; + goto ret; + } + totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE); + memcpy(buf, &xhci_confd, min(l, sizeof(xhci_confd))); + ((usb_config_descriptor_t *)buf)->bDescriptorType = + value >> 8; + buf = (char *)buf + l; + len -= l; + l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE); + totlen += l; + memcpy(buf, &xhci_ifcd, min(l, sizeof(xhci_ifcd))); + buf = (char *)buf + l; + len -= l; + l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE); + totlen += l; + memcpy(buf, &xhci_endpd, min(l, sizeof(xhci_endpd))); + break; + case UDESC_STRING: +#define sd ((usb_string_descriptor_t *)buf) + switch (value & 0xff) { + case 0: /* Language table */ + totlen = usb_makelangtbl(sd, len); + break; + case 1: /* Vendor */ + totlen = usb_makestrdesc(sd, len, "NetBSD"); + break; + case 2: /* Product */ + totlen = usb_makestrdesc(sd, len, + "xHCI Root Hub"); + break; + } +#undef sd + break; + default: + err = USBD_IOERROR; + goto ret; + } + break; + case C(UR_GET_INTERFACE, UT_READ_INTERFACE): + if (len > 0) { + *(uint8_t *)buf = 0; + totlen = 1; + } + break; + case C(UR_GET_STATUS, UT_READ_DEVICE): + if (len > 1) { + USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED); + totlen = 2; + } + break; + case C(UR_GET_STATUS, UT_READ_INTERFACE): + case C(UR_GET_STATUS, UT_READ_ENDPOINT): + if (len > 1) { + USETW(((usb_status_t *)buf)->wStatus, 0); + totlen = 2; + } + break; + case C(UR_SET_ADDRESS, UT_WRITE_DEVICE): + if (value >= USB_MAX_DEVICES) { + err = USBD_IOERROR; + goto ret; + } + //sc->sc_addr = value; + break; + case C(UR_SET_CONFIG, UT_WRITE_DEVICE): + if (value != 0 && value != 1) { + err = USBD_IOERROR; + goto ret; + } + sc->sc_conf = value; + break; + case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): + break; + case C(UR_SET_FEATURE, UT_WRITE_DEVICE): + case C(UR_SET_FEATURE, UT_WRITE_INTERFACE): + case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT): + err = USBD_IOERROR; + goto ret; + case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE): + break; + case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): + break; + /* Hub requests */ + case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): + break; + case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): + DPRINTFN(4, ("xhci_root_ctrl_start: UR_CLEAR_PORT_FEATURE " + "port=%d feature=%d\n", + index, value)); + if (index < 1 || index > sc->sc_hs_port_count) { + err = USBD_IOERROR; + goto ret; + } + port = XHCI_PORTSC(sc->sc_hs_port_start - 1 + index); + v = xhci_op_read_4(sc, port); + DPRINTFN(4, ("xhci_root_ctrl_start: portsc=0x%08x\n", v)); + v &= ~XHCI_PS_CLEAR; + switch (value) { + case UHF_PORT_ENABLE: + xhci_op_write_4(sc, port, v &~ XHCI_PS_PED); + break; + case UHF_PORT_SUSPEND: + err = USBD_IOERROR; + goto ret; + case UHF_PORT_POWER: + break; + case UHF_PORT_TEST: + case UHF_PORT_INDICATOR: + err = USBD_IOERROR; + goto ret; + case UHF_C_PORT_CONNECTION: + xhci_op_write_4(sc, port, v | XHCI_PS_CSC); + break; + case UHF_C_PORT_ENABLE: + case UHF_C_PORT_SUSPEND: + case UHF_C_PORT_OVER_CURRENT: + err = USBD_IOERROR; + goto ret; + case UHF_C_PORT_RESET: + xhci_op_write_4(sc, port, v | XHCI_PS_PRC); + break; + default: + err = USBD_IOERROR; + goto ret; + } + + break; + case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): + if (len == 0) + break; + if ((value & 0xff) != 0) { + err = USBD_IOERROR; + goto ret; + } + hubd = xhci_hubd; + hubd.bNbrPorts = sc->sc_hs_port_count; + USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH); + hubd.bPwrOn2PwrGood = 200; + for (i = 0, l = sc->sc_maxports; l > 0; i++, l -= 8) + hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */ + hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i; + l = min(len, hubd.bDescLength); + totlen = l; + memcpy(buf, &hubd, l); + break; + case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): + if (len != 4) { + err = USBD_IOERROR; + goto ret; + } + memset(buf, 0, len); /* ? XXX */ + totlen = len; + break; + case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): + DPRINTFN(8,("xhci_root_ctrl_start: get port status i=%d\n", + index)); + if (index < 1 || index > sc->sc_maxports) { + err = USBD_IOERROR; + goto ret; + } + if (len != 4) { + err = USBD_IOERROR; + goto ret; + } + v = xhci_op_read_4(sc, XHCI_PORTSC(sc->sc_hs_port_start - 1 + + index)); + DPRINTF(("%s READ_CLASS_OTHER GET_STATUS PORTSC %d (%d) %08x\n", + __func__, index, sc->sc_hs_port_start - 1 + index, v)); + switch (XHCI_PS_SPEED_GET(v)) { + case 1: + i = UPS_FULL_SPEED; + break; + case 2: + i = UPS_LOW_SPEED; + break; + case 3: + i = UPS_HIGH_SPEED; + break; + default: + i = 0; + break; + } + if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS; + if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED; + if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR; + //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND; + if (v & XHCI_PS_PR) i |= UPS_RESET; + if (v & XHCI_PS_PP) i |= UPS_PORT_POWER; + USETW(ps.wPortStatus, i); + i = 0; + if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS; + if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED; + if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR; + if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET; + USETW(ps.wPortChange, i); + l = min(len, sizeof ps); + memcpy(buf, &ps, l); + totlen = l; + break; + case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): + err = USBD_IOERROR; + goto ret; + case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): + break; + case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): + if (index < 1 || index > sc->sc_hs_port_count) { + err = USBD_IOERROR; + goto ret; + } + port = XHCI_PORTSC(sc->sc_hs_port_start - 1 + index); + v = xhci_op_read_4(sc, port); + DPRINTFN(4, ("xhci_root_ctrl_start: portsc=0x%08x\n", v)); + v &= ~XHCI_PS_CLEAR; + switch (value) { + case UHF_PORT_ENABLE: + xhci_op_write_4(sc, port, v | XHCI_PS_PED); + break; + case UHF_PORT_SUSPEND: + /* XXX suspend */ + break; + case UHF_PORT_RESET: + v &= ~ (XHCI_PS_PED | XHCI_PS_PR); + xhci_op_write_4(sc, port, v | XHCI_PS_PR); + /* Wait for reset to complete. */ + usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY); + if (sc->sc_dying) { + err = USBD_IOERROR; + goto ret; + } + v = xhci_op_read_4(sc, port); + if (v & XHCI_PS_PR) { + xhci_op_write_4(sc, port, v & ~XHCI_PS_PR); + usb_delay_ms(&sc->sc_bus, 10); + /* XXX */ + } + break; + case UHF_PORT_POWER: + /* XXX power control */ + break; + /* XXX more */ + case UHF_C_PORT_RESET: + xhci_op_write_4(sc, port, v | XHCI_PS_PRC); + break; + default: + err = USBD_IOERROR; + goto ret; + } + break; + case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): + case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): + case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): + case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): + break; + default: + err = USBD_IOERROR; + goto ret; + } + xfer->actlen = totlen; + err = USBD_NORMAL_COMPLETION; +ret: + xfer->status = err; + mutex_enter(&sc->sc_lock); + usb_transfer_complete(xfer); + mutex_exit(&sc->sc_lock); + return USBD_IN_PROGRESS; +} + + +static void +xhci_root_ctrl_abort(usbd_xfer_handle xfer) +{ + /* Nothing to do, all transfers are synchronous. */ +} + + +static void +xhci_root_ctrl_close(usbd_pipe_handle pipe) +{ + DPRINTF(("%s\n", __func__)); + /* Nothing to do. */ +} + +static void +xhci_root_ctrl_done(usbd_xfer_handle xfer) +{ + xfer->hcpriv = NULL; +} + +/* root hub intrerrupt */ + +static usbd_status +xhci_root_intr_transfer(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + usbd_status err; + + /* Insert last in queue. */ + mutex_enter(&sc->sc_lock); + err = usb_insert_transfer(xfer); + mutex_exit(&sc->sc_lock); + if (err) + return err; + + /* Pipe isn't running, start first */ + return (xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); +} + +static usbd_status +xhci_root_intr_start(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + + if (sc->sc_dying) + return USBD_IOERROR; + + mutex_enter(&sc->sc_lock); + sc->sc_intrxfer = xfer; + mutex_exit(&sc->sc_lock); + + return USBD_IN_PROGRESS; +} + +static void +xhci_root_intr_abort(usbd_xfer_handle xfer) +{ +#ifdef DIAGNOSTIC + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; +#endif + + KASSERT(mutex_owned(&sc->sc_lock)); + if (xfer->pipe->intrxfer == xfer) { + DPRINTF(("%s: remove\n", __func__)); + xfer->pipe->intrxfer = NULL; + } + xfer->status = USBD_CANCELLED; + usb_transfer_complete(xfer); +} + +static void +xhci_root_intr_close(usbd_pipe_handle pipe) +{ + struct xhci_softc * const sc = pipe->device->bus->hci_private; + + KASSERT(mutex_owned(&sc->sc_lock)); + + DPRINTF(("%s\n", __func__)); + + sc->sc_intrxfer = NULL; +} + +static void +xhci_root_intr_done(usbd_xfer_handle xfer) +{ + xfer->hcpriv = NULL; +} + +/* -------------- */ +/* device control */ + +static usbd_status +xhci_device_ctrl_transfer(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + usbd_status err; + + /* Insert last in queue. */ + mutex_enter(&sc->sc_lock); + err = usb_insert_transfer(xfer); + mutex_exit(&sc->sc_lock); + if (err) + return (err); + + /* Pipe isn't running, start first */ + return (xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); +} + +static usbd_status +xhci_device_ctrl_start(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + struct xhci_slot * const xs = xfer->pipe->device->hci_private; + const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); + struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; + struct xhci_xfer * const xx = (void *)xfer; + usb_device_request_t * const req = &xfer->request; + const bool isread = UT_GET_DIR(req->bmRequestType) == UT_READ; + const uint32_t len = UGETW(req->wLength); + usb_dma_t * const dma = &xfer->dmabuf; + uint64_t parameter; + uint32_t status; + uint32_t control; + u_int i; + + DPRINTF(("%s\n", __func__)); + DPRINTF(("req: %02x %02x %04x %04x %04x\n", req->bmRequestType, + req->bRequest, UGETW(req->wValue), UGETW(req->wIndex), + UGETW(req->wLength))); + + /* XXX */ + if (tr->is_halted) { + xhci_reset_endpoint(xfer->pipe); + tr->is_halted = false; + xhci_set_dequeue(xfer->pipe); + } + + /* we rely on the bottom bits for extra info */ + KASSERT(((uintptr_t)xfer & 0x3) == 0x0); + + KASSERT((xfer->rqflags & URQ_REQUEST) != 0); + + i = 0; + + /* setup phase */ + memcpy(¶meter, req, sizeof(*req)); + parameter = le64toh(parameter); + status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req)); + control = ((len == 0) ? XHCI_TRB_3_TRT_NONE : + (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | + XHCI_TRB_3_IDT_BIT; + xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); + + if (len == 0) + goto no_data; + + /* data phase */ + parameter = DMAADDR(dma, 0); + KASSERT(len <= 0x10000); + status = XHCI_TRB_2_IRQ_SET(0) | + XHCI_TRB_2_TDSZ_SET(1) | + XHCI_TRB_2_BYTES_SET(len); + control = (isread ? XHCI_TRB_3_DIR_IN : 0) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) | + XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; + xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); + + parameter = (uintptr_t)xfer | 0x3; + status = XHCI_TRB_2_IRQ_SET(0); + control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | + XHCI_TRB_3_IOC_BIT; + xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); + +no_data: + parameter = 0; + status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_TDSZ_SET(1); + /* the status stage has inverted direction */ + control = (isread ? 0 : XHCI_TRB_3_DIR_IN) | + XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) | + XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; + xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); + + parameter = (uintptr_t)xfer | 0x0; + status = XHCI_TRB_2_IRQ_SET(0); + control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | + XHCI_TRB_3_IOC_BIT; + xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); + + mutex_enter(&tr->xr_lock); + xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); + mutex_exit(&tr->xr_lock); + + xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); + + if (xfer->timeout && !sc->sc_bus.use_polling) { + callout_reset(&xfer->timeout_handle, mstohz(xfer->timeout), + xhci_timeout, xfer); + } + + if (sc->sc_bus.use_polling) { + device_printf(sc->sc_dev, "%s polling\n", __func__); + //xhci_waitintr(sc, xfer); + } + + return USBD_IN_PROGRESS; +} + +static void +xhci_device_ctrl_done(usbd_xfer_handle xfer) +{ + DPRINTF(("%s\n", __func__)); + + callout_stop(&xfer->timeout_handle); /* XXX wrong place */ + +} + +static void +xhci_device_ctrl_abort(usbd_xfer_handle xfer) +{ + DPRINTF(("%s\n", __func__)); +} + +static void +xhci_device_ctrl_close(usbd_pipe_handle pipe) +{ + DPRINTF(("%s\n", __func__)); +} + +/* ----------------- */ +/* device isochronus */ + +/* ----------- */ +/* device bulk */ + +static usbd_status +xhci_device_bulk_transfer(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + usbd_status err; + + /* Insert last in queue. */ + mutex_enter(&sc->sc_lock); + err = usb_insert_transfer(xfer); + mutex_exit(&sc->sc_lock); + if (err) + return err; + + /* + * Pipe isn't running (otherwise err would be USBD_INPROG), + * so start it first. + */ + return (xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); +} + +static usbd_status +xhci_device_bulk_start(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + struct xhci_slot * const xs = xfer->pipe->device->hci_private; + const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); + struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; + struct xhci_xfer * const xx = (void *)xfer; + const uint32_t len = xfer->length; + usb_dma_t * const dma = &xfer->dmabuf; + uint64_t parameter; + uint32_t status; + uint32_t control; + u_int i = 0; + +#if 0 + device_printf(sc->sc_dev, "%s %p slot %u dci %u\n", __func__, xfer, + xs->xs_idx, dci); +#endif + + if (sc->sc_dying) + return USBD_IOERROR; + + KASSERT((xfer->rqflags & URQ_REQUEST) == 0); + + parameter = DMAADDR(dma, 0); + /* + * XXX: (dsl) The physical buffer must not cross a 64k boundary. + * If the user supplied buffer crosses such a boundary then 2 + * (or more) TRB should be used. + * If multiple TRB are used the td_size field must be set correctly. + * For v1.0 devices (like ivy bridge) this is the number of usb data + * blocks needed to complete the transfer. + * Setting it to 1 in the last TRB causes an extra zero-length + * data block be sent. + * The earlier documentation differs, I don't know how it behaves. + */ + KASSERT(len <= 0x10000); + status = XHCI_TRB_2_IRQ_SET(0) | + XHCI_TRB_2_TDSZ_SET(1) | + XHCI_TRB_2_BYTES_SET(len); + control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | + XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; + xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); + + mutex_enter(&tr->xr_lock); + xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); + mutex_exit(&tr->xr_lock); + + xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); + + if (sc->sc_bus.use_polling) { + device_printf(sc->sc_dev, "%s polling\n", __func__); + //xhci_waitintr(sc, xfer); + } + + return USBD_IN_PROGRESS; +} + +static void +xhci_device_bulk_done(usbd_xfer_handle xfer) +{ + //struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + //struct xhci_slot * const xs = xfer->pipe->device->hci_private; + //const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); + const u_int endpt = xfer->pipe->endpoint->edesc->bEndpointAddress; + const bool isread = UE_GET_DIR(endpt) == UE_DIR_IN; + + DPRINTF(("%s\n", __func__)); + +#if 0 + device_printf(sc->sc_dev, "%s %p slot %u dci %u\n", __func__, xfer, + xs->xs_idx, dci); +#endif + + callout_stop(&xfer->timeout_handle); /* XXX wrong place */ + + usb_syncmem(&xfer->dmabuf, 0, xfer->length, + isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); + + +} + +static void +xhci_device_bulk_abort(usbd_xfer_handle xfer) +{ + DPRINTF(("%s\n", __func__)); +} + +static void +xhci_device_bulk_close(usbd_pipe_handle pipe) +{ + DPRINTF(("%s\n", __func__)); +} + +/* --------------- */ +/* device intrrupt */ + +static usbd_status +xhci_device_intr_transfer(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + usbd_status err; + + /* Insert last in queue. */ + mutex_enter(&sc->sc_lock); + err = usb_insert_transfer(xfer); + mutex_exit(&sc->sc_lock); + if (err) + return err; + + /* + * Pipe isn't running (otherwise err would be USBD_INPROG), + * so start it first. + */ + return (xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue))); +} + +static usbd_status +xhci_device_intr_start(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + struct xhci_slot * const xs = xfer->pipe->device->hci_private; + const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); + struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; + struct xhci_xfer * const xx = (void *)xfer; + const uint32_t len = xfer->length; + usb_dma_t * const dma = &xfer->dmabuf; + uint64_t parameter; + uint32_t status; + uint32_t control; + u_int i = 0; + +#if 0 + device_printf(sc->sc_dev, "%s %p slot %u dci %u\n", __func__, xfer, + xs->xs_idx, dci); +#endif + + if (sc->sc_dying) + return USBD_IOERROR; + + KASSERT((xfer->rqflags & URQ_REQUEST) == 0); + + parameter = DMAADDR(dma, 0); + KASSERT(len <= 0x10000); + status = XHCI_TRB_2_IRQ_SET(0) | + XHCI_TRB_2_TDSZ_SET(1) | + XHCI_TRB_2_BYTES_SET(len); + control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | + XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; + xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); + + mutex_enter(&tr->xr_lock); + xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); + mutex_exit(&tr->xr_lock); + + xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); + + if (sc->sc_bus.use_polling) { + device_printf(sc->sc_dev, "%s polling\n", __func__); + //xhci_waitintr(sc, xfer); + } + + return USBD_IN_PROGRESS; +} + +static void +xhci_device_intr_done(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + struct xhci_slot * const xs = xfer->pipe->device->hci_private; + const u_int dci = xhci_ep_get_dci(xfer->pipe->endpoint->edesc); + const u_int endpt = xfer->pipe->endpoint->edesc->bEndpointAddress; + const bool isread = UE_GET_DIR(endpt) == UE_DIR_IN; + DPRINTF(("%s\n", __func__)); + + device_printf(sc->sc_dev, "%s %p slot %u dci %u\n", __func__, xfer, + xs->xs_idx, dci); + + KASSERT(sc->sc_bus.use_polling || mutex_owned(&sc->sc_lock)); + + usb_syncmem(&xfer->dmabuf, 0, xfer->length, + isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); + +#if 0 + device_printf(sc->sc_dev, ""); + for (size_t i = 0; i < xfer->length; i++) { + printf(" %02x", ((uint8_t const *)xfer->buffer)[i]); + } + printf("\n"); +#endif + + if (xfer->pipe->repeat) { + xfer->status = xhci_device_intr_start(xfer); + } else { + callout_stop(&xfer->timeout_handle); /* XXX */ + } + +} + +static void +xhci_device_intr_abort(usbd_xfer_handle xfer) +{ + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + DPRINTF(("%s\n", __func__)); + + KASSERT(mutex_owned(&sc->sc_lock)); + device_printf(sc->sc_dev, "%s %p\n", __func__, xfer); + /* XXX */ + if (xfer->pipe->intrxfer == xfer) { + xfer->pipe->intrxfer = NULL; + } + xfer->status = USBD_CANCELLED; + usb_transfer_complete(xfer); +} + +static void +xhci_device_intr_close(usbd_pipe_handle pipe) +{ + struct xhci_softc * const sc = pipe->device->bus->hci_private; + DPRINTF(("%s\n", __func__)); + device_printf(sc->sc_dev, "%s %p\n", __func__, pipe); + xhci_unconfigure_endpoint(pipe); +} + +/* ------------ */ + +static void +xhci_timeout(void *addr) +{ + struct xhci_xfer * const xx = addr; + usbd_xfer_handle const xfer = &xx->xx_xfer; + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + + if (sc->sc_dying) { + return; + } + + usb_init_task(&xx->xx_abort_task, xhci_timeout_task, addr, + USB_TASKQ_MPSAFE); + usb_add_task(xx->xx_xfer.pipe->device, &xx->xx_abort_task, + USB_TASKQ_HC); +} + +static void +xhci_timeout_task(void *addr) +{ + usbd_xfer_handle const xfer = addr; + struct xhci_softc * const sc = xfer->pipe->device->bus->hci_private; + + mutex_enter(&sc->sc_lock); +#if 0 + xhci_abort_xfer(xfer, USBD_TIMEOUT); +#else + xfer->status = USBD_TIMEOUT; + usb_transfer_complete(xfer); +#endif + mutex_exit(&sc->sc_lock); +}