[BACK]Return to sys_select.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / kern

Annotation of src/sys/kern/sys_select.c, Revision 1.42

1.42    ! christos    1: /*     $NetBSD: sys_select.c,v 1.41 2018/01/30 07:52:23 ozaki-r Exp $  */
1.1       ad          2:
                      3: /*-
1.22      ad          4:  * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc.
1.1       ad          5:  * All rights reserved.
                      6:  *
                      7:  * This code is derived from software contributed to The NetBSD Foundation
1.23      rmind       8:  * by Andrew Doran and Mindaugas Rasiukevicius.
1.1       ad          9:  *
                     10:  * Redistribution and use in source and binary forms, with or without
                     11:  * modification, are permitted provided that the following conditions
                     12:  * are met:
                     13:  * 1. Redistributions of source code must retain the above copyright
                     14:  *    notice, this list of conditions and the following disclaimer.
                     15:  * 2. Redistributions in binary form must reproduce the above copyright
                     16:  *    notice, this list of conditions and the following disclaimer in the
                     17:  *    documentation and/or other materials provided with the distribution.
                     18:  *
                     19:  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     20:  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     21:  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     22:  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     23:  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     24:  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     25:  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     26:  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     27:  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     28:  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     29:  * POSSIBILITY OF SUCH DAMAGE.
                     30:  */
                     31:
                     32: /*
                     33:  * Copyright (c) 1982, 1986, 1989, 1993
                     34:  *     The Regents of the University of California.  All rights reserved.
                     35:  * (c) UNIX System Laboratories, Inc.
                     36:  * All or some portions of this file are derived from material licensed
                     37:  * to the University of California by American Telephone and Telegraph
                     38:  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
                     39:  * the permission of UNIX System Laboratories, Inc.
                     40:  *
                     41:  * Redistribution and use in source and binary forms, with or without
                     42:  * modification, are permitted provided that the following conditions
                     43:  * are met:
                     44:  * 1. Redistributions of source code must retain the above copyright
                     45:  *    notice, this list of conditions and the following disclaimer.
                     46:  * 2. Redistributions in binary form must reproduce the above copyright
                     47:  *    notice, this list of conditions and the following disclaimer in the
                     48:  *    documentation and/or other materials provided with the distribution.
                     49:  * 3. Neither the name of the University nor the names of its contributors
                     50:  *    may be used to endorse or promote products derived from this software
                     51:  *    without specific prior written permission.
                     52:  *
                     53:  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
                     54:  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
                     55:  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
                     56:  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
                     57:  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
                     58:  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
                     59:  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
                     60:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
                     61:  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
                     62:  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
                     63:  * SUCH DAMAGE.
                     64:  *
                     65:  *     @(#)sys_generic.c       8.9 (Berkeley) 2/14/95
                     66:  */
                     67:
                     68: /*
1.21      rmind      69:  * System calls of synchronous I/O multiplexing subsystem.
                     70:  *
                     71:  * Locking
                     72:  *
1.22      ad         73:  * Two locks are used: <object-lock> and selcluster_t::sc_lock.
1.21      rmind      74:  *
                     75:  * The <object-lock> might be a device driver or another subsystem, e.g.
                     76:  * socket or pipe.  This lock is not exported, and thus invisible to this
                     77:  * subsystem.  Mainly, synchronisation between selrecord() and selnotify()
                     78:  * routines depends on this lock, as it will be described in the comments.
                     79:  *
                     80:  * Lock order
                     81:  *
                     82:  *     <object-lock> ->
1.22      ad         83:  *             selcluster_t::sc_lock
1.1       ad         84:  */
                     85:
                     86: #include <sys/cdefs.h>
1.42    ! christos   87: __KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.41 2018/01/30 07:52:23 ozaki-r Exp $");
1.1       ad         88:
                     89: #include <sys/param.h>
                     90: #include <sys/systm.h>
                     91: #include <sys/filedesc.h>
                     92: #include <sys/file.h>
                     93: #include <sys/proc.h>
                     94: #include <sys/socketvar.h>
                     95: #include <sys/signalvar.h>
                     96: #include <sys/uio.h>
                     97: #include <sys/kernel.h>
1.29      rmind      98: #include <sys/lwp.h>
1.1       ad         99: #include <sys/poll.h>
                    100: #include <sys/mount.h>
                    101: #include <sys/syscallargs.h>
                    102: #include <sys/cpu.h>
                    103: #include <sys/atomic.h>
                    104: #include <sys/socketvar.h>
                    105: #include <sys/sleepq.h>
1.36      rmind     106: #include <sys/sysctl.h>
1.1       ad        107:
                    108: /* Flags for lwp::l_selflag. */
                    109: #define        SEL_RESET       0       /* awoken, interrupted, or not yet polling */
                    110: #define        SEL_SCANNING    1       /* polling descriptors */
1.23      rmind     111: #define        SEL_BLOCKING    2       /* blocking and waiting for event */
                    112: #define        SEL_EVENT       3       /* interrupted, events set directly */
                    113:
                    114: /* Operations: either select() or poll(). */
                    115: #define        SELOP_SELECT    1
                    116: #define        SELOP_POLL      2
1.1       ad        117:
1.22      ad        118: /*
                    119:  * Per-cluster state for select()/poll().  For a system with fewer
                    120:  * than 32 CPUs, this gives us per-CPU clusters.
                    121:  */
                    122: #define        SELCLUSTERS     32
                    123: #define        SELCLUSTERMASK  (SELCLUSTERS - 1)
                    124:
                    125: typedef struct selcluster {
1.13      ad        126:        kmutex_t        *sc_lock;
1.1       ad        127:        sleepq_t        sc_sleepq;
                    128:        int             sc_ncoll;
                    129:        uint32_t        sc_mask;
1.22      ad        130: } selcluster_t;
1.1       ad        131:
1.23      rmind     132: static inline int      selscan(char *, const int, const size_t, register_t *);
                    133: static inline int      pollscan(struct pollfd *, const int, register_t *);
1.19      rmind     134: static void            selclear(void);
1.1       ad        135:
1.23      rmind     136: static const int sel_flag[] = {
                    137:        POLLRDNORM | POLLHUP | POLLERR,
                    138:        POLLWRNORM | POLLHUP | POLLERR,
                    139:        POLLRDBAND
                    140: };
                    141:
1.1       ad        142: static syncobj_t select_sobj = {
1.41      ozaki-r   143:        .sobj_flag      = SOBJ_SLEEPQ_FIFO,
                    144:        .sobj_unsleep   = sleepq_unsleep,
                    145:        .sobj_changepri = sleepq_changepri,
                    146:        .sobj_lendpri   = sleepq_lendpri,
                    147:        .sobj_owner     = syncobj_noowner,
1.1       ad        148: };
                    149:
1.23      rmind     150: static selcluster_t    *selcluster[SELCLUSTERS] __read_mostly;
1.36      rmind     151: static int             direct_select __read_mostly = 0;
1.22      ad        152:
1.1       ad        153: /*
                    154:  * Select system call.
                    155:  */
                    156: int
1.12      christos  157: sys___pselect50(struct lwp *l, const struct sys___pselect50_args *uap,
                    158:     register_t *retval)
1.1       ad        159: {
                    160:        /* {
                    161:                syscallarg(int)                         nd;
                    162:                syscallarg(fd_set *)                    in;
                    163:                syscallarg(fd_set *)                    ou;
                    164:                syscallarg(fd_set *)                    ex;
                    165:                syscallarg(const struct timespec *)     ts;
                    166:                syscallarg(sigset_t *)                  mask;
                    167:        } */
1.14      christos  168:        struct timespec ats, *ts = NULL;
1.1       ad        169:        sigset_t        amask, *mask = NULL;
                    170:        int             error;
                    171:
                    172:        if (SCARG(uap, ts)) {
                    173:                error = copyin(SCARG(uap, ts), &ats, sizeof(ats));
                    174:                if (error)
                    175:                        return error;
1.14      christos  176:                ts = &ats;
1.1       ad        177:        }
                    178:        if (SCARG(uap, mask) != NULL) {
                    179:                error = copyin(SCARG(uap, mask), &amask, sizeof(amask));
                    180:                if (error)
                    181:                        return error;
                    182:                mask = &amask;
                    183:        }
                    184:
1.19      rmind     185:        return selcommon(retval, SCARG(uap, nd), SCARG(uap, in),
1.14      christos  186:            SCARG(uap, ou), SCARG(uap, ex), ts, mask);
1.1       ad        187: }
                    188:
                    189: int
1.12      christos  190: sys___select50(struct lwp *l, const struct sys___select50_args *uap,
                    191:     register_t *retval)
1.1       ad        192: {
                    193:        /* {
                    194:                syscallarg(int)                 nd;
                    195:                syscallarg(fd_set *)            in;
                    196:                syscallarg(fd_set *)            ou;
                    197:                syscallarg(fd_set *)            ex;
                    198:                syscallarg(struct timeval *)    tv;
                    199:        } */
1.14      christos  200:        struct timeval atv;
                    201:        struct timespec ats, *ts = NULL;
1.1       ad        202:        int error;
                    203:
                    204:        if (SCARG(uap, tv)) {
1.14      christos  205:                error = copyin(SCARG(uap, tv), (void *)&atv, sizeof(atv));
1.1       ad        206:                if (error)
                    207:                        return error;
1.14      christos  208:                TIMEVAL_TO_TIMESPEC(&atv, &ats);
                    209:                ts = &ats;
1.1       ad        210:        }
                    211:
1.19      rmind     212:        return selcommon(retval, SCARG(uap, nd), SCARG(uap, in),
1.14      christos  213:            SCARG(uap, ou), SCARG(uap, ex), ts, NULL);
1.1       ad        214: }
                    215:
1.17      rmind     216: /*
                    217:  * sel_do_scan: common code to perform the scan on descriptors.
                    218:  */
                    219: static int
1.23      rmind     220: sel_do_scan(const int op, void *fds, const int nf, const size_t ni,
                    221:     struct timespec *ts, sigset_t *mask, register_t *retval)
1.1       ad        222: {
1.17      rmind     223:        lwp_t           * const l = curlwp;
1.22      ad        224:        selcluster_t    *sc;
1.13      ad        225:        kmutex_t        *lock;
1.17      rmind     226:        struct timespec sleepts;
                    227:        int             error, timo;
1.1       ad        228:
                    229:        timo = 0;
1.14      christos  230:        if (ts && inittimeleft(ts, &sleepts) == -1) {
1.17      rmind     231:                return EINVAL;
1.1       ad        232:        }
                    233:
1.32      christos  234:        if (__predict_false(mask))
1.31      christos  235:                sigsuspendsetup(l, mask);
1.1       ad        236:
1.22      ad        237:        sc = curcpu()->ci_data.cpu_selcluster;
1.13      ad        238:        lock = sc->sc_lock;
1.22      ad        239:        l->l_selcluster = sc;
1.23      rmind     240:        if (op == SELOP_SELECT) {
1.30      rmind     241:                l->l_selbits = fds;
1.23      rmind     242:                l->l_selni = ni;
                    243:        } else {
                    244:                l->l_selbits = NULL;
                    245:        }
1.34      hannken   246:
1.1       ad        247:        for (;;) {
1.17      rmind     248:                int ncoll;
                    249:
1.34      hannken   250:                SLIST_INIT(&l->l_selwait);
                    251:                l->l_selret = 0;
                    252:
1.1       ad        253:                /*
1.17      rmind     254:                 * No need to lock.  If this is overwritten by another value
                    255:                 * while scanning, we will retry below.  We only need to see
                    256:                 * exact state from the descriptors that we are about to poll,
                    257:                 * and lock activity resulting from fo_poll is enough to
                    258:                 * provide an up to date value for new polling activity.
1.1       ad        259:                 */
1.17      rmind     260:                l->l_selflag = SEL_SCANNING;
1.1       ad        261:                ncoll = sc->sc_ncoll;
                    262:
1.23      rmind     263:                if (op == SELOP_SELECT) {
                    264:                        error = selscan((char *)fds, nf, ni, retval);
1.17      rmind     265:                } else {
1.23      rmind     266:                        error = pollscan((struct pollfd *)fds, nf, retval);
1.17      rmind     267:                }
1.1       ad        268:                if (error || *retval)
                    269:                        break;
1.14      christos  270:                if (ts && (timo = gettimeleft(ts, &sleepts)) <= 0)
1.1       ad        271:                        break;
1.23      rmind     272:                /*
                    273:                 * Acquire the lock and perform the (re)checks.  Note, if
                    274:                 * collision has occured, then our state does not matter,
                    275:                 * as we must perform re-scan.  Therefore, check it first.
                    276:                 */
                    277: state_check:
1.13      ad        278:                mutex_spin_enter(lock);
1.23      rmind     279:                if (__predict_false(sc->sc_ncoll != ncoll)) {
                    280:                        /* Collision: perform re-scan. */
                    281:                        mutex_spin_exit(lock);
1.34      hannken   282:                        selclear();
1.23      rmind     283:                        continue;
                    284:                }
                    285:                if (__predict_true(l->l_selflag == SEL_EVENT)) {
                    286:                        /* Events occured, they are set directly. */
                    287:                        mutex_spin_exit(lock);
                    288:                        break;
                    289:                }
                    290:                if (__predict_true(l->l_selflag == SEL_RESET)) {
                    291:                        /* Events occured, but re-scan is requested. */
1.13      ad        292:                        mutex_spin_exit(lock);
1.34      hannken   293:                        selclear();
1.1       ad        294:                        continue;
                    295:                }
1.23      rmind     296:                /* Nothing happen, therefore - sleep. */
1.1       ad        297:                l->l_selflag = SEL_BLOCKING;
1.7       ad        298:                l->l_kpriority = true;
1.13      ad        299:                sleepq_enter(&sc->sc_sleepq, l, lock);
1.1       ad        300:                sleepq_enqueue(&sc->sc_sleepq, sc, "select", &select_sobj);
                    301:                error = sleepq_block(timo, true);
1.23      rmind     302:                if (error != 0) {
1.1       ad        303:                        break;
1.23      rmind     304:                }
                    305:                /* Awoken: need to check the state. */
                    306:                goto state_check;
1.1       ad        307:        }
                    308:        selclear();
                    309:
1.34      hannken   310:        /* Add direct events if any. */
                    311:        if (l->l_selflag == SEL_EVENT) {
                    312:                KASSERT(l->l_selret != 0);
                    313:                *retval += l->l_selret;
                    314:        }
                    315:
1.33      christos  316:        if (__predict_false(mask))
                    317:                sigsuspendteardown(l);
                    318:
1.20      dsl       319:        /* select and poll are not restarted after signals... */
                    320:        if (error == ERESTART)
                    321:                return EINTR;
                    322:        if (error == EWOULDBLOCK)
                    323:                return 0;
1.17      rmind     324:        return error;
                    325: }
                    326:
                    327: int
1.19      rmind     328: selcommon(register_t *retval, int nd, fd_set *u_in, fd_set *u_ou,
                    329:     fd_set *u_ex, struct timespec *ts, sigset_t *mask)
1.17      rmind     330: {
                    331:        char            smallbits[howmany(FD_SETSIZE, NFDBITS) *
                    332:                            sizeof(fd_mask) * 6];
                    333:        char            *bits;
                    334:        int             error, nf;
                    335:        size_t          ni;
                    336:
                    337:        if (nd < 0)
                    338:                return (EINVAL);
1.19      rmind     339:        nf = curlwp->l_fd->fd_dt->dt_nfiles;
1.17      rmind     340:        if (nd > nf) {
                    341:                /* forgiving; slightly wrong */
                    342:                nd = nf;
                    343:        }
                    344:        ni = howmany(nd, NFDBITS) * sizeof(fd_mask);
1.40      chs       345:        if (ni * 6 > sizeof(smallbits))
1.17      rmind     346:                bits = kmem_alloc(ni * 6, KM_SLEEP);
1.40      chs       347:        else
1.17      rmind     348:                bits = smallbits;
                    349:
                    350: #define        getbits(name, x)                                                \
                    351:        if (u_ ## name) {                                               \
                    352:                error = copyin(u_ ## name, bits + ni * x, ni);          \
                    353:                if (error)                                              \
1.20      dsl       354:                        goto fail;                                      \
1.17      rmind     355:        } else                                                          \
                    356:                memset(bits + ni * x, 0, ni);
                    357:        getbits(in, 0);
                    358:        getbits(ou, 1);
                    359:        getbits(ex, 2);
                    360: #undef getbits
1.1       ad        361:
1.23      rmind     362:        error = sel_do_scan(SELOP_SELECT, bits, nd, ni, ts, mask, retval);
1.1       ad        363:        if (error == 0 && u_in != NULL)
                    364:                error = copyout(bits + ni * 3, u_in, ni);
                    365:        if (error == 0 && u_ou != NULL)
                    366:                error = copyout(bits + ni * 4, u_ou, ni);
                    367:        if (error == 0 && u_ex != NULL)
                    368:                error = copyout(bits + ni * 5, u_ex, ni);
1.20      dsl       369:  fail:
1.1       ad        370:        if (bits != smallbits)
                    371:                kmem_free(bits, ni * 6);
                    372:        return (error);
                    373: }
                    374:
1.19      rmind     375: static inline int
1.23      rmind     376: selscan(char *bits, const int nfd, const size_t ni, register_t *retval)
1.1       ad        377: {
1.17      rmind     378:        fd_mask *ibitp, *obitp;
1.23      rmind     379:        int msk, i, j, fd, n;
1.1       ad        380:        file_t *fp;
                    381:
1.17      rmind     382:        ibitp = (fd_mask *)(bits + ni * 0);
                    383:        obitp = (fd_mask *)(bits + ni * 3);
1.1       ad        384:        n = 0;
1.17      rmind     385:
1.34      hannken   386:        memset(obitp, 0, ni * 3);
1.1       ad        387:        for (msk = 0; msk < 3; msk++) {
                    388:                for (i = 0; i < nfd; i += NFDBITS) {
1.23      rmind     389:                        fd_mask ibits, obits;
                    390:
1.35      hannken   391:                        ibits = *ibitp;
1.1       ad        392:                        obits = 0;
                    393:                        while ((j = ffs(ibits)) && (fd = i + --j) < nfd) {
                    394:                                ibits &= ~(1 << j);
                    395:                                if ((fp = fd_getfile(fd)) == NULL)
                    396:                                        return (EBADF);
1.23      rmind     397:                                /*
                    398:                                 * Setup an argument to selrecord(), which is
                    399:                                 * a file descriptor number.
                    400:                                 */
                    401:                                curlwp->l_selrec = fd;
                    402:                                if ((*fp->f_ops->fo_poll)(fp, sel_flag[msk])) {
1.1       ad        403:                                        obits |= (1 << j);
                    404:                                        n++;
                    405:                                }
                    406:                                fd_putfile(fd);
                    407:                        }
1.34      hannken   408:                        if (obits != 0) {
1.36      rmind     409:                                if (direct_select) {
                    410:                                        kmutex_t *lock;
                    411:                                        lock = curlwp->l_selcluster->sc_lock;
1.35      hannken   412:                                        mutex_spin_enter(lock);
1.36      rmind     413:                                        *obitp |= obits;
1.35      hannken   414:                                        mutex_spin_exit(lock);
1.36      rmind     415:                                } else {
                    416:                                        *obitp |= obits;
                    417:                                }
1.34      hannken   418:                        }
1.35      hannken   419:                        ibitp++;
1.34      hannken   420:                        obitp++;
1.1       ad        421:                }
                    422:        }
                    423:        *retval = n;
                    424:        return (0);
                    425: }
                    426:
                    427: /*
                    428:  * Poll system call.
                    429:  */
                    430: int
                    431: sys_poll(struct lwp *l, const struct sys_poll_args *uap, register_t *retval)
                    432: {
                    433:        /* {
                    434:                syscallarg(struct pollfd *)     fds;
                    435:                syscallarg(u_int)               nfds;
                    436:                syscallarg(int)                 timeout;
                    437:        } */
1.14      christos  438:        struct timespec ats, *ts = NULL;
1.1       ad        439:
                    440:        if (SCARG(uap, timeout) != INFTIM) {
1.14      christos  441:                ats.tv_sec = SCARG(uap, timeout) / 1000;
                    442:                ats.tv_nsec = (SCARG(uap, timeout) % 1000) * 1000000;
                    443:                ts = &ats;
1.1       ad        444:        }
                    445:
1.19      rmind     446:        return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, NULL);
1.1       ad        447: }
                    448:
                    449: /*
                    450:  * Poll system call.
                    451:  */
                    452: int
1.12      christos  453: sys___pollts50(struct lwp *l, const struct sys___pollts50_args *uap,
                    454:     register_t *retval)
1.1       ad        455: {
                    456:        /* {
                    457:                syscallarg(struct pollfd *)             fds;
                    458:                syscallarg(u_int)                       nfds;
                    459:                syscallarg(const struct timespec *)     ts;
                    460:                syscallarg(const sigset_t *)            mask;
                    461:        } */
1.14      christos  462:        struct timespec ats, *ts = NULL;
1.1       ad        463:        sigset_t        amask, *mask = NULL;
                    464:        int             error;
                    465:
                    466:        if (SCARG(uap, ts)) {
                    467:                error = copyin(SCARG(uap, ts), &ats, sizeof(ats));
                    468:                if (error)
                    469:                        return error;
1.14      christos  470:                ts = &ats;
1.1       ad        471:        }
                    472:        if (SCARG(uap, mask)) {
                    473:                error = copyin(SCARG(uap, mask), &amask, sizeof(amask));
                    474:                if (error)
                    475:                        return error;
                    476:                mask = &amask;
                    477:        }
                    478:
1.19      rmind     479:        return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, mask);
1.1       ad        480: }
                    481:
                    482: int
1.19      rmind     483: pollcommon(register_t *retval, struct pollfd *u_fds, u_int nfds,
1.14      christos  484:     struct timespec *ts, sigset_t *mask)
1.1       ad        485: {
1.11      yamt      486:        struct pollfd   smallfds[32];
                    487:        struct pollfd   *fds;
1.17      rmind     488:        int             error;
1.20      dsl       489:        size_t          ni;
1.1       ad        490:
1.42    ! christos  491:        if (nfds > MAX(1000 + curlwp->l_fd->fd_dt->dt_nfiles,
        !           492:            curlwp->l_proc->p_rlimit[RLIMIT_NOFILE].rlim_cur)) {
1.20      dsl       493:                /*
                    494:                 * Either the user passed in a very sparse 'fds' or junk!
                    495:                 * The kmem_alloc() call below would be bad news.
                    496:                 * We could process the 'fds' array in chunks, but that
                    497:                 * is a lot of code that isn't normally useful.
                    498:                 * (Or just move the copyin/out into pollscan().)
                    499:                 * Historically the code silently truncated 'fds' to
                    500:                 * dt_nfiles entries - but that does cause issues.
                    501:                 */
                    502:                return EINVAL;
1.1       ad        503:        }
                    504:        ni = nfds * sizeof(struct pollfd);
1.40      chs       505:        if (ni > sizeof(smallfds))
1.11      yamt      506:                fds = kmem_alloc(ni, KM_SLEEP);
1.40      chs       507:        else
1.11      yamt      508:                fds = smallfds;
1.1       ad        509:
1.11      yamt      510:        error = copyin(u_fds, fds, ni);
1.1       ad        511:        if (error)
1.20      dsl       512:                goto fail;
1.1       ad        513:
1.23      rmind     514:        error = sel_do_scan(SELOP_POLL, fds, nfds, ni, ts, mask, retval);
1.1       ad        515:        if (error == 0)
1.11      yamt      516:                error = copyout(fds, u_fds, ni);
1.20      dsl       517:  fail:
1.11      yamt      518:        if (fds != smallfds)
                    519:                kmem_free(fds, ni);
1.1       ad        520:        return (error);
                    521: }
                    522:
1.19      rmind     523: static inline int
1.23      rmind     524: pollscan(struct pollfd *fds, const int nfd, register_t *retval)
1.1       ad        525: {
                    526:        file_t *fp;
1.34      hannken   527:        int i, n = 0, revents;
1.1       ad        528:
                    529:        for (i = 0; i < nfd; i++, fds++) {
1.34      hannken   530:                fds->revents = 0;
1.1       ad        531:                if (fds->fd < 0) {
1.34      hannken   532:                        revents = 0;
1.1       ad        533:                } else if ((fp = fd_getfile(fds->fd)) == NULL) {
1.34      hannken   534:                        revents = POLLNVAL;
1.1       ad        535:                } else {
1.23      rmind     536:                        /*
                    537:                         * Perform poll: registers select request or returns
                    538:                         * the events which are set.  Setup an argument for
                    539:                         * selrecord(), which is a pointer to struct pollfd.
                    540:                         */
                    541:                        curlwp->l_selrec = (uintptr_t)fds;
1.34      hannken   542:                        revents = (*fp->f_ops->fo_poll)(fp,
1.1       ad        543:                            fds->events | POLLERR | POLLHUP);
                    544:                        fd_putfile(fds->fd);
                    545:                }
1.34      hannken   546:                if (revents) {
                    547:                        fds->revents = revents;
                    548:                        n++;
                    549:                }
1.1       ad        550:        }
                    551:        *retval = n;
                    552:        return (0);
                    553: }
                    554:
                    555: int
                    556: seltrue(dev_t dev, int events, lwp_t *l)
                    557: {
                    558:
                    559:        return (events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM));
                    560: }
                    561:
                    562: /*
                    563:  * Record a select request.  Concurrency issues:
                    564:  *
                    565:  * The caller holds the same lock across calls to selrecord() and
1.4       yamt      566:  * selnotify(), so we don't need to consider a concurrent wakeup
1.1       ad        567:  * while in this routine.
                    568:  *
                    569:  * The only activity we need to guard against is selclear(), called by
1.17      rmind     570:  * another thread that is exiting sel_do_scan().
1.1       ad        571:  * `sel_lwp' can only become non-NULL while the caller's lock is held,
                    572:  * so it cannot become non-NULL due to a change made by another thread
                    573:  * while we are in this routine.  It can only become _NULL_ due to a
                    574:  * call to selclear().
                    575:  *
                    576:  * If it is non-NULL and != selector there is the potential for
                    577:  * selclear() to be called by another thread.  If either of those
                    578:  * conditions are true, we're not interested in touching the `named
                    579:  * waiter' part of the selinfo record because we need to record a
                    580:  * collision.  Hence there is no need for additional locking in this
                    581:  * routine.
                    582:  */
                    583: void
                    584: selrecord(lwp_t *selector, struct selinfo *sip)
                    585: {
1.22      ad        586:        selcluster_t *sc;
1.1       ad        587:        lwp_t *other;
                    588:
                    589:        KASSERT(selector == curlwp);
                    590:
1.22      ad        591:        sc = selector->l_selcluster;
1.1       ad        592:        other = sip->sel_lwp;
                    593:
                    594:        if (other == selector) {
1.23      rmind     595:                /* 1. We (selector) already claimed to be the first LWP. */
1.37      riastrad  596:                KASSERT(sip->sel_cluster == sc);
1.1       ad        597:        } else if (other == NULL) {
                    598:                /*
1.23      rmind     599:                 * 2. No first LWP, therefore we (selector) are the first.
                    600:                 *
                    601:                 * There may be unnamed waiters (collisions).  Issue a memory
                    602:                 * barrier to ensure that we access sel_lwp (above) before
                    603:                 * other fields - this guards against a call to selclear().
1.1       ad        604:                 */
                    605:                membar_enter();
                    606:                sip->sel_lwp = selector;
                    607:                SLIST_INSERT_HEAD(&selector->l_selwait, sip, sel_chain);
1.23      rmind     608:                /* Copy the argument, which is for selnotify(). */
                    609:                sip->sel_fdinfo = selector->l_selrec;
1.22      ad        610:                /* Replace selinfo's lock with the chosen cluster's lock. */
                    611:                sip->sel_cluster = sc;
1.1       ad        612:        } else {
1.23      rmind     613:                /* 3. Multiple waiters: record a collision. */
1.1       ad        614:                sip->sel_collision |= sc->sc_mask;
1.22      ad        615:                KASSERT(sip->sel_cluster != NULL);
1.1       ad        616:        }
                    617: }
                    618:
                    619: /*
1.23      rmind     620:  * sel_setevents: a helper function for selnotify(), to set the events
                    621:  * for LWP sleeping in selcommon() or pollcommon().
                    622:  */
1.30      rmind     623: static inline bool
1.23      rmind     624: sel_setevents(lwp_t *l, struct selinfo *sip, const int events)
                    625: {
                    626:        const int oflag = l->l_selflag;
1.30      rmind     627:        int ret = 0;
1.23      rmind     628:
                    629:        /*
                    630:         * If we require re-scan or it was required by somebody else,
                    631:         * then just (re)set SEL_RESET and return.
                    632:         */
                    633:        if (__predict_false(events == 0 || oflag == SEL_RESET)) {
                    634:                l->l_selflag = SEL_RESET;
1.30      rmind     635:                return true;
1.23      rmind     636:        }
                    637:        /*
                    638:         * Direct set.  Note: select state of LWP is locked.  First,
                    639:         * determine whether it is selcommon() or pollcommon().
                    640:         */
                    641:        if (l->l_selbits != NULL) {
1.30      rmind     642:                const size_t ni = l->l_selni;
1.23      rmind     643:                fd_mask *fds = (fd_mask *)l->l_selbits;
1.30      rmind     644:                fd_mask *ofds = (fd_mask *)((char *)fds + ni * 3);
                    645:                const int fd = sip->sel_fdinfo, fbit = 1 << (fd & __NFDMASK);
1.25      rmind     646:                const int idx = fd >> __NFDSHIFT;
1.23      rmind     647:                int n;
                    648:
                    649:                for (n = 0; n < 3; n++) {
1.34      hannken   650:                        if ((fds[idx] & fbit) != 0 &&
                    651:                            (ofds[idx] & fbit) == 0 &&
                    652:                            (sel_flag[n] & events)) {
1.30      rmind     653:                                ofds[idx] |= fbit;
                    654:                                ret++;
1.23      rmind     655:                        }
                    656:                        fds = (fd_mask *)((char *)fds + ni);
1.30      rmind     657:                        ofds = (fd_mask *)((char *)ofds + ni);
1.23      rmind     658:                }
                    659:        } else {
                    660:                struct pollfd *pfd = (void *)sip->sel_fdinfo;
1.30      rmind     661:                int revents = events & (pfd->events | POLLERR | POLLHUP);
                    662:
                    663:                if (revents) {
1.34      hannken   664:                        if (pfd->revents == 0)
                    665:                                ret = 1;
1.30      rmind     666:                        pfd->revents |= revents;
                    667:                }
                    668:        }
                    669:        /* Check whether there are any events to return. */
                    670:        if (!ret) {
                    671:                return false;
1.23      rmind     672:        }
                    673:        /* Indicate direct set and note the event (cluster lock is held). */
                    674:        l->l_selflag = SEL_EVENT;
1.30      rmind     675:        l->l_selret += ret;
                    676:        return true;
1.23      rmind     677: }
                    678:
                    679: /*
1.1       ad        680:  * Do a wakeup when a selectable event occurs.  Concurrency issues:
                    681:  *
                    682:  * As per selrecord(), the caller's object lock is held.  If there
1.22      ad        683:  * is a named waiter, we must acquire the associated selcluster's lock
1.1       ad        684:  * in order to synchronize with selclear() and pollers going to sleep
1.17      rmind     685:  * in sel_do_scan().
1.1       ad        686:  *
1.22      ad        687:  * sip->sel_cluser cannot change at this point, as it is only changed
1.1       ad        688:  * in selrecord(), and concurrent calls to selrecord() are locked
                    689:  * out by the caller.
                    690:  */
                    691: void
                    692: selnotify(struct selinfo *sip, int events, long knhint)
                    693: {
1.22      ad        694:        selcluster_t *sc;
1.1       ad        695:        uint32_t mask;
1.16      rmind     696:        int index, oflag;
1.1       ad        697:        lwp_t *l;
1.13      ad        698:        kmutex_t *lock;
1.1       ad        699:
                    700:        KNOTE(&sip->sel_klist, knhint);
                    701:
                    702:        if (sip->sel_lwp != NULL) {
                    703:                /* One named LWP is waiting. */
1.22      ad        704:                sc = sip->sel_cluster;
1.13      ad        705:                lock = sc->sc_lock;
                    706:                mutex_spin_enter(lock);
1.1       ad        707:                /* Still there? */
                    708:                if (sip->sel_lwp != NULL) {
1.23      rmind     709:                        /*
                    710:                         * Set the events for our LWP and indicate that.
                    711:                         * Otherwise, request for a full re-scan.
                    712:                         */
1.1       ad        713:                        l = sip->sel_lwp;
1.23      rmind     714:                        oflag = l->l_selflag;
1.36      rmind     715:
                    716:                        if (!direct_select) {
                    717:                                l->l_selflag = SEL_RESET;
                    718:                        } else if (!sel_setevents(l, sip, events)) {
1.30      rmind     719:                                /* No events to return. */
                    720:                                mutex_spin_exit(lock);
                    721:                                return;
                    722:                        }
1.36      rmind     723:
1.1       ad        724:                        /*
                    725:                         * If thread is sleeping, wake it up.  If it's not
                    726:                         * yet asleep, it will notice the change in state
                    727:                         * and will re-poll the descriptors.
                    728:                         */
1.13      ad        729:                        if (oflag == SEL_BLOCKING && l->l_mutex == lock) {
1.1       ad        730:                                KASSERT(l->l_wchan == sc);
1.16      rmind     731:                                sleepq_unsleep(l, false);
1.1       ad        732:                        }
                    733:                }
1.13      ad        734:                mutex_spin_exit(lock);
1.1       ad        735:        }
                    736:
                    737:        if ((mask = sip->sel_collision) != 0) {
                    738:                /*
                    739:                 * There was a collision (multiple waiters): we must
                    740:                 * inform all potentially interested waiters.
                    741:                 */
                    742:                sip->sel_collision = 0;
1.3       ad        743:                do {
1.1       ad        744:                        index = ffs(mask) - 1;
                    745:                        mask &= ~(1 << index);
1.22      ad        746:                        sc = selcluster[index];
1.13      ad        747:                        lock = sc->sc_lock;
                    748:                        mutex_spin_enter(lock);
1.1       ad        749:                        sc->sc_ncoll++;
1.13      ad        750:                        sleepq_wake(&sc->sc_sleepq, sc, (u_int)-1, lock);
1.3       ad        751:                } while (__predict_false(mask != 0));
1.1       ad        752:        }
                    753: }
                    754:
                    755: /*
                    756:  * Remove an LWP from all objects that it is waiting for.  Concurrency
                    757:  * issues:
                    758:  *
                    759:  * The object owner's (e.g. device driver) lock is not held here.  Calls
                    760:  * can be made to selrecord() and we do not synchronize against those
                    761:  * directly using locks.  However, we use `sel_lwp' to lock out changes.
                    762:  * Before clearing it we must use memory barriers to ensure that we can
                    763:  * safely traverse the list of selinfo records.
                    764:  */
                    765: static void
                    766: selclear(void)
                    767: {
                    768:        struct selinfo *sip, *next;
1.22      ad        769:        selcluster_t *sc;
1.1       ad        770:        lwp_t *l;
1.13      ad        771:        kmutex_t *lock;
1.1       ad        772:
                    773:        l = curlwp;
1.22      ad        774:        sc = l->l_selcluster;
1.13      ad        775:        lock = sc->sc_lock;
1.1       ad        776:
1.13      ad        777:        mutex_spin_enter(lock);
1.1       ad        778:        for (sip = SLIST_FIRST(&l->l_selwait); sip != NULL; sip = next) {
                    779:                KASSERT(sip->sel_lwp == l);
1.22      ad        780:                KASSERT(sip->sel_cluster == l->l_selcluster);
                    781:
1.1       ad        782:                /*
                    783:                 * Read link to next selinfo record, if any.
                    784:                 * It's no longer safe to touch `sip' after clearing
                    785:                 * `sel_lwp', so ensure that the read of `sel_chain'
                    786:                 * completes before the clearing of sel_lwp becomes
                    787:                 * globally visible.
                    788:                 */
                    789:                next = SLIST_NEXT(sip, sel_chain);
                    790:                membar_exit();
                    791:                /* Release the record for another named waiter to use. */
                    792:                sip->sel_lwp = NULL;
                    793:        }
1.13      ad        794:        mutex_spin_exit(lock);
1.1       ad        795: }
                    796:
                    797: /*
                    798:  * Initialize the select/poll system calls.  Called once for each
                    799:  * CPU in the system, as they are attached.
                    800:  */
                    801: void
                    802: selsysinit(struct cpu_info *ci)
                    803: {
1.22      ad        804:        selcluster_t *sc;
                    805:        u_int index;
1.1       ad        806:
1.22      ad        807:        /* If already a cluster in place for this bit, re-use. */
                    808:        index = cpu_index(ci) & SELCLUSTERMASK;
                    809:        sc = selcluster[index];
                    810:        if (sc == NULL) {
                    811:                sc = kmem_alloc(roundup2(sizeof(selcluster_t),
                    812:                    coherency_unit) + coherency_unit, KM_SLEEP);
                    813:                sc = (void *)roundup2((uintptr_t)sc, coherency_unit);
                    814:                sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
                    815:                sleepq_init(&sc->sc_sleepq);
                    816:                sc->sc_ncoll = 0;
                    817:                sc->sc_mask = (1 << index);
                    818:                selcluster[index] = sc;
                    819:        }
                    820:        ci->ci_data.cpu_selcluster = sc;
1.1       ad        821: }
                    822:
                    823: /*
                    824:  * Initialize a selinfo record.
                    825:  */
                    826: void
                    827: selinit(struct selinfo *sip)
                    828: {
                    829:
                    830:        memset(sip, 0, sizeof(*sip));
                    831: }
                    832:
                    833: /*
                    834:  * Destroy a selinfo record.  The owning object must not gain new
                    835:  * references while this is in progress: all activity on the record
                    836:  * must be stopped.
                    837:  *
                    838:  * Concurrency issues: we only need guard against a call to selclear()
1.17      rmind     839:  * by a thread exiting sel_do_scan().  The caller has prevented further
                    840:  * references being made to the selinfo record via selrecord(), and it
1.23      rmind     841:  * will not call selnotify() again.
1.1       ad        842:  */
                    843: void
                    844: seldestroy(struct selinfo *sip)
                    845: {
1.22      ad        846:        selcluster_t *sc;
1.13      ad        847:        kmutex_t *lock;
1.1       ad        848:        lwp_t *l;
                    849:
                    850:        if (sip->sel_lwp == NULL)
                    851:                return;
                    852:
                    853:        /*
1.22      ad        854:         * Lock out selclear().  The selcluster pointer can't change while
1.1       ad        855:         * we are here since it is only ever changed in selrecord(),
                    856:         * and that will not be entered again for this record because
                    857:         * it is dying.
                    858:         */
1.22      ad        859:        KASSERT(sip->sel_cluster != NULL);
                    860:        sc = sip->sel_cluster;
1.13      ad        861:        lock = sc->sc_lock;
                    862:        mutex_spin_enter(lock);
1.1       ad        863:        if ((l = sip->sel_lwp) != NULL) {
                    864:                /*
                    865:                 * This should rarely happen, so although SLIST_REMOVE()
                    866:                 * is slow, using it here is not a problem.
                    867:                 */
1.22      ad        868:                KASSERT(l->l_selcluster == sc);
1.1       ad        869:                SLIST_REMOVE(&l->l_selwait, sip, selinfo, sel_chain);
                    870:                sip->sel_lwp = NULL;
                    871:        }
1.13      ad        872:        mutex_spin_exit(lock);
1.1       ad        873: }
                    874:
1.36      rmind     875: /*
                    876:  * System control nodes.
                    877:  */
                    878: SYSCTL_SETUP(sysctl_select_setup, "sysctl select setup")
                    879: {
                    880:
1.38      pooka     881:        sysctl_createv(clog, 0, NULL, NULL,
1.36      rmind     882:                CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
                    883:                CTLTYPE_INT, "direct_select",
                    884:                SYSCTL_DESCR("Enable/disable direct select (for testing)"),
                    885:                NULL, 0, &direct_select, 0,
1.38      pooka     886:                CTL_KERN, CTL_CREATE, CTL_EOL);
1.36      rmind     887: }

CVSweb <webmaster@jp.NetBSD.org>