Annotation of src/sys/kern/sys_select.c, Revision 1.42
1.42 ! christos 1: /* $NetBSD: sys_select.c,v 1.41 2018/01/30 07:52:23 ozaki-r Exp $ */
1.1 ad 2:
3: /*-
1.22 ad 4: * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc.
1.1 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
1.23 rmind 8: * by Andrew Doran and Mindaugas Rasiukevicius.
1.1 ad 9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: /*
33: * Copyright (c) 1982, 1986, 1989, 1993
34: * The Regents of the University of California. All rights reserved.
35: * (c) UNIX System Laboratories, Inc.
36: * All or some portions of this file are derived from material licensed
37: * to the University of California by American Telephone and Telegraph
38: * Co. or Unix System Laboratories, Inc. and are reproduced herein with
39: * the permission of UNIX System Laboratories, Inc.
40: *
41: * Redistribution and use in source and binary forms, with or without
42: * modification, are permitted provided that the following conditions
43: * are met:
44: * 1. Redistributions of source code must retain the above copyright
45: * notice, this list of conditions and the following disclaimer.
46: * 2. Redistributions in binary form must reproduce the above copyright
47: * notice, this list of conditions and the following disclaimer in the
48: * documentation and/or other materials provided with the distribution.
49: * 3. Neither the name of the University nor the names of its contributors
50: * may be used to endorse or promote products derived from this software
51: * without specific prior written permission.
52: *
53: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63: * SUCH DAMAGE.
64: *
65: * @(#)sys_generic.c 8.9 (Berkeley) 2/14/95
66: */
67:
68: /*
1.21 rmind 69: * System calls of synchronous I/O multiplexing subsystem.
70: *
71: * Locking
72: *
1.22 ad 73: * Two locks are used: <object-lock> and selcluster_t::sc_lock.
1.21 rmind 74: *
75: * The <object-lock> might be a device driver or another subsystem, e.g.
76: * socket or pipe. This lock is not exported, and thus invisible to this
77: * subsystem. Mainly, synchronisation between selrecord() and selnotify()
78: * routines depends on this lock, as it will be described in the comments.
79: *
80: * Lock order
81: *
82: * <object-lock> ->
1.22 ad 83: * selcluster_t::sc_lock
1.1 ad 84: */
85:
86: #include <sys/cdefs.h>
1.42 ! christos 87: __KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.41 2018/01/30 07:52:23 ozaki-r Exp $");
1.1 ad 88:
89: #include <sys/param.h>
90: #include <sys/systm.h>
91: #include <sys/filedesc.h>
92: #include <sys/file.h>
93: #include <sys/proc.h>
94: #include <sys/socketvar.h>
95: #include <sys/signalvar.h>
96: #include <sys/uio.h>
97: #include <sys/kernel.h>
1.29 rmind 98: #include <sys/lwp.h>
1.1 ad 99: #include <sys/poll.h>
100: #include <sys/mount.h>
101: #include <sys/syscallargs.h>
102: #include <sys/cpu.h>
103: #include <sys/atomic.h>
104: #include <sys/socketvar.h>
105: #include <sys/sleepq.h>
1.36 rmind 106: #include <sys/sysctl.h>
1.1 ad 107:
108: /* Flags for lwp::l_selflag. */
109: #define SEL_RESET 0 /* awoken, interrupted, or not yet polling */
110: #define SEL_SCANNING 1 /* polling descriptors */
1.23 rmind 111: #define SEL_BLOCKING 2 /* blocking and waiting for event */
112: #define SEL_EVENT 3 /* interrupted, events set directly */
113:
114: /* Operations: either select() or poll(). */
115: #define SELOP_SELECT 1
116: #define SELOP_POLL 2
1.1 ad 117:
1.22 ad 118: /*
119: * Per-cluster state for select()/poll(). For a system with fewer
120: * than 32 CPUs, this gives us per-CPU clusters.
121: */
122: #define SELCLUSTERS 32
123: #define SELCLUSTERMASK (SELCLUSTERS - 1)
124:
125: typedef struct selcluster {
1.13 ad 126: kmutex_t *sc_lock;
1.1 ad 127: sleepq_t sc_sleepq;
128: int sc_ncoll;
129: uint32_t sc_mask;
1.22 ad 130: } selcluster_t;
1.1 ad 131:
1.23 rmind 132: static inline int selscan(char *, const int, const size_t, register_t *);
133: static inline int pollscan(struct pollfd *, const int, register_t *);
1.19 rmind 134: static void selclear(void);
1.1 ad 135:
1.23 rmind 136: static const int sel_flag[] = {
137: POLLRDNORM | POLLHUP | POLLERR,
138: POLLWRNORM | POLLHUP | POLLERR,
139: POLLRDBAND
140: };
141:
1.1 ad 142: static syncobj_t select_sobj = {
1.41 ozaki-r 143: .sobj_flag = SOBJ_SLEEPQ_FIFO,
144: .sobj_unsleep = sleepq_unsleep,
145: .sobj_changepri = sleepq_changepri,
146: .sobj_lendpri = sleepq_lendpri,
147: .sobj_owner = syncobj_noowner,
1.1 ad 148: };
149:
1.23 rmind 150: static selcluster_t *selcluster[SELCLUSTERS] __read_mostly;
1.36 rmind 151: static int direct_select __read_mostly = 0;
1.22 ad 152:
1.1 ad 153: /*
154: * Select system call.
155: */
156: int
1.12 christos 157: sys___pselect50(struct lwp *l, const struct sys___pselect50_args *uap,
158: register_t *retval)
1.1 ad 159: {
160: /* {
161: syscallarg(int) nd;
162: syscallarg(fd_set *) in;
163: syscallarg(fd_set *) ou;
164: syscallarg(fd_set *) ex;
165: syscallarg(const struct timespec *) ts;
166: syscallarg(sigset_t *) mask;
167: } */
1.14 christos 168: struct timespec ats, *ts = NULL;
1.1 ad 169: sigset_t amask, *mask = NULL;
170: int error;
171:
172: if (SCARG(uap, ts)) {
173: error = copyin(SCARG(uap, ts), &ats, sizeof(ats));
174: if (error)
175: return error;
1.14 christos 176: ts = &ats;
1.1 ad 177: }
178: if (SCARG(uap, mask) != NULL) {
179: error = copyin(SCARG(uap, mask), &amask, sizeof(amask));
180: if (error)
181: return error;
182: mask = &amask;
183: }
184:
1.19 rmind 185: return selcommon(retval, SCARG(uap, nd), SCARG(uap, in),
1.14 christos 186: SCARG(uap, ou), SCARG(uap, ex), ts, mask);
1.1 ad 187: }
188:
189: int
1.12 christos 190: sys___select50(struct lwp *l, const struct sys___select50_args *uap,
191: register_t *retval)
1.1 ad 192: {
193: /* {
194: syscallarg(int) nd;
195: syscallarg(fd_set *) in;
196: syscallarg(fd_set *) ou;
197: syscallarg(fd_set *) ex;
198: syscallarg(struct timeval *) tv;
199: } */
1.14 christos 200: struct timeval atv;
201: struct timespec ats, *ts = NULL;
1.1 ad 202: int error;
203:
204: if (SCARG(uap, tv)) {
1.14 christos 205: error = copyin(SCARG(uap, tv), (void *)&atv, sizeof(atv));
1.1 ad 206: if (error)
207: return error;
1.14 christos 208: TIMEVAL_TO_TIMESPEC(&atv, &ats);
209: ts = &ats;
1.1 ad 210: }
211:
1.19 rmind 212: return selcommon(retval, SCARG(uap, nd), SCARG(uap, in),
1.14 christos 213: SCARG(uap, ou), SCARG(uap, ex), ts, NULL);
1.1 ad 214: }
215:
1.17 rmind 216: /*
217: * sel_do_scan: common code to perform the scan on descriptors.
218: */
219: static int
1.23 rmind 220: sel_do_scan(const int op, void *fds, const int nf, const size_t ni,
221: struct timespec *ts, sigset_t *mask, register_t *retval)
1.1 ad 222: {
1.17 rmind 223: lwp_t * const l = curlwp;
1.22 ad 224: selcluster_t *sc;
1.13 ad 225: kmutex_t *lock;
1.17 rmind 226: struct timespec sleepts;
227: int error, timo;
1.1 ad 228:
229: timo = 0;
1.14 christos 230: if (ts && inittimeleft(ts, &sleepts) == -1) {
1.17 rmind 231: return EINVAL;
1.1 ad 232: }
233:
1.32 christos 234: if (__predict_false(mask))
1.31 christos 235: sigsuspendsetup(l, mask);
1.1 ad 236:
1.22 ad 237: sc = curcpu()->ci_data.cpu_selcluster;
1.13 ad 238: lock = sc->sc_lock;
1.22 ad 239: l->l_selcluster = sc;
1.23 rmind 240: if (op == SELOP_SELECT) {
1.30 rmind 241: l->l_selbits = fds;
1.23 rmind 242: l->l_selni = ni;
243: } else {
244: l->l_selbits = NULL;
245: }
1.34 hannken 246:
1.1 ad 247: for (;;) {
1.17 rmind 248: int ncoll;
249:
1.34 hannken 250: SLIST_INIT(&l->l_selwait);
251: l->l_selret = 0;
252:
1.1 ad 253: /*
1.17 rmind 254: * No need to lock. If this is overwritten by another value
255: * while scanning, we will retry below. We only need to see
256: * exact state from the descriptors that we are about to poll,
257: * and lock activity resulting from fo_poll is enough to
258: * provide an up to date value for new polling activity.
1.1 ad 259: */
1.17 rmind 260: l->l_selflag = SEL_SCANNING;
1.1 ad 261: ncoll = sc->sc_ncoll;
262:
1.23 rmind 263: if (op == SELOP_SELECT) {
264: error = selscan((char *)fds, nf, ni, retval);
1.17 rmind 265: } else {
1.23 rmind 266: error = pollscan((struct pollfd *)fds, nf, retval);
1.17 rmind 267: }
1.1 ad 268: if (error || *retval)
269: break;
1.14 christos 270: if (ts && (timo = gettimeleft(ts, &sleepts)) <= 0)
1.1 ad 271: break;
1.23 rmind 272: /*
273: * Acquire the lock and perform the (re)checks. Note, if
274: * collision has occured, then our state does not matter,
275: * as we must perform re-scan. Therefore, check it first.
276: */
277: state_check:
1.13 ad 278: mutex_spin_enter(lock);
1.23 rmind 279: if (__predict_false(sc->sc_ncoll != ncoll)) {
280: /* Collision: perform re-scan. */
281: mutex_spin_exit(lock);
1.34 hannken 282: selclear();
1.23 rmind 283: continue;
284: }
285: if (__predict_true(l->l_selflag == SEL_EVENT)) {
286: /* Events occured, they are set directly. */
287: mutex_spin_exit(lock);
288: break;
289: }
290: if (__predict_true(l->l_selflag == SEL_RESET)) {
291: /* Events occured, but re-scan is requested. */
1.13 ad 292: mutex_spin_exit(lock);
1.34 hannken 293: selclear();
1.1 ad 294: continue;
295: }
1.23 rmind 296: /* Nothing happen, therefore - sleep. */
1.1 ad 297: l->l_selflag = SEL_BLOCKING;
1.7 ad 298: l->l_kpriority = true;
1.13 ad 299: sleepq_enter(&sc->sc_sleepq, l, lock);
1.1 ad 300: sleepq_enqueue(&sc->sc_sleepq, sc, "select", &select_sobj);
301: error = sleepq_block(timo, true);
1.23 rmind 302: if (error != 0) {
1.1 ad 303: break;
1.23 rmind 304: }
305: /* Awoken: need to check the state. */
306: goto state_check;
1.1 ad 307: }
308: selclear();
309:
1.34 hannken 310: /* Add direct events if any. */
311: if (l->l_selflag == SEL_EVENT) {
312: KASSERT(l->l_selret != 0);
313: *retval += l->l_selret;
314: }
315:
1.33 christos 316: if (__predict_false(mask))
317: sigsuspendteardown(l);
318:
1.20 dsl 319: /* select and poll are not restarted after signals... */
320: if (error == ERESTART)
321: return EINTR;
322: if (error == EWOULDBLOCK)
323: return 0;
1.17 rmind 324: return error;
325: }
326:
327: int
1.19 rmind 328: selcommon(register_t *retval, int nd, fd_set *u_in, fd_set *u_ou,
329: fd_set *u_ex, struct timespec *ts, sigset_t *mask)
1.17 rmind 330: {
331: char smallbits[howmany(FD_SETSIZE, NFDBITS) *
332: sizeof(fd_mask) * 6];
333: char *bits;
334: int error, nf;
335: size_t ni;
336:
337: if (nd < 0)
338: return (EINVAL);
1.19 rmind 339: nf = curlwp->l_fd->fd_dt->dt_nfiles;
1.17 rmind 340: if (nd > nf) {
341: /* forgiving; slightly wrong */
342: nd = nf;
343: }
344: ni = howmany(nd, NFDBITS) * sizeof(fd_mask);
1.40 chs 345: if (ni * 6 > sizeof(smallbits))
1.17 rmind 346: bits = kmem_alloc(ni * 6, KM_SLEEP);
1.40 chs 347: else
1.17 rmind 348: bits = smallbits;
349:
350: #define getbits(name, x) \
351: if (u_ ## name) { \
352: error = copyin(u_ ## name, bits + ni * x, ni); \
353: if (error) \
1.20 dsl 354: goto fail; \
1.17 rmind 355: } else \
356: memset(bits + ni * x, 0, ni);
357: getbits(in, 0);
358: getbits(ou, 1);
359: getbits(ex, 2);
360: #undef getbits
1.1 ad 361:
1.23 rmind 362: error = sel_do_scan(SELOP_SELECT, bits, nd, ni, ts, mask, retval);
1.1 ad 363: if (error == 0 && u_in != NULL)
364: error = copyout(bits + ni * 3, u_in, ni);
365: if (error == 0 && u_ou != NULL)
366: error = copyout(bits + ni * 4, u_ou, ni);
367: if (error == 0 && u_ex != NULL)
368: error = copyout(bits + ni * 5, u_ex, ni);
1.20 dsl 369: fail:
1.1 ad 370: if (bits != smallbits)
371: kmem_free(bits, ni * 6);
372: return (error);
373: }
374:
1.19 rmind 375: static inline int
1.23 rmind 376: selscan(char *bits, const int nfd, const size_t ni, register_t *retval)
1.1 ad 377: {
1.17 rmind 378: fd_mask *ibitp, *obitp;
1.23 rmind 379: int msk, i, j, fd, n;
1.1 ad 380: file_t *fp;
381:
1.17 rmind 382: ibitp = (fd_mask *)(bits + ni * 0);
383: obitp = (fd_mask *)(bits + ni * 3);
1.1 ad 384: n = 0;
1.17 rmind 385:
1.34 hannken 386: memset(obitp, 0, ni * 3);
1.1 ad 387: for (msk = 0; msk < 3; msk++) {
388: for (i = 0; i < nfd; i += NFDBITS) {
1.23 rmind 389: fd_mask ibits, obits;
390:
1.35 hannken 391: ibits = *ibitp;
1.1 ad 392: obits = 0;
393: while ((j = ffs(ibits)) && (fd = i + --j) < nfd) {
394: ibits &= ~(1 << j);
395: if ((fp = fd_getfile(fd)) == NULL)
396: return (EBADF);
1.23 rmind 397: /*
398: * Setup an argument to selrecord(), which is
399: * a file descriptor number.
400: */
401: curlwp->l_selrec = fd;
402: if ((*fp->f_ops->fo_poll)(fp, sel_flag[msk])) {
1.1 ad 403: obits |= (1 << j);
404: n++;
405: }
406: fd_putfile(fd);
407: }
1.34 hannken 408: if (obits != 0) {
1.36 rmind 409: if (direct_select) {
410: kmutex_t *lock;
411: lock = curlwp->l_selcluster->sc_lock;
1.35 hannken 412: mutex_spin_enter(lock);
1.36 rmind 413: *obitp |= obits;
1.35 hannken 414: mutex_spin_exit(lock);
1.36 rmind 415: } else {
416: *obitp |= obits;
417: }
1.34 hannken 418: }
1.35 hannken 419: ibitp++;
1.34 hannken 420: obitp++;
1.1 ad 421: }
422: }
423: *retval = n;
424: return (0);
425: }
426:
427: /*
428: * Poll system call.
429: */
430: int
431: sys_poll(struct lwp *l, const struct sys_poll_args *uap, register_t *retval)
432: {
433: /* {
434: syscallarg(struct pollfd *) fds;
435: syscallarg(u_int) nfds;
436: syscallarg(int) timeout;
437: } */
1.14 christos 438: struct timespec ats, *ts = NULL;
1.1 ad 439:
440: if (SCARG(uap, timeout) != INFTIM) {
1.14 christos 441: ats.tv_sec = SCARG(uap, timeout) / 1000;
442: ats.tv_nsec = (SCARG(uap, timeout) % 1000) * 1000000;
443: ts = &ats;
1.1 ad 444: }
445:
1.19 rmind 446: return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, NULL);
1.1 ad 447: }
448:
449: /*
450: * Poll system call.
451: */
452: int
1.12 christos 453: sys___pollts50(struct lwp *l, const struct sys___pollts50_args *uap,
454: register_t *retval)
1.1 ad 455: {
456: /* {
457: syscallarg(struct pollfd *) fds;
458: syscallarg(u_int) nfds;
459: syscallarg(const struct timespec *) ts;
460: syscallarg(const sigset_t *) mask;
461: } */
1.14 christos 462: struct timespec ats, *ts = NULL;
1.1 ad 463: sigset_t amask, *mask = NULL;
464: int error;
465:
466: if (SCARG(uap, ts)) {
467: error = copyin(SCARG(uap, ts), &ats, sizeof(ats));
468: if (error)
469: return error;
1.14 christos 470: ts = &ats;
1.1 ad 471: }
472: if (SCARG(uap, mask)) {
473: error = copyin(SCARG(uap, mask), &amask, sizeof(amask));
474: if (error)
475: return error;
476: mask = &amask;
477: }
478:
1.19 rmind 479: return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, mask);
1.1 ad 480: }
481:
482: int
1.19 rmind 483: pollcommon(register_t *retval, struct pollfd *u_fds, u_int nfds,
1.14 christos 484: struct timespec *ts, sigset_t *mask)
1.1 ad 485: {
1.11 yamt 486: struct pollfd smallfds[32];
487: struct pollfd *fds;
1.17 rmind 488: int error;
1.20 dsl 489: size_t ni;
1.1 ad 490:
1.42 ! christos 491: if (nfds > MAX(1000 + curlwp->l_fd->fd_dt->dt_nfiles,
! 492: curlwp->l_proc->p_rlimit[RLIMIT_NOFILE].rlim_cur)) {
1.20 dsl 493: /*
494: * Either the user passed in a very sparse 'fds' or junk!
495: * The kmem_alloc() call below would be bad news.
496: * We could process the 'fds' array in chunks, but that
497: * is a lot of code that isn't normally useful.
498: * (Or just move the copyin/out into pollscan().)
499: * Historically the code silently truncated 'fds' to
500: * dt_nfiles entries - but that does cause issues.
501: */
502: return EINVAL;
1.1 ad 503: }
504: ni = nfds * sizeof(struct pollfd);
1.40 chs 505: if (ni > sizeof(smallfds))
1.11 yamt 506: fds = kmem_alloc(ni, KM_SLEEP);
1.40 chs 507: else
1.11 yamt 508: fds = smallfds;
1.1 ad 509:
1.11 yamt 510: error = copyin(u_fds, fds, ni);
1.1 ad 511: if (error)
1.20 dsl 512: goto fail;
1.1 ad 513:
1.23 rmind 514: error = sel_do_scan(SELOP_POLL, fds, nfds, ni, ts, mask, retval);
1.1 ad 515: if (error == 0)
1.11 yamt 516: error = copyout(fds, u_fds, ni);
1.20 dsl 517: fail:
1.11 yamt 518: if (fds != smallfds)
519: kmem_free(fds, ni);
1.1 ad 520: return (error);
521: }
522:
1.19 rmind 523: static inline int
1.23 rmind 524: pollscan(struct pollfd *fds, const int nfd, register_t *retval)
1.1 ad 525: {
526: file_t *fp;
1.34 hannken 527: int i, n = 0, revents;
1.1 ad 528:
529: for (i = 0; i < nfd; i++, fds++) {
1.34 hannken 530: fds->revents = 0;
1.1 ad 531: if (fds->fd < 0) {
1.34 hannken 532: revents = 0;
1.1 ad 533: } else if ((fp = fd_getfile(fds->fd)) == NULL) {
1.34 hannken 534: revents = POLLNVAL;
1.1 ad 535: } else {
1.23 rmind 536: /*
537: * Perform poll: registers select request or returns
538: * the events which are set. Setup an argument for
539: * selrecord(), which is a pointer to struct pollfd.
540: */
541: curlwp->l_selrec = (uintptr_t)fds;
1.34 hannken 542: revents = (*fp->f_ops->fo_poll)(fp,
1.1 ad 543: fds->events | POLLERR | POLLHUP);
544: fd_putfile(fds->fd);
545: }
1.34 hannken 546: if (revents) {
547: fds->revents = revents;
548: n++;
549: }
1.1 ad 550: }
551: *retval = n;
552: return (0);
553: }
554:
555: int
556: seltrue(dev_t dev, int events, lwp_t *l)
557: {
558:
559: return (events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM));
560: }
561:
562: /*
563: * Record a select request. Concurrency issues:
564: *
565: * The caller holds the same lock across calls to selrecord() and
1.4 yamt 566: * selnotify(), so we don't need to consider a concurrent wakeup
1.1 ad 567: * while in this routine.
568: *
569: * The only activity we need to guard against is selclear(), called by
1.17 rmind 570: * another thread that is exiting sel_do_scan().
1.1 ad 571: * `sel_lwp' can only become non-NULL while the caller's lock is held,
572: * so it cannot become non-NULL due to a change made by another thread
573: * while we are in this routine. It can only become _NULL_ due to a
574: * call to selclear().
575: *
576: * If it is non-NULL and != selector there is the potential for
577: * selclear() to be called by another thread. If either of those
578: * conditions are true, we're not interested in touching the `named
579: * waiter' part of the selinfo record because we need to record a
580: * collision. Hence there is no need for additional locking in this
581: * routine.
582: */
583: void
584: selrecord(lwp_t *selector, struct selinfo *sip)
585: {
1.22 ad 586: selcluster_t *sc;
1.1 ad 587: lwp_t *other;
588:
589: KASSERT(selector == curlwp);
590:
1.22 ad 591: sc = selector->l_selcluster;
1.1 ad 592: other = sip->sel_lwp;
593:
594: if (other == selector) {
1.23 rmind 595: /* 1. We (selector) already claimed to be the first LWP. */
1.37 riastrad 596: KASSERT(sip->sel_cluster == sc);
1.1 ad 597: } else if (other == NULL) {
598: /*
1.23 rmind 599: * 2. No first LWP, therefore we (selector) are the first.
600: *
601: * There may be unnamed waiters (collisions). Issue a memory
602: * barrier to ensure that we access sel_lwp (above) before
603: * other fields - this guards against a call to selclear().
1.1 ad 604: */
605: membar_enter();
606: sip->sel_lwp = selector;
607: SLIST_INSERT_HEAD(&selector->l_selwait, sip, sel_chain);
1.23 rmind 608: /* Copy the argument, which is for selnotify(). */
609: sip->sel_fdinfo = selector->l_selrec;
1.22 ad 610: /* Replace selinfo's lock with the chosen cluster's lock. */
611: sip->sel_cluster = sc;
1.1 ad 612: } else {
1.23 rmind 613: /* 3. Multiple waiters: record a collision. */
1.1 ad 614: sip->sel_collision |= sc->sc_mask;
1.22 ad 615: KASSERT(sip->sel_cluster != NULL);
1.1 ad 616: }
617: }
618:
619: /*
1.23 rmind 620: * sel_setevents: a helper function for selnotify(), to set the events
621: * for LWP sleeping in selcommon() or pollcommon().
622: */
1.30 rmind 623: static inline bool
1.23 rmind 624: sel_setevents(lwp_t *l, struct selinfo *sip, const int events)
625: {
626: const int oflag = l->l_selflag;
1.30 rmind 627: int ret = 0;
1.23 rmind 628:
629: /*
630: * If we require re-scan or it was required by somebody else,
631: * then just (re)set SEL_RESET and return.
632: */
633: if (__predict_false(events == 0 || oflag == SEL_RESET)) {
634: l->l_selflag = SEL_RESET;
1.30 rmind 635: return true;
1.23 rmind 636: }
637: /*
638: * Direct set. Note: select state of LWP is locked. First,
639: * determine whether it is selcommon() or pollcommon().
640: */
641: if (l->l_selbits != NULL) {
1.30 rmind 642: const size_t ni = l->l_selni;
1.23 rmind 643: fd_mask *fds = (fd_mask *)l->l_selbits;
1.30 rmind 644: fd_mask *ofds = (fd_mask *)((char *)fds + ni * 3);
645: const int fd = sip->sel_fdinfo, fbit = 1 << (fd & __NFDMASK);
1.25 rmind 646: const int idx = fd >> __NFDSHIFT;
1.23 rmind 647: int n;
648:
649: for (n = 0; n < 3; n++) {
1.34 hannken 650: if ((fds[idx] & fbit) != 0 &&
651: (ofds[idx] & fbit) == 0 &&
652: (sel_flag[n] & events)) {
1.30 rmind 653: ofds[idx] |= fbit;
654: ret++;
1.23 rmind 655: }
656: fds = (fd_mask *)((char *)fds + ni);
1.30 rmind 657: ofds = (fd_mask *)((char *)ofds + ni);
1.23 rmind 658: }
659: } else {
660: struct pollfd *pfd = (void *)sip->sel_fdinfo;
1.30 rmind 661: int revents = events & (pfd->events | POLLERR | POLLHUP);
662:
663: if (revents) {
1.34 hannken 664: if (pfd->revents == 0)
665: ret = 1;
1.30 rmind 666: pfd->revents |= revents;
667: }
668: }
669: /* Check whether there are any events to return. */
670: if (!ret) {
671: return false;
1.23 rmind 672: }
673: /* Indicate direct set and note the event (cluster lock is held). */
674: l->l_selflag = SEL_EVENT;
1.30 rmind 675: l->l_selret += ret;
676: return true;
1.23 rmind 677: }
678:
679: /*
1.1 ad 680: * Do a wakeup when a selectable event occurs. Concurrency issues:
681: *
682: * As per selrecord(), the caller's object lock is held. If there
1.22 ad 683: * is a named waiter, we must acquire the associated selcluster's lock
1.1 ad 684: * in order to synchronize with selclear() and pollers going to sleep
1.17 rmind 685: * in sel_do_scan().
1.1 ad 686: *
1.22 ad 687: * sip->sel_cluser cannot change at this point, as it is only changed
1.1 ad 688: * in selrecord(), and concurrent calls to selrecord() are locked
689: * out by the caller.
690: */
691: void
692: selnotify(struct selinfo *sip, int events, long knhint)
693: {
1.22 ad 694: selcluster_t *sc;
1.1 ad 695: uint32_t mask;
1.16 rmind 696: int index, oflag;
1.1 ad 697: lwp_t *l;
1.13 ad 698: kmutex_t *lock;
1.1 ad 699:
700: KNOTE(&sip->sel_klist, knhint);
701:
702: if (sip->sel_lwp != NULL) {
703: /* One named LWP is waiting. */
1.22 ad 704: sc = sip->sel_cluster;
1.13 ad 705: lock = sc->sc_lock;
706: mutex_spin_enter(lock);
1.1 ad 707: /* Still there? */
708: if (sip->sel_lwp != NULL) {
1.23 rmind 709: /*
710: * Set the events for our LWP and indicate that.
711: * Otherwise, request for a full re-scan.
712: */
1.1 ad 713: l = sip->sel_lwp;
1.23 rmind 714: oflag = l->l_selflag;
1.36 rmind 715:
716: if (!direct_select) {
717: l->l_selflag = SEL_RESET;
718: } else if (!sel_setevents(l, sip, events)) {
1.30 rmind 719: /* No events to return. */
720: mutex_spin_exit(lock);
721: return;
722: }
1.36 rmind 723:
1.1 ad 724: /*
725: * If thread is sleeping, wake it up. If it's not
726: * yet asleep, it will notice the change in state
727: * and will re-poll the descriptors.
728: */
1.13 ad 729: if (oflag == SEL_BLOCKING && l->l_mutex == lock) {
1.1 ad 730: KASSERT(l->l_wchan == sc);
1.16 rmind 731: sleepq_unsleep(l, false);
1.1 ad 732: }
733: }
1.13 ad 734: mutex_spin_exit(lock);
1.1 ad 735: }
736:
737: if ((mask = sip->sel_collision) != 0) {
738: /*
739: * There was a collision (multiple waiters): we must
740: * inform all potentially interested waiters.
741: */
742: sip->sel_collision = 0;
1.3 ad 743: do {
1.1 ad 744: index = ffs(mask) - 1;
745: mask &= ~(1 << index);
1.22 ad 746: sc = selcluster[index];
1.13 ad 747: lock = sc->sc_lock;
748: mutex_spin_enter(lock);
1.1 ad 749: sc->sc_ncoll++;
1.13 ad 750: sleepq_wake(&sc->sc_sleepq, sc, (u_int)-1, lock);
1.3 ad 751: } while (__predict_false(mask != 0));
1.1 ad 752: }
753: }
754:
755: /*
756: * Remove an LWP from all objects that it is waiting for. Concurrency
757: * issues:
758: *
759: * The object owner's (e.g. device driver) lock is not held here. Calls
760: * can be made to selrecord() and we do not synchronize against those
761: * directly using locks. However, we use `sel_lwp' to lock out changes.
762: * Before clearing it we must use memory barriers to ensure that we can
763: * safely traverse the list of selinfo records.
764: */
765: static void
766: selclear(void)
767: {
768: struct selinfo *sip, *next;
1.22 ad 769: selcluster_t *sc;
1.1 ad 770: lwp_t *l;
1.13 ad 771: kmutex_t *lock;
1.1 ad 772:
773: l = curlwp;
1.22 ad 774: sc = l->l_selcluster;
1.13 ad 775: lock = sc->sc_lock;
1.1 ad 776:
1.13 ad 777: mutex_spin_enter(lock);
1.1 ad 778: for (sip = SLIST_FIRST(&l->l_selwait); sip != NULL; sip = next) {
779: KASSERT(sip->sel_lwp == l);
1.22 ad 780: KASSERT(sip->sel_cluster == l->l_selcluster);
781:
1.1 ad 782: /*
783: * Read link to next selinfo record, if any.
784: * It's no longer safe to touch `sip' after clearing
785: * `sel_lwp', so ensure that the read of `sel_chain'
786: * completes before the clearing of sel_lwp becomes
787: * globally visible.
788: */
789: next = SLIST_NEXT(sip, sel_chain);
790: membar_exit();
791: /* Release the record for another named waiter to use. */
792: sip->sel_lwp = NULL;
793: }
1.13 ad 794: mutex_spin_exit(lock);
1.1 ad 795: }
796:
797: /*
798: * Initialize the select/poll system calls. Called once for each
799: * CPU in the system, as they are attached.
800: */
801: void
802: selsysinit(struct cpu_info *ci)
803: {
1.22 ad 804: selcluster_t *sc;
805: u_int index;
1.1 ad 806:
1.22 ad 807: /* If already a cluster in place for this bit, re-use. */
808: index = cpu_index(ci) & SELCLUSTERMASK;
809: sc = selcluster[index];
810: if (sc == NULL) {
811: sc = kmem_alloc(roundup2(sizeof(selcluster_t),
812: coherency_unit) + coherency_unit, KM_SLEEP);
813: sc = (void *)roundup2((uintptr_t)sc, coherency_unit);
814: sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
815: sleepq_init(&sc->sc_sleepq);
816: sc->sc_ncoll = 0;
817: sc->sc_mask = (1 << index);
818: selcluster[index] = sc;
819: }
820: ci->ci_data.cpu_selcluster = sc;
1.1 ad 821: }
822:
823: /*
824: * Initialize a selinfo record.
825: */
826: void
827: selinit(struct selinfo *sip)
828: {
829:
830: memset(sip, 0, sizeof(*sip));
831: }
832:
833: /*
834: * Destroy a selinfo record. The owning object must not gain new
835: * references while this is in progress: all activity on the record
836: * must be stopped.
837: *
838: * Concurrency issues: we only need guard against a call to selclear()
1.17 rmind 839: * by a thread exiting sel_do_scan(). The caller has prevented further
840: * references being made to the selinfo record via selrecord(), and it
1.23 rmind 841: * will not call selnotify() again.
1.1 ad 842: */
843: void
844: seldestroy(struct selinfo *sip)
845: {
1.22 ad 846: selcluster_t *sc;
1.13 ad 847: kmutex_t *lock;
1.1 ad 848: lwp_t *l;
849:
850: if (sip->sel_lwp == NULL)
851: return;
852:
853: /*
1.22 ad 854: * Lock out selclear(). The selcluster pointer can't change while
1.1 ad 855: * we are here since it is only ever changed in selrecord(),
856: * and that will not be entered again for this record because
857: * it is dying.
858: */
1.22 ad 859: KASSERT(sip->sel_cluster != NULL);
860: sc = sip->sel_cluster;
1.13 ad 861: lock = sc->sc_lock;
862: mutex_spin_enter(lock);
1.1 ad 863: if ((l = sip->sel_lwp) != NULL) {
864: /*
865: * This should rarely happen, so although SLIST_REMOVE()
866: * is slow, using it here is not a problem.
867: */
1.22 ad 868: KASSERT(l->l_selcluster == sc);
1.1 ad 869: SLIST_REMOVE(&l->l_selwait, sip, selinfo, sel_chain);
870: sip->sel_lwp = NULL;
871: }
1.13 ad 872: mutex_spin_exit(lock);
1.1 ad 873: }
874:
1.36 rmind 875: /*
876: * System control nodes.
877: */
878: SYSCTL_SETUP(sysctl_select_setup, "sysctl select setup")
879: {
880:
1.38 pooka 881: sysctl_createv(clog, 0, NULL, NULL,
1.36 rmind 882: CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
883: CTLTYPE_INT, "direct_select",
884: SYSCTL_DESCR("Enable/disable direct select (for testing)"),
885: NULL, 0, &direct_select, 0,
1.38 pooka 886: CTL_KERN, CTL_CREATE, CTL_EOL);
1.36 rmind 887: }
CVSweb <webmaster@jp.NetBSD.org>