Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/uipc_socket.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/uipc_socket.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.67 retrieving revision 1.79 diff -u -p -r1.67 -r1.79 --- src/sys/kern/uipc_socket.c 2002/06/10 20:43:16 1.67 +++ src/sys/kern/uipc_socket.c 2003/04/09 18:38:03 1.79 @@ -1,4 +1,4 @@ -/* $NetBSD: uipc_socket.c,v 1.67 2002/06/10 20:43:16 he Exp $ */ +/* $NetBSD: uipc_socket.c,v 1.79 2003/04/09 18:38:03 thorpej Exp $ */ /*- * Copyright (c) 2002 The NetBSD Foundation, Inc. @@ -72,7 +72,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.67 2002/06/10 20:43:16 he Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.79 2003/04/09 18:38:03 thorpej Exp $"); #include "opt_sock_counters.h" #include "opt_sosend_loan.h" @@ -91,11 +91,15 @@ __KERNEL_RCSID(0, "$NetBSD: uipc_socket. #include #include #include +#include #include struct pool socket_pool; +MALLOC_DEFINE(M_SOOPTS, "soopts", "socket options"); +MALLOC_DEFINE(M_SONAME, "soname", "socket name"); + extern int somaxconn; /* patchable (XXX sysctl) */ int somaxconn = SOMAXCONN; @@ -134,10 +138,10 @@ soinit(void) #endif /* SOSEND_COUNTERS */ } -#ifdef SOSEND_LOAN -int use_sosend_loan = 1; -#else +#ifdef SOSEND_NO_LOAN int use_sosend_loan = 0; +#else +int use_sosend_loan = 1; #endif struct mbuf *so_pendfree; @@ -150,9 +154,8 @@ int sokvawaiters; #define SOCK_LOAN_CHUNK 65536 static void -sodoloanfree(caddr_t buf, u_int size) +sodoloanfree(struct vm_page **pgs, caddr_t buf, size_t size) { - struct vm_page **pgs; vaddr_t va, sva, eva; vsize_t len; paddr_t pa; @@ -163,12 +166,14 @@ sodoloanfree(caddr_t buf, u_int size) len = eva - sva; npgs = len >> PAGE_SHIFT; - pgs = alloca(npgs * sizeof(*pgs)); + if (__predict_false(pgs == NULL)) { + pgs = alloca(npgs * sizeof(*pgs)); - for (i = 0, va = sva; va < eva; i++, va += PAGE_SIZE) { - if (pmap_extract(pmap_kernel(), va, &pa) == FALSE) - panic("sodoloanfree: va 0x%lx not mapped", va); - pgs[i] = PHYS_TO_VM_PAGE(pa); + for (i = 0, va = sva; va < eva; i++, va += PAGE_SIZE) { + if (pmap_extract(pmap_kernel(), va, &pa) == FALSE) + panic("sodoloanfree: va 0x%lx not mapped", va); + pgs[i] = PHYS_TO_VM_PAGE(pa); + } } pmap_kremove(sva, len); @@ -197,7 +202,9 @@ sodopendfree(struct socket *so) splx(s); rv += m->m_ext.ext_size; - sodoloanfree(m->m_ext.ext_buf, m->m_ext.ext_size); + sodoloanfree((m->m_flags & M_EXT_PAGES) ? + m->m_ext.ext_pgs : NULL, m->m_ext.ext_buf, + m->m_ext.ext_size); s = splvm(); pool_cache_put(&mbpool_cache, m); } @@ -210,7 +217,9 @@ sodopendfree(struct socket *so) splx(s); rv += m->m_ext.ext_size; - sodoloanfree(m->m_ext.ext_buf, m->m_ext.ext_size); + sodoloanfree((m->m_flags & M_EXT_PAGES) ? + m->m_ext.ext_pgs : NULL, m->m_ext.ext_buf, + m->m_ext.ext_size); s = splvm(); pool_cache_put(&mbpool_cache, m); } @@ -220,13 +229,13 @@ sodopendfree(struct socket *so) } static void -soloanfree(struct mbuf *m, caddr_t buf, u_int size, void *arg) +soloanfree(struct mbuf *m, caddr_t buf, size_t size, void *arg) { struct socket *so = arg; int s; if (m == NULL) { - sodoloanfree(buf, size); + sodoloanfree(NULL, buf, size); return; } @@ -244,7 +253,6 @@ sosend_loan(struct socket *so, struct ui struct iovec *iov = uio->uio_iov; vaddr_t sva, eva; vsize_t len; - struct vm_page **pgs; vaddr_t lva, va; int npgs, s, i, error; @@ -261,6 +269,9 @@ sosend_loan(struct socket *so, struct ui len = eva - sva; npgs = len >> PAGE_SHIFT; + /* XXX KDASSERT */ + KASSERT(npgs <= M_EXT_MAXPAGES); + while (socurkva + len > somaxkva) { if (sodopendfree(so)) continue; @@ -277,10 +288,8 @@ sosend_loan(struct socket *so, struct ui return (0); socurkva += len; - pgs = alloca(npgs * sizeof(*pgs)); - error = uvm_loan(&uio->uio_procp->p_vmspace->vm_map, sva, len, - pgs, UVM_LOAN_TOPAGE); + m->m_ext.ext_pgs, UVM_LOAN_TOPAGE); if (error) { uvm_km_free(kernel_map, lva, len); socurkva -= len; @@ -288,12 +297,14 @@ sosend_loan(struct socket *so, struct ui } for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE) - pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pgs[i]), VM_PROT_READ); + pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]), + VM_PROT_READ); pmap_update(pmap_kernel()); lva += (vaddr_t) iov->iov_base & PAGE_MASK; MEXTADD(m, (caddr_t) lva, space, M_MBUF, soloanfree, so); + m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP; uio->uio_resid -= space; /* uio_offset not updated, not set/used for write(2) */ @@ -341,6 +352,11 @@ socreate(int dom, struct socket **aso, i so->so_proto = prp; so->so_send = sosend; so->so_receive = soreceive; +#ifdef MBUFTRACE + so->so_rcv.sb_mowner = &prp->pr_domain->dom_mowner; + so->so_snd.sb_mowner = &prp->pr_domain->dom_mowner; + so->so_mowner = &prp->pr_domain->dom_mowner; +#endif if (p != 0) so->so_uid = p->p_ucred->cr_uid; error = (*prp->pr_usrreq)(so, PRU_ATTACH, (struct mbuf *)0, @@ -677,14 +693,15 @@ sosend(struct socket *so, struct mbuf *a top->m_flags |= M_EOR; } else do { if (top == 0) { - MGETHDR(m, M_WAIT, MT_DATA); + m = m_gethdr(M_WAIT, MT_DATA); mlen = MHLEN; m->m_pkthdr.len = 0; m->m_pkthdr.rcvif = (struct ifnet *)0; } else { - MGET(m, M_WAIT, MT_DATA); + m = m_get(M_WAIT, MT_DATA); mlen = MLEN; } + MCLAIM(m, so->so_snd.sb_mowner); if (use_sosend_loan && uio->uio_iov->iov_len >= SOCK_LOAN_THRESH && space >= SOCK_LOAN_THRESH && @@ -696,7 +713,7 @@ sosend(struct socket *so, struct mbuf *a } if (resid >= MINCLSIZE && space >= MCLBYTES) { SOSEND_COUNTER_INCR(&sosend_copy_big); - MCLGET(m, M_WAIT); + m_clget(m, M_WAIT); if ((m->m_flags & M_EXT) == 0) goto nopages; mlen = MCLBYTES; @@ -894,6 +911,8 @@ soreceive(struct socket *so, struct mbuf error = EWOULDBLOCK; goto release; } + SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1"); sbunlock(&so->so_rcv); error = sbwait(&so->so_rcv); splx(s); @@ -902,10 +921,18 @@ soreceive(struct socket *so, struct mbuf goto restart; } dontblock: + /* + * On entry here, m points to the first record of the socket buffer. + * While we process the initial mbufs containing address and control + * info, we save a copy of m->m_nextpkt into nextrecord. + */ #ifdef notyet /* XXXX */ if (uio->uio_procp) uio->uio_procp->p_stats->p_ru.ru_msgrcv++; #endif + KASSERT(m == so->so_rcv.sb_mb); + SBLASTRECORDCHK(&so->so_rcv, "soreceive 1"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 1"); nextrecord = m->m_nextpkt; if (pr->pr_flags & PR_ADDR) { #ifdef DIAGNOSTIC @@ -958,13 +985,39 @@ soreceive(struct socket *so, struct mbuf controlp = &(*controlp)->m_next; } } + + /* + * If m is non-NULL, we have some data to read. From now on, + * make sure to keep sb_lastrecord consistent when working on + * the last packet on the chain (nextrecord == NULL) and we + * change m->m_nextpkt. + */ if (m) { - if ((flags & MSG_PEEK) == 0) + if ((flags & MSG_PEEK) == 0) { m->m_nextpkt = nextrecord; + /* + * If nextrecord == NULL (this is a single chain), + * then sb_lastrecord may not be valid here if m + * was changed earlier. + */ + if (nextrecord == NULL) { + KASSERT(so->so_rcv.sb_mb == m); + so->so_rcv.sb_lastrecord = m; + } + } type = m->m_type; if (type == MT_OOBDATA) flags |= MSG_OOB; + } else { + if ((flags & MSG_PEEK) == 0) { + KASSERT(so->so_rcv.sb_mb == m); + so->so_rcv.sb_mb = nextrecord; + SB_EMPTY_FIXUP(&so->so_rcv); + } } + SBLASTRECORDCHK(&so->so_rcv, "soreceive 2"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 2"); + moff = 0; offset = 0; while (m && uio->uio_resid > 0 && error == 0) { @@ -992,6 +1045,8 @@ soreceive(struct socket *so, struct mbuf * block interrupts again. */ if (mp == 0) { + SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove"); splx(s); error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio); s = splsoftnet(); @@ -1033,8 +1088,21 @@ soreceive(struct socket *so, struct mbuf MFREE(m, so->so_rcv.sb_mb); m = so->so_rcv.sb_mb; } - if (m) + /* + * If m != NULL, we also know that + * so->so_rcv.sb_mb != NULL. + */ + KASSERT(so->so_rcv.sb_mb == m); + if (m) { m->m_nextpkt = nextrecord; + if (nextrecord == NULL) + so->so_rcv.sb_lastrecord = m; + } else { + so->so_rcv.sb_mb = nextrecord; + SB_EMPTY_FIXUP(&so->so_rcv); + } + SBLASTRECORDCHK(&so->so_rcv, "soreceive 3"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 3"); } } else { if (flags & MSG_PEEK) @@ -1073,6 +1141,25 @@ soreceive(struct socket *so, struct mbuf !sosendallatonce(so) && !nextrecord) { if (so->so_error || so->so_state & SS_CANTRCVMORE) break; + /* + * If we are peeking and the socket receive buffer is + * full, stop since we can't get more data to peek at. + */ + if ((flags & MSG_PEEK) && sbspace(&so->so_rcv) <= 0) + break; + /* + * If we've drained the socket buffer, tell the + * protocol in case it needs to do something to + * get it filled again. + */ + if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb) + (*pr->pr_usrreq)(so, PRU_RCVD, + (struct mbuf *)0, + (struct mbuf *)(long)flags, + (struct mbuf *)0, + (struct proc *)0); + SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2"); error = sbwait(&so->so_rcv); if (error) { sbunlock(&so->so_rcv); @@ -1090,8 +1177,21 @@ soreceive(struct socket *so, struct mbuf (void) sbdroprecord(&so->so_rcv); } if ((flags & MSG_PEEK) == 0) { - if (m == 0) + if (m == 0) { + /* + * First part is an inline SB_EMPTY_FIXUP(). Second + * part makes sure sb_lastrecord is up-to-date if + * there is still data in the socket buffer. + */ so->so_rcv.sb_mb = nextrecord; + if (so->so_rcv.sb_mb == NULL) { + so->so_rcv.sb_mbtail = NULL; + so->so_rcv.sb_lastrecord = NULL; + } else if (nextrecord->m_nextpkt == NULL) + so->so_rcv.sb_lastrecord = nextrecord; + } + SBLASTRECORDCHK(&so->so_rcv, "soreceive 4"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 4"); if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) (*pr->pr_usrreq)(so, PRU_RCVD, (struct mbuf *)0, (struct mbuf *)(long)flags, (struct mbuf *)0, @@ -1257,11 +1357,13 @@ sosetopt(struct socket *so, int level, i goto bad; } tv = mtod(m, struct timeval *); - if (tv->tv_sec * hz + tv->tv_usec / tick > SHRT_MAX) { + if (tv->tv_sec > (SHRT_MAX - tv->tv_usec / tick) / hz) { error = EDOM; goto bad; } val = tv->tv_sec * hz + tv->tv_usec / tick; + if (val == 0 && tv->tv_usec != 0) + val = 1; switch (optname) { @@ -1385,3 +1487,119 @@ sohasoutofband(struct socket *so) psignal(p, SIGURG); selwakeup(&so->so_rcv.sb_sel); } + +static void +filt_sordetach(struct knote *kn) +{ + struct socket *so; + + so = (struct socket *)kn->kn_fp->f_data; + SLIST_REMOVE(&so->so_rcv.sb_sel.sel_klist, kn, knote, kn_selnext); + if (SLIST_EMPTY(&so->so_rcv.sb_sel.sel_klist)) + so->so_rcv.sb_flags &= ~SB_KNOTE; +} + +/*ARGSUSED*/ +static int +filt_soread(struct knote *kn, long hint) +{ + struct socket *so; + + so = (struct socket *)kn->kn_fp->f_data; + kn->kn_data = so->so_rcv.sb_cc; + if (so->so_state & SS_CANTRCVMORE) { + kn->kn_flags |= EV_EOF; + kn->kn_fflags = so->so_error; + return (1); + } + if (so->so_error) /* temporary udp error */ + return (1); + if (kn->kn_sfflags & NOTE_LOWAT) + return (kn->kn_data >= kn->kn_sdata); + return (kn->kn_data >= so->so_rcv.sb_lowat); +} + +static void +filt_sowdetach(struct knote *kn) +{ + struct socket *so; + + so = (struct socket *)kn->kn_fp->f_data; + SLIST_REMOVE(&so->so_snd.sb_sel.sel_klist, kn, knote, kn_selnext); + if (SLIST_EMPTY(&so->so_snd.sb_sel.sel_klist)) + so->so_snd.sb_flags &= ~SB_KNOTE; +} + +/*ARGSUSED*/ +static int +filt_sowrite(struct knote *kn, long hint) +{ + struct socket *so; + + so = (struct socket *)kn->kn_fp->f_data; + kn->kn_data = sbspace(&so->so_snd); + if (so->so_state & SS_CANTSENDMORE) { + kn->kn_flags |= EV_EOF; + kn->kn_fflags = so->so_error; + return (1); + } + if (so->so_error) /* temporary udp error */ + return (1); + if (((so->so_state & SS_ISCONNECTED) == 0) && + (so->so_proto->pr_flags & PR_CONNREQUIRED)) + return (0); + if (kn->kn_sfflags & NOTE_LOWAT) + return (kn->kn_data >= kn->kn_sdata); + return (kn->kn_data >= so->so_snd.sb_lowat); +} + +/*ARGSUSED*/ +static int +filt_solisten(struct knote *kn, long hint) +{ + struct socket *so; + + so = (struct socket *)kn->kn_fp->f_data; + + /* + * Set kn_data to number of incoming connections, not + * counting partial (incomplete) connections. + */ + kn->kn_data = so->so_qlen; + return (kn->kn_data > 0); +} + +static const struct filterops solisten_filtops = + { 1, NULL, filt_sordetach, filt_solisten }; +static const struct filterops soread_filtops = + { 1, NULL, filt_sordetach, filt_soread }; +static const struct filterops sowrite_filtops = + { 1, NULL, filt_sowdetach, filt_sowrite }; + +int +soo_kqfilter(struct file *fp, struct knote *kn) +{ + struct socket *so; + struct sockbuf *sb; + + so = (struct socket *)kn->kn_fp->f_data; + switch (kn->kn_filter) { + case EVFILT_READ: + if (so->so_options & SO_ACCEPTCONN) + kn->kn_fop = &solisten_filtops; + else + kn->kn_fop = &soread_filtops; + sb = &so->so_rcv; + break; + case EVFILT_WRITE: + kn->kn_fop = &sowrite_filtops; + sb = &so->so_snd; + break; + default: + return (1); + } + SLIST_INSERT_HEAD(&sb->sb_sel.sel_klist, kn, kn_selnext); + sb->sb_flags |= SB_KNOTE; + return (0); +} +