version 1.87, 2003/09/06 22:03:10 |
version 1.201, 2010/10/14 03:07:51 |
|
|
/* $NetBSD$ */ |
/* $NetBSD$ */ |
|
|
/*- |
/*- |
* Copyright (c) 2002 The NetBSD Foundation, Inc. |
* Copyright (c) 2002, 2007, 2008, 2009 The NetBSD Foundation, Inc. |
* All rights reserved. |
* All rights reserved. |
* |
* |
* This code is derived from software contributed to The NetBSD Foundation |
* This code is derived from software contributed to The NetBSD Foundation |
* by Jason R. Thorpe of Wasabi Systems, Inc. |
* by Jason R. Thorpe of Wasabi Systems, Inc, and by Andrew Doran. |
* |
* |
* Redistribution and use in source and binary forms, with or without |
* Redistribution and use in source and binary forms, with or without |
* modification, are permitted provided that the following conditions |
* modification, are permitted provided that the following conditions |
|
|
* 2. Redistributions in binary form must reproduce the above copyright |
* 2. Redistributions in binary form must reproduce the above copyright |
* notice, this list of conditions and the following disclaimer in the |
* notice, this list of conditions and the following disclaimer in the |
* documentation and/or other materials provided with the distribution. |
* documentation and/or other materials provided with the distribution. |
* 3. All advertising materials mentioning features or use of this software |
|
* must display the following acknowledgement: |
|
* This product includes software developed by the NetBSD |
|
* Foundation, Inc. and its contributors. |
|
* 4. Neither the name of The NetBSD Foundation nor the names of its |
|
* contributors may be used to endorse or promote products derived |
|
* from this software without specific prior written permission. |
|
* |
* |
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
|
|
*/ |
*/ |
|
|
/* |
/* |
|
* Copyright (c) 2004 The FreeBSD Foundation |
|
* Copyright (c) 2004 Robert Watson |
* Copyright (c) 1982, 1986, 1988, 1990, 1993 |
* Copyright (c) 1982, 1986, 1988, 1990, 1993 |
* The Regents of the University of California. All rights reserved. |
* The Regents of the University of California. All rights reserved. |
* |
* |
|
|
#include <sys/cdefs.h> |
#include <sys/cdefs.h> |
__KERNEL_RCSID(0, "$NetBSD$"); |
__KERNEL_RCSID(0, "$NetBSD$"); |
|
|
|
#include "opt_compat_netbsd.h" |
#include "opt_sock_counters.h" |
#include "opt_sock_counters.h" |
#include "opt_sosend_loan.h" |
#include "opt_sosend_loan.h" |
#include "opt_mbuftrace.h" |
#include "opt_mbuftrace.h" |
#include "opt_somaxkva.h" |
#include "opt_somaxkva.h" |
|
#include "opt_multiprocessor.h" /* XXX */ |
|
|
#include <sys/param.h> |
#include <sys/param.h> |
#include <sys/systm.h> |
#include <sys/systm.h> |
#include <sys/proc.h> |
#include <sys/proc.h> |
#include <sys/file.h> |
#include <sys/file.h> |
#include <sys/malloc.h> |
#include <sys/filedesc.h> |
|
#include <sys/kmem.h> |
#include <sys/mbuf.h> |
#include <sys/mbuf.h> |
#include <sys/domain.h> |
#include <sys/domain.h> |
#include <sys/kernel.h> |
#include <sys/kernel.h> |
Line 88 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 86 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include <sys/socketvar.h> |
#include <sys/socketvar.h> |
#include <sys/signalvar.h> |
#include <sys/signalvar.h> |
#include <sys/resourcevar.h> |
#include <sys/resourcevar.h> |
#include <sys/pool.h> |
#include <sys/uidinfo.h> |
#include <sys/event.h> |
#include <sys/event.h> |
|
#include <sys/poll.h> |
|
#include <sys/kauth.h> |
|
#include <sys/mutex.h> |
|
#include <sys/condvar.h> |
|
|
|
#ifdef COMPAT_50 |
|
#include <compat/sys/time.h> |
|
#include <compat/sys/socket.h> |
|
#endif |
|
|
#include <uvm/uvm.h> |
#include <uvm/uvm.h> |
|
|
struct pool socket_pool; |
|
|
|
MALLOC_DEFINE(M_SOOPTS, "soopts", "socket options"); |
MALLOC_DEFINE(M_SOOPTS, "soopts", "socket options"); |
MALLOC_DEFINE(M_SONAME, "soname", "socket name"); |
MALLOC_DEFINE(M_SONAME, "soname", "socket name"); |
|
|
|
extern const struct fileops socketops; |
|
|
extern int somaxconn; /* patchable (XXX sysctl) */ |
extern int somaxconn; /* patchable (XXX sysctl) */ |
int somaxconn = SOMAXCONN; |
int somaxconn = SOMAXCONN; |
|
kmutex_t *softnet_lock; |
|
|
#ifdef SOSEND_COUNTERS |
#ifdef SOSEND_COUNTERS |
#include <sys/device.h> |
#include <sys/device.h> |
|
|
struct evcnt sosend_loan_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
static struct evcnt sosend_loan_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
NULL, "sosend", "loan big"); |
NULL, "sosend", "loan big"); |
struct evcnt sosend_copy_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
static struct evcnt sosend_copy_big = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
NULL, "sosend", "copy big"); |
NULL, "sosend", "copy big"); |
struct evcnt sosend_copy_small = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
static struct evcnt sosend_copy_small = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
NULL, "sosend", "copy small"); |
NULL, "sosend", "copy small"); |
struct evcnt sosend_kvalimit = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
static struct evcnt sosend_kvalimit = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, |
NULL, "sosend", "kva limit"); |
NULL, "sosend", "kva limit"); |
|
|
#define SOSEND_COUNTER_INCR(ev) (ev)->ev_count++ |
#define SOSEND_COUNTER_INCR(ev) (ev)->ev_count++ |
|
|
|
EVCNT_ATTACH_STATIC(sosend_loan_big); |
|
EVCNT_ATTACH_STATIC(sosend_copy_big); |
|
EVCNT_ATTACH_STATIC(sosend_copy_small); |
|
EVCNT_ATTACH_STATIC(sosend_kvalimit); |
#else |
#else |
|
|
#define SOSEND_COUNTER_INCR(ev) /* nothing */ |
#define SOSEND_COUNTER_INCR(ev) /* nothing */ |
|
|
#endif /* SOSEND_COUNTERS */ |
#endif /* SOSEND_COUNTERS */ |
|
|
void |
static struct callback_entry sokva_reclaimerentry; |
soinit(void) |
|
{ |
|
|
|
pool_init(&socket_pool, sizeof(struct socket), 0, 0, 0, |
|
"sockpl", NULL); |
|
|
|
#ifdef SOSEND_COUNTERS |
|
evcnt_attach_static(&sosend_loan_big); |
|
evcnt_attach_static(&sosend_copy_big); |
|
evcnt_attach_static(&sosend_copy_small); |
|
evcnt_attach_static(&sosend_kvalimit); |
|
#endif /* SOSEND_COUNTERS */ |
|
} |
|
|
|
#ifdef SOSEND_NO_LOAN |
#if defined(SOSEND_NO_LOAN) || defined(MULTIPROCESSOR) |
int use_sosend_loan = 0; |
int sock_loan_thresh = -1; |
#else |
#else |
int use_sosend_loan = 1; |
int sock_loan_thresh = 4096; |
#endif |
#endif |
|
|
struct mbuf *so_pendfree; |
static kmutex_t so_pendfree_lock; |
|
static struct mbuf *so_pendfree; |
|
|
#ifndef SOMAXKVA |
#ifndef SOMAXKVA |
#define SOMAXKVA (16 * 1024 * 1024) |
#define SOMAXKVA (16 * 1024 * 1024) |
#endif |
#endif |
int somaxkva = SOMAXKVA; |
int somaxkva = SOMAXKVA; |
int socurkva; |
static int socurkva; |
int sokvawaiters; |
static kcondvar_t socurkva_cv; |
|
|
|
static kauth_listener_t socket_listener; |
|
|
#define SOCK_LOAN_THRESH 4096 |
|
#define SOCK_LOAN_CHUNK 65536 |
#define SOCK_LOAN_CHUNK 65536 |
|
|
static size_t sodopendfree(struct socket *); |
static size_t sodopendfree(void); |
|
static size_t sodopendfreel(void); |
|
|
vaddr_t |
static void sysctl_kern_somaxkva_setup(void); |
sokvaalloc(vsize_t len, struct socket *so) |
static struct sysctllog *socket_sysctllog; |
|
|
|
static vsize_t |
|
sokvareserve(struct socket *so, vsize_t len) |
{ |
{ |
vaddr_t lva; |
int error; |
int s; |
|
|
|
|
mutex_enter(&so_pendfree_lock); |
while (socurkva + len > somaxkva) { |
while (socurkva + len > somaxkva) { |
if (sodopendfree(so)) |
size_t freed; |
|
|
|
/* |
|
* try to do pendfree. |
|
*/ |
|
|
|
freed = sodopendfreel(); |
|
|
|
/* |
|
* if some kva was freed, try again. |
|
*/ |
|
|
|
if (freed) |
continue; |
continue; |
|
|
SOSEND_COUNTER_INCR(&sosend_kvalimit); |
SOSEND_COUNTER_INCR(&sosend_kvalimit); |
s = splvm(); |
error = cv_wait_sig(&socurkva_cv, &so_pendfree_lock); |
sokvawaiters++; |
if (error) { |
(void) tsleep(&socurkva, PVM, "sokva", 0); |
len = 0; |
sokvawaiters--; |
break; |
splx(s); |
} |
} |
} |
|
socurkva += len; |
|
mutex_exit(&so_pendfree_lock); |
|
return len; |
|
} |
|
|
lva = uvm_km_valloc_wait(kernel_map, len); |
static void |
if (lva == 0) |
sokvaunreserve(vsize_t len) |
|
{ |
|
|
|
mutex_enter(&so_pendfree_lock); |
|
socurkva -= len; |
|
cv_broadcast(&socurkva_cv); |
|
mutex_exit(&so_pendfree_lock); |
|
} |
|
|
|
/* |
|
* sokvaalloc: allocate kva for loan. |
|
*/ |
|
|
|
vaddr_t |
|
sokvaalloc(vsize_t len, struct socket *so) |
|
{ |
|
vaddr_t lva; |
|
|
|
/* |
|
* reserve kva. |
|
*/ |
|
|
|
if (sokvareserve(so, len) == 0) |
|
return 0; |
|
|
|
/* |
|
* allocate kva. |
|
*/ |
|
|
|
lva = uvm_km_alloc(kernel_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA); |
|
if (lva == 0) { |
|
sokvaunreserve(len); |
return (0); |
return (0); |
socurkva += len; |
} |
|
|
return lva; |
return lva; |
} |
} |
|
|
|
/* |
|
* sokvafree: free kva for loan. |
|
*/ |
|
|
void |
void |
sokvafree(vaddr_t sva, vsize_t len) |
sokvafree(vaddr_t sva, vsize_t len) |
{ |
{ |
|
|
uvm_km_free(kernel_map, sva, len); |
/* |
socurkva -= len; |
* free kva. |
if (sokvawaiters) |
*/ |
wakeup(&socurkva); |
|
|
uvm_km_free(kernel_map, sva, len, UVM_KMF_VAONLY); |
|
|
|
/* |
|
* unreserve kva. |
|
*/ |
|
|
|
sokvaunreserve(len); |
} |
} |
|
|
static void |
static void |
sodoloanfree(struct vm_page **pgs, caddr_t buf, size_t size) |
sodoloanfree(struct vm_page **pgs, void *buf, size_t size) |
{ |
{ |
vaddr_t va, sva, eva; |
vaddr_t sva, eva; |
vsize_t len; |
vsize_t len; |
paddr_t pa; |
int npgs; |
int i, npgs; |
|
|
KASSERT(pgs != NULL); |
|
|
eva = round_page((vaddr_t) buf + size); |
eva = round_page((vaddr_t) buf + size); |
sva = trunc_page((vaddr_t) buf); |
sva = trunc_page((vaddr_t) buf); |
len = eva - sva; |
len = eva - sva; |
npgs = len >> PAGE_SHIFT; |
npgs = len >> PAGE_SHIFT; |
|
|
if (__predict_false(pgs == NULL)) { |
|
pgs = alloca(npgs * sizeof(*pgs)); |
|
|
|
for (i = 0, va = sva; va < eva; i++, va += PAGE_SIZE) { |
|
if (pmap_extract(pmap_kernel(), va, &pa) == FALSE) |
|
panic("sodoloanfree: va 0x%lx not mapped", va); |
|
pgs[i] = PHYS_TO_VM_PAGE(pa); |
|
} |
|
} |
|
|
|
pmap_kremove(sva, len); |
pmap_kremove(sva, len); |
pmap_update(pmap_kernel()); |
pmap_update(pmap_kernel()); |
uvm_unloan(pgs, npgs, UVM_LOAN_TOPAGE); |
uvm_unloan(pgs, npgs, UVM_LOAN_TOPAGE); |
Line 221 sodoloanfree(struct vm_page **pgs, caddr |
|
Line 276 sodoloanfree(struct vm_page **pgs, caddr |
|
} |
} |
|
|
static size_t |
static size_t |
sodopendfree(struct socket *so) |
sodopendfree(void) |
{ |
{ |
struct mbuf *m; |
size_t rv; |
|
|
|
if (__predict_true(so_pendfree == NULL)) |
|
return 0; |
|
|
|
mutex_enter(&so_pendfree_lock); |
|
rv = sodopendfreel(); |
|
mutex_exit(&so_pendfree_lock); |
|
|
|
return rv; |
|
} |
|
|
|
/* |
|
* sodopendfreel: free mbufs on "pendfree" list. |
|
* unlock and relock so_pendfree_lock when freeing mbufs. |
|
* |
|
* => called with so_pendfree_lock held. |
|
*/ |
|
|
|
static size_t |
|
sodopendfreel(void) |
|
{ |
|
struct mbuf *m, *next; |
size_t rv = 0; |
size_t rv = 0; |
int s; |
|
|
|
s = splvm(); |
KASSERT(mutex_owned(&so_pendfree_lock)); |
|
|
for (;;) { |
while (so_pendfree != NULL) { |
m = so_pendfree; |
m = so_pendfree; |
if (m == NULL) |
so_pendfree = NULL; |
break; |
mutex_exit(&so_pendfree_lock); |
so_pendfree = m->m_next; |
|
splx(s); |
|
|
|
rv += m->m_ext.ext_size; |
for (; m != NULL; m = next) { |
sodoloanfree((m->m_flags & M_EXT_PAGES) ? |
next = m->m_next; |
m->m_ext.ext_pgs : NULL, m->m_ext.ext_buf, |
KASSERT((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0); |
m->m_ext.ext_size); |
KASSERT(m->m_ext.ext_refcnt == 0); |
s = splvm(); |
|
pool_cache_put(&mbpool_cache, m); |
|
} |
|
|
|
for (;;) { |
rv += m->m_ext.ext_size; |
m = so->so_pendfree; |
sodoloanfree(m->m_ext.ext_pgs, m->m_ext.ext_buf, |
if (m == NULL) |
m->m_ext.ext_size); |
break; |
pool_cache_put(mb_cache, m); |
so->so_pendfree = m->m_next; |
} |
splx(s); |
|
|
|
rv += m->m_ext.ext_size; |
mutex_enter(&so_pendfree_lock); |
sodoloanfree((m->m_flags & M_EXT_PAGES) ? |
|
m->m_ext.ext_pgs : NULL, m->m_ext.ext_buf, |
|
m->m_ext.ext_size); |
|
s = splvm(); |
|
pool_cache_put(&mbpool_cache, m); |
|
} |
} |
|
|
splx(s); |
|
return (rv); |
return (rv); |
} |
} |
|
|
void |
void |
soloanfree(struct mbuf *m, caddr_t buf, size_t size, void *arg) |
soloanfree(struct mbuf *m, void *buf, size_t size, void *arg) |
{ |
{ |
struct socket *so = arg; |
|
int s; |
|
|
|
if (m == NULL) { |
KASSERT(m != NULL); |
sodoloanfree(NULL, buf, size); |
|
return; |
|
} |
|
|
|
s = splvm(); |
/* |
m->m_next = so->so_pendfree; |
* postpone freeing mbuf. |
so->so_pendfree = m; |
* |
splx(s); |
* we can't do it in interrupt context |
if (sokvawaiters) |
* because we need to put kva back to kernel_map. |
wakeup(&socurkva); |
*/ |
|
|
|
mutex_enter(&so_pendfree_lock); |
|
m->m_next = so_pendfree; |
|
so_pendfree = m; |
|
cv_broadcast(&socurkva_cv); |
|
mutex_exit(&so_pendfree_lock); |
} |
} |
|
|
static long |
static long |
Line 288 sosend_loan(struct socket *so, struct ui |
|
Line 353 sosend_loan(struct socket *so, struct ui |
|
struct iovec *iov = uio->uio_iov; |
struct iovec *iov = uio->uio_iov; |
vaddr_t sva, eva; |
vaddr_t sva, eva; |
vsize_t len; |
vsize_t len; |
vaddr_t lva, va; |
vaddr_t lva; |
int npgs, i, error; |
int npgs, error; |
|
vaddr_t va; |
|
int i; |
|
|
if (uio->uio_segflg != UIO_USERSPACE) |
if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) |
return (0); |
return (0); |
|
|
if (iov->iov_len < (size_t) space) |
if (iov->iov_len < (size_t) space) |
Line 304 sosend_loan(struct socket *so, struct ui |
|
Line 371 sosend_loan(struct socket *so, struct ui |
|
len = eva - sva; |
len = eva - sva; |
npgs = len >> PAGE_SHIFT; |
npgs = len >> PAGE_SHIFT; |
|
|
/* XXX KDASSERT */ |
|
KASSERT(npgs <= M_EXT_MAXPAGES); |
KASSERT(npgs <= M_EXT_MAXPAGES); |
|
|
lva = sokvaalloc(len, so); |
lva = sokvaalloc(len, so); |
if (lva == 0) |
if (lva == 0) |
return 0; |
return 0; |
|
|
error = uvm_loan(&uio->uio_procp->p_vmspace->vm_map, sva, len, |
error = uvm_loan(&uio->uio_vmspace->vm_map, sva, len, |
m->m_ext.ext_pgs, UVM_LOAN_TOPAGE); |
m->m_ext.ext_pgs, UVM_LOAN_TOPAGE); |
if (error) { |
if (error) { |
sokvafree(lva, len); |
sokvafree(lva, len); |
Line 320 sosend_loan(struct socket *so, struct ui |
|
Line 386 sosend_loan(struct socket *so, struct ui |
|
|
|
for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE) |
for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE) |
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]), |
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]), |
VM_PROT_READ); |
VM_PROT_READ, 0); |
pmap_update(pmap_kernel()); |
pmap_update(pmap_kernel()); |
|
|
lva += (vaddr_t) iov->iov_base & PAGE_MASK; |
lva += (vaddr_t) iov->iov_base & PAGE_MASK; |
|
|
MEXTADD(m, (caddr_t) lva, space, M_MBUF, soloanfree, so); |
MEXTADD(m, (void *) lva, space, M_MBUF, soloanfree, so); |
m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP; |
m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP; |
|
|
uio->uio_resid -= space; |
uio->uio_resid -= space; |
/* uio_offset not updated, not set/used for write(2) */ |
/* uio_offset not updated, not set/used for write(2) */ |
uio->uio_iov->iov_base = (caddr_t) uio->uio_iov->iov_base + space; |
uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + space; |
uio->uio_iov->iov_len -= space; |
uio->uio_iov->iov_len -= space; |
if (uio->uio_iov->iov_len == 0) { |
if (uio->uio_iov->iov_len == 0) { |
uio->uio_iov++; |
uio->uio_iov++; |
Line 340 sosend_loan(struct socket *so, struct ui |
|
Line 406 sosend_loan(struct socket *so, struct ui |
|
return (space); |
return (space); |
} |
} |
|
|
|
static int |
|
sokva_reclaim_callback(struct callback_entry *ce, void *obj, void *arg) |
|
{ |
|
|
|
KASSERT(ce == &sokva_reclaimerentry); |
|
KASSERT(obj == NULL); |
|
|
|
sodopendfree(); |
|
if (!vm_map_starved_p(kernel_map)) { |
|
return CALLBACK_CHAIN_ABORT; |
|
} |
|
return CALLBACK_CHAIN_CONTINUE; |
|
} |
|
|
|
struct mbuf * |
|
getsombuf(struct socket *so, int type) |
|
{ |
|
struct mbuf *m; |
|
|
|
m = m_get(M_WAIT, type); |
|
MCLAIM(m, so->so_mowner); |
|
return m; |
|
} |
|
|
|
static int |
|
socket_listener_cb(kauth_cred_t cred, kauth_action_t action, void *cookie, |
|
void *arg0, void *arg1, void *arg2, void *arg3) |
|
{ |
|
int result; |
|
enum kauth_network_req req; |
|
|
|
result = KAUTH_RESULT_DEFER; |
|
req = (enum kauth_network_req)arg0; |
|
|
|
if ((action != KAUTH_NETWORK_SOCKET) && |
|
(action != KAUTH_NETWORK_BIND)) |
|
return result; |
|
|
|
switch (req) { |
|
case KAUTH_REQ_NETWORK_BIND_PORT: |
|
result = KAUTH_RESULT_ALLOW; |
|
break; |
|
|
|
case KAUTH_REQ_NETWORK_SOCKET_DROP: { |
|
/* Normal users can only drop their own connections. */ |
|
struct socket *so = (struct socket *)arg1; |
|
|
|
if (proc_uidmatch(cred, so->so_cred)) |
|
result = KAUTH_RESULT_ALLOW; |
|
|
|
break; |
|
} |
|
|
|
case KAUTH_REQ_NETWORK_SOCKET_OPEN: |
|
/* We allow "raw" routing/bluetooth sockets to anyone. */ |
|
if ((u_long)arg1 == PF_ROUTE || (u_long)arg1 == PF_BLUETOOTH) |
|
result = KAUTH_RESULT_ALLOW; |
|
else { |
|
/* Privileged, let secmodel handle this. */ |
|
if ((u_long)arg2 == SOCK_RAW) |
|
break; |
|
} |
|
|
|
result = KAUTH_RESULT_ALLOW; |
|
|
|
break; |
|
|
|
case KAUTH_REQ_NETWORK_SOCKET_CANSEE: |
|
result = KAUTH_RESULT_ALLOW; |
|
|
|
break; |
|
|
|
default: |
|
break; |
|
} |
|
|
|
return result; |
|
} |
|
|
|
void |
|
soinit(void) |
|
{ |
|
|
|
sysctl_kern_somaxkva_setup(); |
|
|
|
mutex_init(&so_pendfree_lock, MUTEX_DEFAULT, IPL_VM); |
|
softnet_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); |
|
cv_init(&socurkva_cv, "sokva"); |
|
soinit2(); |
|
|
|
/* Set the initial adjusted socket buffer size. */ |
|
if (sb_max_set(sb_max)) |
|
panic("bad initial sb_max value: %lu", sb_max); |
|
|
|
callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback, |
|
&sokva_reclaimerentry, NULL, sokva_reclaim_callback); |
|
|
|
socket_listener = kauth_listen_scope(KAUTH_SCOPE_NETWORK, |
|
socket_listener_cb, NULL); |
|
} |
|
|
/* |
/* |
* Socket operation routines. |
* Socket operation routines. |
* These routines are called by the routines in |
* These routines are called by the routines in |
Line 349 sosend_loan(struct socket *so, struct ui |
|
Line 516 sosend_loan(struct socket *so, struct ui |
|
*/ |
*/ |
/*ARGSUSED*/ |
/*ARGSUSED*/ |
int |
int |
socreate(int dom, struct socket **aso, int type, int proto) |
socreate(int dom, struct socket **aso, int type, int proto, struct lwp *l, |
|
struct socket *lockso) |
{ |
{ |
struct proc *p; |
const struct protosw *prp; |
struct protosw *prp; |
|
struct socket *so; |
struct socket *so; |
int error, s; |
uid_t uid; |
|
int error; |
|
kmutex_t *lock; |
|
|
|
error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_SOCKET, |
|
KAUTH_REQ_NETWORK_SOCKET_OPEN, KAUTH_ARG(dom), KAUTH_ARG(type), |
|
KAUTH_ARG(proto)); |
|
if (error != 0) |
|
return error; |
|
|
p = curproc; /* XXX */ |
|
if (proto) |
if (proto) |
prp = pffindproto(dom, proto, type); |
prp = pffindproto(dom, proto, type); |
else |
else |
prp = pffindtype(dom, type); |
prp = pffindtype(dom, type); |
if (prp == 0 || prp->pr_usrreq == 0) |
if (prp == NULL) { |
return (EPROTONOSUPPORT); |
/* no support for domain */ |
|
if (pffinddomain(dom) == 0) |
|
return EAFNOSUPPORT; |
|
/* no support for socket type */ |
|
if (proto == 0 && type != 0) |
|
return EPROTOTYPE; |
|
return EPROTONOSUPPORT; |
|
} |
|
if (prp->pr_usrreq == NULL) |
|
return EPROTONOSUPPORT; |
if (prp->pr_type != type) |
if (prp->pr_type != type) |
return (EPROTOTYPE); |
return EPROTOTYPE; |
s = splsoftnet(); |
|
so = pool_get(&socket_pool, PR_WAITOK); |
so = soget(true); |
memset((caddr_t)so, 0, sizeof(*so)); |
|
TAILQ_INIT(&so->so_q0); |
|
TAILQ_INIT(&so->so_q); |
|
so->so_type = type; |
so->so_type = type; |
so->so_proto = prp; |
so->so_proto = prp; |
so->so_send = sosend; |
so->so_send = sosend; |
Line 379 socreate(int dom, struct socket **aso, i |
|
Line 559 socreate(int dom, struct socket **aso, i |
|
so->so_snd.sb_mowner = &prp->pr_domain->dom_mowner; |
so->so_snd.sb_mowner = &prp->pr_domain->dom_mowner; |
so->so_mowner = &prp->pr_domain->dom_mowner; |
so->so_mowner = &prp->pr_domain->dom_mowner; |
#endif |
#endif |
if (p != 0) |
uid = kauth_cred_geteuid(l->l_cred); |
so->so_uid = p->p_ucred->cr_uid; |
so->so_uidinfo = uid_find(uid); |
error = (*prp->pr_usrreq)(so, PRU_ATTACH, (struct mbuf *)0, |
so->so_cpid = l->l_proc->p_pid; |
(struct mbuf *)(long)proto, (struct mbuf *)0, p); |
if (lockso != NULL) { |
if (error) { |
/* Caller wants us to share a lock. */ |
|
lock = lockso->so_lock; |
|
so->so_lock = lock; |
|
mutex_obj_hold(lock); |
|
mutex_enter(lock); |
|
} else { |
|
/* Lock assigned and taken during PRU_ATTACH. */ |
|
} |
|
error = (*prp->pr_usrreq)(so, PRU_ATTACH, NULL, |
|
(struct mbuf *)(long)proto, NULL, l); |
|
KASSERT(solocked(so)); |
|
if (error != 0) { |
so->so_state |= SS_NOFDREF; |
so->so_state |= SS_NOFDREF; |
sofree(so); |
sofree(so); |
splx(s); |
return error; |
return (error); |
|
} |
} |
splx(s); |
so->so_cred = kauth_cred_dup(l->l_cred); |
|
sounlock(so); |
*aso = so; |
*aso = so; |
return (0); |
return 0; |
|
} |
|
|
|
/* On success, write file descriptor to fdout and return zero. On |
|
* failure, return non-zero; *fdout will be undefined. |
|
*/ |
|
int |
|
fsocreate(int domain, struct socket **sop, int type, int protocol, |
|
struct lwp *l, int *fdout) |
|
{ |
|
struct socket *so; |
|
struct file *fp; |
|
int fd, error; |
|
|
|
if ((error = fd_allocfile(&fp, &fd)) != 0) |
|
return (error); |
|
fp->f_flag = FREAD|FWRITE; |
|
fp->f_type = DTYPE_SOCKET; |
|
fp->f_ops = &socketops; |
|
error = socreate(domain, &so, type, protocol, l, NULL); |
|
if (error != 0) { |
|
fd_abort(curproc, fp, fd); |
|
} else { |
|
if (sop != NULL) |
|
*sop = so; |
|
fp->f_data = so; |
|
fd_affix(curproc, fp, fd); |
|
*fdout = fd; |
|
} |
|
return error; |
} |
} |
|
|
int |
int |
sobind(struct socket *so, struct mbuf *nam, struct proc *p) |
sofamily(const struct socket *so) |
{ |
{ |
int s, error; |
const struct protosw *pr; |
|
const struct domain *dom; |
|
|
s = splsoftnet(); |
if ((pr = so->so_proto) == NULL) |
error = (*so->so_proto->pr_usrreq)(so, PRU_BIND, (struct mbuf *)0, |
return AF_UNSPEC; |
nam, (struct mbuf *)0, p); |
if ((dom = pr->pr_domain) == NULL) |
splx(s); |
return AF_UNSPEC; |
return (error); |
return dom->dom_family; |
} |
} |
|
|
int |
int |
solisten(struct socket *so, int backlog) |
sobind(struct socket *so, struct mbuf *nam, struct lwp *l) |
{ |
{ |
int s, error; |
int error; |
|
|
s = splsoftnet(); |
solock(so); |
error = (*so->so_proto->pr_usrreq)(so, PRU_LISTEN, (struct mbuf *)0, |
error = (*so->so_proto->pr_usrreq)(so, PRU_BIND, NULL, nam, NULL, l); |
(struct mbuf *)0, (struct mbuf *)0, (struct proc *)0); |
sounlock(so); |
if (error) { |
return error; |
splx(s); |
} |
return (error); |
|
|
int |
|
solisten(struct socket *so, int backlog, struct lwp *l) |
|
{ |
|
int error; |
|
|
|
solock(so); |
|
if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | |
|
SS_ISDISCONNECTING)) != 0) { |
|
sounlock(so); |
|
return (EOPNOTSUPP); |
|
} |
|
error = (*so->so_proto->pr_usrreq)(so, PRU_LISTEN, NULL, |
|
NULL, NULL, l); |
|
if (error != 0) { |
|
sounlock(so); |
|
return error; |
} |
} |
if (TAILQ_EMPTY(&so->so_q)) |
if (TAILQ_EMPTY(&so->so_q)) |
so->so_options |= SO_ACCEPTCONN; |
so->so_options |= SO_ACCEPTCONN; |
if (backlog < 0) |
if (backlog < 0) |
backlog = 0; |
backlog = 0; |
so->so_qlimit = min(backlog, somaxconn); |
so->so_qlimit = min(backlog, somaxconn); |
splx(s); |
sounlock(so); |
return (0); |
return 0; |
} |
} |
|
|
void |
void |
sofree(struct socket *so) |
sofree(struct socket *so) |
{ |
{ |
struct mbuf *m; |
u_int refs; |
|
|
if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) |
KASSERT(solocked(so)); |
|
|
|
if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) { |
|
sounlock(so); |
return; |
return; |
|
} |
if (so->so_head) { |
if (so->so_head) { |
/* |
/* |
* We must not decommission a socket that's on the accept(2) |
* We must not decommission a socket that's on the accept(2) |
* queue. If we do, then accept(2) may hang after select(2) |
* queue. If we do, then accept(2) may hang after select(2) |
* indicated that the listening socket was ready. |
* indicated that the listening socket was ready. |
*/ |
*/ |
if (!soqremque(so, 0)) |
if (!soqremque(so, 0)) { |
|
sounlock(so); |
return; |
return; |
|
} |
} |
} |
sbrelease(&so->so_snd); |
if (so->so_rcv.sb_hiwat) |
|
(void)chgsbsize(so->so_uidinfo, &so->so_rcv.sb_hiwat, 0, |
|
RLIM_INFINITY); |
|
if (so->so_snd.sb_hiwat) |
|
(void)chgsbsize(so->so_uidinfo, &so->so_snd.sb_hiwat, 0, |
|
RLIM_INFINITY); |
|
sbrelease(&so->so_snd, so); |
|
KASSERT(!cv_has_waiters(&so->so_cv)); |
|
KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv)); |
|
KASSERT(!cv_has_waiters(&so->so_snd.sb_cv)); |
sorflush(so); |
sorflush(so); |
while ((m = so->so_pendfree) != NULL) { |
refs = so->so_aborting; /* XXX */ |
so->so_pendfree = m->m_next; |
/* Remove acccept filter if one is present. */ |
m->m_next = so_pendfree; |
if (so->so_accf != NULL) |
so_pendfree = m; |
(void)accept_filt_clear(so); |
} |
sounlock(so); |
pool_put(&socket_pool, so); |
if (refs == 0) /* XXX */ |
|
soput(so); |
} |
} |
|
|
/* |
/* |
|
|
soclose(struct socket *so) |
soclose(struct socket *so) |
{ |
{ |
struct socket *so2; |
struct socket *so2; |
int s, error; |
int error; |
|
int error2; |
|
|
error = 0; |
error = 0; |
s = splsoftnet(); /* conservative */ |
solock(so); |
if (so->so_options & SO_ACCEPTCONN) { |
if (so->so_options & SO_ACCEPTCONN) { |
while ((so2 = TAILQ_FIRST(&so->so_q0)) != 0) { |
for (;;) { |
(void) soqremque(so2, 0); |
if ((so2 = TAILQ_FIRST(&so->so_q0)) != 0) { |
(void) soabort(so2); |
KASSERT(solocked2(so, so2)); |
} |
(void) soqremque(so2, 0); |
while ((so2 = TAILQ_FIRST(&so->so_q)) != 0) { |
/* soabort drops the lock. */ |
(void) soqremque(so2, 1); |
(void) soabort(so2); |
(void) soabort(so2); |
solock(so); |
|
continue; |
|
} |
|
if ((so2 = TAILQ_FIRST(&so->so_q)) != 0) { |
|
KASSERT(solocked2(so, so2)); |
|
(void) soqremque(so2, 1); |
|
/* soabort drops the lock. */ |
|
(void) soabort(so2); |
|
solock(so); |
|
continue; |
|
} |
|
break; |
} |
} |
} |
} |
if (so->so_pcb == 0) |
if (so->so_pcb == 0) |
Line 485 soclose(struct socket *so) |
|
Line 750 soclose(struct socket *so) |
|
goto drop; |
goto drop; |
} |
} |
if (so->so_options & SO_LINGER) { |
if (so->so_options & SO_LINGER) { |
if ((so->so_state & SS_ISDISCONNECTING) && |
if ((so->so_state & SS_ISDISCONNECTING) && so->so_nbio) |
(so->so_state & SS_NBIO)) |
|
goto drop; |
goto drop; |
while (so->so_state & SS_ISCONNECTED) { |
while (so->so_state & SS_ISCONNECTED) { |
error = tsleep((caddr_t)&so->so_timeo, |
error = sowait(so, true, so->so_linger * hz); |
PSOCK | PCATCH, netcls, |
|
so->so_linger * hz); |
|
if (error) |
if (error) |
break; |
break; |
} |
} |
Line 499 soclose(struct socket *so) |
|
Line 761 soclose(struct socket *so) |
|
} |
} |
drop: |
drop: |
if (so->so_pcb) { |
if (so->so_pcb) { |
int error2 = (*so->so_proto->pr_usrreq)(so, PRU_DETACH, |
error2 = (*so->so_proto->pr_usrreq)(so, PRU_DETACH, |
(struct mbuf *)0, (struct mbuf *)0, (struct mbuf *)0, |
NULL, NULL, NULL, NULL); |
(struct proc *)0); |
|
if (error == 0) |
if (error == 0) |
error = error2; |
error = error2; |
} |
} |
discard: |
discard: |
if (so->so_state & SS_NOFDREF) |
if (so->so_state & SS_NOFDREF) |
panic("soclose: NOFDREF"); |
panic("soclose: NOFDREF"); |
|
kauth_cred_free(so->so_cred); |
so->so_state |= SS_NOFDREF; |
so->so_state |= SS_NOFDREF; |
sofree(so); |
sofree(so); |
splx(s); |
|
return (error); |
return (error); |
} |
} |
|
|
/* |
/* |
* Must be called at splsoftnet... |
* Must be called with the socket locked.. Will return with it unlocked. |
*/ |
*/ |
int |
int |
soabort(struct socket *so) |
soabort(struct socket *so) |
{ |
{ |
|
u_int refs; |
return (*so->so_proto->pr_usrreq)(so, PRU_ABORT, (struct mbuf *)0, |
int error; |
(struct mbuf *)0, (struct mbuf *)0, (struct proc *)0); |
|
|
KASSERT(solocked(so)); |
|
KASSERT(so->so_head == NULL); |
|
|
|
so->so_aborting++; /* XXX */ |
|
error = (*so->so_proto->pr_usrreq)(so, PRU_ABORT, NULL, |
|
NULL, NULL, NULL); |
|
refs = --so->so_aborting; /* XXX */ |
|
if (error || (refs == 0)) { |
|
sofree(so); |
|
} else { |
|
sounlock(so); |
|
} |
|
return error; |
} |
} |
|
|
int |
int |
soaccept(struct socket *so, struct mbuf *nam) |
soaccept(struct socket *so, struct mbuf *nam) |
{ |
{ |
int s, error; |
int error; |
|
|
|
KASSERT(solocked(so)); |
|
|
error = 0; |
error = 0; |
s = splsoftnet(); |
|
if ((so->so_state & SS_NOFDREF) == 0) |
if ((so->so_state & SS_NOFDREF) == 0) |
panic("soaccept: !NOFDREF"); |
panic("soaccept: !NOFDREF"); |
so->so_state &= ~SS_NOFDREF; |
so->so_state &= ~SS_NOFDREF; |
if ((so->so_state & SS_ISDISCONNECTED) == 0 || |
if ((so->so_state & SS_ISDISCONNECTED) == 0 || |
(so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0) |
(so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0) |
error = (*so->so_proto->pr_usrreq)(so, PRU_ACCEPT, |
error = (*so->so_proto->pr_usrreq)(so, PRU_ACCEPT, |
(struct mbuf *)0, nam, (struct mbuf *)0, (struct proc *)0); |
NULL, nam, NULL, NULL); |
else |
else |
error = ECONNABORTED; |
error = ECONNABORTED; |
|
|
splx(s); |
|
return (error); |
return (error); |
} |
} |
|
|
int |
int |
soconnect(struct socket *so, struct mbuf *nam) |
soconnect(struct socket *so, struct mbuf *nam, struct lwp *l) |
{ |
{ |
struct proc *p; |
int error; |
int s, error; |
|
|
KASSERT(solocked(so)); |
|
|
p = curproc; /* XXX */ |
|
if (so->so_options & SO_ACCEPTCONN) |
if (so->so_options & SO_ACCEPTCONN) |
return (EOPNOTSUPP); |
return (EOPNOTSUPP); |
s = splsoftnet(); |
|
/* |
/* |
* If protocol is connection-based, can only connect once. |
* If protocol is connection-based, can only connect once. |
* Otherwise, if connected, try to disconnect first. |
* Otherwise, if connected, try to disconnect first. |
Line 568 soconnect(struct socket *so, struct mbuf |
|
Line 841 soconnect(struct socket *so, struct mbuf |
|
error = EISCONN; |
error = EISCONN; |
else |
else |
error = (*so->so_proto->pr_usrreq)(so, PRU_CONNECT, |
error = (*so->so_proto->pr_usrreq)(so, PRU_CONNECT, |
(struct mbuf *)0, nam, (struct mbuf *)0, p); |
NULL, nam, NULL, l); |
splx(s); |
|
return (error); |
return (error); |
} |
} |
|
|
int |
int |
soconnect2(struct socket *so1, struct socket *so2) |
soconnect2(struct socket *so1, struct socket *so2) |
{ |
{ |
int s, error; |
int error; |
|
|
|
KASSERT(solocked2(so1, so2)); |
|
|
s = splsoftnet(); |
|
error = (*so1->so_proto->pr_usrreq)(so1, PRU_CONNECT2, |
error = (*so1->so_proto->pr_usrreq)(so1, PRU_CONNECT2, |
(struct mbuf *)0, (struct mbuf *)so2, (struct mbuf *)0, |
NULL, (struct mbuf *)so2, NULL, NULL); |
(struct proc *)0); |
|
splx(s); |
|
return (error); |
return (error); |
} |
} |
|
|
int |
int |
sodisconnect(struct socket *so) |
sodisconnect(struct socket *so) |
{ |
{ |
int s, error; |
int error; |
|
|
|
KASSERT(solocked(so)); |
|
|
s = splsoftnet(); |
|
if ((so->so_state & SS_ISCONNECTED) == 0) { |
if ((so->so_state & SS_ISCONNECTED) == 0) { |
error = ENOTCONN; |
error = ENOTCONN; |
goto bad; |
} else if (so->so_state & SS_ISDISCONNECTING) { |
} |
|
if (so->so_state & SS_ISDISCONNECTING) { |
|
error = EALREADY; |
error = EALREADY; |
goto bad; |
} else { |
|
error = (*so->so_proto->pr_usrreq)(so, PRU_DISCONNECT, |
|
NULL, NULL, NULL, NULL); |
} |
} |
error = (*so->so_proto->pr_usrreq)(so, PRU_DISCONNECT, |
sodopendfree(); |
(struct mbuf *)0, (struct mbuf *)0, (struct mbuf *)0, |
|
(struct proc *)0); |
|
bad: |
|
splx(s); |
|
sodopendfree(so); |
|
return (error); |
return (error); |
} |
} |
|
|
Line 629 sodisconnect(struct socket *so) |
|
Line 896 sodisconnect(struct socket *so) |
|
*/ |
*/ |
int |
int |
sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top, |
sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top, |
struct mbuf *control, int flags) |
struct mbuf *control, int flags, struct lwp *l) |
{ |
{ |
struct proc *p; |
|
struct mbuf **mp, *m; |
struct mbuf **mp, *m; |
|
struct proc *p; |
long space, len, resid, clen, mlen; |
long space, len, resid, clen, mlen; |
int error, s, dontroute, atomic; |
int error, s, dontroute, atomic; |
|
short wakeup_state = 0; |
|
|
sodopendfree(so); |
p = l->l_proc; |
|
sodopendfree(); |
p = curproc; /* XXX */ |
|
clen = 0; |
clen = 0; |
|
|
|
/* |
|
* solock() provides atomicity of access. splsoftnet() prevents |
|
* protocol processing soft interrupts from interrupting us and |
|
* blocking (expensive). |
|
*/ |
|
s = splsoftnet(); |
|
solock(so); |
atomic = sosendallatonce(so) || top; |
atomic = sosendallatonce(so) || top; |
if (uio) |
if (uio) |
resid = uio->uio_resid; |
resid = uio->uio_resid; |
Line 659 sosend(struct socket *so, struct mbuf *a |
|
Line 934 sosend(struct socket *so, struct mbuf *a |
|
dontroute = |
dontroute = |
(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && |
(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && |
(so->so_proto->pr_flags & PR_ATOMIC); |
(so->so_proto->pr_flags & PR_ATOMIC); |
p->p_stats->p_ru.ru_msgsnd++; |
l->l_ru.ru_msgsnd++; |
if (control) |
if (control) |
clen = control->m_len; |
clen = control->m_len; |
#define snderr(errno) { error = errno; splx(s); goto release; } |
|
|
|
restart: |
restart: |
if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0) |
if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0) |
goto out; |
goto out; |
do { |
do { |
s = splsoftnet(); |
if (so->so_state & SS_CANTSENDMORE) { |
if (so->so_state & SS_CANTSENDMORE) |
error = EPIPE; |
snderr(EPIPE); |
goto release; |
|
} |
if (so->so_error) { |
if (so->so_error) { |
error = so->so_error; |
error = so->so_error; |
so->so_error = 0; |
so->so_error = 0; |
splx(s); |
|
goto release; |
goto release; |
} |
} |
if ((so->so_state & SS_ISCONNECTED) == 0) { |
if ((so->so_state & SS_ISCONNECTED) == 0) { |
if (so->so_proto->pr_flags & PR_CONNREQUIRED) { |
if (so->so_proto->pr_flags & PR_CONNREQUIRED) { |
if ((so->so_state & SS_ISCONFIRMING) == 0 && |
if ((so->so_state & SS_ISCONFIRMING) == 0 && |
!(resid == 0 && clen != 0)) |
!(resid == 0 && clen != 0)) { |
snderr(ENOTCONN); |
error = ENOTCONN; |
} else if (addr == 0) |
goto release; |
snderr(EDESTADDRREQ); |
} |
|
} else if (addr == 0) { |
|
error = EDESTADDRREQ; |
|
goto release; |
|
} |
} |
} |
space = sbspace(&so->so_snd); |
space = sbspace(&so->so_snd); |
if (flags & MSG_OOB) |
if (flags & MSG_OOB) |
space += 1024; |
space += 1024; |
if ((atomic && resid > so->so_snd.sb_hiwat) || |
if ((atomic && resid > so->so_snd.sb_hiwat) || |
clen > so->so_snd.sb_hiwat) |
clen > so->so_snd.sb_hiwat) { |
snderr(EMSGSIZE); |
error = EMSGSIZE; |
if (space < resid + clen && uio && |
goto release; |
|
} |
|
if (space < resid + clen && |
(atomic || space < so->so_snd.sb_lowat || space < clen)) { |
(atomic || space < so->so_snd.sb_lowat || space < clen)) { |
if (so->so_state & SS_NBIO) |
if (so->so_nbio) { |
snderr(EWOULDBLOCK); |
error = EWOULDBLOCK; |
|
goto release; |
|
} |
sbunlock(&so->so_snd); |
sbunlock(&so->so_snd); |
|
if (wakeup_state & SS_RESTARTSYS) { |
|
error = ERESTART; |
|
goto out; |
|
} |
error = sbwait(&so->so_snd); |
error = sbwait(&so->so_snd); |
splx(s); |
|
if (error) |
if (error) |
goto out; |
goto out; |
|
wakeup_state = so->so_state; |
goto restart; |
goto restart; |
} |
} |
splx(s); |
wakeup_state = 0; |
mp = ⊤ |
mp = ⊤ |
space -= clen; |
space -= clen; |
do { |
do { |
Line 714 sosend(struct socket *so, struct mbuf *a |
|
Line 999 sosend(struct socket *so, struct mbuf *a |
|
if (flags & MSG_EOR) |
if (flags & MSG_EOR) |
top->m_flags |= M_EOR; |
top->m_flags |= M_EOR; |
} else do { |
} else do { |
if (top == 0) { |
sounlock(so); |
|
splx(s); |
|
if (top == NULL) { |
m = m_gethdr(M_WAIT, MT_DATA); |
m = m_gethdr(M_WAIT, MT_DATA); |
mlen = MHLEN; |
mlen = MHLEN; |
m->m_pkthdr.len = 0; |
m->m_pkthdr.len = 0; |
m->m_pkthdr.rcvif = (struct ifnet *)0; |
m->m_pkthdr.rcvif = NULL; |
} else { |
} else { |
m = m_get(M_WAIT, MT_DATA); |
m = m_get(M_WAIT, MT_DATA); |
mlen = MLEN; |
mlen = MLEN; |
} |
} |
MCLAIM(m, so->so_snd.sb_mowner); |
MCLAIM(m, so->so_snd.sb_mowner); |
if (use_sosend_loan && |
if (sock_loan_thresh >= 0 && |
uio->uio_iov->iov_len >= SOCK_LOAN_THRESH && |
uio->uio_iov->iov_len >= sock_loan_thresh && |
space >= SOCK_LOAN_THRESH && |
space >= sock_loan_thresh && |
(len = sosend_loan(so, uio, m, |
(len = sosend_loan(so, uio, m, |
space)) != 0) { |
space)) != 0) { |
SOSEND_COUNTER_INCR(&sosend_loan_big); |
SOSEND_COUNTER_INCR(&sosend_loan_big); |
Line 735 sosend(struct socket *so, struct mbuf *a |
|
Line 1022 sosend(struct socket *so, struct mbuf *a |
|
} |
} |
if (resid >= MINCLSIZE && space >= MCLBYTES) { |
if (resid >= MINCLSIZE && space >= MCLBYTES) { |
SOSEND_COUNTER_INCR(&sosend_copy_big); |
SOSEND_COUNTER_INCR(&sosend_copy_big); |
m_clget(m, M_WAIT); |
m_clget(m, M_DONTWAIT); |
if ((m->m_flags & M_EXT) == 0) |
if ((m->m_flags & M_EXT) == 0) |
goto nopages; |
goto nopages; |
mlen = MCLBYTES; |
mlen = MCLBYTES; |
Line 758 sosend(struct socket *so, struct mbuf *a |
|
Line 1045 sosend(struct socket *so, struct mbuf *a |
|
if (atomic && top == 0 && len < mlen) |
if (atomic && top == 0 && len < mlen) |
MH_ALIGN(m, len); |
MH_ALIGN(m, len); |
} |
} |
error = uiomove(mtod(m, caddr_t), (int)len, |
error = uiomove(mtod(m, void *), (int)len, uio); |
uio); |
|
have_data: |
have_data: |
resid = uio->uio_resid; |
resid = uio->uio_resid; |
m->m_len = len; |
m->m_len = len; |
*mp = m; |
*mp = m; |
top->m_pkthdr.len += len; |
top->m_pkthdr.len += len; |
if (error) |
s = splsoftnet(); |
|
solock(so); |
|
if (error != 0) |
goto release; |
goto release; |
mp = &m->m_next; |
mp = &m->m_next; |
if (resid <= 0) { |
if (resid <= 0) { |
Line 774 sosend(struct socket *so, struct mbuf *a |
|
Line 1062 sosend(struct socket *so, struct mbuf *a |
|
break; |
break; |
} |
} |
} while (space > 0 && atomic); |
} while (space > 0 && atomic); |
|
|
s = splsoftnet(); |
|
|
|
if (so->so_state & SS_CANTSENDMORE) |
|
snderr(EPIPE); |
|
|
|
|
if (so->so_state & SS_CANTSENDMORE) { |
|
error = EPIPE; |
|
goto release; |
|
} |
if (dontroute) |
if (dontroute) |
so->so_options |= SO_DONTROUTE; |
so->so_options |= SO_DONTROUTE; |
if (resid > 0) |
if (resid > 0) |
so->so_state |= SS_MORETOCOME; |
so->so_state |= SS_MORETOCOME; |
error = (*so->so_proto->pr_usrreq)(so, |
error = (*so->so_proto->pr_usrreq)(so, |
(flags & MSG_OOB) ? PRU_SENDOOB : PRU_SEND, |
(flags & MSG_OOB) ? PRU_SENDOOB : PRU_SEND, |
top, addr, control, p); |
top, addr, control, curlwp); |
if (dontroute) |
if (dontroute) |
so->so_options &= ~SO_DONTROUTE; |
so->so_options &= ~SO_DONTROUTE; |
if (resid > 0) |
if (resid > 0) |
so->so_state &= ~SS_MORETOCOME; |
so->so_state &= ~SS_MORETOCOME; |
splx(s); |
|
|
|
clen = 0; |
clen = 0; |
control = 0; |
control = NULL; |
top = 0; |
top = NULL; |
mp = ⊤ |
mp = ⊤ |
if (error) |
if (error != 0) |
goto release; |
goto release; |
} while (resid && space > 0); |
} while (resid && space > 0); |
} while (resid); |
} while (resid); |
Line 805 sosend(struct socket *so, struct mbuf *a |
|
Line 1090 sosend(struct socket *so, struct mbuf *a |
|
release: |
release: |
sbunlock(&so->so_snd); |
sbunlock(&so->so_snd); |
out: |
out: |
|
sounlock(so); |
|
splx(s); |
if (top) |
if (top) |
m_freem(top); |
m_freem(top); |
if (control) |
if (control) |
Line 813 sosend(struct socket *so, struct mbuf *a |
|
Line 1100 sosend(struct socket *so, struct mbuf *a |
|
} |
} |
|
|
/* |
/* |
* Implement receive operations on a socket. |
* Following replacement or removal of the first mbuf on the first |
* We depend on the way that records are added to the sockbuf |
* mbuf chain of a socket buffer, push necessary state changes back |
* by sbappend*. In particular, each record (mbufs linked through m_next) |
* into the socket buffer so that other consumers see the values |
* must begin with an address if the protocol so specifies, |
* consistently. 'nextrecord' is the callers locally stored value of |
* followed by an optional mbuf or mbufs containing ancillary data, |
* the original value of sb->sb_mb->m_nextpkt which must be restored |
* and then zero or more mbufs of data. |
* when the lead mbuf changes. NOTE: 'nextrecord' may be NULL. |
* In order to avoid blocking network interrupts for the entire time here, |
*/ |
|
static void |
|
sbsync(struct sockbuf *sb, struct mbuf *nextrecord) |
|
{ |
|
|
|
KASSERT(solocked(sb->sb_so)); |
|
|
|
/* |
|
* First, update for the new value of nextrecord. If necessary, |
|
* make it the first record. |
|
*/ |
|
if (sb->sb_mb != NULL) |
|
sb->sb_mb->m_nextpkt = nextrecord; |
|
else |
|
sb->sb_mb = nextrecord; |
|
|
|
/* |
|
* Now update any dependent socket buffer fields to reflect |
|
* the new state. This is an inline of SB_EMPTY_FIXUP, with |
|
* the addition of a second clause that takes care of the |
|
* case where sb_mb has been updated, but remains the last |
|
* record. |
|
*/ |
|
if (sb->sb_mb == NULL) { |
|
sb->sb_mbtail = NULL; |
|
sb->sb_lastrecord = NULL; |
|
} else if (sb->sb_mb->m_nextpkt == NULL) |
|
sb->sb_lastrecord = sb->sb_mb; |
|
} |
|
|
|
/* |
|
* Implement receive operations on a socket. |
|
* We depend on the way that records are added to the sockbuf |
|
* by sbappend*. In particular, each record (mbufs linked through m_next) |
|
* must begin with an address if the protocol so specifies, |
|
* followed by an optional mbuf or mbufs containing ancillary data, |
|
* and then zero or more mbufs of data. |
|
* In order to avoid blocking network interrupts for the entire time here, |
* we splx() while doing the actual copy to user space. |
* we splx() while doing the actual copy to user space. |
* Although the sockbuf is locked, new data may still be appended, |
* Although the sockbuf is locked, new data may still be appended, |
* and thus we must maintain consistency of the sockbuf during that time. |
* and thus we must maintain consistency of the sockbuf during that time. |
|
|
soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, |
soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, |
struct mbuf **mp0, struct mbuf **controlp, int *flagsp) |
struct mbuf **mp0, struct mbuf **controlp, int *flagsp) |
{ |
{ |
struct mbuf *m, **mp; |
struct lwp *l = curlwp; |
int flags, len, error, s, offset, moff, type, orig_resid; |
struct mbuf *m, **mp, *mt; |
struct protosw *pr; |
int atomic, flags, len, error, s, offset, moff, type, orig_resid; |
|
const struct protosw *pr; |
struct mbuf *nextrecord; |
struct mbuf *nextrecord; |
int mbuf_removed = 0; |
int mbuf_removed = 0; |
|
const struct domain *dom; |
|
short wakeup_state = 0; |
|
|
pr = so->so_proto; |
pr = so->so_proto; |
|
atomic = pr->pr_flags & PR_ATOMIC; |
|
dom = pr->pr_domain; |
mp = mp0; |
mp = mp0; |
type = 0; |
type = 0; |
orig_resid = uio->uio_resid; |
orig_resid = uio->uio_resid; |
if (paddr) |
|
*paddr = 0; |
if (paddr != NULL) |
if (controlp) |
*paddr = NULL; |
*controlp = 0; |
if (controlp != NULL) |
if (flagsp) |
*controlp = NULL; |
|
if (flagsp != NULL) |
flags = *flagsp &~ MSG_EOR; |
flags = *flagsp &~ MSG_EOR; |
else |
else |
flags = 0; |
flags = 0; |
|
|
if ((flags & MSG_DONTWAIT) == 0) |
if ((flags & MSG_DONTWAIT) == 0) |
sodopendfree(so); |
sodopendfree(); |
|
|
if (flags & MSG_OOB) { |
if (flags & MSG_OOB) { |
m = m_get(M_WAIT, MT_DATA); |
m = m_get(M_WAIT, MT_DATA); |
|
solock(so); |
error = (*pr->pr_usrreq)(so, PRU_RCVOOB, m, |
error = (*pr->pr_usrreq)(so, PRU_RCVOOB, m, |
(struct mbuf *)(long)(flags & MSG_PEEK), (struct mbuf *)0, |
(struct mbuf *)(long)(flags & MSG_PEEK), NULL, l); |
(struct proc *)0); |
sounlock(so); |
if (error) |
if (error) |
goto bad; |
goto bad; |
do { |
do { |
error = uiomove(mtod(m, caddr_t), |
error = uiomove(mtod(m, void *), |
(int) min(uio->uio_resid, m->m_len), uio); |
(int) min(uio->uio_resid, m->m_len), uio); |
m = m_free(m); |
m = m_free(m); |
} while (uio->uio_resid && error == 0 && m); |
} while (uio->uio_resid > 0 && error == 0 && m); |
bad: |
bad: |
if (m) |
if (m != NULL) |
m_freem(m); |
m_freem(m); |
return (error); |
return error; |
} |
} |
if (mp) |
if (mp != NULL) |
*mp = (struct mbuf *)0; |
*mp = NULL; |
|
|
|
/* |
|
* solock() provides atomicity of access. splsoftnet() prevents |
|
* protocol processing soft interrupts from interrupting us and |
|
* blocking (expensive). |
|
*/ |
|
s = splsoftnet(); |
|
solock(so); |
if (so->so_state & SS_ISCONFIRMING && uio->uio_resid) |
if (so->so_state & SS_ISCONFIRMING && uio->uio_resid) |
(*pr->pr_usrreq)(so, PRU_RCVD, (struct mbuf *)0, |
(*pr->pr_usrreq)(so, PRU_RCVD, NULL, NULL, NULL, l); |
(struct mbuf *)0, (struct mbuf *)0, (struct proc *)0); |
|
|
|
restart: |
restart: |
if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0) |
if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0) { |
return (error); |
sounlock(so); |
s = splsoftnet(); |
splx(s); |
|
return error; |
|
} |
|
|
m = so->so_rcv.sb_mb; |
m = so->so_rcv.sb_mb; |
/* |
/* |
Line 894 soreceive(struct socket *so, struct mbuf |
|
Line 1234 soreceive(struct socket *so, struct mbuf |
|
* we have to do the receive in sections, and thus risk returning |
* we have to do the receive in sections, and thus risk returning |
* a short count if a timeout or signal occurs after we start. |
* a short count if a timeout or signal occurs after we start. |
*/ |
*/ |
if (m == 0 || (((flags & MSG_DONTWAIT) == 0 && |
if (m == NULL || |
so->so_rcv.sb_cc < uio->uio_resid) && |
((flags & MSG_DONTWAIT) == 0 && |
(so->so_rcv.sb_cc < so->so_rcv.sb_lowat || |
so->so_rcv.sb_cc < uio->uio_resid && |
((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) && |
(so->so_rcv.sb_cc < so->so_rcv.sb_lowat || |
m->m_nextpkt == 0 && (pr->pr_flags & PR_ATOMIC) == 0)) { |
((flags & MSG_WAITALL) && |
|
uio->uio_resid <= so->so_rcv.sb_hiwat)) && |
|
m->m_nextpkt == NULL && !atomic)) { |
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
if (m == 0 && so->so_rcv.sb_cc) |
if (m == NULL && so->so_rcv.sb_cc) |
panic("receive 1"); |
panic("receive 1"); |
#endif |
#endif |
if (so->so_error) { |
if (so->so_error) { |
if (m) |
if (m != NULL) |
goto dontblock; |
goto dontblock; |
error = so->so_error; |
error = so->so_error; |
if ((flags & MSG_PEEK) == 0) |
if ((flags & MSG_PEEK) == 0) |
Line 912 soreceive(struct socket *so, struct mbuf |
|
Line 1254 soreceive(struct socket *so, struct mbuf |
|
goto release; |
goto release; |
} |
} |
if (so->so_state & SS_CANTRCVMORE) { |
if (so->so_state & SS_CANTRCVMORE) { |
if (m) |
if (m != NULL) |
goto dontblock; |
goto dontblock; |
else |
else |
goto release; |
goto release; |
} |
} |
for (; m; m = m->m_next) |
for (; m != NULL; m = m->m_next) |
if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { |
if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { |
m = so->so_rcv.sb_mb; |
m = so->so_rcv.sb_mb; |
goto dontblock; |
goto dontblock; |
Line 929 soreceive(struct socket *so, struct mbuf |
|
Line 1271 soreceive(struct socket *so, struct mbuf |
|
} |
} |
if (uio->uio_resid == 0) |
if (uio->uio_resid == 0) |
goto release; |
goto release; |
if ((so->so_state & SS_NBIO) || (flags & MSG_DONTWAIT)) { |
if (so->so_nbio || (flags & MSG_DONTWAIT)) { |
error = EWOULDBLOCK; |
error = EWOULDBLOCK; |
goto release; |
goto release; |
} |
} |
SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1"); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1"); |
sbunlock(&so->so_rcv); |
sbunlock(&so->so_rcv); |
error = sbwait(&so->so_rcv); |
if (wakeup_state & SS_RESTARTSYS) |
splx(s); |
error = ERESTART; |
if (error) |
else |
return (error); |
error = sbwait(&so->so_rcv); |
|
if (error != 0) { |
|
sounlock(so); |
|
splx(s); |
|
return error; |
|
} |
|
wakeup_state = so->so_state; |
goto restart; |
goto restart; |
} |
} |
dontblock: |
dontblock: |
/* |
/* |
* On entry here, m points to the first record of the socket buffer. |
* On entry here, m points to the first record of the socket buffer. |
* While we process the initial mbufs containing address and control |
* From this point onward, we maintain 'nextrecord' as a cache of the |
* info, we save a copy of m->m_nextpkt into nextrecord. |
* pointer to the next record in the socket buffer. We must keep the |
|
* various socket buffer pointers and local stack versions of the |
|
* pointers in sync, pushing out modifications before dropping the |
|
* socket lock, and re-reading them when picking it up. |
|
* |
|
* Otherwise, we will race with the network stack appending new data |
|
* or records onto the socket buffer by using inconsistent/stale |
|
* versions of the field, possibly resulting in socket buffer |
|
* corruption. |
|
* |
|
* By holding the high-level sblock(), we prevent simultaneous |
|
* readers from pulling off the front of the socket buffer. |
*/ |
*/ |
#ifdef notyet /* XXXX */ |
if (l != NULL) |
if (uio->uio_procp) |
l->l_ru.ru_msgrcv++; |
uio->uio_procp->p_stats->p_ru.ru_msgrcv++; |
|
#endif |
|
KASSERT(m == so->so_rcv.sb_mb); |
KASSERT(m == so->so_rcv.sb_mb); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive 1"); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive 1"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 1"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 1"); |
Line 969 soreceive(struct socket *so, struct mbuf |
|
Line 1326 soreceive(struct socket *so, struct mbuf |
|
} else { |
} else { |
sbfree(&so->so_rcv, m); |
sbfree(&so->so_rcv, m); |
mbuf_removed = 1; |
mbuf_removed = 1; |
if (paddr) { |
if (paddr != NULL) { |
*paddr = m; |
*paddr = m; |
so->so_rcv.sb_mb = m->m_next; |
so->so_rcv.sb_mb = m->m_next; |
m->m_next = 0; |
m->m_next = NULL; |
m = so->so_rcv.sb_mb; |
m = so->so_rcv.sb_mb; |
} else { |
} else { |
MFREE(m, so->so_rcv.sb_mb); |
MFREE(m, so->so_rcv.sb_mb); |
m = so->so_rcv.sb_mb; |
m = so->so_rcv.sb_mb; |
} |
} |
|
sbsync(&so->so_rcv, nextrecord); |
} |
} |
} |
} |
while (m && m->m_type == MT_CONTROL && error == 0) { |
|
if (flags & MSG_PEEK) { |
/* |
if (controlp) |
* Process one or more MT_CONTROL mbufs present before any data mbufs |
*controlp = m_copy(m, 0, m->m_len); |
* in the first mbuf chain on the socket buffer. If MSG_PEEK, we |
m = m->m_next; |
* just copy the data; if !MSG_PEEK, we call into the protocol to |
} else { |
* perform externalization (or freeing if controlp == NULL). |
sbfree(&so->so_rcv, m); |
*/ |
mbuf_removed = 1; |
if (__predict_false(m != NULL && m->m_type == MT_CONTROL)) { |
if (controlp) { |
struct mbuf *cm = NULL, *cmn; |
if (pr->pr_domain->dom_externalize && |
struct mbuf **cme = &cm; |
mtod(m, struct cmsghdr *)->cmsg_type == |
|
SCM_RIGHTS) |
do { |
error = (*pr->pr_domain->dom_externalize)(m); |
if (flags & MSG_PEEK) { |
*controlp = m; |
if (controlp != NULL) { |
|
*controlp = m_copy(m, 0, m->m_len); |
|
controlp = &(*controlp)->m_next; |
|
} |
|
m = m->m_next; |
|
} else { |
|
sbfree(&so->so_rcv, m); |
so->so_rcv.sb_mb = m->m_next; |
so->so_rcv.sb_mb = m->m_next; |
m->m_next = 0; |
m->m_next = NULL; |
|
*cme = m; |
|
cme = &(*cme)->m_next; |
m = so->so_rcv.sb_mb; |
m = so->so_rcv.sb_mb; |
|
} |
|
} while (m != NULL && m->m_type == MT_CONTROL); |
|
if ((flags & MSG_PEEK) == 0) |
|
sbsync(&so->so_rcv, nextrecord); |
|
for (; cm != NULL; cm = cmn) { |
|
cmn = cm->m_next; |
|
cm->m_next = NULL; |
|
type = mtod(cm, struct cmsghdr *)->cmsg_type; |
|
if (controlp != NULL) { |
|
if (dom->dom_externalize != NULL && |
|
type == SCM_RIGHTS) { |
|
sounlock(so); |
|
splx(s); |
|
error = (*dom->dom_externalize)(cm, l); |
|
s = splsoftnet(); |
|
solock(so); |
|
} |
|
*controlp = cm; |
|
while (*controlp != NULL) |
|
controlp = &(*controlp)->m_next; |
} else { |
} else { |
MFREE(m, so->so_rcv.sb_mb); |
/* |
m = so->so_rcv.sb_mb; |
* Dispose of any SCM_RIGHTS message that went |
|
* through the read path rather than recv. |
|
*/ |
|
if (dom->dom_dispose != NULL && |
|
type == SCM_RIGHTS) { |
|
sounlock(so); |
|
(*dom->dom_dispose)(cm); |
|
solock(so); |
|
} |
|
m_freem(cm); |
} |
} |
} |
} |
if (controlp) { |
if (m != NULL) |
orig_resid = 0; |
nextrecord = so->so_rcv.sb_mb->m_nextpkt; |
controlp = &(*controlp)->m_next; |
else |
} |
nextrecord = so->so_rcv.sb_mb; |
|
orig_resid = 0; |
} |
} |
|
|
/* |
/* If m is non-NULL, we have some data to read. */ |
* If m is non-NULL, we have some data to read. From now on, |
if (__predict_true(m != NULL)) { |
* make sure to keep sb_lastrecord consistent when working on |
|
* the last packet on the chain (nextrecord == NULL) and we |
|
* change m->m_nextpkt. |
|
*/ |
|
if (m) { |
|
if ((flags & MSG_PEEK) == 0) { |
|
m->m_nextpkt = nextrecord; |
|
/* |
|
* If nextrecord == NULL (this is a single chain), |
|
* then sb_lastrecord may not be valid here if m |
|
* was changed earlier. |
|
*/ |
|
if (nextrecord == NULL) { |
|
KASSERT(so->so_rcv.sb_mb == m); |
|
so->so_rcv.sb_lastrecord = m; |
|
} |
|
} |
|
type = m->m_type; |
type = m->m_type; |
if (type == MT_OOBDATA) |
if (type == MT_OOBDATA) |
flags |= MSG_OOB; |
flags |= MSG_OOB; |
} else { |
|
if ((flags & MSG_PEEK) == 0) { |
|
KASSERT(so->so_rcv.sb_mb == m); |
|
so->so_rcv.sb_mb = nextrecord; |
|
SB_EMPTY_FIXUP(&so->so_rcv); |
|
} |
|
} |
} |
SBLASTRECORDCHK(&so->so_rcv, "soreceive 2"); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive 2"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 2"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 2"); |
|
|
moff = 0; |
moff = 0; |
offset = 0; |
offset = 0; |
while (m && uio->uio_resid > 0 && error == 0) { |
while (m != NULL && uio->uio_resid > 0 && error == 0) { |
if (m->m_type == MT_OOBDATA) { |
if (m->m_type == MT_OOBDATA) { |
if (type != MT_OOBDATA) |
if (type != MT_OOBDATA) |
break; |
break; |
Line 1053 soreceive(struct socket *so, struct mbuf |
|
Line 1426 soreceive(struct socket *so, struct mbuf |
|
panic("receive 3"); |
panic("receive 3"); |
#endif |
#endif |
so->so_state &= ~SS_RCVATMARK; |
so->so_state &= ~SS_RCVATMARK; |
|
wakeup_state = 0; |
len = uio->uio_resid; |
len = uio->uio_resid; |
if (so->so_oobmark && len > so->so_oobmark - offset) |
if (so->so_oobmark && len > so->so_oobmark - offset) |
len = so->so_oobmark - offset; |
len = so->so_oobmark - offset; |
Line 1066 soreceive(struct socket *so, struct mbuf |
|
Line 1440 soreceive(struct socket *so, struct mbuf |
|
* we must note any additions to the sockbuf when we |
* we must note any additions to the sockbuf when we |
* block interrupts again. |
* block interrupts again. |
*/ |
*/ |
if (mp == 0) { |
if (mp == NULL) { |
SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove"); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove"); |
|
sounlock(so); |
splx(s); |
splx(s); |
error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio); |
error = uiomove(mtod(m, char *) + moff, (int)len, uio); |
s = splsoftnet(); |
s = splsoftnet(); |
if (error) { |
solock(so); |
|
if (error != 0) { |
/* |
/* |
* If any part of the record has been removed |
* If any part of the record has been removed |
* (such as the MT_SONAME mbuf, which will |
* (such as the MT_SONAME mbuf, which will |
Line 1084 soreceive(struct socket *so, struct mbuf |
|
Line 1460 soreceive(struct socket *so, struct mbuf |
|
* This avoids a later panic("receive 1a") |
* This avoids a later panic("receive 1a") |
* when compiled with DIAGNOSTIC. |
* when compiled with DIAGNOSTIC. |
*/ |
*/ |
if (m && mbuf_removed |
if (m && mbuf_removed && atomic) |
&& (pr->pr_flags & PR_ATOMIC)) |
|
(void) sbdroprecord(&so->so_rcv); |
(void) sbdroprecord(&so->so_rcv); |
|
|
goto release; |
goto release; |
Line 1105 soreceive(struct socket *so, struct mbuf |
|
Line 1480 soreceive(struct socket *so, struct mbuf |
|
*mp = m; |
*mp = m; |
mp = &m->m_next; |
mp = &m->m_next; |
so->so_rcv.sb_mb = m = m->m_next; |
so->so_rcv.sb_mb = m = m->m_next; |
*mp = (struct mbuf *)0; |
*mp = NULL; |
} else { |
} else { |
MFREE(m, so->so_rcv.sb_mb); |
MFREE(m, so->so_rcv.sb_mb); |
m = so->so_rcv.sb_mb; |
m = so->so_rcv.sb_mb; |
Line 1126 soreceive(struct socket *so, struct mbuf |
|
Line 1501 soreceive(struct socket *so, struct mbuf |
|
SBLASTRECORDCHK(&so->so_rcv, "soreceive 3"); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive 3"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 3"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 3"); |
} |
} |
} else { |
} else if (flags & MSG_PEEK) |
if (flags & MSG_PEEK) |
moff += len; |
moff += len; |
else { |
else { |
if (mp != NULL) { |
if (mp) |
mt = m_copym(m, 0, len, M_NOWAIT); |
*mp = m_copym(m, 0, len, M_WAIT); |
if (__predict_false(mt == NULL)) { |
m->m_data += len; |
sounlock(so); |
m->m_len -= len; |
mt = m_copym(m, 0, len, M_WAIT); |
so->so_rcv.sb_cc -= len; |
solock(so); |
|
} |
|
*mp = mt; |
} |
} |
|
m->m_data += len; |
|
m->m_len -= len; |
|
so->so_rcv.sb_cc -= len; |
} |
} |
if (so->so_oobmark) { |
if (so->so_oobmark) { |
if ((flags & MSG_PEEK) == 0) { |
if ((flags & MSG_PEEK) == 0) { |
Line 1159 soreceive(struct socket *so, struct mbuf |
|
Line 1539 soreceive(struct socket *so, struct mbuf |
|
* with a short count but without error. |
* with a short count but without error. |
* Keep sockbuf locked against other readers. |
* Keep sockbuf locked against other readers. |
*/ |
*/ |
while (flags & MSG_WAITALL && m == 0 && uio->uio_resid > 0 && |
while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && |
!sosendallatonce(so) && !nextrecord) { |
!sosendallatonce(so) && !nextrecord) { |
if (so->so_error || so->so_state & SS_CANTRCVMORE) |
if (so->so_error || so->so_state & SS_CANTRCVMORE) |
break; |
break; |
Line 1176 soreceive(struct socket *so, struct mbuf |
|
Line 1556 soreceive(struct socket *so, struct mbuf |
|
*/ |
*/ |
if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb) |
if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb) |
(*pr->pr_usrreq)(so, PRU_RCVD, |
(*pr->pr_usrreq)(so, PRU_RCVD, |
(struct mbuf *)0, |
NULL, (struct mbuf *)(long)flags, NULL, l); |
(struct mbuf *)(long)flags, |
|
(struct mbuf *)0, |
|
(struct proc *)0); |
|
SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2"); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2"); |
error = sbwait(&so->so_rcv); |
if (wakeup_state & SS_RESTARTSYS) |
if (error) { |
error = ERESTART; |
|
else |
|
error = sbwait(&so->so_rcv); |
|
if (error != 0) { |
sbunlock(&so->so_rcv); |
sbunlock(&so->so_rcv); |
|
sounlock(so); |
splx(s); |
splx(s); |
return (0); |
return 0; |
} |
} |
if ((m = so->so_rcv.sb_mb) != NULL) |
if ((m = so->so_rcv.sb_mb) != NULL) |
nextrecord = m->m_nextpkt; |
nextrecord = m->m_nextpkt; |
|
wakeup_state = so->so_state; |
} |
} |
} |
} |
|
|
if (m && pr->pr_flags & PR_ATOMIC) { |
if (m && atomic) { |
flags |= MSG_TRUNC; |
flags |= MSG_TRUNC; |
if ((flags & MSG_PEEK) == 0) |
if ((flags & MSG_PEEK) == 0) |
(void) sbdroprecord(&so->so_rcv); |
(void) sbdroprecord(&so->so_rcv); |
} |
} |
if ((flags & MSG_PEEK) == 0) { |
if ((flags & MSG_PEEK) == 0) { |
if (m == 0) { |
if (m == NULL) { |
/* |
/* |
* First part is an inline SB_EMPTY_FIXUP(). Second |
* First part is an inline SB_EMPTY_FIXUP(). Second |
* part makes sure sb_lastrecord is up-to-date if |
* part makes sure sb_lastrecord is up-to-date if |
Line 1215 soreceive(struct socket *so, struct mbuf |
|
Line 1597 soreceive(struct socket *so, struct mbuf |
|
SBLASTRECORDCHK(&so->so_rcv, "soreceive 4"); |
SBLASTRECORDCHK(&so->so_rcv, "soreceive 4"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 4"); |
SBLASTMBUFCHK(&so->so_rcv, "soreceive 4"); |
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) |
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) |
(*pr->pr_usrreq)(so, PRU_RCVD, (struct mbuf *)0, |
(*pr->pr_usrreq)(so, PRU_RCVD, NULL, |
(struct mbuf *)(long)flags, (struct mbuf *)0, |
(struct mbuf *)(long)flags, NULL, l); |
(struct proc *)0); |
|
} |
} |
if (orig_resid == uio->uio_resid && orig_resid && |
if (orig_resid == uio->uio_resid && orig_resid && |
(flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) { |
(flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) { |
sbunlock(&so->so_rcv); |
sbunlock(&so->so_rcv); |
splx(s); |
|
goto restart; |
goto restart; |
} |
} |
|
|
if (flagsp) |
if (flagsp != NULL) |
*flagsp |= flags; |
*flagsp |= flags; |
release: |
release: |
sbunlock(&so->so_rcv); |
sbunlock(&so->so_rcv); |
|
sounlock(so); |
splx(s); |
splx(s); |
return (error); |
return error; |
} |
} |
|
|
int |
int |
soshutdown(struct socket *so, int how) |
soshutdown(struct socket *so, int how) |
{ |
{ |
struct protosw *pr; |
const struct protosw *pr; |
|
int error; |
|
|
|
KASSERT(solocked(so)); |
|
|
pr = so->so_proto; |
pr = so->so_proto; |
if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) |
if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) |
return (EINVAL); |
return (EINVAL); |
|
|
if (how == SHUT_RD || how == SHUT_RDWR) |
if (how == SHUT_RD || how == SHUT_RDWR) { |
sorflush(so); |
sorflush(so); |
|
error = 0; |
|
} |
if (how == SHUT_WR || how == SHUT_RDWR) |
if (how == SHUT_WR || how == SHUT_RDWR) |
return (*pr->pr_usrreq)(so, PRU_SHUTDOWN, (struct mbuf *)0, |
error = (*pr->pr_usrreq)(so, PRU_SHUTDOWN, NULL, |
(struct mbuf *)0, (struct mbuf *)0, (struct proc *)0); |
NULL, NULL, NULL); |
return (0); |
|
|
return error; |
|
} |
|
|
|
void |
|
sorestart(struct socket *so) |
|
{ |
|
/* |
|
* An application has called close() on an fd on which another |
|
* of its threads has called a socket system call. |
|
* Mark this and wake everyone up, and code that would block again |
|
* instead returns ERESTART. |
|
* On system call re-entry the fd is validated and EBADF returned. |
|
* Any other fd will block again on the 2nd syscall. |
|
*/ |
|
solock(so); |
|
so->so_state |= SS_RESTARTSYS; |
|
cv_broadcast(&so->so_cv); |
|
cv_broadcast(&so->so_snd.sb_cv); |
|
cv_broadcast(&so->so_rcv.sb_cv); |
|
sounlock(so); |
} |
} |
|
|
void |
void |
sorflush(struct socket *so) |
sorflush(struct socket *so) |
{ |
{ |
struct sockbuf *sb, asb; |
struct sockbuf *sb, asb; |
struct protosw *pr; |
const struct protosw *pr; |
int s; |
|
|
KASSERT(solocked(so)); |
|
|
sb = &so->so_rcv; |
sb = &so->so_rcv; |
pr = so->so_proto; |
pr = so->so_proto; |
sb->sb_flags |= SB_NOINTR; |
|
(void) sblock(sb, M_WAITOK); |
|
s = splnet(); |
|
socantrcvmore(so); |
socantrcvmore(so); |
|
sb->sb_flags |= SB_NOINTR; |
|
(void )sblock(sb, M_WAITOK); |
sbunlock(sb); |
sbunlock(sb); |
asb = *sb; |
asb = *sb; |
/* |
/* |
Line 1272 sorflush(struct socket *so) |
|
Line 1678 sorflush(struct socket *so) |
|
*/ |
*/ |
memset(&sb->sb_startzero, 0, |
memset(&sb->sb_startzero, 0, |
sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); |
sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); |
splx(s); |
if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose) { |
if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose) |
sounlock(so); |
(*pr->pr_domain->dom_dispose)(asb.sb_mb); |
(*pr->pr_domain->dom_dispose)(asb.sb_mb); |
sbrelease(&asb); |
solock(so); |
|
} |
|
sbrelease(&asb, so); |
} |
} |
|
|
int |
/* |
sosetopt(struct socket *so, int level, int optname, struct mbuf *m0) |
* internal set SOL_SOCKET options |
|
*/ |
|
static int |
|
sosetopt1(struct socket *so, const struct sockopt *sopt) |
{ |
{ |
int error; |
int error = EINVAL, optval, opt; |
struct mbuf *m; |
struct linger l; |
|
struct timeval tv; |
|
|
error = 0; |
switch ((opt = sopt->sopt_name)) { |
m = m0; |
|
if (level != SOL_SOCKET) { |
|
if (so->so_proto && so->so_proto->pr_ctloutput) |
|
return ((*so->so_proto->pr_ctloutput) |
|
(PRCO_SETOPT, so, level, optname, &m0)); |
|
error = ENOPROTOOPT; |
|
} else { |
|
switch (optname) { |
|
|
|
case SO_LINGER: |
case SO_ACCEPTFILTER: |
if (m == NULL || m->m_len != sizeof(struct linger)) { |
error = accept_filt_setopt(so, sopt); |
error = EINVAL; |
KASSERT(solocked(so)); |
goto bad; |
break; |
} |
|
so->so_linger = mtod(m, struct linger *)->l_linger; |
|
/* fall thru... */ |
|
|
|
case SO_DEBUG: |
case SO_LINGER: |
case SO_KEEPALIVE: |
error = sockopt_get(sopt, &l, sizeof(l)); |
case SO_DONTROUTE: |
solock(so); |
case SO_USELOOPBACK: |
if (error) |
case SO_BROADCAST: |
break; |
case SO_REUSEADDR: |
if (l.l_linger < 0 || l.l_linger > USHRT_MAX || |
case SO_REUSEPORT: |
l.l_linger > (INT_MAX / hz)) { |
case SO_OOBINLINE: |
error = EDOM; |
case SO_TIMESTAMP: |
|
if (m == NULL || m->m_len < sizeof(int)) { |
|
error = EINVAL; |
|
goto bad; |
|
} |
|
if (*mtod(m, int *)) |
|
so->so_options |= optname; |
|
else |
|
so->so_options &= ~optname; |
|
break; |
break; |
|
} |
|
so->so_linger = l.l_linger; |
|
if (l.l_onoff) |
|
so->so_options |= SO_LINGER; |
|
else |
|
so->so_options &= ~SO_LINGER; |
|
break; |
|
|
|
case SO_DEBUG: |
|
case SO_KEEPALIVE: |
|
case SO_DONTROUTE: |
|
case SO_USELOOPBACK: |
|
case SO_BROADCAST: |
|
case SO_REUSEADDR: |
|
case SO_REUSEPORT: |
|
case SO_OOBINLINE: |
|
case SO_TIMESTAMP: |
|
#ifdef SO_OTIMESTAMP |
|
case SO_OTIMESTAMP: |
|
#endif |
|
error = sockopt_getint(sopt, &optval); |
|
solock(so); |
|
if (error) |
|
break; |
|
if (optval) |
|
so->so_options |= opt; |
|
else |
|
so->so_options &= ~opt; |
|
break; |
|
|
case SO_SNDBUF: |
case SO_SNDBUF: |
case SO_RCVBUF: |
case SO_RCVBUF: |
case SO_SNDLOWAT: |
case SO_SNDLOWAT: |
case SO_RCVLOWAT: |
case SO_RCVLOWAT: |
{ |
error = sockopt_getint(sopt, &optval); |
int optval; |
solock(so); |
|
if (error) |
if (m == NULL || m->m_len < sizeof(int)) { |
break; |
error = EINVAL; |
|
goto bad; |
|
} |
|
|
|
/* |
|
* Values < 1 make no sense for any of these |
|
* options, so disallow them. |
|
*/ |
|
optval = *mtod(m, int *); |
|
if (optval < 1) { |
|
error = EINVAL; |
|
goto bad; |
|
} |
|
|
|
switch (optname) { |
/* |
|
* Values < 1 make no sense for any of these |
|
* options, so disallow them. |
|
*/ |
|
if (optval < 1) { |
|
error = EINVAL; |
|
break; |
|
} |
|
|
case SO_SNDBUF: |
switch (opt) { |
case SO_RCVBUF: |
case SO_SNDBUF: |
if (sbreserve(optname == SO_SNDBUF ? |
if (sbreserve(&so->so_snd, (u_long)optval, so) == 0) { |
&so->so_snd : &so->so_rcv, |
error = ENOBUFS; |
(u_long) optval) == 0) { |
|
error = ENOBUFS; |
|
goto bad; |
|
} |
|
break; |
break; |
|
} |
|
so->so_snd.sb_flags &= ~SB_AUTOSIZE; |
|
break; |
|
|
/* |
case SO_RCVBUF: |
* Make sure the low-water is never greater than |
if (sbreserve(&so->so_rcv, (u_long)optval, so) == 0) { |
* the high-water. |
error = ENOBUFS; |
*/ |
|
case SO_SNDLOWAT: |
|
so->so_snd.sb_lowat = |
|
(optval > so->so_snd.sb_hiwat) ? |
|
so->so_snd.sb_hiwat : optval; |
|
break; |
|
case SO_RCVLOWAT: |
|
so->so_rcv.sb_lowat = |
|
(optval > so->so_rcv.sb_hiwat) ? |
|
so->so_rcv.sb_hiwat : optval; |
|
break; |
break; |
} |
} |
|
so->so_rcv.sb_flags &= ~SB_AUTOSIZE; |
break; |
break; |
} |
|
|
|
case SO_SNDTIMEO: |
/* |
case SO_RCVTIMEO: |
* Make sure the low-water is never greater than |
{ |
* the high-water. |
struct timeval *tv; |
*/ |
short val; |
case SO_SNDLOWAT: |
|
if (optval > so->so_snd.sb_hiwat) |
if (m == NULL || m->m_len < sizeof(*tv)) { |
optval = so->so_snd.sb_hiwat; |
error = EINVAL; |
|
goto bad; |
|
} |
|
tv = mtod(m, struct timeval *); |
|
if (tv->tv_sec > (SHRT_MAX - tv->tv_usec / tick) / hz) { |
|
error = EDOM; |
|
goto bad; |
|
} |
|
val = tv->tv_sec * hz + tv->tv_usec / tick; |
|
if (val == 0 && tv->tv_usec != 0) |
|
val = 1; |
|
|
|
switch (optname) { |
so->so_snd.sb_lowat = optval; |
|
break; |
|
|
case SO_SNDTIMEO: |
case SO_RCVLOWAT: |
so->so_snd.sb_timeo = val; |
if (optval > so->so_rcv.sb_hiwat) |
break; |
optval = so->so_rcv.sb_hiwat; |
case SO_RCVTIMEO: |
|
so->so_rcv.sb_timeo = val; |
so->so_rcv.sb_lowat = optval; |
break; |
break; |
} |
} |
|
break; |
|
|
|
#ifdef COMPAT_50 |
|
case SO_OSNDTIMEO: |
|
case SO_ORCVTIMEO: { |
|
struct timeval50 otv; |
|
error = sockopt_get(sopt, &otv, sizeof(otv)); |
|
if (error) { |
|
solock(so); |
|
break; |
|
} |
|
timeval50_to_timeval(&otv, &tv); |
|
opt = opt == SO_OSNDTIMEO ? SO_SNDTIMEO : SO_RCVTIMEO; |
|
error = 0; |
|
/*FALLTHROUGH*/ |
|
} |
|
#endif /* COMPAT_50 */ |
|
|
|
case SO_SNDTIMEO: |
|
case SO_RCVTIMEO: |
|
if (error) |
|
error = sockopt_get(sopt, &tv, sizeof(tv)); |
|
solock(so); |
|
if (error) |
break; |
break; |
} |
|
|
|
default: |
if (tv.tv_sec > (INT_MAX - tv.tv_usec / tick) / hz) { |
error = ENOPROTOOPT; |
error = EDOM; |
break; |
break; |
} |
} |
if (error == 0 && so->so_proto && so->so_proto->pr_ctloutput) { |
|
(void) ((*so->so_proto->pr_ctloutput) |
optval = tv.tv_sec * hz + tv.tv_usec / tick; |
(PRCO_SETOPT, so, level, optname, &m0)); |
if (optval == 0 && tv.tv_usec != 0) |
m = NULL; /* freed by protocol */ |
optval = 1; |
|
|
|
switch (opt) { |
|
case SO_SNDTIMEO: |
|
so->so_snd.sb_timeo = optval; |
|
break; |
|
case SO_RCVTIMEO: |
|
so->so_rcv.sb_timeo = optval; |
|
break; |
} |
} |
|
break; |
|
|
|
default: |
|
solock(so); |
|
error = ENOPROTOOPT; |
|
break; |
} |
} |
bad: |
KASSERT(solocked(so)); |
if (m) |
return error; |
(void) m_free(m); |
} |
|
|
|
int |
|
sosetopt(struct socket *so, struct sockopt *sopt) |
|
{ |
|
int error, prerr; |
|
|
|
if (sopt->sopt_level == SOL_SOCKET) { |
|
error = sosetopt1(so, sopt); |
|
KASSERT(solocked(so)); |
|
} else { |
|
error = ENOPROTOOPT; |
|
solock(so); |
|
} |
|
|
|
if ((error == 0 || error == ENOPROTOOPT) && |
|
so->so_proto != NULL && so->so_proto->pr_ctloutput != NULL) { |
|
/* give the protocol stack a shot */ |
|
prerr = (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so, sopt); |
|
if (prerr == 0) |
|
error = 0; |
|
else if (prerr != ENOPROTOOPT) |
|
error = prerr; |
|
} |
|
sounlock(so); |
|
return error; |
|
} |
|
|
|
/* |
|
* so_setsockopt() is a wrapper providing a sockopt structure for sosetopt() |
|
*/ |
|
int |
|
so_setsockopt(struct lwp *l, struct socket *so, int level, int name, |
|
const void *val, size_t valsize) |
|
{ |
|
struct sockopt sopt; |
|
int error; |
|
|
|
KASSERT(valsize == 0 || val != NULL); |
|
|
|
sockopt_init(&sopt, level, name, valsize); |
|
sockopt_set(&sopt, val, valsize); |
|
|
|
error = sosetopt(so, &sopt); |
|
|
|
sockopt_destroy(&sopt); |
|
|
|
return error; |
|
} |
|
|
|
/* |
|
* internal get SOL_SOCKET options |
|
*/ |
|
static int |
|
sogetopt1(struct socket *so, struct sockopt *sopt) |
|
{ |
|
int error, optval, opt; |
|
struct linger l; |
|
struct timeval tv; |
|
|
|
switch ((opt = sopt->sopt_name)) { |
|
|
|
case SO_ACCEPTFILTER: |
|
error = accept_filt_getopt(so, sopt); |
|
break; |
|
|
|
case SO_LINGER: |
|
l.l_onoff = (so->so_options & SO_LINGER) ? 1 : 0; |
|
l.l_linger = so->so_linger; |
|
|
|
error = sockopt_set(sopt, &l, sizeof(l)); |
|
break; |
|
|
|
case SO_USELOOPBACK: |
|
case SO_DONTROUTE: |
|
case SO_DEBUG: |
|
case SO_KEEPALIVE: |
|
case SO_REUSEADDR: |
|
case SO_REUSEPORT: |
|
case SO_BROADCAST: |
|
case SO_OOBINLINE: |
|
case SO_TIMESTAMP: |
|
#ifdef SO_OTIMESTAMP |
|
case SO_OTIMESTAMP: |
|
#endif |
|
error = sockopt_setint(sopt, (so->so_options & opt) ? 1 : 0); |
|
break; |
|
|
|
case SO_TYPE: |
|
error = sockopt_setint(sopt, so->so_type); |
|
break; |
|
|
|
case SO_ERROR: |
|
error = sockopt_setint(sopt, so->so_error); |
|
so->so_error = 0; |
|
break; |
|
|
|
case SO_SNDBUF: |
|
error = sockopt_setint(sopt, so->so_snd.sb_hiwat); |
|
break; |
|
|
|
case SO_RCVBUF: |
|
error = sockopt_setint(sopt, so->so_rcv.sb_hiwat); |
|
break; |
|
|
|
case SO_SNDLOWAT: |
|
error = sockopt_setint(sopt, so->so_snd.sb_lowat); |
|
break; |
|
|
|
case SO_RCVLOWAT: |
|
error = sockopt_setint(sopt, so->so_rcv.sb_lowat); |
|
break; |
|
|
|
#ifdef COMPAT_50 |
|
case SO_OSNDTIMEO: |
|
case SO_ORCVTIMEO: { |
|
struct timeval50 otv; |
|
|
|
optval = (opt == SO_OSNDTIMEO ? |
|
so->so_snd.sb_timeo : so->so_rcv.sb_timeo); |
|
|
|
otv.tv_sec = optval / hz; |
|
otv.tv_usec = (optval % hz) * tick; |
|
|
|
error = sockopt_set(sopt, &otv, sizeof(otv)); |
|
break; |
|
} |
|
#endif /* COMPAT_50 */ |
|
|
|
case SO_SNDTIMEO: |
|
case SO_RCVTIMEO: |
|
optval = (opt == SO_SNDTIMEO ? |
|
so->so_snd.sb_timeo : so->so_rcv.sb_timeo); |
|
|
|
tv.tv_sec = optval / hz; |
|
tv.tv_usec = (optval % hz) * tick; |
|
|
|
error = sockopt_set(sopt, &tv, sizeof(tv)); |
|
break; |
|
|
|
case SO_OVERFLOWED: |
|
error = sockopt_setint(sopt, so->so_rcv.sb_overflowed); |
|
break; |
|
|
|
default: |
|
error = ENOPROTOOPT; |
|
break; |
|
} |
|
|
return (error); |
return (error); |
} |
} |
|
|
int |
int |
sogetopt(struct socket *so, int level, int optname, struct mbuf **mp) |
sogetopt(struct socket *so, struct sockopt *sopt) |
{ |
{ |
struct mbuf *m; |
int error; |
|
|
if (level != SOL_SOCKET) { |
solock(so); |
|
if (sopt->sopt_level != SOL_SOCKET) { |
if (so->so_proto && so->so_proto->pr_ctloutput) { |
if (so->so_proto && so->so_proto->pr_ctloutput) { |
return ((*so->so_proto->pr_ctloutput) |
error = ((*so->so_proto->pr_ctloutput) |
(PRCO_GETOPT, so, level, optname, mp)); |
(PRCO_GETOPT, so, sopt)); |
} else |
} else |
return (ENOPROTOOPT); |
error = (ENOPROTOOPT); |
} else { |
} else { |
m = m_get(M_WAIT, MT_SOOPTS); |
error = sogetopt1(so, sopt); |
m->m_len = sizeof(int); |
} |
|
sounlock(so); |
|
return (error); |
|
} |
|
|
switch (optname) { |
/* |
|
* alloc sockopt data buffer buffer |
|
* - will be released at destroy |
|
*/ |
|
static int |
|
sockopt_alloc(struct sockopt *sopt, size_t len, km_flag_t kmflag) |
|
{ |
|
|
case SO_LINGER: |
KASSERT(sopt->sopt_size == 0); |
m->m_len = sizeof(struct linger); |
|
mtod(m, struct linger *)->l_onoff = |
|
so->so_options & SO_LINGER; |
|
mtod(m, struct linger *)->l_linger = so->so_linger; |
|
break; |
|
|
|
case SO_USELOOPBACK: |
if (len > sizeof(sopt->sopt_buf)) { |
case SO_DONTROUTE: |
sopt->sopt_data = kmem_zalloc(len, kmflag); |
case SO_DEBUG: |
if (sopt->sopt_data == NULL) |
case SO_KEEPALIVE: |
return ENOMEM; |
case SO_REUSEADDR: |
} else |
case SO_REUSEPORT: |
sopt->sopt_data = sopt->sopt_buf; |
case SO_BROADCAST: |
|
case SO_OOBINLINE: |
|
case SO_TIMESTAMP: |
|
*mtod(m, int *) = so->so_options & optname; |
|
break; |
|
|
|
case SO_TYPE: |
sopt->sopt_size = len; |
*mtod(m, int *) = so->so_type; |
return 0; |
break; |
} |
|
|
case SO_ERROR: |
/* |
*mtod(m, int *) = so->so_error; |
* initialise sockopt storage |
so->so_error = 0; |
* - MAY sleep during allocation |
break; |
*/ |
|
void |
|
sockopt_init(struct sockopt *sopt, int level, int name, size_t size) |
|
{ |
|
|
case SO_SNDBUF: |
memset(sopt, 0, sizeof(*sopt)); |
*mtod(m, int *) = so->so_snd.sb_hiwat; |
|
break; |
|
|
|
case SO_RCVBUF: |
sopt->sopt_level = level; |
*mtod(m, int *) = so->so_rcv.sb_hiwat; |
sopt->sopt_name = name; |
break; |
(void)sockopt_alloc(sopt, size, KM_SLEEP); |
|
} |
|
|
case SO_SNDLOWAT: |
/* |
*mtod(m, int *) = so->so_snd.sb_lowat; |
* destroy sockopt storage |
break; |
* - will release any held memory references |
|
*/ |
|
void |
|
sockopt_destroy(struct sockopt *sopt) |
|
{ |
|
|
case SO_RCVLOWAT: |
if (sopt->sopt_data != sopt->sopt_buf) |
*mtod(m, int *) = so->so_rcv.sb_lowat; |
kmem_free(sopt->sopt_data, sopt->sopt_size); |
break; |
|
|
|
case SO_SNDTIMEO: |
memset(sopt, 0, sizeof(*sopt)); |
case SO_RCVTIMEO: |
} |
{ |
|
int val = (optname == SO_SNDTIMEO ? |
|
so->so_snd.sb_timeo : so->so_rcv.sb_timeo); |
|
|
|
m->m_len = sizeof(struct timeval); |
|
mtod(m, struct timeval *)->tv_sec = val / hz; |
|
mtod(m, struct timeval *)->tv_usec = |
|
(val % hz) * tick; |
|
break; |
|
} |
|
|
|
default: |
/* |
(void)m_free(m); |
* set sockopt value |
return (ENOPROTOOPT); |
* - value is copied into sockopt |
|
* - memory is allocated when necessary, will not sleep |
|
*/ |
|
int |
|
sockopt_set(struct sockopt *sopt, const void *buf, size_t len) |
|
{ |
|
int error; |
|
|
|
if (sopt->sopt_size == 0) { |
|
error = sockopt_alloc(sopt, len, KM_NOSLEEP); |
|
if (error) |
|
return error; |
|
} |
|
|
|
KASSERT(sopt->sopt_size == len); |
|
memcpy(sopt->sopt_data, buf, len); |
|
return 0; |
|
} |
|
|
|
/* |
|
* common case of set sockopt integer value |
|
*/ |
|
int |
|
sockopt_setint(struct sockopt *sopt, int val) |
|
{ |
|
|
|
return sockopt_set(sopt, &val, sizeof(int)); |
|
} |
|
|
|
/* |
|
* get sockopt value |
|
* - correct size must be given |
|
*/ |
|
int |
|
sockopt_get(const struct sockopt *sopt, void *buf, size_t len) |
|
{ |
|
|
|
if (sopt->sopt_size != len) |
|
return EINVAL; |
|
|
|
memcpy(buf, sopt->sopt_data, len); |
|
return 0; |
|
} |
|
|
|
/* |
|
* common case of get sockopt integer value |
|
*/ |
|
int |
|
sockopt_getint(const struct sockopt *sopt, int *valp) |
|
{ |
|
|
|
return sockopt_get(sopt, valp, sizeof(int)); |
|
} |
|
|
|
/* |
|
* set sockopt value from mbuf |
|
* - ONLY for legacy code |
|
* - mbuf is released by sockopt |
|
* - will not sleep |
|
*/ |
|
int |
|
sockopt_setmbuf(struct sockopt *sopt, struct mbuf *m) |
|
{ |
|
size_t len; |
|
int error; |
|
|
|
len = m_length(m); |
|
|
|
if (sopt->sopt_size == 0) { |
|
error = sockopt_alloc(sopt, len, KM_NOSLEEP); |
|
if (error) |
|
return error; |
|
} |
|
|
|
KASSERT(sopt->sopt_size == len); |
|
m_copydata(m, 0, len, sopt->sopt_data); |
|
m_freem(m); |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* get sockopt value into mbuf |
|
* - ONLY for legacy code |
|
* - mbuf to be released by the caller |
|
* - will not sleep |
|
*/ |
|
struct mbuf * |
|
sockopt_getmbuf(const struct sockopt *sopt) |
|
{ |
|
struct mbuf *m; |
|
|
|
if (sopt->sopt_size > MCLBYTES) |
|
return NULL; |
|
|
|
m = m_get(M_DONTWAIT, MT_SOOPTS); |
|
if (m == NULL) |
|
return NULL; |
|
|
|
if (sopt->sopt_size > MLEN) { |
|
MCLGET(m, M_DONTWAIT); |
|
if ((m->m_flags & M_EXT) == 0) { |
|
m_free(m); |
|
return NULL; |
} |
} |
*mp = m; |
|
return (0); |
|
} |
} |
|
|
|
memcpy(mtod(m, void *), sopt->sopt_data, sopt->sopt_size); |
|
m->m_len = sopt->sopt_size; |
|
|
|
return m; |
} |
} |
|
|
void |
void |
sohasoutofband(struct socket *so) |
sohasoutofband(struct socket *so) |
{ |
{ |
struct proc *p; |
|
ksiginfo_t ksi; |
fownsignal(so->so_pgid, SIGURG, POLL_PRI, POLLPRI|POLLRDBAND, so); |
memset(&ksi, 0, sizeof(ksi)); |
selnotify(&so->so_rcv.sb_sel, POLLPRI | POLLRDBAND, NOTE_SUBMIT); |
ksi.ksi_signo = SIGURG; |
|
|
|
if (so->so_pgid < 0) |
|
kgsignal(-so->so_pgid, &ksi, NULL); |
|
else if (so->so_pgid > 0 && (p = pfind(so->so_pgid)) != 0) |
|
kpsignal(p, &ksi, NULL); |
|
selwakeup(&so->so_rcv.sb_sel); |
|
} |
} |
|
|
static void |
static void |
Line 1523 filt_sordetach(struct knote *kn) |
|
Line 2196 filt_sordetach(struct knote *kn) |
|
{ |
{ |
struct socket *so; |
struct socket *so; |
|
|
so = (struct socket *)kn->kn_fp->f_data; |
so = ((file_t *)kn->kn_obj)->f_data; |
|
solock(so); |
SLIST_REMOVE(&so->so_rcv.sb_sel.sel_klist, kn, knote, kn_selnext); |
SLIST_REMOVE(&so->so_rcv.sb_sel.sel_klist, kn, knote, kn_selnext); |
if (SLIST_EMPTY(&so->so_rcv.sb_sel.sel_klist)) |
if (SLIST_EMPTY(&so->so_rcv.sb_sel.sel_klist)) |
so->so_rcv.sb_flags &= ~SB_KNOTE; |
so->so_rcv.sb_flags &= ~SB_KNOTE; |
|
sounlock(so); |
} |
} |
|
|
/*ARGSUSED*/ |
/*ARGSUSED*/ |
|
|
filt_soread(struct knote *kn, long hint) |
filt_soread(struct knote *kn, long hint) |
{ |
{ |
struct socket *so; |
struct socket *so; |
|
int rv; |
|
|
so = (struct socket *)kn->kn_fp->f_data; |
so = ((file_t *)kn->kn_obj)->f_data; |
|
if (hint != NOTE_SUBMIT) |
|
solock(so); |
kn->kn_data = so->so_rcv.sb_cc; |
kn->kn_data = so->so_rcv.sb_cc; |
if (so->so_state & SS_CANTRCVMORE) { |
if (so->so_state & SS_CANTRCVMORE) { |
kn->kn_flags |= EV_EOF; |
kn->kn_flags |= EV_EOF; |
kn->kn_fflags = so->so_error; |
kn->kn_fflags = so->so_error; |
return (1); |
rv = 1; |
} |
} else if (so->so_error) /* temporary udp error */ |
if (so->so_error) /* temporary udp error */ |
rv = 1; |
return (1); |
else if (kn->kn_sfflags & NOTE_LOWAT) |
if (kn->kn_sfflags & NOTE_LOWAT) |
rv = (kn->kn_data >= kn->kn_sdata); |
return (kn->kn_data >= kn->kn_sdata); |
else |
return (kn->kn_data >= so->so_rcv.sb_lowat); |
rv = (kn->kn_data >= so->so_rcv.sb_lowat); |
|
if (hint != NOTE_SUBMIT) |
|
sounlock(so); |
|
return rv; |
} |
} |
|
|
static void |
static void |
Line 1554 filt_sowdetach(struct knote *kn) |
|
Line 2235 filt_sowdetach(struct knote *kn) |
|
{ |
{ |
struct socket *so; |
struct socket *so; |
|
|
so = (struct socket *)kn->kn_fp->f_data; |
so = ((file_t *)kn->kn_obj)->f_data; |
|
solock(so); |
SLIST_REMOVE(&so->so_snd.sb_sel.sel_klist, kn, knote, kn_selnext); |
SLIST_REMOVE(&so->so_snd.sb_sel.sel_klist, kn, knote, kn_selnext); |
if (SLIST_EMPTY(&so->so_snd.sb_sel.sel_klist)) |
if (SLIST_EMPTY(&so->so_snd.sb_sel.sel_klist)) |
so->so_snd.sb_flags &= ~SB_KNOTE; |
so->so_snd.sb_flags &= ~SB_KNOTE; |
|
sounlock(so); |
} |
} |
|
|
/*ARGSUSED*/ |
/*ARGSUSED*/ |
|
|
filt_sowrite(struct knote *kn, long hint) |
filt_sowrite(struct knote *kn, long hint) |
{ |
{ |
struct socket *so; |
struct socket *so; |
|
int rv; |
|
|
so = (struct socket *)kn->kn_fp->f_data; |
so = ((file_t *)kn->kn_obj)->f_data; |
|
if (hint != NOTE_SUBMIT) |
|
solock(so); |
kn->kn_data = sbspace(&so->so_snd); |
kn->kn_data = sbspace(&so->so_snd); |
if (so->so_state & SS_CANTSENDMORE) { |
if (so->so_state & SS_CANTSENDMORE) { |
kn->kn_flags |= EV_EOF; |
kn->kn_flags |= EV_EOF; |
kn->kn_fflags = so->so_error; |
kn->kn_fflags = so->so_error; |
return (1); |
rv = 1; |
} |
} else if (so->so_error) /* temporary udp error */ |
if (so->so_error) /* temporary udp error */ |
rv = 1; |
return (1); |
else if (((so->so_state & SS_ISCONNECTED) == 0) && |
if (((so->so_state & SS_ISCONNECTED) == 0) && |
|
(so->so_proto->pr_flags & PR_CONNREQUIRED)) |
(so->so_proto->pr_flags & PR_CONNREQUIRED)) |
return (0); |
rv = 0; |
if (kn->kn_sfflags & NOTE_LOWAT) |
else if (kn->kn_sfflags & NOTE_LOWAT) |
return (kn->kn_data >= kn->kn_sdata); |
rv = (kn->kn_data >= kn->kn_sdata); |
return (kn->kn_data >= so->so_snd.sb_lowat); |
else |
|
rv = (kn->kn_data >= so->so_snd.sb_lowat); |
|
if (hint != NOTE_SUBMIT) |
|
sounlock(so); |
|
return rv; |
} |
} |
|
|
/*ARGSUSED*/ |
/*ARGSUSED*/ |
|
|
filt_solisten(struct knote *kn, long hint) |
filt_solisten(struct knote *kn, long hint) |
{ |
{ |
struct socket *so; |
struct socket *so; |
|
int rv; |
|
|
so = (struct socket *)kn->kn_fp->f_data; |
so = ((file_t *)kn->kn_obj)->f_data; |
|
|
/* |
/* |
* Set kn_data to number of incoming connections, not |
* Set kn_data to number of incoming connections, not |
* counting partial (incomplete) connections. |
* counting partial (incomplete) connections. |
*/ |
*/ |
|
if (hint != NOTE_SUBMIT) |
|
solock(so); |
kn->kn_data = so->so_qlen; |
kn->kn_data = so->so_qlen; |
return (kn->kn_data > 0); |
rv = (kn->kn_data > 0); |
|
if (hint != NOTE_SUBMIT) |
|
sounlock(so); |
|
return rv; |
} |
} |
|
|
static const struct filterops solisten_filtops = |
static const struct filterops solisten_filtops = |
Line 1612 soo_kqfilter(struct file *fp, struct kno |
|
Line 2307 soo_kqfilter(struct file *fp, struct kno |
|
struct socket *so; |
struct socket *so; |
struct sockbuf *sb; |
struct sockbuf *sb; |
|
|
so = (struct socket *)kn->kn_fp->f_data; |
so = ((file_t *)kn->kn_obj)->f_data; |
|
solock(so); |
switch (kn->kn_filter) { |
switch (kn->kn_filter) { |
case EVFILT_READ: |
case EVFILT_READ: |
if (so->so_options & SO_ACCEPTCONN) |
if (so->so_options & SO_ACCEPTCONN) |
Line 1626 soo_kqfilter(struct file *fp, struct kno |
|
Line 2322 soo_kqfilter(struct file *fp, struct kno |
|
sb = &so->so_snd; |
sb = &so->so_snd; |
break; |
break; |
default: |
default: |
return (1); |
sounlock(so); |
|
return (EINVAL); |
} |
} |
SLIST_INSERT_HEAD(&sb->sb_sel.sel_klist, kn, kn_selnext); |
SLIST_INSERT_HEAD(&sb->sb_sel.sel_klist, kn, kn_selnext); |
sb->sb_flags |= SB_KNOTE; |
sb->sb_flags |= SB_KNOTE; |
|
sounlock(so); |
return (0); |
return (0); |
} |
} |
|
|
|
static int |
|
sodopoll(struct socket *so, int events) |
|
{ |
|
int revents; |
|
|
|
revents = 0; |
|
|
|
if (events & (POLLIN | POLLRDNORM)) |
|
if (soreadable(so)) |
|
revents |= events & (POLLIN | POLLRDNORM); |
|
|
|
if (events & (POLLOUT | POLLWRNORM)) |
|
if (sowritable(so)) |
|
revents |= events & (POLLOUT | POLLWRNORM); |
|
|
|
if (events & (POLLPRI | POLLRDBAND)) |
|
if (so->so_oobmark || (so->so_state & SS_RCVATMARK)) |
|
revents |= events & (POLLPRI | POLLRDBAND); |
|
|
|
return revents; |
|
} |
|
|
|
int |
|
sopoll(struct socket *so, int events) |
|
{ |
|
int revents = 0; |
|
|
|
#ifndef DIAGNOSTIC |
|
/* |
|
* Do a quick, unlocked check in expectation that the socket |
|
* will be ready for I/O. Don't do this check if DIAGNOSTIC, |
|
* as the solocked() assertions will fail. |
|
*/ |
|
if ((revents = sodopoll(so, events)) != 0) |
|
return revents; |
|
#endif |
|
|
|
solock(so); |
|
if ((revents = sodopoll(so, events)) == 0) { |
|
if (events & (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) { |
|
selrecord(curlwp, &so->so_rcv.sb_sel); |
|
so->so_rcv.sb_flags |= SB_NOTIFY; |
|
} |
|
|
|
if (events & (POLLOUT | POLLWRNORM)) { |
|
selrecord(curlwp, &so->so_snd.sb_sel); |
|
so->so_snd.sb_flags |= SB_NOTIFY; |
|
} |
|
} |
|
sounlock(so); |
|
|
|
return revents; |
|
} |
|
|
|
|
|
#include <sys/sysctl.h> |
|
|
|
static int sysctl_kern_somaxkva(SYSCTLFN_PROTO); |
|
|
|
/* |
|
* sysctl helper routine for kern.somaxkva. ensures that the given |
|
* value is not too small. |
|
* (XXX should we maybe make sure it's not too large as well?) |
|
*/ |
|
static int |
|
sysctl_kern_somaxkva(SYSCTLFN_ARGS) |
|
{ |
|
int error, new_somaxkva; |
|
struct sysctlnode node; |
|
|
|
new_somaxkva = somaxkva; |
|
node = *rnode; |
|
node.sysctl_data = &new_somaxkva; |
|
error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
|
if (error || newp == NULL) |
|
return (error); |
|
|
|
if (new_somaxkva < (16 * 1024 * 1024)) /* sanity */ |
|
return (EINVAL); |
|
|
|
mutex_enter(&so_pendfree_lock); |
|
somaxkva = new_somaxkva; |
|
cv_broadcast(&socurkva_cv); |
|
mutex_exit(&so_pendfree_lock); |
|
|
|
return (error); |
|
} |
|
|
|
static void |
|
sysctl_kern_somaxkva_setup(void) |
|
{ |
|
|
|
KASSERT(socket_sysctllog == NULL); |
|
sysctl_createv(&socket_sysctllog, 0, NULL, NULL, |
|
CTLFLAG_PERMANENT, |
|
CTLTYPE_NODE, "kern", NULL, |
|
NULL, 0, NULL, 0, |
|
CTL_KERN, CTL_EOL); |
|
|
|
sysctl_createv(&socket_sysctllog, 0, NULL, NULL, |
|
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, |
|
CTLTYPE_INT, "somaxkva", |
|
SYSCTL_DESCR("Maximum amount of kernel memory to be " |
|
"used for socket buffers"), |
|
sysctl_kern_somaxkva, 0, NULL, 0, |
|
CTL_KERN, KERN_SOMAXKVA, CTL_EOL); |
|
} |