version 1.111.2.3, 2006/06/21 15:09:39 |
version 1.116.4.1, 2006/03/08 00:53:41 |
Line 91 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 91 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include <sys/pool.h> |
#include <sys/pool.h> |
#include <sys/event.h> |
#include <sys/event.h> |
#include <sys/poll.h> |
#include <sys/poll.h> |
#include <sys/kauth.h> |
|
|
|
#include <uvm/uvm.h> |
#include <uvm/uvm.h> |
|
|
Line 127 EVCNT_ATTACH_STATIC(sosend_kvalimit); |
|
Line 126 EVCNT_ATTACH_STATIC(sosend_kvalimit); |
|
|
|
#endif /* SOSEND_COUNTERS */ |
#endif /* SOSEND_COUNTERS */ |
|
|
static struct callback_entry sokva_reclaimerentry; |
void |
|
soinit(void) |
|
{ |
|
|
|
/* Set the initial adjusted socket buffer size. */ |
|
if (sb_max_set(sb_max)) |
|
panic("bad initial sb_max value: %lu", sb_max); |
|
|
|
} |
|
|
#ifdef SOSEND_NO_LOAN |
#ifdef SOSEND_NO_LOAN |
int sock_loan_thresh = -1; |
int use_sosend_loan = 0; |
#else |
#else |
int sock_loan_thresh = 4096; |
int use_sosend_loan = 1; |
#endif |
#endif |
|
|
static struct simplelock so_pendfree_slock = SIMPLELOCK_INITIALIZER; |
static struct simplelock so_pendfree_slock = SIMPLELOCK_INITIALIZER; |
Line 145 int somaxkva = SOMAXKVA; |
|
Line 152 int somaxkva = SOMAXKVA; |
|
static int socurkva; |
static int socurkva; |
static int sokvawaiters; |
static int sokvawaiters; |
|
|
|
#define SOCK_LOAN_THRESH 4096 |
#define SOCK_LOAN_CHUNK 65536 |
#define SOCK_LOAN_CHUNK 65536 |
|
|
static size_t sodopendfree(void); |
static size_t sodopendfree(struct socket *); |
static size_t sodopendfreel(void); |
static size_t sodopendfreel(struct socket *); |
|
|
static vsize_t |
static vsize_t |
sokvareserve(struct socket *so, vsize_t len) |
sokvareserve(struct socket *so, vsize_t len) |
Line 165 sokvareserve(struct socket *so, vsize_t |
|
Line 173 sokvareserve(struct socket *so, vsize_t |
|
* try to do pendfree. |
* try to do pendfree. |
*/ |
*/ |
|
|
freed = sodopendfreel(); |
freed = sodopendfreel(so); |
|
|
/* |
/* |
* if some kva was freed, try again. |
* if some kva was freed, try again. |
Line 255 sokvafree(vaddr_t sva, vsize_t len) |
|
Line 263 sokvafree(vaddr_t sva, vsize_t len) |
|
} |
} |
|
|
static void |
static void |
sodoloanfree(struct vm_page **pgs, caddr_t buf, size_t size, boolean_t mapped) |
sodoloanfree(struct vm_page **pgs, caddr_t buf, size_t size) |
{ |
{ |
vaddr_t sva, eva; |
vaddr_t va, sva, eva; |
vsize_t len; |
vsize_t len; |
int npgs; |
paddr_t pa; |
|
int i, npgs; |
KASSERT(pgs != NULL); |
|
|
|
eva = round_page((vaddr_t) buf + size); |
eva = round_page((vaddr_t) buf + size); |
sva = trunc_page((vaddr_t) buf); |
sva = trunc_page((vaddr_t) buf); |
len = eva - sva; |
len = eva - sva; |
npgs = len >> PAGE_SHIFT; |
npgs = len >> PAGE_SHIFT; |
|
|
if (mapped) { |
if (__predict_false(pgs == NULL)) { |
pmap_kremove(sva, len); |
pgs = alloca(npgs * sizeof(*pgs)); |
pmap_update(pmap_kernel()); |
|
|
for (i = 0, va = sva; va < eva; i++, va += PAGE_SIZE) { |
|
if (pmap_extract(pmap_kernel(), va, &pa) == FALSE) |
|
panic("sodoloanfree: va 0x%lx not mapped", va); |
|
pgs[i] = PHYS_TO_VM_PAGE(pa); |
|
} |
} |
} |
|
|
|
pmap_kremove(sva, len); |
|
pmap_update(pmap_kernel()); |
uvm_unloan(pgs, npgs, UVM_LOAN_TOPAGE); |
uvm_unloan(pgs, npgs, UVM_LOAN_TOPAGE); |
sokvafree(sva, len); |
sokvafree(sva, len); |
} |
} |
|
|
static size_t |
static size_t |
sodopendfree() |
sodopendfree(struct socket *so) |
{ |
{ |
int s; |
int s; |
size_t rv; |
size_t rv; |
|
|
s = splvm(); |
s = splvm(); |
simple_lock(&so_pendfree_slock); |
simple_lock(&so_pendfree_slock); |
rv = sodopendfreel(); |
rv = sodopendfreel(so); |
simple_unlock(&so_pendfree_slock); |
simple_unlock(&so_pendfree_slock); |
splx(s); |
splx(s); |
|
|
|
|
*/ |
*/ |
|
|
static size_t |
static size_t |
sodopendfreel() |
sodopendfreel(struct socket *so) |
{ |
{ |
size_t rv = 0; |
size_t rv = 0; |
|
|
|
|
|
|
for (; m != NULL; m = next) { |
for (; m != NULL; m = next) { |
next = m->m_next; |
next = m->m_next; |
KASSERT((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0); |
|
KASSERT(!MCLISREFERENCED(m)); |
|
|
|
rv += m->m_ext.ext_size; |
rv += m->m_ext.ext_size; |
sodoloanfree(m->m_ext.ext_pgs, m->m_ext.ext_buf, |
sodoloanfree((m->m_flags & M_EXT_PAGES) ? |
m->m_ext.ext_size, |
m->m_ext.ext_pgs : NULL, m->m_ext.ext_buf, |
(m->m_ext.ext_flags & M_EXT_LAZY) == 0); |
m->m_ext.ext_size); |
pool_cache_put(&mbpool_cache, m); |
pool_cache_put(&mbpool_cache, m); |
} |
} |
|
|
Line 341 soloanfree(struct mbuf *m, caddr_t buf, |
|
Line 354 soloanfree(struct mbuf *m, caddr_t buf, |
|
{ |
{ |
int s; |
int s; |
|
|
KASSERT(m != NULL); |
if (m == NULL) { |
|
|
|
/* |
|
* called from MEXTREMOVE. |
|
*/ |
|
|
|
sodoloanfree(NULL, buf, size); |
|
return; |
|
} |
|
|
/* |
/* |
* postpone freeing mbuf. |
* postpone freeing mbuf. |
Line 366 sosend_loan(struct socket *so, struct ui |
|
Line 387 sosend_loan(struct socket *so, struct ui |
|
struct iovec *iov = uio->uio_iov; |
struct iovec *iov = uio->uio_iov; |
vaddr_t sva, eva; |
vaddr_t sva, eva; |
vsize_t len; |
vsize_t len; |
vaddr_t lva; |
vaddr_t lva, va; |
int npgs, error; |
int npgs, i, error; |
#if !defined(__HAVE_LAZY_MBUF) |
|
vaddr_t va; |
|
int i; |
|
#endif /* !defined(__HAVE_LAZY_MBUF) */ |
|
|
|
if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) |
if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) |
return (0); |
return (0); |
Line 400 sosend_loan(struct socket *so, struct ui |
|
Line 417 sosend_loan(struct socket *so, struct ui |
|
return (0); |
return (0); |
} |
} |
|
|
#if !defined(__HAVE_LAZY_MBUF) |
|
for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE) |
for (i = 0, va = lva; i < npgs; i++, va += PAGE_SIZE) |
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]), |
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(m->m_ext.ext_pgs[i]), |
VM_PROT_READ); |
VM_PROT_READ); |
pmap_update(pmap_kernel()); |
pmap_update(pmap_kernel()); |
#endif /* !defined(__HAVE_LAZY_MBUF) */ |
|
|
|
lva += (vaddr_t) iov->iov_base & PAGE_MASK; |
lva += (vaddr_t) iov->iov_base & PAGE_MASK; |
|
|
MEXTADD(m, (caddr_t) lva, space, M_MBUF, soloanfree, so); |
MEXTADD(m, (caddr_t) lva, space, M_MBUF, soloanfree, so); |
m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP; |
m->m_flags |= M_EXT_PAGES | M_EXT_ROMAP; |
|
|
#if defined(__HAVE_LAZY_MBUF) |
|
m->m_flags |= M_EXT_LAZY; |
|
m->m_ext.ext_flags |= M_EXT_LAZY; |
|
#endif /* defined(__HAVE_LAZY_MBUF) */ |
|
|
|
uio->uio_resid -= space; |
uio->uio_resid -= space; |
/* uio_offset not updated, not set/used for write(2) */ |
/* uio_offset not updated, not set/used for write(2) */ |
uio->uio_iov->iov_base = (caddr_t) uio->uio_iov->iov_base + space; |
uio->uio_iov->iov_base = (caddr_t) uio->uio_iov->iov_base + space; |
Line 429 sosend_loan(struct socket *so, struct ui |
|
Line 439 sosend_loan(struct socket *so, struct ui |
|
return (space); |
return (space); |
} |
} |
|
|
static int |
|
sokva_reclaim_callback(struct callback_entry *ce, void *obj, void *arg) |
|
{ |
|
|
|
KASSERT(ce == &sokva_reclaimerentry); |
|
KASSERT(obj == NULL); |
|
|
|
sodopendfree(); |
|
if (!vm_map_starved_p(kernel_map)) { |
|
return CALLBACK_CHAIN_ABORT; |
|
} |
|
return CALLBACK_CHAIN_CONTINUE; |
|
} |
|
|
|
void |
|
soinit(void) |
|
{ |
|
|
|
/* Set the initial adjusted socket buffer size. */ |
|
if (sb_max_set(sb_max)) |
|
panic("bad initial sb_max value: %lu", sb_max); |
|
|
|
callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback, |
|
&sokva_reclaimerentry, NULL, sokva_reclaim_callback); |
|
} |
|
|
|
/* |
/* |
* Socket operation routines. |
* Socket operation routines. |
* These routines are called by the routines in |
* These routines are called by the routines in |
Line 475 socreate(int dom, struct socket **aso, i |
|
Line 459 socreate(int dom, struct socket **aso, i |
|
prp = pffindproto(dom, proto, type); |
prp = pffindproto(dom, proto, type); |
else |
else |
prp = pffindtype(dom, type); |
prp = pffindtype(dom, type); |
if (prp == 0) { |
if (prp == 0 || prp->pr_usrreq == 0) |
/* no support for domain */ |
|
if (pffinddomain(dom) == 0) |
|
return (EAFNOSUPPORT); |
|
/* no support for socket type */ |
|
if (proto == 0 && type != 0) |
|
return (EPROTOTYPE); |
|
return (EPROTONOSUPPORT); |
|
} |
|
if (prp->pr_usrreq == 0) |
|
return (EPROTONOSUPPORT); |
return (EPROTONOSUPPORT); |
if (prp->pr_type != type) |
if (prp->pr_type != type) |
return (EPROTOTYPE); |
return (EPROTOTYPE); |
Line 730 sodisconnect(struct socket *so) |
|
Line 705 sodisconnect(struct socket *so) |
|
(struct lwp *)0); |
(struct lwp *)0); |
bad: |
bad: |
splx(s); |
splx(s); |
sodopendfree(); |
sodopendfree(so); |
return (error); |
return (error); |
} |
} |
|
|
Line 762 sosend(struct socket *so, struct mbuf *a |
|
Line 737 sosend(struct socket *so, struct mbuf *a |
|
int error, s, dontroute, atomic; |
int error, s, dontroute, atomic; |
|
|
p = l->l_proc; |
p = l->l_proc; |
sodopendfree(); |
sodopendfree(so); |
|
|
clen = 0; |
clen = 0; |
atomic = sosendallatonce(so) || top; |
atomic = sosendallatonce(so) || top; |
Line 850 sosend(struct socket *so, struct mbuf *a |
|
Line 825 sosend(struct socket *so, struct mbuf *a |
|
mlen = MLEN; |
mlen = MLEN; |
} |
} |
MCLAIM(m, so->so_snd.sb_mowner); |
MCLAIM(m, so->so_snd.sb_mowner); |
if (sock_loan_thresh >= 0 && |
if (use_sosend_loan && |
uio->uio_iov->iov_len >= sock_loan_thresh && |
uio->uio_iov->iov_len >= SOCK_LOAN_THRESH && |
space >= sock_loan_thresh && |
space >= SOCK_LOAN_THRESH && |
(len = sosend_loan(so, uio, m, |
(len = sosend_loan(so, uio, m, |
space)) != 0) { |
space)) != 0) { |
SOSEND_COUNTER_INCR(&sosend_loan_big); |
SOSEND_COUNTER_INCR(&sosend_loan_big); |
Line 980 soreceive(struct socket *so, struct mbuf |
|
Line 955 soreceive(struct socket *so, struct mbuf |
|
flags = 0; |
flags = 0; |
|
|
if ((flags & MSG_DONTWAIT) == 0) |
if ((flags & MSG_DONTWAIT) == 0) |
sodopendfree(); |
sodopendfree(so); |
|
|
if (flags & MSG_OOB) { |
if (flags & MSG_OOB) { |
m = m_get(M_WAIT, MT_DATA); |
m = m_get(M_WAIT, MT_DATA); |