version 1.160.2.2, 2008/09/18 04:31:43 |
version 1.161, 2008/05/31 13:31:25 |
Line 179 static struct pool cache_cpu_pool; |
|
Line 179 static struct pool cache_cpu_pool; |
|
TAILQ_HEAD(,pool_cache) pool_cache_head = |
TAILQ_HEAD(,pool_cache) pool_cache_head = |
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
|
|
int pool_cache_disable; /* global disable for caching */ |
int pool_cache_disable; |
static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ |
|
|
|
static bool pool_cache_put_slow(pool_cache_cpu_t *, int, |
|
void *); |
static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, |
static bool pool_cache_get_slow(pool_cache_cpu_t *, int, |
void *, paddr_t); |
void **, paddr_t *, int); |
static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, |
|
void **, paddr_t *, int); |
static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t); |
static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t); |
static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *); |
static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *); |
static void pool_cache_xcall(pool_cache_t); |
static void pool_cache_xcall(pool_cache_t); |
Line 864 pool_init(struct pool *pp, size_t size, |
|
Line 864 pool_init(struct pool *pp, size_t size, |
|
if (__predict_true(!cold)) |
if (__predict_true(!cold)) |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* Insert this into the list of pools using this allocator. */ |
/* Insert this into the list of pools using this allocator. */ |
if (__predict_true(!cold)) |
if (__predict_true(!cold)) |
mutex_enter(&palloc->pa_lock); |
mutex_enter(&palloc->pa_lock); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
Line 1253 pool_do_put(struct pool *pp, void *v, st |
|
Line 1253 pool_do_put(struct pool *pp, void *v, st |
|
|
|
if (pp->pr_flags & PR_WANTED) { |
if (pp->pr_flags & PR_WANTED) { |
pp->pr_flags &= ~PR_WANTED; |
pp->pr_flags &= ~PR_WANTED; |
|
if (ph->ph_nmissing == 0) |
|
pp->pr_nidle++; |
cv_broadcast(&pp->pr_cv); |
cv_broadcast(&pp->pr_cv); |
|
return; |
} |
} |
|
|
/* |
/* |
Line 1527 pool_update_curpage(struct pool *pp) |
|
Line 1530 pool_update_curpage(struct pool *pp) |
|
if (pp->pr_curpage == NULL) { |
if (pp->pr_curpage == NULL) { |
pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); |
pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); |
} |
} |
KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) || |
|
(pp->pr_curpage != NULL && pp->pr_nitems > 0)); |
|
} |
} |
|
|
void |
void |
Line 2084 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2085 pool_cache_bootstrap(pool_cache_t pc, si |
|
|
|
if ((flags & PR_LARGECACHE) != 0) { |
if ((flags & PR_LARGECACHE) != 0) { |
pc->pc_pcgsize = PCG_NOBJECTS_LARGE; |
pc->pc_pcgsize = PCG_NOBJECTS_LARGE; |
pc->pc_pcgpool = &pcg_large_pool; |
|
} else { |
} else { |
pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; |
pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; |
pc->pc_pcgpool = &pcg_normal_pool; |
|
} |
} |
|
|
/* Allocate per-CPU caches. */ |
/* Allocate per-CPU caches. */ |
Line 2152 pool_cache_destroy(pool_cache_t pc) |
|
Line 2151 pool_cache_destroy(pool_cache_t pc) |
|
for (i = 0; i < MAXCPUS; i++) { |
for (i = 0; i < MAXCPUS; i++) { |
if ((cc = pc->pc_cpus[i]) == NULL) |
if ((cc = pc->pc_cpus[i]) == NULL) |
continue; |
continue; |
if ((pcg = cc->cc_current) != &pcg_dummy) { |
if ((pcg = cc->cc_current) != NULL) { |
pcg->pcg_next = NULL; |
pcg->pcg_next = NULL; |
pool_cache_invalidate_groups(pc, pcg); |
pool_cache_invalidate_groups(pc, pcg); |
} |
} |
if ((pcg = cc->cc_previous) != &pcg_dummy) { |
if ((pcg = cc->cc_previous) != NULL) { |
pcg->pcg_next = NULL; |
pcg->pcg_next = NULL; |
pool_cache_invalidate_groups(pc, pcg); |
pool_cache_invalidate_groups(pc, pcg); |
} |
} |
Line 2210 pool_cache_cpu_init1(struct cpu_info *ci |
|
Line 2209 pool_cache_cpu_init1(struct cpu_info *ci |
|
cc->cc_cpuindex = index; |
cc->cc_cpuindex = index; |
cc->cc_hits = 0; |
cc->cc_hits = 0; |
cc->cc_misses = 0; |
cc->cc_misses = 0; |
cc->cc_current = __UNCONST(&pcg_dummy); |
cc->cc_current = NULL; |
cc->cc_previous = __UNCONST(&pcg_dummy); |
cc->cc_previous = NULL; |
|
|
pc->pc_cpus[index] = cc; |
pc->pc_cpus[index] = cc; |
} |
} |
Line 2360 pool_cache_sethardlimit(pool_cache_t pc, |
|
Line 2359 pool_cache_sethardlimit(pool_cache_t pc, |
|
pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap); |
pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap); |
} |
} |
|
|
static bool __noinline |
static inline pool_cache_cpu_t * |
pool_cache_get_slow(pool_cache_cpu_t *cc, int s, void **objectp, |
pool_cache_cpu_enter(pool_cache_t pc, int *s) |
|
{ |
|
pool_cache_cpu_t *cc; |
|
|
|
/* |
|
* Prevent other users of the cache from accessing our |
|
* CPU-local data. To avoid touching shared state, we |
|
* pull the neccessary information from CPU local data. |
|
*/ |
|
KPREEMPT_DISABLE(curlwp); |
|
cc = pc->pc_cpus[curcpu()->ci_index]; |
|
KASSERT(cc->cc_cache == pc); |
|
if (cc->cc_ipl != IPL_NONE) { |
|
*s = splraiseipl(cc->cc_iplcookie); |
|
} |
|
|
|
return cc; |
|
} |
|
|
|
static inline void |
|
pool_cache_cpu_exit(pool_cache_cpu_t *cc, int *s) |
|
{ |
|
|
|
/* No longer need exclusive access to the per-CPU data. */ |
|
if (cc->cc_ipl != IPL_NONE) { |
|
splx(*s); |
|
} |
|
KPREEMPT_ENABLE(curlwp); |
|
} |
|
|
|
pool_cache_cpu_t * __noinline |
|
pool_cache_get_slow(pool_cache_cpu_t *cc, int *s, void **objectp, |
paddr_t *pap, int flags) |
paddr_t *pap, int flags) |
{ |
{ |
pcg_t *pcg, *cur; |
pcg_t *pcg, *cur; |
Line 2369 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2399 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pool_cache_t pc; |
pool_cache_t pc; |
void *object; |
void *object; |
|
|
KASSERT(cc->cc_current->pcg_avail == 0); |
|
KASSERT(cc->cc_previous->pcg_avail == 0); |
|
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
|
|
Line 2379 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2406 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
* Nothing was available locally. Try and grab a group |
* Nothing was available locally. Try and grab a group |
* from the cache. |
* from the cache. |
*/ |
*/ |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
if (!mutex_tryenter(&pc->pc_lock)) { |
ncsw = curlwp->l_ncsw; |
ncsw = curlwp->l_ncsw; |
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
pc->pc_contended++; |
pc->pc_contended++; |
Line 2391 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2418 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
*/ |
*/ |
if (curlwp->l_ncsw != ncsw) { |
if (curlwp->l_ncsw != ncsw) { |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
return true; |
pool_cache_cpu_exit(cc, s); |
|
return pool_cache_cpu_enter(pc, s); |
} |
} |
} |
} |
|
|
if (__predict_true((pcg = pc->pc_fullgroups) != NULL)) { |
if ((pcg = pc->pc_fullgroups) != NULL) { |
/* |
/* |
* If there's a full group, release our empty |
* If there's a full group, release our empty |
* group back to the cache. Install the full |
* group back to the cache. Install the full |
* group as cc_current and return. |
* group as cc_current and return. |
*/ |
*/ |
if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) { |
if ((cur = cc->cc_current) != NULL) { |
KASSERT(cur->pcg_avail == 0); |
KASSERT(cur->pcg_avail == 0); |
cur->pcg_next = pc->pc_emptygroups; |
cur->pcg_next = pc->pc_emptygroups; |
pc->pc_emptygroups = cur; |
pc->pc_emptygroups = cur; |
Line 2413 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2441 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pc->pc_hits++; |
pc->pc_hits++; |
pc->pc_nfull--; |
pc->pc_nfull--; |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
return true; |
return cc; |
} |
} |
|
|
/* |
/* |
Line 2423 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2451 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
*/ |
*/ |
pc->pc_misses++; |
pc->pc_misses++; |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
splx(s); |
pool_cache_cpu_exit(cc, s); |
|
|
object = pool_get(&pc->pc_pool, flags); |
object = pool_get(&pc->pc_pool, flags); |
*objectp = object; |
*objectp = object; |
if (__predict_false(object == NULL)) |
if (object == NULL) |
return false; |
return NULL; |
|
|
if (__predict_false((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0)) { |
if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) { |
pool_put(&pc->pc_pool, object); |
pool_put(&pc->pc_pool, object); |
*objectp = NULL; |
*objectp = NULL; |
return false; |
return NULL; |
} |
} |
|
|
KASSERT((((vaddr_t)object + pc->pc_pool.pr_itemoffset) & |
KASSERT((((vaddr_t)object + pc->pc_pool.pr_itemoffset) & |
Line 2448 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2476 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
} |
} |
|
|
FREECHECK_OUT(&pc->pc_freecheck, object); |
FREECHECK_OUT(&pc->pc_freecheck, object); |
return false; |
return NULL; |
} |
} |
|
|
/* |
/* |
Line 2471 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2499 pool_cache_get_paddr(pool_cache_t pc, in |
|
} |
} |
#endif |
#endif |
|
|
/* Lock out interrupts and disable preemption. */ |
cc = pool_cache_cpu_enter(pc, &s); |
s = splvm(); |
do { |
while (/* CONSTCOND */ true) { |
|
/* Try and allocate an object from the current group. */ |
/* Try and allocate an object from the current group. */ |
cc = pc->pc_cpus[curcpu()->ci_index]; |
|
KASSERT(cc->cc_cache == pc); |
|
pcg = cc->cc_current; |
pcg = cc->cc_current; |
if (__predict_true(pcg->pcg_avail > 0)) { |
if (pcg != NULL && pcg->pcg_avail > 0) { |
object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va; |
object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va; |
if (__predict_false(pap != NULL)) |
if (pap != NULL) |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
#if defined(DIAGNOSTIC) |
#if defined(DIAGNOSTIC) |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
KASSERT(pcg->pcg_avail < pcg->pcg_size); |
#endif /* defined(DIAGNOSTIC) */ |
|
KASSERT(pcg->pcg_avail <= pcg->pcg_size); |
KASSERT(object != NULL); |
KASSERT(object != NULL); |
#endif |
|
cc->cc_hits++; |
cc->cc_hits++; |
splx(s); |
pool_cache_cpu_exit(cc, &s); |
FREECHECK_OUT(&pc->pc_freecheck, object); |
FREECHECK_OUT(&pc->pc_freecheck, object); |
return object; |
return object; |
} |
} |
Line 2498 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2523 pool_cache_get_paddr(pool_cache_t pc, in |
|
* it with the current group and allocate from there. |
* it with the current group and allocate from there. |
*/ |
*/ |
pcg = cc->cc_previous; |
pcg = cc->cc_previous; |
if (__predict_true(pcg->pcg_avail > 0)) { |
if (pcg != NULL && pcg->pcg_avail > 0) { |
cc->cc_previous = cc->cc_current; |
cc->cc_previous = cc->cc_current; |
cc->cc_current = pcg; |
cc->cc_current = pcg; |
continue; |
continue; |
Line 2507 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2532 pool_cache_get_paddr(pool_cache_t pc, in |
|
/* |
/* |
* Can't allocate from either group: try the slow path. |
* Can't allocate from either group: try the slow path. |
* If get_slow() allocated an object for us, or if |
* If get_slow() allocated an object for us, or if |
* no more objects are available, it will return false. |
* no more objects are available, it will return NULL. |
* Otherwise, we need to retry. |
* Otherwise, we need to retry. |
*/ |
*/ |
if (!pool_cache_get_slow(cc, s, &object, pap, flags)) |
cc = pool_cache_get_slow(cc, &s, &object, pap, flags); |
break; |
} while (cc != NULL); |
} |
|
|
|
return object; |
return object; |
} |
} |
|
|
static bool __noinline |
pool_cache_cpu_t * __noinline |
pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object) |
pool_cache_put_slow(pool_cache_cpu_t *cc, int *s, void *object, paddr_t pa) |
{ |
{ |
pcg_t *pcg, *cur; |
pcg_t *pcg, *cur; |
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
|
u_int nobj; |
KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size); |
|
KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size); |
|
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
|
|
/* Lock the cache. */ |
/* |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
* No free slots locally. Try to grab an empty, unused |
|
* group from the cache. |
|
*/ |
|
if (!mutex_tryenter(&pc->pc_lock)) { |
ncsw = curlwp->l_ncsw; |
ncsw = curlwp->l_ncsw; |
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
pc->pc_contended++; |
pc->pc_contended++; |
|
|
/* |
/* |
* If we context switched while locking, then our view of |
* If we context switched while locking, then |
* the per-CPU data is invalid: retry. |
* our view of the per-CPU data is invalid: |
|
* retry. |
*/ |
*/ |
if (__predict_false(curlwp->l_ncsw != ncsw)) { |
if (curlwp->l_ncsw != ncsw) { |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
return true; |
pool_cache_cpu_exit(cc, s); |
|
return pool_cache_cpu_enter(pc, s); |
} |
} |
} |
} |
|
|
/* If there are no empty groups in the cache then allocate one. */ |
if ((pcg = pc->pc_emptygroups) != NULL) { |
if (__predict_false((pcg = pc->pc_emptygroups) == NULL)) { |
/* |
if (__predict_true(!pool_cache_disable)) { |
* If there's a empty group, release our full |
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
* group back to the cache. Install the empty |
} |
* group and return. |
if (__predict_true(pcg != NULL)) { |
*/ |
pcg->pcg_avail = 0; |
|
pcg->pcg_size = pc->pc_pcgsize; |
|
} |
|
} else { |
|
pc->pc_emptygroups = pcg->pcg_next; |
|
pc->pc_nempty--; |
|
} |
|
|
|
/* |
|
* If there's a empty group, release our full group back |
|
* to the cache. Install the empty group to the local CPU |
|
* and return. |
|
*/ |
|
if (pcg != NULL) { |
|
KASSERT(pcg->pcg_avail == 0); |
KASSERT(pcg->pcg_avail == 0); |
if (__predict_false(cc->cc_previous == &pcg_dummy)) { |
pc->pc_emptygroups = pcg->pcg_next; |
|
if (cc->cc_previous == NULL) { |
cc->cc_previous = pcg; |
cc->cc_previous = pcg; |
} else { |
} else { |
cur = cc->cc_current; |
if ((cur = cc->cc_current) != NULL) { |
if (__predict_true(cur != &pcg_dummy)) { |
KASSERT(cur->pcg_avail == pcg->pcg_size); |
KASSERT(cur->pcg_avail == cur->pcg_size); |
|
cur->pcg_next = pc->pc_fullgroups; |
cur->pcg_next = pc->pc_fullgroups; |
pc->pc_fullgroups = cur; |
pc->pc_fullgroups = cur; |
pc->pc_nfull++; |
pc->pc_nfull++; |
Line 2580 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2593 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
cc->cc_current = pcg; |
cc->cc_current = pcg; |
} |
} |
pc->pc_hits++; |
pc->pc_hits++; |
|
pc->pc_nempty--; |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
return true; |
return cc; |
} |
} |
|
|
/* |
/* |
* Nothing available locally or in cache, and we didn't |
* Nothing available locally or in cache. Take the |
* allocate an empty group. Take the slow path and destroy |
* slow path and try to allocate a new group that we |
* the object here and now. |
* can release to. |
*/ |
*/ |
pc->pc_misses++; |
pc->pc_misses++; |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
splx(s); |
pool_cache_cpu_exit(cc, s); |
pool_cache_destruct_object(pc, object); |
|
|
|
return false; |
/* |
|
* If we can't allocate a new group, just throw the |
|
* object away. |
|
*/ |
|
nobj = pc->pc_pcgsize; |
|
if (pool_cache_disable) { |
|
pcg = NULL; |
|
} else if (nobj == PCG_NOBJECTS_LARGE) { |
|
pcg = pool_get(&pcg_large_pool, PR_NOWAIT); |
|
} else { |
|
pcg = pool_get(&pcg_normal_pool, PR_NOWAIT); |
|
} |
|
if (pcg == NULL) { |
|
pool_cache_destruct_object(pc, object); |
|
return NULL; |
|
} |
|
pcg->pcg_avail = 0; |
|
pcg->pcg_size = nobj; |
|
|
|
/* |
|
* Add the empty group to the cache and try again. |
|
*/ |
|
mutex_enter(&pc->pc_lock); |
|
pcg->pcg_next = pc->pc_emptygroups; |
|
pc->pc_emptygroups = pcg; |
|
pc->pc_nempty++; |
|
mutex_exit(&pc->pc_lock); |
|
|
|
return pool_cache_cpu_enter(pc, s); |
} |
} |
|
|
/* |
/* |
Line 2612 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2653 pool_cache_put_paddr(pool_cache_t pc, vo |
|
|
|
FREECHECK_IN(&pc->pc_freecheck, object); |
FREECHECK_IN(&pc->pc_freecheck, object); |
|
|
/* Lock out interrupts and disable preemption. */ |
cc = pool_cache_cpu_enter(pc, &s); |
s = splvm(); |
do { |
while (/* CONSTCOND */ true) { |
|
/* If the current group isn't full, release it there. */ |
/* If the current group isn't full, release it there. */ |
cc = pc->pc_cpus[curcpu()->ci_index]; |
|
KASSERT(cc->cc_cache == pc); |
|
pcg = cc->cc_current; |
pcg = cc->cc_current; |
if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { |
if (pcg != NULL && pcg->pcg_avail < pcg->pcg_size) { |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_avail++; |
pcg->pcg_avail++; |
cc->cc_hits++; |
cc->cc_hits++; |
splx(s); |
pool_cache_cpu_exit(cc, &s); |
return; |
return; |
} |
} |
|
|
/* |
/* |
* That failed. If the previous group isn't full, swap |
* That failed. If the previous group is empty, swap |
* it with the current group and try again. |
* it with the current group and try again. |
*/ |
*/ |
pcg = cc->cc_previous; |
pcg = cc->cc_previous; |
if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { |
if (pcg != NULL && pcg->pcg_avail == 0) { |
cc->cc_previous = cc->cc_current; |
cc->cc_previous = cc->cc_current; |
cc->cc_current = pcg; |
cc->cc_current = pcg; |
continue; |
continue; |
Line 2642 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2680 pool_cache_put_paddr(pool_cache_t pc, vo |
|
/* |
/* |
* Can't free to either group: try the slow path. |
* Can't free to either group: try the slow path. |
* If put_slow() releases the object for us, it |
* If put_slow() releases the object for us, it |
* will return false. Otherwise we need to retry. |
* will return NULL. Otherwise we need to retry. |
*/ |
*/ |
if (!pool_cache_put_slow(cc, s, object)) |
cc = pool_cache_put_slow(cc, &s, object, pa); |
break; |
} while (cc != NULL); |
} |
|
} |
} |
|
|
/* |
/* |
Line 2660 pool_cache_xcall(pool_cache_t pc) |
|
Line 2697 pool_cache_xcall(pool_cache_t pc) |
|
{ |
{ |
pool_cache_cpu_t *cc; |
pool_cache_cpu_t *cc; |
pcg_t *prev, *cur, **list; |
pcg_t *prev, *cur, **list; |
int s; |
int s = 0; /* XXXgcc */ |
|
|
s = splvm(); |
cc = pool_cache_cpu_enter(pc, &s); |
mutex_enter(&pc->pc_lock); |
|
cc = pc->pc_cpus[curcpu()->ci_index]; |
|
cur = cc->cc_current; |
cur = cc->cc_current; |
cc->cc_current = __UNCONST(&pcg_dummy); |
cc->cc_current = NULL; |
prev = cc->cc_previous; |
prev = cc->cc_previous; |
cc->cc_previous = __UNCONST(&pcg_dummy); |
cc->cc_previous = NULL; |
if (cur != &pcg_dummy) { |
pool_cache_cpu_exit(cc, &s); |
|
|
|
/* |
|
* XXXSMP Go to splvm to prevent kernel_lock from being taken, |
|
* because locks at IPL_SOFTXXX are still spinlocks. Does not |
|
* apply to IPL_SOFTBIO. Cross-call threads do not take the |
|
* kernel_lock. |
|
*/ |
|
s = splvm(); |
|
mutex_enter(&pc->pc_lock); |
|
if (cur != NULL) { |
if (cur->pcg_avail == cur->pcg_size) { |
if (cur->pcg_avail == cur->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
Line 2683 pool_cache_xcall(pool_cache_t pc) |
|
Line 2728 pool_cache_xcall(pool_cache_t pc) |
|
cur->pcg_next = *list; |
cur->pcg_next = *list; |
*list = cur; |
*list = cur; |
} |
} |
if (prev != &pcg_dummy) { |
if (prev != NULL) { |
if (prev->pcg_avail == prev->pcg_size) { |
if (prev->pcg_avail == prev->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |