version 1.151.6.4, 2009/01/17 13:29:19 |
version 1.164, 2008/07/04 16:41:00 |
Line 180 TAILQ_HEAD(,pool_cache) pool_cache_head |
|
Line 180 TAILQ_HEAD(,pool_cache) pool_cache_head |
|
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
|
|
int pool_cache_disable; /* global disable for caching */ |
int pool_cache_disable; /* global disable for caching */ |
static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ |
static pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ |
|
|
static bool pool_cache_put_slow(pool_cache_cpu_t *, int, |
static bool pool_cache_put_slow(pool_cache_cpu_t *, int, |
void *); |
void *); |
Line 864 pool_init(struct pool *pp, size_t size, |
|
Line 864 pool_init(struct pool *pp, size_t size, |
|
if (__predict_true(!cold)) |
if (__predict_true(!cold)) |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* Insert this into the list of pools using this allocator. */ |
/* Insert this into the list of pools using this allocator. */ |
if (__predict_true(!cold)) |
if (__predict_true(!cold)) |
mutex_enter(&palloc->pa_lock); |
mutex_enter(&palloc->pa_lock); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
Line 1253 pool_do_put(struct pool *pp, void *v, st |
|
Line 1253 pool_do_put(struct pool *pp, void *v, st |
|
|
|
if (pp->pr_flags & PR_WANTED) { |
if (pp->pr_flags & PR_WANTED) { |
pp->pr_flags &= ~PR_WANTED; |
pp->pr_flags &= ~PR_WANTED; |
|
if (ph->ph_nmissing == 0) |
|
pp->pr_nidle++; |
cv_broadcast(&pp->pr_cv); |
cv_broadcast(&pp->pr_cv); |
|
return; |
} |
} |
|
|
/* |
/* |
Line 1527 pool_update_curpage(struct pool *pp) |
|
Line 1530 pool_update_curpage(struct pool *pp) |
|
if (pp->pr_curpage == NULL) { |
if (pp->pr_curpage == NULL) { |
pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); |
pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); |
} |
} |
KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) || |
|
(pp->pr_curpage != NULL && pp->pr_nitems > 0)); |
|
} |
} |
|
|
void |
void |
Line 2210 pool_cache_cpu_init1(struct cpu_info *ci |
|
Line 2211 pool_cache_cpu_init1(struct cpu_info *ci |
|
cc->cc_cpuindex = index; |
cc->cc_cpuindex = index; |
cc->cc_hits = 0; |
cc->cc_hits = 0; |
cc->cc_misses = 0; |
cc->cc_misses = 0; |
cc->cc_current = __UNCONST(&pcg_dummy); |
cc->cc_current = &pcg_dummy; |
cc->cc_previous = __UNCONST(&pcg_dummy); |
cc->cc_previous = &pcg_dummy; |
|
|
pc->pc_cpus[index] = cc; |
pc->pc_cpus[index] = cc; |
} |
} |
Line 2369 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2370 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pool_cache_t pc; |
pool_cache_t pc; |
void *object; |
void *object; |
|
|
KASSERT(cc->cc_current->pcg_avail == 0); |
|
KASSERT(cc->cc_previous->pcg_avail == 0); |
|
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
|
|
Line 2473 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2471 pool_cache_get_paddr(pool_cache_t pc, in |
|
|
|
/* Lock out interrupts and disable preemption. */ |
/* Lock out interrupts and disable preemption. */ |
s = splvm(); |
s = splvm(); |
while (/* CONSTCOND */ true) { |
do { |
/* Try and allocate an object from the current group. */ |
/* Try and allocate an object from the current group. */ |
cc = pc->pc_cpus[curcpu()->ci_index]; |
cc = pc->pc_cpus[curcpu()->ci_index]; |
KASSERT(cc->cc_cache == pc); |
KASSERT(cc->cc_cache == pc); |
Line 2510 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2508 pool_cache_get_paddr(pool_cache_t pc, in |
|
* no more objects are available, it will return false. |
* no more objects are available, it will return false. |
* Otherwise, we need to retry. |
* Otherwise, we need to retry. |
*/ |
*/ |
if (!pool_cache_get_slow(cc, s, &object, pap, flags)) |
} while (pool_cache_get_slow(cc, s, &object, pap, flags)); |
break; |
|
} |
|
|
|
return object; |
return object; |
} |
} |
Line 2524 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2520 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
|
|
KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size); |
|
KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size); |
|
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
pcg = NULL; |
|
cc->cc_misses++; |
cc->cc_misses++; |
|
|
/* |
|
* If there are no empty groups in the cache then allocate one |
|
* while still unlocked. |
|
*/ |
|
if (__predict_false(pc->pc_emptygroups == NULL)) { |
|
if (__predict_true(!pool_cache_disable)) { |
|
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
|
} |
|
if (__predict_true(pcg != NULL)) { |
|
pcg->pcg_avail = 0; |
|
pcg->pcg_size = pc->pc_pcgsize; |
|
} |
|
} |
|
|
|
/* Lock the cache. */ |
/* Lock the cache. */ |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
ncsw = curlwp->l_ncsw; |
ncsw = curlwp->l_ncsw; |
Line 2557 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2535 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
*/ |
*/ |
if (__predict_false(curlwp->l_ncsw != ncsw)) { |
if (__predict_false(curlwp->l_ncsw != ncsw)) { |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
if (pcg != NULL) { |
|
pool_put(pc->pc_pcgpool, pcg); |
|
} |
|
return true; |
return true; |
} |
} |
} |
} |
|
|
/* If there are no empty groups in the cache then allocate one. */ |
/* If there are no empty groups in the cache then allocate one. */ |
if (pcg == NULL && pc->pc_emptygroups != NULL) { |
if (__predict_false((pcg = pc->pc_emptygroups) == NULL)) { |
pcg = pc->pc_emptygroups; |
if (__predict_true(!pool_cache_disable)) { |
|
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
|
} |
|
if (__predict_true(pcg != NULL)) { |
|
pcg->pcg_avail = 0; |
|
pcg->pcg_size = pc->pc_pcgsize; |
|
} |
|
} else { |
pc->pc_emptygroups = pcg->pcg_next; |
pc->pc_emptygroups = pcg->pcg_next; |
pc->pc_nempty--; |
pc->pc_nempty--; |
} |
} |
Line 2625 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2607 pool_cache_put_paddr(pool_cache_t pc, vo |
|
|
|
/* Lock out interrupts and disable preemption. */ |
/* Lock out interrupts and disable preemption. */ |
s = splvm(); |
s = splvm(); |
while (/* CONSTCOND */ true) { |
do { |
/* If the current group isn't full, release it there. */ |
/* If the current group isn't full, release it there. */ |
cc = pc->pc_cpus[curcpu()->ci_index]; |
cc = pc->pc_cpus[curcpu()->ci_index]; |
KASSERT(cc->cc_cache == pc); |
KASSERT(cc->cc_cache == pc); |
Line 2655 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2637 pool_cache_put_paddr(pool_cache_t pc, vo |
|
* If put_slow() releases the object for us, it |
* If put_slow() releases the object for us, it |
* will return false. Otherwise we need to retry. |
* will return false. Otherwise we need to retry. |
*/ |
*/ |
if (!pool_cache_put_slow(cc, s, object)) |
} while (pool_cache_put_slow(cc, s, object)); |
break; |
|
} |
|
} |
} |
|
|
/* |
/* |
Line 2677 pool_cache_xcall(pool_cache_t pc) |
|
Line 2657 pool_cache_xcall(pool_cache_t pc) |
|
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
cc = pc->pc_cpus[curcpu()->ci_index]; |
cc = pc->pc_cpus[curcpu()->ci_index]; |
cur = cc->cc_current; |
cur = cc->cc_current; |
cc->cc_current = __UNCONST(&pcg_dummy); |
cc->cc_current = &pcg_dummy; |
prev = cc->cc_previous; |
prev = cc->cc_previous; |
cc->cc_previous = __UNCONST(&pcg_dummy); |
cc->cc_previous = &pcg_dummy; |
if (cur != &pcg_dummy) { |
if (cur != &pcg_dummy) { |
if (cur->pcg_avail == cur->pcg_size) { |
if (cur->pcg_avail == cur->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
|
|
if (pool_in_cg(pp, cc->cc_current, addr) || |
if (pool_in_cg(pp, cc->cc_current, addr) || |
pool_in_cg(pp, cc->cc_previous, addr)) { |
pool_in_cg(pp, cc->cc_previous, addr)) { |
struct cpu_info *ci = |
struct cpu_info *ci = |
cpu_lookup(i); |
cpu_lookup_byindex(i); |
|
|
incpucache = true; |
incpucache = true; |
snprintf(cpucachestr, |
snprintf(cpucachestr, |