version 1.161.2.1, 2008/07/18 16:37:49 |
version 1.172, 2009/04/15 11:45:18 |
Line 180 TAILQ_HEAD(,pool_cache) pool_cache_head |
|
Line 180 TAILQ_HEAD(,pool_cache) pool_cache_head |
|
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
|
|
int pool_cache_disable; /* global disable for caching */ |
int pool_cache_disable; /* global disable for caching */ |
static pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ |
static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ |
|
|
static bool pool_cache_put_slow(pool_cache_cpu_t *, int, |
static bool pool_cache_put_slow(pool_cache_cpu_t *, int, |
void *); |
void *); |
Line 864 pool_init(struct pool *pp, size_t size, |
|
Line 864 pool_init(struct pool *pp, size_t size, |
|
if (__predict_true(!cold)) |
if (__predict_true(!cold)) |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* Insert this into the list of pools using this allocator. */ |
/* Insert this into the list of pools using this allocator. */ |
if (__predict_true(!cold)) |
if (__predict_true(!cold)) |
mutex_enter(&palloc->pa_lock); |
mutex_enter(&palloc->pa_lock); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
Line 1527 pool_update_curpage(struct pool *pp) |
|
Line 1527 pool_update_curpage(struct pool *pp) |
|
if (pp->pr_curpage == NULL) { |
if (pp->pr_curpage == NULL) { |
pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); |
pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); |
} |
} |
|
KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) || |
|
(pp->pr_curpage != NULL && pp->pr_nitems > 0)); |
} |
} |
|
|
void |
void |
Line 2208 pool_cache_cpu_init1(struct cpu_info *ci |
|
Line 2210 pool_cache_cpu_init1(struct cpu_info *ci |
|
cc->cc_cpuindex = index; |
cc->cc_cpuindex = index; |
cc->cc_hits = 0; |
cc->cc_hits = 0; |
cc->cc_misses = 0; |
cc->cc_misses = 0; |
cc->cc_current = &pcg_dummy; |
cc->cc_current = __UNCONST(&pcg_dummy); |
cc->cc_previous = &pcg_dummy; |
cc->cc_previous = __UNCONST(&pcg_dummy); |
|
|
pc->pc_cpus[index] = cc; |
pc->pc_cpus[index] = cc; |
} |
} |
Line 2367 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2369 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pool_cache_t pc; |
pool_cache_t pc; |
void *object; |
void *object; |
|
|
|
KASSERT(cc->cc_current->pcg_avail == 0); |
|
KASSERT(cc->cc_previous->pcg_avail == 0); |
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
|
|
Line 2519 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2524 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
|
|
|
KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size); |
|
KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size); |
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
|
pcg = NULL; |
cc->cc_misses++; |
cc->cc_misses++; |
|
|
|
/* |
|
* If there are no empty groups in the cache then allocate one |
|
* while still unlocked. |
|
*/ |
|
if (__predict_false(pc->pc_emptygroups == NULL)) { |
|
if (__predict_true(!pool_cache_disable)) { |
|
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
|
} |
|
if (__predict_true(pcg != NULL)) { |
|
pcg->pcg_avail = 0; |
|
pcg->pcg_size = pc->pc_pcgsize; |
|
} |
|
} |
|
|
/* Lock the cache. */ |
/* Lock the cache. */ |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
ncsw = curlwp->l_ncsw; |
ncsw = curlwp->l_ncsw; |
Line 2534 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2557 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
*/ |
*/ |
if (__predict_false(curlwp->l_ncsw != ncsw)) { |
if (__predict_false(curlwp->l_ncsw != ncsw)) { |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
|
if (pcg != NULL) { |
|
pool_put(pc->pc_pcgpool, pcg); |
|
} |
return true; |
return true; |
} |
} |
} |
} |
|
|
/* If there are no empty groups in the cache then allocate one. */ |
/* If there are no empty groups in the cache then allocate one. */ |
if (__predict_false((pcg = pc->pc_emptygroups) == NULL)) { |
if (pcg == NULL && pc->pc_emptygroups != NULL) { |
if (__predict_true(!pool_cache_disable)) { |
pcg = pc->pc_emptygroups; |
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
|
} |
|
if (__predict_true(pcg != NULL)) { |
|
pcg->pcg_avail = 0; |
|
pcg->pcg_size = pc->pc_pcgsize; |
|
} |
|
} else { |
|
pc->pc_emptygroups = pcg->pcg_next; |
pc->pc_emptygroups = pcg->pcg_next; |
pc->pc_nempty--; |
pc->pc_nempty--; |
} |
} |
Line 2602 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2621 pool_cache_put_paddr(pool_cache_t pc, vo |
|
pcg_t *pcg; |
pcg_t *pcg; |
int s; |
int s; |
|
|
|
KASSERT(object != NULL); |
FREECHECK_IN(&pc->pc_freecheck, object); |
FREECHECK_IN(&pc->pc_freecheck, object); |
|
|
/* Lock out interrupts and disable preemption. */ |
/* Lock out interrupts and disable preemption. */ |
Line 2658 pool_cache_xcall(pool_cache_t pc) |
|
Line 2678 pool_cache_xcall(pool_cache_t pc) |
|
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
cc = pc->pc_cpus[curcpu()->ci_index]; |
cc = pc->pc_cpus[curcpu()->ci_index]; |
cur = cc->cc_current; |
cur = cc->cc_current; |
cc->cc_current = &pcg_dummy; |
cc->cc_current = __UNCONST(&pcg_dummy); |
prev = cc->cc_previous; |
prev = cc->cc_previous; |
cc->cc_previous = &pcg_dummy; |
cc->cc_previous = __UNCONST(&pcg_dummy); |
if (cur != &pcg_dummy) { |
if (cur != &pcg_dummy) { |
if (cur->pcg_avail == cur->pcg_size) { |
if (cur->pcg_avail == cur->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
|
|
if (pool_in_cg(pp, cc->cc_current, addr) || |
if (pool_in_cg(pp, cc->cc_current, addr) || |
pool_in_cg(pp, cc->cc_previous, addr)) { |
pool_in_cg(pp, cc->cc_previous, addr)) { |
struct cpu_info *ci = |
struct cpu_info *ci = |
cpu_lookup_byindex(i); |
cpu_lookup(i); |
|
|
incpucache = true; |
incpucache = true; |
snprintf(cpucachestr, |
snprintf(cpucachestr, |