version 1.101.2.7, 2007/12/07 17:33:07 |
version 1.129.12.3, 2007/11/11 16:48:08 |
Line 802 pool_init(struct pool *pp, size_t size, |
|
Line 802 pool_init(struct pool *pp, size_t size, |
|
pp->pr_entered_file = NULL; |
pp->pr_entered_file = NULL; |
pp->pr_entered_line = 0; |
pp->pr_entered_line = 0; |
|
|
/* |
mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); |
* XXXAD hack to prevent IP input processing from blocking. |
|
*/ |
|
if (ipl == IPL_SOFTNET) { |
|
mutex_init(&pp->pr_lock, MUTEX_DEFAULT, IPL_VM); |
|
} else { |
|
mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); |
|
} |
|
cv_init(&pp->pr_cv, wchan); |
cv_init(&pp->pr_cv, wchan); |
pp->pr_ipl = ipl; |
pp->pr_ipl = ipl; |
|
|
Line 2047 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2040 pool_cache_bootstrap(pool_cache_t pc, si |
|
palloc = &pool_allocator_nointr; |
palloc = &pool_allocator_nointr; |
pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl); |
pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl); |
|
|
/* |
mutex_init(&pc->pc_lock, MUTEX_DEFAULT, pp->pr_ipl); |
* XXXAD hack to prevent IP input processing from blocking. |
|
*/ |
|
if (ipl == IPL_SOFTNET) { |
|
mutex_init(&pc->pc_lock, MUTEX_DEFAULT, IPL_VM); |
|
} else { |
|
mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl); |
|
} |
|
|
|
if (ctor == NULL) { |
if (ctor == NULL) { |
ctor = (int (*)(void *, void *, int))nullop; |
ctor = (int (*)(void *, void *, int))nullop; |
Line 2076 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2062 pool_cache_bootstrap(pool_cache_t pc, si |
|
pc->pc_nfull = 0; |
pc->pc_nfull = 0; |
pc->pc_contended = 0; |
pc->pc_contended = 0; |
pc->pc_refcnt = 0; |
pc->pc_refcnt = 0; |
pc->pc_freecheck = NULL; |
|
|
|
/* Allocate per-CPU caches. */ |
/* Allocate per-CPU caches. */ |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
pc->pc_ncpu = 0; |
pc->pc_ncpu = 0; |
if (ncpu == 0) { |
for (CPU_INFO_FOREACH(cii, ci)) { |
/* XXX For sparc: boot CPU is not attached yet. */ |
pool_cache_cpu_init1(ci, pc); |
pool_cache_cpu_init1(curcpu(), pc); |
|
} else { |
|
for (CPU_INFO_FOREACH(cii, ci)) { |
|
pool_cache_cpu_init1(ci, pc); |
|
} |
|
} |
} |
|
|
if (__predict_true(!cold)) { |
if (__predict_true(!cold)) { |
|
|
pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) |
pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) |
{ |
{ |
pool_cache_cpu_t *cc; |
pool_cache_cpu_t *cc; |
int index; |
|
|
|
index = ci->ci_index; |
|
|
|
KASSERT(index < MAXCPUS); |
|
KASSERT(((uintptr_t)pc->pc_cpus & (CACHE_LINE_SIZE - 1)) == 0); |
KASSERT(((uintptr_t)pc->pc_cpus & (CACHE_LINE_SIZE - 1)) == 0); |
|
|
if ((cc = pc->pc_cpus[index]) != NULL) { |
if ((cc = pc->pc_cpus[ci->ci_index]) != NULL) { |
KASSERT(cc->cc_cpuindex == index); |
KASSERT(cc->cc_cpu = ci); |
return; |
return; |
} |
} |
|
|
Line 2191 pool_cache_cpu_init1(struct cpu_info *ci |
|
Line 2167 pool_cache_cpu_init1(struct cpu_info *ci |
|
cc->cc_ipl = pc->pc_pool.pr_ipl; |
cc->cc_ipl = pc->pc_pool.pr_ipl; |
cc->cc_iplcookie = makeiplcookie(cc->cc_ipl); |
cc->cc_iplcookie = makeiplcookie(cc->cc_ipl); |
cc->cc_cache = pc; |
cc->cc_cache = pc; |
cc->cc_cpuindex = index; |
cc->cc_cpu = ci; |
cc->cc_hits = 0; |
cc->cc_hits = 0; |
cc->cc_misses = 0; |
cc->cc_misses = 0; |
cc->cc_current = NULL; |
cc->cc_current = NULL; |
cc->cc_previous = NULL; |
cc->cc_previous = NULL; |
|
|
pc->pc_cpus[index] = cc; |
pc->pc_cpus[ci->ci_index] = cc; |
} |
} |
|
|
/* |
/* |
Line 2236 pool_cache_reclaim(pool_cache_t pc) |
|
Line 2212 pool_cache_reclaim(pool_cache_t pc) |
|
return pool_reclaim(&pc->pc_pool); |
return pool_reclaim(&pc->pc_pool); |
} |
} |
|
|
static void |
|
pool_cache_destruct_object1(pool_cache_t pc, void *object) |
|
{ |
|
|
|
(*pc->pc_dtor)(pc->pc_arg, object); |
|
pool_put(&pc->pc_pool, object); |
|
} |
|
|
|
/* |
/* |
* pool_cache_destruct_object: |
* pool_cache_destruct_object: |
* |
* |
|
|
pool_cache_destruct_object(pool_cache_t pc, void *object) |
pool_cache_destruct_object(pool_cache_t pc, void *object) |
{ |
{ |
|
|
FREECHECK_IN(&pc->pc_freecheck, object); |
(*pc->pc_dtor)(pc->pc_arg, object); |
|
pool_put(&pc->pc_pool, object); |
pool_cache_destruct_object1(pc, object); |
|
} |
} |
|
|
/* |
/* |
Line 2276 pool_cache_invalidate_groups(pool_cache_ |
|
Line 2243 pool_cache_invalidate_groups(pool_cache_ |
|
|
|
for (i = 0; i < pcg->pcg_avail; i++) { |
for (i = 0; i < pcg->pcg_avail; i++) { |
object = pcg->pcg_objects[i].pcgo_va; |
object = pcg->pcg_objects[i].pcgo_va; |
pool_cache_destruct_object1(pc, object); |
pool_cache_destruct_object(pc, object); |
} |
} |
|
|
pool_put(&pcgpool, pcg); |
pool_put(&pcgpool, pcg); |
Line 2343 static inline pool_cache_cpu_t * |
|
Line 2310 static inline pool_cache_cpu_t * |
|
pool_cache_cpu_enter(pool_cache_t pc, int *s) |
pool_cache_cpu_enter(pool_cache_t pc, int *s) |
{ |
{ |
pool_cache_cpu_t *cc; |
pool_cache_cpu_t *cc; |
|
struct cpu_info *ci; |
|
|
/* |
/* |
* Prevent other users of the cache from accessing our |
* Prevent other users of the cache from accessing our |
* CPU-local data. To avoid touching shared state, we |
* CPU-local data. To avoid touching shared state, we |
* pull the neccessary information from CPU local data. |
* pull the neccessary information from CPU local data. |
*/ |
*/ |
crit_enter(); |
ci = curcpu(); |
cc = pc->pc_cpus[curcpu()->ci_index]; |
KASSERT(ci->ci_data.cpu_index < MAXCPUS); |
|
cc = pc->pc_cpus[ci->ci_data.cpu_index]; |
KASSERT(cc->cc_cache == pc); |
KASSERT(cc->cc_cache == pc); |
if (cc->cc_ipl != IPL_NONE) { |
if (cc->cc_ipl == IPL_NONE) { |
|
crit_enter(); |
|
} else { |
*s = splraiseipl(cc->cc_iplcookie); |
*s = splraiseipl(cc->cc_iplcookie); |
} |
} |
|
|
|
/* Moved to another CPU before disabling preemption? */ |
|
if (__predict_false(ci != curcpu())) { |
|
ci = curcpu(); |
|
cc = pc->pc_cpus[ci->ci_data.cpu_index]; |
|
} |
|
|
|
#ifdef DIAGNOSTIC |
|
KASSERT(cc->cc_cpu == ci); |
KASSERT(((uintptr_t)cc & (CACHE_LINE_SIZE - 1)) == 0); |
KASSERT(((uintptr_t)cc & (CACHE_LINE_SIZE - 1)) == 0); |
|
#endif |
|
|
return cc; |
return cc; |
} |
} |
Line 2365 pool_cache_cpu_exit(pool_cache_cpu_t *cc |
|
Line 2346 pool_cache_cpu_exit(pool_cache_cpu_t *cc |
|
{ |
{ |
|
|
/* No longer need exclusive access to the per-CPU data. */ |
/* No longer need exclusive access to the per-CPU data. */ |
if (cc->cc_ipl != IPL_NONE) { |
if (cc->cc_ipl == IPL_NONE) { |
|
crit_exit(); |
|
} else { |
splx(*s); |
splx(*s); |
} |
} |
crit_exit(); |
|
} |
} |
|
|
#if __GNUC_PREREQ__(3, 0) |
#if __GNUC_PREREQ__(3, 0) |