version 1.133.4.3, 2007/12/27 00:46:08 |
version 1.138.4.1, 2007/12/13 21:56:54 |
Line 181 struct pool_item { |
|
Line 181 struct pool_item { |
|
* from it. |
* from it. |
*/ |
*/ |
|
|
static struct pool pcg_normal_pool; |
static struct pool pcgpool; |
static struct pool pcg_large_pool; |
|
static struct pool cache_pool; |
static struct pool cache_pool; |
static struct pool cache_cpu_pool; |
static struct pool cache_cpu_pool; |
|
|
Line 851 pool_init(struct pool *pp, size_t size, |
|
Line 850 pool_init(struct pool *pp, size_t size, |
|
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
#endif |
#endif |
|
pool_init(&pcgpool, sizeof(pcg_t), CACHE_LINE_SIZE, 0, 0, |
size = sizeof(pcg_t) + |
"cachegrp", &pool_allocator_meta, IPL_VM); |
(PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_normal_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcgnormal", &pool_allocator_meta, IPL_VM); |
|
|
|
size = sizeof(pcg_t) + |
|
(PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_large_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcglarge", &pool_allocator_meta, IPL_VM); |
|
} |
} |
|
|
if (__predict_true(!cold)) { |
if (__predict_true(!cold)) { |
Line 1886 pool_print1(struct pool *pp, const char |
|
Line 1877 pool_print1(struct pool *pp, const char |
|
|
|
#define PR_GROUPLIST(pcg) \ |
#define PR_GROUPLIST(pcg) \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
for (i = 0; i < pcg->pcg_size; i++) { \ |
for (i = 0; i < PCG_NOBJECTS; i++) { \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
POOL_PADDR_INVALID) { \ |
POOL_PADDR_INVALID) { \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
Line 2098 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2089 pool_cache_bootstrap(pool_cache_t pc, si |
|
pc->pc_refcnt = 0; |
pc->pc_refcnt = 0; |
pc->pc_freecheck = NULL; |
pc->pc_freecheck = NULL; |
|
|
if ((flags & PR_LARGECACHE) != 0) { |
|
pc->pc_pcgsize = PCG_NOBJECTS_LARGE; |
|
} else { |
|
pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; |
|
} |
|
|
|
/* Allocate per-CPU caches. */ |
/* Allocate per-CPU caches. */ |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
pc->pc_ncpu = 0; |
pc->pc_ncpu = 0; |
Line 2305 pool_cache_invalidate_groups(pool_cache_ |
|
Line 2290 pool_cache_invalidate_groups(pool_cache_ |
|
pool_cache_destruct_object1(pc, object); |
pool_cache_destruct_object1(pc, object); |
} |
} |
|
|
if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { |
pool_put(&pcgpool, pcg); |
pool_put(&pcg_large_pool, pcg); |
|
} else { |
|
KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); |
|
pool_put(&pcg_normal_pool, pcg); |
|
} |
|
} |
} |
} |
} |
|
|
Line 2450 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2430 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pc->pc_emptygroups = cur; |
pc->pc_emptygroups = cur; |
pc->pc_nempty++; |
pc->pc_nempty++; |
} |
} |
KASSERT(pcg->pcg_avail == pcg->pcg_size); |
KASSERT(pcg->pcg_avail == PCG_NOBJECTS); |
cc->cc_current = pcg; |
cc->cc_current = pcg; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_hits++; |
pc->pc_hits++; |
Line 2522 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2502 pool_cache_get_paddr(pool_cache_t pc, in |
|
if (pap != NULL) |
if (pap != NULL) |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
KASSERT(pcg->pcg_avail <= pcg->pcg_size); |
KASSERT(pcg->pcg_avail <= PCG_NOBJECTS); |
KASSERT(object != NULL); |
KASSERT(object != NULL); |
cc->cc_hits++; |
cc->cc_hits++; |
pool_cache_cpu_exit(cc, &s); |
pool_cache_cpu_exit(cc, &s); |
Line 2562 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2542 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
pcg_t *pcg, *cur; |
pcg_t *pcg, *cur; |
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
u_int nobj; |
|
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
Line 2595 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2574 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* group as cc_current and return. |
* group as cc_current and return. |
*/ |
*/ |
if ((cur = cc->cc_current) != NULL) { |
if ((cur = cc->cc_current) != NULL) { |
KASSERT(cur->pcg_avail == pcg->pcg_size); |
KASSERT(cur->pcg_avail == PCG_NOBJECTS); |
cur->pcg_next = pc->pc_fullgroups; |
cur->pcg_next = pc->pc_fullgroups; |
pc->pc_fullgroups = cur; |
pc->pc_fullgroups = cur; |
pc->pc_nfull++; |
pc->pc_nfull++; |
Line 2622 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2601 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* If we can't allocate a new group, just throw the |
* If we can't allocate a new group, just throw the |
* object away. |
* object away. |
*/ |
*/ |
nobj = pc->pc_pcgsize; |
pcg = pool_get(&pcgpool, PR_NOWAIT); |
if (nobj == PCG_NOBJECTS_LARGE) { |
|
pcg = pool_get(&pcg_large_pool, PR_NOWAIT); |
|
} else { |
|
pcg = pool_get(&pcg_normal_pool, PR_NOWAIT); |
|
} |
|
if (pcg == NULL) { |
if (pcg == NULL) { |
pool_cache_destruct_object(pc, object); |
pool_cache_destruct_object(pc, object); |
return NULL; |
return NULL; |
} |
} |
|
#ifdef DIAGNOSTIC |
|
memset(pcg, 0, sizeof(*pcg)); |
|
#else |
pcg->pcg_avail = 0; |
pcg->pcg_avail = 0; |
pcg->pcg_size = nobj; |
#endif |
|
|
/* |
/* |
* Add the empty group to the cache and try again. |
* Add the empty group to the cache and try again. |
Line 2666 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2643 pool_cache_put_paddr(pool_cache_t pc, vo |
|
do { |
do { |
/* If the current group isn't full, release it there. */ |
/* If the current group isn't full, release it there. */ |
pcg = cc->cc_current; |
pcg = cc->cc_current; |
if (pcg != NULL && pcg->pcg_avail < pcg->pcg_size) { |
if (pcg != NULL && pcg->pcg_avail < PCG_NOBJECTS) { |
|
KASSERT(pcg->pcg_objects[pcg->pcg_avail].pcgo_va |
|
== NULL); |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_avail++; |
pcg->pcg_avail++; |
Line 2724 pool_cache_xcall(pool_cache_t pc) |
|
Line 2703 pool_cache_xcall(pool_cache_t pc) |
|
s = splvm(); |
s = splvm(); |
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
if (cur != NULL) { |
if (cur != NULL) { |
if (cur->pcg_avail == cur->pcg_size) { |
if (cur->pcg_avail == PCG_NOBJECTS) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (cur->pcg_avail == 0) { |
} else if (cur->pcg_avail == 0) { |
Line 2738 pool_cache_xcall(pool_cache_t pc) |
|
Line 2717 pool_cache_xcall(pool_cache_t pc) |
|
*list = cur; |
*list = cur; |
} |
} |
if (prev != NULL) { |
if (prev != NULL) { |
if (prev->pcg_avail == prev->pcg_size) { |
if (prev->pcg_avail == PCG_NOBJECTS) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (prev->pcg_avail == 0) { |
} else if (prev->pcg_avail == 0) { |
Line 2928 pool_in_page(struct pool *pp, struct poo |
|
Line 2907 pool_in_page(struct pool *pp, struct poo |
|
addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
} |
} |
|
|
static bool |
|
pool_in_item(struct pool *pp, void *item, uintptr_t addr) |
|
{ |
|
|
|
return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; |
|
} |
|
|
|
static bool |
|
pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) |
|
{ |
|
int i; |
|
|
|
if (pcg == NULL) { |
|
return false; |
|
} |
|
for (i = 0; i < pcg->pcg_avail; i++) { |
|
if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { |
|
return true; |
|
} |
|
} |
|
return false; |
|
} |
|
|
|
static bool |
|
pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) |
|
{ |
|
|
|
if ((pp->pr_roflags & PR_NOTOUCH) != 0) { |
|
unsigned int idx = pr_item_notouch_index(pp, ph, (void *)addr); |
|
pool_item_bitmap_t *bitmap = |
|
ph->ph_bitmap + (idx / BITMAP_SIZE); |
|
pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); |
|
|
|
return (*bitmap & mask) == 0; |
|
} else { |
|
struct pool_item *pi; |
|
|
|
LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { |
|
if (pool_in_item(pp, pi, addr)) { |
|
return false; |
|
} |
|
} |
|
return true; |
|
} |
|
} |
|
|
|
void |
void |
pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
{ |
{ |
Line 2982 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2915 pool_whatis(uintptr_t addr, void (*pr)(c |
|
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
struct pool_item_header *ph; |
struct pool_item_header *ph; |
uintptr_t item; |
uintptr_t item; |
bool allocated = true; |
|
bool incache = false; |
|
bool incpucache = false; |
|
char cpucachestr[32]; |
|
|
|
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
Line 2995 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2924 pool_whatis(uintptr_t addr, void (*pr)(c |
|
} |
} |
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
if (pool_in_page(pp, ph, addr)) { |
if (pool_in_page(pp, ph, addr)) { |
allocated = |
|
pool_allocated(pp, ph, addr); |
|
goto found; |
|
} |
|
} |
|
LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { |
|
if (pool_in_page(pp, ph, addr)) { |
|
allocated = false; |
|
goto found; |
goto found; |
} |
} |
} |
} |
Line 3012 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2933 pool_whatis(uintptr_t addr, void (*pr)(c |
|
if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
continue; |
continue; |
} |
} |
allocated = pool_allocated(pp, ph, addr); |
|
} |
} |
found: |
found: |
if (allocated && pp->pr_cache) { |
|
pool_cache_t pc = pp->pr_cache; |
|
struct pool_cache_group *pcg; |
|
int i; |
|
|
|
for (pcg = pc->pc_fullgroups; pcg != NULL; |
|
pcg = pcg->pcg_next) { |
|
if (pool_in_cg(pp, pcg, addr)) { |
|
incache = true; |
|
goto print; |
|
} |
|
} |
|
for (i = 0; i < MAXCPUS; i++) { |
|
pool_cache_cpu_t *cc; |
|
|
|
if ((cc = pc->pc_cpus[i]) == NULL) { |
|
continue; |
|
} |
|
if (pool_in_cg(pp, cc->cc_current, addr) || |
|
pool_in_cg(pp, cc->cc_previous, addr)) { |
|
struct cpu_info *ci = |
|
cpu_lookup_byindex(i); |
|
|
|
incpucache = true; |
|
snprintf(cpucachestr, |
|
sizeof(cpucachestr), |
|
"cached by CPU %u", |
|
(u_int)ci->ci_cpuid); |
|
goto print; |
|
} |
|
} |
|
} |
|
print: |
|
item = (uintptr_t)ph->ph_page + ph->ph_off; |
item = (uintptr_t)ph->ph_page + ph->ph_off; |
item = item + rounddown(addr - item, pp->pr_size); |
item = item + rounddown(addr - item, pp->pr_size); |
(*pr)("%p is %p+%zu in POOL '%s' (%s)\n", |
(*pr)("%p is %p+%zu from POOL '%s'\n", |
(void *)addr, item, (size_t)(addr - item), |
(void *)addr, item, (size_t)(addr - item), |
pp->pr_wchan, |
pp->pr_wchan); |
incpucache ? cpucachestr : |
|
incache ? "cached" : allocated ? "allocated" : "free"); |
|
} |
} |
} |
} |
#endif /* defined(DDB) */ |
#endif /* defined(DDB) */ |