version 1.138.2.3, 2007/12/13 05:06:01 |
version 1.144, 2007/12/22 04:31:34 |
Line 102 static void pool_page_free_meta(struct p |
|
Line 102 static void pool_page_free_meta(struct p |
|
/* allocator for pool metadata */ |
/* allocator for pool metadata */ |
struct pool_allocator pool_allocator_meta = { |
struct pool_allocator pool_allocator_meta = { |
pool_page_alloc_meta, pool_page_free_meta, |
pool_page_alloc_meta, pool_page_free_meta, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
|
|
/* # of seconds to retain page after last use */ |
/* # of seconds to retain page after last use */ |
Line 181 struct pool_item { |
|
Line 181 struct pool_item { |
|
* from it. |
* from it. |
*/ |
*/ |
|
|
static struct pool pcgpool; |
static struct pool pcg_normal_pool; |
|
static struct pool pcg_large_pool; |
static struct pool cache_pool; |
static struct pool cache_pool; |
static struct pool cache_cpu_pool; |
static struct pool cache_cpu_pool; |
|
|
Line 800 pool_init(struct pool *pp, size_t size, |
|
Line 801 pool_init(struct pool *pp, size_t size, |
|
|
|
#ifdef POOL_DIAGNOSTIC |
#ifdef POOL_DIAGNOSTIC |
if (flags & PR_LOGGING) { |
if (flags & PR_LOGGING) { |
if (kernel_map == NULL || |
if (kmem_map == NULL || |
(pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), |
(pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), |
M_TEMP, M_NOWAIT)) == NULL) |
M_TEMP, M_NOWAIT)) == NULL) |
pp->pr_roflags &= ~PR_LOGGING; |
pp->pr_roflags &= ~PR_LOGGING; |
Line 850 pool_init(struct pool *pp, size_t size, |
|
Line 851 pool_init(struct pool *pp, size_t size, |
|
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
#endif |
#endif |
pool_init(&pcgpool, sizeof(pcg_t), CACHE_LINE_SIZE, 0, 0, |
|
"cachegrp", &pool_allocator_meta, IPL_VM); |
size = sizeof(pcg_t) + |
|
(PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_normal_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcgnormal", &pool_allocator_meta, IPL_VM); |
|
|
|
size = sizeof(pcg_t) + |
|
(PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_large_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcglarge", &pool_allocator_meta, IPL_VM); |
} |
} |
|
|
if (__predict_true(!cold)) { |
if (__predict_true(!cold)) { |
Line 1877 pool_print1(struct pool *pp, const char |
|
Line 1886 pool_print1(struct pool *pp, const char |
|
|
|
#define PR_GROUPLIST(pcg) \ |
#define PR_GROUPLIST(pcg) \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
for (i = 0; i < PCG_NOBJECTS; i++) { \ |
for (i = 0; i < pcg->pcg_size; i++) { \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
POOL_PADDR_INVALID) { \ |
POOL_PADDR_INVALID) { \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
Line 2089 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2098 pool_cache_bootstrap(pool_cache_t pc, si |
|
pc->pc_refcnt = 0; |
pc->pc_refcnt = 0; |
pc->pc_freecheck = NULL; |
pc->pc_freecheck = NULL; |
|
|
|
if ((flags & PR_LARGECACHE) != 0) { |
|
pc->pc_pcgsize = PCG_NOBJECTS_LARGE; |
|
} else { |
|
pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; |
|
} |
|
|
/* Allocate per-CPU caches. */ |
/* Allocate per-CPU caches. */ |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
pc->pc_ncpu = 0; |
pc->pc_ncpu = 0; |
Line 2122 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2137 pool_cache_bootstrap(pool_cache_t pc, si |
|
void |
void |
pool_cache_destroy(pool_cache_t pc) |
pool_cache_destroy(pool_cache_t pc) |
{ |
{ |
|
|
pool_cache_bootstrap_destroy(pc); |
|
pool_put(&cache_pool, pc); |
|
} |
|
|
|
/* |
|
* pool_cache_bootstrap_destroy: |
|
* |
|
* Kernel-private version of pool_cache_destroy(). |
|
* Destroy a pool cache initialized by pool_cache_bootstrap. |
|
*/ |
|
void |
|
pool_cache_bootstrap_destroy(pool_cache_t pc) |
|
{ |
|
struct pool *pp = &pc->pc_pool; |
struct pool *pp = &pc->pc_pool; |
pool_cache_cpu_t *cc; |
pool_cache_cpu_t *cc; |
pcg_t *pcg; |
pcg_t *pcg; |
Line 2175 pool_cache_bootstrap_destroy(pool_cache_ |
|
Line 2176 pool_cache_bootstrap_destroy(pool_cache_ |
|
/* Finally, destroy it. */ |
/* Finally, destroy it. */ |
mutex_destroy(&pc->pc_lock); |
mutex_destroy(&pc->pc_lock); |
pool_destroy(pp); |
pool_destroy(pp); |
|
pool_put(&cache_pool, pc); |
} |
} |
|
|
/* |
/* |
Line 2303 pool_cache_invalidate_groups(pool_cache_ |
|
Line 2305 pool_cache_invalidate_groups(pool_cache_ |
|
pool_cache_destruct_object1(pc, object); |
pool_cache_destruct_object1(pc, object); |
} |
} |
|
|
pool_put(&pcgpool, pcg); |
if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { |
|
pool_put(&pcg_large_pool, pcg); |
|
} else { |
|
KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); |
|
pool_put(&pcg_normal_pool, pcg); |
|
} |
} |
} |
} |
} |
|
|
Line 2443 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2450 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pc->pc_emptygroups = cur; |
pc->pc_emptygroups = cur; |
pc->pc_nempty++; |
pc->pc_nempty++; |
} |
} |
KASSERT(pcg->pcg_avail == PCG_NOBJECTS); |
KASSERT(pcg->pcg_avail == pcg->pcg_size); |
cc->cc_current = pcg; |
cc->cc_current = pcg; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_hits++; |
pc->pc_hits++; |
Line 2515 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2522 pool_cache_get_paddr(pool_cache_t pc, in |
|
if (pap != NULL) |
if (pap != NULL) |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
KASSERT(pcg->pcg_avail <= PCG_NOBJECTS); |
KASSERT(pcg->pcg_avail <= pcg->pcg_size); |
KASSERT(object != NULL); |
KASSERT(object != NULL); |
cc->cc_hits++; |
cc->cc_hits++; |
pool_cache_cpu_exit(cc, &s); |
pool_cache_cpu_exit(cc, &s); |
Line 2555 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2562 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
pcg_t *pcg, *cur; |
pcg_t *pcg, *cur; |
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
|
u_int nobj; |
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
Line 2587 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2595 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* group as cc_current and return. |
* group as cc_current and return. |
*/ |
*/ |
if ((cur = cc->cc_current) != NULL) { |
if ((cur = cc->cc_current) != NULL) { |
KASSERT(cur->pcg_avail == PCG_NOBJECTS); |
KASSERT(cur->pcg_avail == pcg->pcg_size); |
cur->pcg_next = pc->pc_fullgroups; |
cur->pcg_next = pc->pc_fullgroups; |
pc->pc_fullgroups = cur; |
pc->pc_fullgroups = cur; |
pc->pc_nfull++; |
pc->pc_nfull++; |
Line 2614 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2622 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* If we can't allocate a new group, just throw the |
* If we can't allocate a new group, just throw the |
* object away. |
* object away. |
*/ |
*/ |
pcg = pool_get(&pcgpool, PR_NOWAIT); |
nobj = pc->pc_pcgsize; |
|
if (nobj == PCG_NOBJECTS_LARGE) { |
|
pcg = pool_get(&pcg_large_pool, PR_NOWAIT); |
|
} else { |
|
pcg = pool_get(&pcg_normal_pool, PR_NOWAIT); |
|
} |
if (pcg == NULL) { |
if (pcg == NULL) { |
pool_cache_destruct_object(pc, object); |
pool_cache_destruct_object(pc, object); |
return NULL; |
return NULL; |
} |
} |
#ifdef DIAGNOSTIC |
|
memset(pcg, 0, sizeof(*pcg)); |
|
#else |
|
pcg->pcg_avail = 0; |
pcg->pcg_avail = 0; |
#endif |
pcg->pcg_size = nobj; |
|
|
/* |
/* |
* Add the empty group to the cache and try again. |
* Add the empty group to the cache and try again. |
Line 2656 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2666 pool_cache_put_paddr(pool_cache_t pc, vo |
|
do { |
do { |
/* If the current group isn't full, release it there. */ |
/* If the current group isn't full, release it there. */ |
pcg = cc->cc_current; |
pcg = cc->cc_current; |
if (pcg != NULL && pcg->pcg_avail < PCG_NOBJECTS) { |
if (pcg != NULL && pcg->pcg_avail < pcg->pcg_size) { |
KASSERT(pcg->pcg_objects[pcg->pcg_avail].pcgo_va |
|
== NULL); |
|
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_avail++; |
pcg->pcg_avail++; |
Line 2716 pool_cache_xcall(pool_cache_t pc) |
|
Line 2724 pool_cache_xcall(pool_cache_t pc) |
|
s = splvm(); |
s = splvm(); |
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
if (cur != NULL) { |
if (cur != NULL) { |
if (cur->pcg_avail == PCG_NOBJECTS) { |
if (cur->pcg_avail == cur->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (cur->pcg_avail == 0) { |
} else if (cur->pcg_avail == 0) { |
Line 2730 pool_cache_xcall(pool_cache_t pc) |
|
Line 2738 pool_cache_xcall(pool_cache_t pc) |
|
*list = cur; |
*list = cur; |
} |
} |
if (prev != NULL) { |
if (prev != NULL) { |
if (prev->pcg_avail == PCG_NOBJECTS) { |
if (prev->pcg_avail == prev->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (prev->pcg_avail == 0) { |
} else if (prev->pcg_avail == 0) { |
Line 2766 void pool_page_free(struct pool *, void |
|
Line 2774 void pool_page_free(struct pool *, void |
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
struct pool_allocator pool_allocator_kmem_fullpage = { |
struct pool_allocator pool_allocator_kmem_fullpage = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
#else |
#else |
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
#endif |
#endif |
|
|
Line 2796 void pool_subpage_free(struct pool *, vo |
|
Line 2804 void pool_subpage_free(struct pool *, vo |
|
|
|
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
|
|
void *pool_subpage_alloc_nointr(struct pool *, int); |
void *pool_subpage_alloc_nointr(struct pool *, int); |
Line 2804 void pool_subpage_free_nointr(struct poo |
|
Line 2812 void pool_subpage_free_nointr(struct poo |
|
|
|
struct pool_allocator pool_allocator_nointr = { |
struct pool_allocator pool_allocator_nointr = { |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
#endif /* POOL_SUBPAGE */ |
#endif /* POOL_SUBPAGE */ |
|
|
Line 2842 pool_page_alloc(struct pool *pp, int fla |
|
Line 2850 pool_page_alloc(struct pool *pp, int fla |
|
{ |
{ |
bool waitok = (flags & PR_WAITOK) ? true : false; |
bool waitok = (flags & PR_WAITOK) ? true : false; |
|
|
return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); |
return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); |
} |
} |
|
|
void |
void |
pool_page_free(struct pool *pp, void *v) |
pool_page_free(struct pool *pp, void *v) |
{ |
{ |
|
|
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v); |
} |
} |
|
|
static void * |
static void * |
Line 2857 pool_page_alloc_meta(struct pool *pp, in |
|
Line 2865 pool_page_alloc_meta(struct pool *pp, in |
|
{ |
{ |
bool waitok = (flags & PR_WAITOK) ? true : false; |
bool waitok = (flags & PR_WAITOK) ? true : false; |
|
|
return ((void *) uvm_km_alloc_poolpage(kernel_map, waitok)); |
return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); |
} |
} |
|
|
static void |
static void |
pool_page_free_meta(struct pool *pp, void *v) |
pool_page_free_meta(struct pool *pp, void *v) |
{ |
{ |
|
|
uvm_km_free_poolpage(kernel_map, (vaddr_t) v); |
uvm_km_free_poolpage(kmem_map, (vaddr_t) v); |
} |
} |
|
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
Line 2920 pool_in_page(struct pool *pp, struct poo |
|
Line 2928 pool_in_page(struct pool *pp, struct poo |
|
addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
} |
} |
|
|
|
static bool |
|
pool_in_item(struct pool *pp, void *item, uintptr_t addr) |
|
{ |
|
|
|
return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; |
|
} |
|
|
|
static bool |
|
pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) |
|
{ |
|
int i; |
|
|
|
if (pcg == NULL) { |
|
return false; |
|
} |
|
for (i = 0; i < pcg->pcg_avail; i++) { |
|
if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { |
|
return true; |
|
} |
|
} |
|
return false; |
|
} |
|
|
|
static bool |
|
pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) |
|
{ |
|
|
|
if ((pp->pr_roflags & PR_NOTOUCH) != 0) { |
|
unsigned int idx = pr_item_notouch_index(pp, ph, (void *)addr); |
|
pool_item_bitmap_t *bitmap = |
|
ph->ph_bitmap + (idx / BITMAP_SIZE); |
|
pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); |
|
|
|
return (*bitmap & mask) == 0; |
|
} else { |
|
struct pool_item *pi; |
|
|
|
LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { |
|
if (pool_in_item(pp, pi, addr)) { |
|
return false; |
|
} |
|
} |
|
return true; |
|
} |
|
} |
|
|
void |
void |
pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
{ |
{ |
Line 2928 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2982 pool_whatis(uintptr_t addr, void (*pr)(c |
|
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
struct pool_item_header *ph; |
struct pool_item_header *ph; |
uintptr_t item; |
uintptr_t item; |
|
bool allocated = true; |
|
bool incache = false; |
|
bool incpucache = false; |
|
char cpucachestr[32]; |
|
|
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
Line 2937 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2995 pool_whatis(uintptr_t addr, void (*pr)(c |
|
} |
} |
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
if (pool_in_page(pp, ph, addr)) { |
if (pool_in_page(pp, ph, addr)) { |
|
allocated = |
|
pool_allocated(pp, ph, addr); |
|
goto found; |
|
} |
|
} |
|
LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { |
|
if (pool_in_page(pp, ph, addr)) { |
|
allocated = false; |
goto found; |
goto found; |
} |
} |
} |
} |
Line 2946 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 3012 pool_whatis(uintptr_t addr, void (*pr)(c |
|
if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
continue; |
continue; |
} |
} |
|
allocated = pool_allocated(pp, ph, addr); |
} |
} |
found: |
found: |
|
if (allocated && pp->pr_cache) { |
|
pool_cache_t pc = pp->pr_cache; |
|
struct pool_cache_group *pcg; |
|
int i; |
|
|
|
for (pcg = pc->pc_fullgroups; pcg != NULL; |
|
pcg = pcg->pcg_next) { |
|
if (pool_in_cg(pp, pcg, addr)) { |
|
incache = true; |
|
goto print; |
|
} |
|
} |
|
for (i = 0; i < MAXCPUS; i++) { |
|
pool_cache_cpu_t *cc; |
|
|
|
if ((cc = pc->pc_cpus[i]) == NULL) { |
|
continue; |
|
} |
|
if (pool_in_cg(pp, cc->cc_current, addr) || |
|
pool_in_cg(pp, cc->cc_previous, addr)) { |
|
struct cpu_info *ci = |
|
cpu_lookup_byindex(i); |
|
|
|
incpucache = true; |
|
snprintf(cpucachestr, |
|
sizeof(cpucachestr), |
|
"cached by CPU %u", |
|
(u_int)ci->ci_cpuid); |
|
goto print; |
|
} |
|
} |
|
} |
|
print: |
item = (uintptr_t)ph->ph_page + ph->ph_off; |
item = (uintptr_t)ph->ph_page + ph->ph_off; |
item = item + rounddown(addr - item, pp->pr_size); |
item = item + rounddown(addr - item, pp->pr_size); |
(*pr)("%p is %p+%zu from POOL '%s'\n", |
(*pr)("%p is %p+%zu in POOL '%s' (%s)\n", |
(void *)addr, item, (size_t)(addr - item), |
(void *)addr, item, (size_t)(addr - item), |
pp->pr_wchan); |
pp->pr_wchan, |
|
incpucache ? cpucachestr : |
|
incache ? "cached" : allocated ? "allocated" : "free"); |
} |
} |
} |
} |
#endif /* defined(DDB) */ |
#endif /* defined(DDB) */ |