version 1.137.2.6, 2007/12/28 15:06:20 |
version 1.138.2.3, 2007/12/13 05:06:01 |
Line 59 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 59 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include <sys/lockdebug.h> |
#include <sys/lockdebug.h> |
#include <sys/xcall.h> |
#include <sys/xcall.h> |
#include <sys/cpu.h> |
#include <sys/cpu.h> |
#include <sys/atomic.h> |
|
|
|
#include <uvm/uvm.h> |
#include <uvm/uvm.h> |
|
|
Line 77 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 76 __KERNEL_RCSID(0, "$NetBSD$"); |
|
*/ |
*/ |
|
|
/* List of all pools */ |
/* List of all pools */ |
TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head); |
|
|
|
/* List of all caches. */ |
|
LIST_HEAD(,pool_cache) pool_cache_head = |
|
LIST_HEAD_INITIALIZER(pool_cache_head); |
|
|
/* Private pool for page header structures */ |
/* Private pool for page header structures */ |
#define PHPOOL_MAX 8 |
#define PHPOOL_MAX 8 |
Line 99 static void pool_page_free_meta(struct p |
|
Line 102 static void pool_page_free_meta(struct p |
|
/* allocator for pool metadata */ |
/* allocator for pool metadata */ |
struct pool_allocator pool_allocator_meta = { |
struct pool_allocator pool_allocator_meta = { |
pool_page_alloc_meta, pool_page_free_meta, |
pool_page_alloc_meta, pool_page_free_meta, |
.pa_backingmapptr = &kmem_map, |
.pa_backingmapptr = &kernel_map, |
}; |
}; |
|
|
/* # of seconds to retain page after last use */ |
/* # of seconds to retain page after last use */ |
Line 178 struct pool_item { |
|
Line 181 struct pool_item { |
|
* from it. |
* from it. |
*/ |
*/ |
|
|
static struct pool pcg_normal_pool; |
static struct pool pcgpool; |
static struct pool pcg_large_pool; |
|
static struct pool cache_pool; |
static struct pool cache_pool; |
static struct pool cache_cpu_pool; |
static struct pool cache_cpu_pool; |
|
|
/* List of all caches. */ |
|
TAILQ_HEAD(,pool_cache) pool_cache_head = |
|
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
|
|
|
int pool_cache_disable; |
|
|
|
|
|
static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, |
static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, |
void *, paddr_t); |
void *, paddr_t); |
static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, |
static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, |
|
|
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
const char *wchan, struct pool_allocator *palloc, int ipl) |
const char *wchan, struct pool_allocator *palloc, int ipl) |
{ |
{ |
|
#ifdef DEBUG |
struct pool *pp1; |
struct pool *pp1; |
|
#endif |
size_t trysize, phsize; |
size_t trysize, phsize; |
int off, slack; |
int off, slack; |
|
|
Line 639 pool_init(struct pool *pp, size_t size, |
|
Line 636 pool_init(struct pool *pp, size_t size, |
|
* Check that the pool hasn't already been initialised and |
* Check that the pool hasn't already been initialised and |
* added to the list of all pools. |
* added to the list of all pools. |
*/ |
*/ |
TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
LIST_FOREACH(pp1, &pool_head, pr_poollist) { |
if (pp == pp1) |
if (pp == pp1) |
panic("pool_init: pool %s already initialised", |
panic("pool_init: pool %s already initialised", |
wchan); |
wchan); |
Line 803 pool_init(struct pool *pp, size_t size, |
|
Line 800 pool_init(struct pool *pp, size_t size, |
|
|
|
#ifdef POOL_DIAGNOSTIC |
#ifdef POOL_DIAGNOSTIC |
if (flags & PR_LOGGING) { |
if (flags & PR_LOGGING) { |
if (kmem_map == NULL || |
if (kernel_map == NULL || |
(pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), |
(pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), |
M_TEMP, M_NOWAIT)) == NULL) |
M_TEMP, M_NOWAIT)) == NULL) |
pp->pr_roflags &= ~PR_LOGGING; |
pp->pr_roflags &= ~PR_LOGGING; |
Line 853 pool_init(struct pool *pp, size_t size, |
|
Line 850 pool_init(struct pool *pp, size_t size, |
|
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
#endif |
#endif |
|
pool_init(&pcgpool, sizeof(pcg_t), CACHE_LINE_SIZE, 0, 0, |
size = sizeof(pcg_t) + |
"cachegrp", &pool_allocator_meta, IPL_VM); |
(PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_normal_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcgnormal", &pool_allocator_meta, IPL_VM); |
|
|
|
size = sizeof(pcg_t) + |
|
(PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_large_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcglarge", &pool_allocator_meta, IPL_VM); |
|
} |
} |
|
|
/* Insert into the list of all pools. */ |
if (__predict_true(!cold)) { |
if (__predict_true(!cold)) |
/* Insert into the list of all pools. */ |
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); |
if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) |
|
break; |
|
} |
|
if (pp1 == NULL) |
|
TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); |
|
else |
|
TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); |
|
if (__predict_true(!cold)) |
|
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* Insert this into the list of pools using this allocator. */ |
/* Insert this into the list of pools using this allocator. */ |
if (__predict_true(!cold)) |
|
mutex_enter(&palloc->pa_lock); |
mutex_enter(&palloc->pa_lock); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
if (__predict_true(!cold)) |
|
mutex_exit(&palloc->pa_lock); |
mutex_exit(&palloc->pa_lock); |
|
} else { |
|
LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); |
|
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
|
} |
|
|
pool_reclaim_register(pp); |
pool_reclaim_register(pp); |
} |
} |
Line 902 pool_destroy(struct pool *pp) |
|
Line 885 pool_destroy(struct pool *pp) |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
while (pp->pr_refcnt != 0) |
while (pp->pr_refcnt != 0) |
cv_wait(&pool_busy, &pool_head_lock); |
cv_wait(&pool_busy, &pool_head_lock); |
TAILQ_REMOVE(&pool_head, pp, pr_poollist); |
LIST_REMOVE(pp, pr_poollist); |
if (drainpp == pp) |
if (drainpp == pp) |
drainpp = NULL; |
drainpp = NULL; |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
Line 1709 pool_drain_start(struct pool **ppp, uint |
|
Line 1692 pool_drain_start(struct pool **ppp, uint |
|
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
KASSERT(!TAILQ_EMPTY(&pool_head)); |
KASSERT(!LIST_EMPTY(&pool_head)); |
|
|
pp = NULL; |
pp = NULL; |
|
|
Line 1717 pool_drain_start(struct pool **ppp, uint |
|
Line 1700 pool_drain_start(struct pool **ppp, uint |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
do { |
do { |
if (drainpp == NULL) { |
if (drainpp == NULL) { |
drainpp = TAILQ_FIRST(&pool_head); |
drainpp = LIST_FIRST(&pool_head); |
} |
} |
if (drainpp != NULL) { |
if (drainpp != NULL) { |
pp = drainpp; |
pp = drainpp; |
drainpp = TAILQ_NEXT(pp, pr_poollist); |
drainpp = LIST_NEXT(pp, pr_poollist); |
} |
} |
/* |
/* |
* Skip completely idle pools. We depend on at least |
* Skip completely idle pools. We depend on at least |
Line 1777 pool_printall(const char *modif, void (* |
|
Line 1760 pool_printall(const char *modif, void (* |
|
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
pool_printit(pp, modif, pr); |
pool_printit(pp, modif, pr); |
} |
} |
} |
} |
Line 1894 pool_print1(struct pool *pp, const char |
|
Line 1877 pool_print1(struct pool *pp, const char |
|
|
|
#define PR_GROUPLIST(pcg) \ |
#define PR_GROUPLIST(pcg) \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
for (i = 0; i < pcg->pcg_size; i++) { \ |
for (i = 0; i < PCG_NOBJECTS; i++) { \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
POOL_PADDR_INVALID) { \ |
POOL_PADDR_INVALID) { \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
Line 2067 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2050 pool_cache_bootstrap(pool_cache_t pc, si |
|
void *arg) |
void *arg) |
{ |
{ |
CPU_INFO_ITERATOR cii; |
CPU_INFO_ITERATOR cii; |
pool_cache_t pc1; |
|
struct cpu_info *ci; |
struct cpu_info *ci; |
struct pool *pp; |
struct pool *pp; |
|
|
Line 2107 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2089 pool_cache_bootstrap(pool_cache_t pc, si |
|
pc->pc_refcnt = 0; |
pc->pc_refcnt = 0; |
pc->pc_freecheck = NULL; |
pc->pc_freecheck = NULL; |
|
|
if ((flags & PR_LARGECACHE) != 0) { |
|
pc->pc_pcgsize = PCG_NOBJECTS_LARGE; |
|
} else { |
|
pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; |
|
} |
|
|
|
/* Allocate per-CPU caches. */ |
/* Allocate per-CPU caches. */ |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
pc->pc_ncpu = 0; |
pc->pc_ncpu = 0; |
Line 2124 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2100 pool_cache_bootstrap(pool_cache_t pc, si |
|
pool_cache_cpu_init1(ci, pc); |
pool_cache_cpu_init1(ci, pc); |
} |
} |
} |
} |
|
|
/* Add to list of all pools. */ |
if (__predict_true(!cold)) { |
if (__predict_true(!cold)) |
mutex_enter(&pp->pr_lock); |
|
pp->pr_cache = pc; |
|
mutex_exit(&pp->pr_lock); |
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { |
LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); |
if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) |
|
break; |
|
} |
|
if (pc1 == NULL) |
|
TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); |
|
else |
|
TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); |
|
if (__predict_true(!cold)) |
|
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
} else { |
membar_sync(); |
pp->pr_cache = pc; |
pp->pr_cache = pc; |
LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); |
|
} |
} |
} |
|
|
/* |
/* |
Line 2151 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2122 pool_cache_bootstrap(pool_cache_t pc, si |
|
void |
void |
pool_cache_destroy(pool_cache_t pc) |
pool_cache_destroy(pool_cache_t pc) |
{ |
{ |
|
|
|
pool_cache_bootstrap_destroy(pc); |
|
pool_put(&cache_pool, pc); |
|
} |
|
|
|
/* |
|
* pool_cache_bootstrap_destroy: |
|
* |
|
* Kernel-private version of pool_cache_destroy(). |
|
* Destroy a pool cache initialized by pool_cache_bootstrap. |
|
*/ |
|
void |
|
pool_cache_bootstrap_destroy(pool_cache_t pc) |
|
{ |
struct pool *pp = &pc->pc_pool; |
struct pool *pp = &pc->pc_pool; |
pool_cache_cpu_t *cc; |
pool_cache_cpu_t *cc; |
pcg_t *pcg; |
pcg_t *pcg; |
Line 2160 pool_cache_destroy(pool_cache_t pc) |
|
Line 2145 pool_cache_destroy(pool_cache_t pc) |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
while (pc->pc_refcnt != 0) |
while (pc->pc_refcnt != 0) |
cv_wait(&pool_busy, &pool_head_lock); |
cv_wait(&pool_busy, &pool_head_lock); |
TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); |
LIST_REMOVE(pc, pc_cachelist); |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* First, invalidate the entire cache. */ |
/* First, invalidate the entire cache. */ |
Line 2190 pool_cache_destroy(pool_cache_t pc) |
|
Line 2175 pool_cache_destroy(pool_cache_t pc) |
|
/* Finally, destroy it. */ |
/* Finally, destroy it. */ |
mutex_destroy(&pc->pc_lock); |
mutex_destroy(&pc->pc_lock); |
pool_destroy(pp); |
pool_destroy(pp); |
pool_put(&cache_pool, pc); |
|
} |
} |
|
|
/* |
/* |
Line 2251 pool_cache_cpu_init(struct cpu_info *ci) |
|
Line 2235 pool_cache_cpu_init(struct cpu_info *ci) |
|
pool_cache_t pc; |
pool_cache_t pc; |
|
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { |
LIST_FOREACH(pc, &pool_cache_head, pc_cachelist) { |
pc->pc_refcnt++; |
pc->pc_refcnt++; |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
Line 2319 pool_cache_invalidate_groups(pool_cache_ |
|
Line 2303 pool_cache_invalidate_groups(pool_cache_ |
|
pool_cache_destruct_object1(pc, object); |
pool_cache_destruct_object1(pc, object); |
} |
} |
|
|
if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { |
pool_put(&pcgpool, pcg); |
pool_put(&pcg_large_pool, pcg); |
|
} else { |
|
KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); |
|
pool_put(&pcg_normal_pool, pcg); |
|
} |
|
} |
} |
} |
} |
|
|
Line 2464 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2443 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pc->pc_emptygroups = cur; |
pc->pc_emptygroups = cur; |
pc->pc_nempty++; |
pc->pc_nempty++; |
} |
} |
KASSERT(pcg->pcg_avail == pcg->pcg_size); |
KASSERT(pcg->pcg_avail == PCG_NOBJECTS); |
cc->cc_current = pcg; |
cc->cc_current = pcg; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_hits++; |
pc->pc_hits++; |
Line 2536 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2515 pool_cache_get_paddr(pool_cache_t pc, in |
|
if (pap != NULL) |
if (pap != NULL) |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
KASSERT(pcg->pcg_avail <= pcg->pcg_size); |
KASSERT(pcg->pcg_avail <= PCG_NOBJECTS); |
KASSERT(object != NULL); |
KASSERT(object != NULL); |
cc->cc_hits++; |
cc->cc_hits++; |
pool_cache_cpu_exit(cc, &s); |
pool_cache_cpu_exit(cc, &s); |
Line 2576 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2555 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
pcg_t *pcg, *cur; |
pcg_t *pcg, *cur; |
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
u_int nobj; |
|
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
Line 2606 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2584 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
/* |
/* |
* If there's a empty group, release our full |
* If there's a empty group, release our full |
* group back to the cache. Install the empty |
* group back to the cache. Install the empty |
* group and return. |
* group as cc_current and return. |
*/ |
*/ |
|
if ((cur = cc->cc_current) != NULL) { |
|
KASSERT(cur->pcg_avail == PCG_NOBJECTS); |
|
cur->pcg_next = pc->pc_fullgroups; |
|
pc->pc_fullgroups = cur; |
|
pc->pc_nfull++; |
|
} |
KASSERT(pcg->pcg_avail == 0); |
KASSERT(pcg->pcg_avail == 0); |
|
cc->cc_current = pcg; |
pc->pc_emptygroups = pcg->pcg_next; |
pc->pc_emptygroups = pcg->pcg_next; |
if (cc->cc_previous == NULL) { |
|
cc->cc_previous = pcg; |
|
} else { |
|
if ((cur = cc->cc_current) != NULL) { |
|
KASSERT(cur->pcg_avail == pcg->pcg_size); |
|
cur->pcg_next = pc->pc_fullgroups; |
|
pc->pc_fullgroups = cur; |
|
pc->pc_nfull++; |
|
} |
|
cc->cc_current = pcg; |
|
} |
|
pc->pc_hits++; |
pc->pc_hits++; |
pc->pc_nempty--; |
pc->pc_nempty--; |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
Line 2640 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2614 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* If we can't allocate a new group, just throw the |
* If we can't allocate a new group, just throw the |
* object away. |
* object away. |
*/ |
*/ |
nobj = pc->pc_pcgsize; |
pcg = pool_get(&pcgpool, PR_NOWAIT); |
if (nobj == PCG_NOBJECTS_LARGE) { |
|
pcg = pool_get(&pcg_large_pool, PR_NOWAIT); |
|
} else { |
|
pcg = pool_get(&pcg_normal_pool, PR_NOWAIT); |
|
} |
|
if (pcg == NULL) { |
if (pcg == NULL) { |
pool_cache_destruct_object(pc, object); |
pool_cache_destruct_object(pc, object); |
return NULL; |
return NULL; |
} |
} |
|
#ifdef DIAGNOSTIC |
|
memset(pcg, 0, sizeof(*pcg)); |
|
#else |
pcg->pcg_avail = 0; |
pcg->pcg_avail = 0; |
pcg->pcg_size = nobj; |
#endif |
|
|
/* |
/* |
* Add the empty group to the cache and try again. |
* Add the empty group to the cache and try again. |
Line 2684 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2656 pool_cache_put_paddr(pool_cache_t pc, vo |
|
do { |
do { |
/* If the current group isn't full, release it there. */ |
/* If the current group isn't full, release it there. */ |
pcg = cc->cc_current; |
pcg = cc->cc_current; |
if (pcg != NULL && pcg->pcg_avail < pcg->pcg_size) { |
if (pcg != NULL && pcg->pcg_avail < PCG_NOBJECTS) { |
|
KASSERT(pcg->pcg_objects[pcg->pcg_avail].pcgo_va |
|
== NULL); |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_avail++; |
pcg->pcg_avail++; |
Line 2742 pool_cache_xcall(pool_cache_t pc) |
|
Line 2716 pool_cache_xcall(pool_cache_t pc) |
|
s = splvm(); |
s = splvm(); |
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
if (cur != NULL) { |
if (cur != NULL) { |
if (cur->pcg_avail == cur->pcg_size) { |
if (cur->pcg_avail == PCG_NOBJECTS) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (cur->pcg_avail == 0) { |
} else if (cur->pcg_avail == 0) { |
Line 2756 pool_cache_xcall(pool_cache_t pc) |
|
Line 2730 pool_cache_xcall(pool_cache_t pc) |
|
*list = cur; |
*list = cur; |
} |
} |
if (prev != NULL) { |
if (prev != NULL) { |
if (prev->pcg_avail == prev->pcg_size) { |
if (prev->pcg_avail == PCG_NOBJECTS) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (prev->pcg_avail == 0) { |
} else if (prev->pcg_avail == 0) { |
Line 2792 void pool_page_free(struct pool *, void |
|
Line 2766 void pool_page_free(struct pool *, void |
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
struct pool_allocator pool_allocator_kmem_fullpage = { |
struct pool_allocator pool_allocator_kmem_fullpage = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
.pa_backingmapptr = &kmem_map, |
.pa_backingmapptr = &kernel_map, |
}; |
}; |
#else |
#else |
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
.pa_backingmapptr = &kmem_map, |
.pa_backingmapptr = &kernel_map, |
}; |
}; |
#endif |
#endif |
|
|
Line 2822 void pool_subpage_free(struct pool *, vo |
|
Line 2796 void pool_subpage_free(struct pool *, vo |
|
|
|
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
.pa_backingmapptr = &kmem_map, |
.pa_backingmapptr = &kernel_map, |
}; |
}; |
|
|
void *pool_subpage_alloc_nointr(struct pool *, int); |
void *pool_subpage_alloc_nointr(struct pool *, int); |
Line 2830 void pool_subpage_free_nointr(struct poo |
|
Line 2804 void pool_subpage_free_nointr(struct poo |
|
|
|
struct pool_allocator pool_allocator_nointr = { |
struct pool_allocator pool_allocator_nointr = { |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
.pa_backingmapptr = &kmem_map, |
.pa_backingmapptr = &kernel_map, |
}; |
}; |
#endif /* POOL_SUBPAGE */ |
#endif /* POOL_SUBPAGE */ |
|
|
Line 2868 pool_page_alloc(struct pool *pp, int fla |
|
Line 2842 pool_page_alloc(struct pool *pp, int fla |
|
{ |
{ |
bool waitok = (flags & PR_WAITOK) ? true : false; |
bool waitok = (flags & PR_WAITOK) ? true : false; |
|
|
return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); |
return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); |
} |
} |
|
|
void |
void |
pool_page_free(struct pool *pp, void *v) |
pool_page_free(struct pool *pp, void *v) |
{ |
{ |
|
|
uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v); |
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
} |
} |
|
|
static void * |
static void * |
Line 2883 pool_page_alloc_meta(struct pool *pp, in |
|
Line 2857 pool_page_alloc_meta(struct pool *pp, in |
|
{ |
{ |
bool waitok = (flags & PR_WAITOK) ? true : false; |
bool waitok = (flags & PR_WAITOK) ? true : false; |
|
|
return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); |
return ((void *) uvm_km_alloc_poolpage(kernel_map, waitok)); |
} |
} |
|
|
static void |
static void |
pool_page_free_meta(struct pool *pp, void *v) |
pool_page_free_meta(struct pool *pp, void *v) |
{ |
{ |
|
|
uvm_km_free_poolpage(kmem_map, (vaddr_t) v); |
uvm_km_free_poolpage(kernel_map, (vaddr_t) v); |
} |
} |
|
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
Line 2946 pool_in_page(struct pool *pp, struct poo |
|
Line 2920 pool_in_page(struct pool *pp, struct poo |
|
addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
} |
} |
|
|
static bool |
|
pool_in_item(struct pool *pp, void *item, uintptr_t addr) |
|
{ |
|
|
|
return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; |
|
} |
|
|
|
static bool |
|
pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) |
|
{ |
|
int i; |
|
|
|
if (pcg == NULL) { |
|
return false; |
|
} |
|
for (i = 0; i < pcg->pcg_avail; i++) { |
|
if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { |
|
return true; |
|
} |
|
} |
|
return false; |
|
} |
|
|
|
static bool |
|
pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) |
|
{ |
|
|
|
if ((pp->pr_roflags & PR_NOTOUCH) != 0) { |
|
unsigned int idx = pr_item_notouch_index(pp, ph, (void *)addr); |
|
pool_item_bitmap_t *bitmap = |
|
ph->ph_bitmap + (idx / BITMAP_SIZE); |
|
pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); |
|
|
|
return (*bitmap & mask) == 0; |
|
} else { |
|
struct pool_item *pi; |
|
|
|
LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { |
|
if (pool_in_item(pp, pi, addr)) { |
|
return false; |
|
} |
|
} |
|
return true; |
|
} |
|
} |
|
|
|
void |
void |
pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
struct pool_item_header *ph; |
struct pool_item_header *ph; |
uintptr_t item; |
uintptr_t item; |
bool allocated = true; |
|
bool incache = false; |
|
bool incpucache = false; |
|
char cpucachestr[32]; |
|
|
|
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
Line 3013 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2937 pool_whatis(uintptr_t addr, void (*pr)(c |
|
} |
} |
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
if (pool_in_page(pp, ph, addr)) { |
if (pool_in_page(pp, ph, addr)) { |
allocated = |
|
pool_allocated(pp, ph, addr); |
|
goto found; |
|
} |
|
} |
|
LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { |
|
if (pool_in_page(pp, ph, addr)) { |
|
allocated = false; |
|
goto found; |
goto found; |
} |
} |
} |
} |
Line 3030 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2946 pool_whatis(uintptr_t addr, void (*pr)(c |
|
if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
continue; |
continue; |
} |
} |
allocated = pool_allocated(pp, ph, addr); |
|
} |
} |
found: |
found: |
if (allocated && pp->pr_cache) { |
|
pool_cache_t pc = pp->pr_cache; |
|
struct pool_cache_group *pcg; |
|
int i; |
|
|
|
for (pcg = pc->pc_fullgroups; pcg != NULL; |
|
pcg = pcg->pcg_next) { |
|
if (pool_in_cg(pp, pcg, addr)) { |
|
incache = true; |
|
goto print; |
|
} |
|
} |
|
for (i = 0; i < MAXCPUS; i++) { |
|
pool_cache_cpu_t *cc; |
|
|
|
if ((cc = pc->pc_cpus[i]) == NULL) { |
|
continue; |
|
} |
|
if (pool_in_cg(pp, cc->cc_current, addr) || |
|
pool_in_cg(pp, cc->cc_previous, addr)) { |
|
struct cpu_info *ci = |
|
cpu_lookup_byindex(i); |
|
|
|
incpucache = true; |
|
snprintf(cpucachestr, |
|
sizeof(cpucachestr), |
|
"cached by CPU %u", |
|
(u_int)ci->ci_cpuid); |
|
goto print; |
|
} |
|
} |
|
} |
|
print: |
|
item = (uintptr_t)ph->ph_page + ph->ph_off; |
item = (uintptr_t)ph->ph_page + ph->ph_off; |
item = item + rounddown(addr - item, pp->pr_size); |
item = item + rounddown(addr - item, pp->pr_size); |
(*pr)("%p is %p+%zu in POOL '%s' (%s)\n", |
(*pr)("%p is %p+%zu from POOL '%s'\n", |
(void *)addr, item, (size_t)(addr - item), |
(void *)addr, item, (size_t)(addr - item), |
pp->pr_wchan, |
pp->pr_wchan); |
incpucache ? cpucachestr : |
|
incache ? "cached" : allocated ? "allocated" : "free"); |
|
} |
} |
} |
} |
#endif /* defined(DDB) */ |
#endif /* defined(DDB) */ |