version 1.138.2.2, 2007/12/10 12:56:10 |
version 1.145, 2007/12/26 16:01:36 |
|
|
#include <sys/cdefs.h> |
#include <sys/cdefs.h> |
__KERNEL_RCSID(0, "$NetBSD$"); |
__KERNEL_RCSID(0, "$NetBSD$"); |
|
|
|
#include "opt_ddb.h" |
#include "opt_pool.h" |
#include "opt_pool.h" |
#include "opt_poollog.h" |
#include "opt_poollog.h" |
#include "opt_lockdebug.h" |
#include "opt_lockdebug.h" |
Line 58 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 59 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include <sys/lockdebug.h> |
#include <sys/lockdebug.h> |
#include <sys/xcall.h> |
#include <sys/xcall.h> |
#include <sys/cpu.h> |
#include <sys/cpu.h> |
|
#include <sys/atomic.h> |
|
|
#include <uvm/uvm.h> |
#include <uvm/uvm.h> |
|
|
Line 75 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 77 __KERNEL_RCSID(0, "$NetBSD$"); |
|
*/ |
*/ |
|
|
/* List of all pools */ |
/* List of all pools */ |
LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head); |
TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
|
|
/* List of all caches. */ |
|
LIST_HEAD(,pool_cache) pool_cache_head = |
|
LIST_HEAD_INITIALIZER(pool_cache_head); |
|
|
|
/* Private pool for page header structures */ |
/* Private pool for page header structures */ |
#define PHPOOL_MAX 8 |
#define PHPOOL_MAX 8 |
Line 101 static void pool_page_free_meta(struct p |
|
Line 99 static void pool_page_free_meta(struct p |
|
/* allocator for pool metadata */ |
/* allocator for pool metadata */ |
struct pool_allocator pool_allocator_meta = { |
struct pool_allocator pool_allocator_meta = { |
pool_page_alloc_meta, pool_page_free_meta, |
pool_page_alloc_meta, pool_page_free_meta, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
|
|
/* # of seconds to retain page after last use */ |
/* # of seconds to retain page after last use */ |
Line 127 struct pool_item_header { |
|
Line 125 struct pool_item_header { |
|
void * ph_page; /* this page's address */ |
void * ph_page; /* this page's address */ |
struct timeval ph_time; /* last referenced */ |
struct timeval ph_time; /* last referenced */ |
uint16_t ph_nmissing; /* # of chunks in use */ |
uint16_t ph_nmissing; /* # of chunks in use */ |
|
uint16_t ph_off; /* start offset in page */ |
union { |
union { |
/* !PR_NOTOUCH */ |
/* !PR_NOTOUCH */ |
struct { |
struct { |
Line 135 struct pool_item_header { |
|
Line 134 struct pool_item_header { |
|
} phu_normal; |
} phu_normal; |
/* PR_NOTOUCH */ |
/* PR_NOTOUCH */ |
struct { |
struct { |
uint16_t phu_off; /* start offset in page */ |
pool_item_bitmap_t phu_bitmap[1]; |
pool_item_bitmap_t phu_bitmap[]; |
|
} phu_notouch; |
} phu_notouch; |
} ph_u; |
} ph_u; |
}; |
}; |
#define ph_itemlist ph_u.phu_normal.phu_itemlist |
#define ph_itemlist ph_u.phu_normal.phu_itemlist |
#define ph_off ph_u.phu_notouch.phu_off |
|
#define ph_bitmap ph_u.phu_notouch.phu_bitmap |
#define ph_bitmap ph_u.phu_notouch.phu_bitmap |
|
|
struct pool_item { |
struct pool_item { |
Line 181 struct pool_item { |
|
Line 178 struct pool_item { |
|
* from it. |
* from it. |
*/ |
*/ |
|
|
static struct pool pcgpool; |
static struct pool pcg_normal_pool; |
|
static struct pool pcg_large_pool; |
static struct pool cache_pool; |
static struct pool cache_pool; |
static struct pool cache_cpu_pool; |
static struct pool cache_cpu_pool; |
|
|
|
/* List of all caches. */ |
|
TAILQ_HEAD(,pool_cache) pool_cache_head = |
|
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
|
|
|
int pool_cache_disable; |
|
|
|
|
static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, |
static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, |
void *, paddr_t); |
void *, paddr_t); |
static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, |
static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, |
Line 412 phtree_compare(struct pool_item_header * |
|
Line 417 phtree_compare(struct pool_item_header * |
|
SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare); |
SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare); |
SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); |
SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); |
|
|
|
static inline struct pool_item_header * |
|
pr_find_pagehead_noalign(struct pool *pp, void *v) |
|
{ |
|
struct pool_item_header *ph, tmp; |
|
|
|
tmp.ph_page = (void *)(uintptr_t)v; |
|
ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
|
if (ph == NULL) { |
|
ph = SPLAY_ROOT(&pp->pr_phtree); |
|
if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { |
|
ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); |
|
} |
|
KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); |
|
} |
|
|
|
return ph; |
|
} |
|
|
/* |
/* |
* Return the pool page header based on item address. |
* Return the pool page header based on item address. |
*/ |
*/ |
Line 421 pr_find_pagehead(struct pool *pp, void * |
|
Line 444 pr_find_pagehead(struct pool *pp, void * |
|
struct pool_item_header *ph, tmp; |
struct pool_item_header *ph, tmp; |
|
|
if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
tmp.ph_page = (void *)(uintptr_t)v; |
ph = pr_find_pagehead_noalign(pp, v); |
ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
|
if (ph == NULL) { |
|
ph = SPLAY_ROOT(&pp->pr_phtree); |
|
if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { |
|
ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); |
|
} |
|
KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); |
|
} |
|
} else { |
} else { |
void *page = |
void *page = |
(void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask); |
(void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask); |
|
|
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
const char *wchan, struct pool_allocator *palloc, int ipl) |
const char *wchan, struct pool_allocator *palloc, int ipl) |
{ |
{ |
#ifdef DEBUG |
|
struct pool *pp1; |
struct pool *pp1; |
#endif |
|
size_t trysize, phsize; |
size_t trysize, phsize; |
int off, slack; |
int off, slack; |
|
|
Line 626 pool_init(struct pool *pp, size_t size, |
|
Line 639 pool_init(struct pool *pp, size_t size, |
|
* Check that the pool hasn't already been initialised and |
* Check that the pool hasn't already been initialised and |
* added to the list of all pools. |
* added to the list of all pools. |
*/ |
*/ |
LIST_FOREACH(pp1, &pool_head, pr_poollist) { |
TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
if (pp == pp1) |
if (pp == pp1) |
panic("pool_init: pool %s already initialised", |
panic("pool_init: pool %s already initialised", |
wchan); |
wchan); |
Line 790 pool_init(struct pool *pp, size_t size, |
|
Line 803 pool_init(struct pool *pp, size_t size, |
|
|
|
#ifdef POOL_DIAGNOSTIC |
#ifdef POOL_DIAGNOSTIC |
if (flags & PR_LOGGING) { |
if (flags & PR_LOGGING) { |
if (kernel_map == NULL || |
if (kmem_map == NULL || |
(pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), |
(pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), |
M_TEMP, M_NOWAIT)) == NULL) |
M_TEMP, M_NOWAIT)) == NULL) |
pp->pr_roflags &= ~PR_LOGGING; |
pp->pr_roflags &= ~PR_LOGGING; |
Line 840 pool_init(struct pool *pp, size_t size, |
|
Line 853 pool_init(struct pool *pp, size_t size, |
|
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
#endif |
#endif |
pool_init(&pcgpool, sizeof(pcg_t), CACHE_LINE_SIZE, 0, 0, |
|
"cachegrp", &pool_allocator_meta, IPL_VM); |
size = sizeof(pcg_t) + |
|
(PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_normal_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcgnormal", &pool_allocator_meta, IPL_VM); |
|
|
|
size = sizeof(pcg_t) + |
|
(PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); |
|
pool_init(&pcg_large_pool, size, CACHE_LINE_SIZE, 0, 0, |
|
"pcglarge", &pool_allocator_meta, IPL_VM); |
} |
} |
|
|
if (__predict_true(!cold)) { |
/* Insert into the list of all pools. */ |
/* Insert into the list of all pools. */ |
if (__predict_true(!cold)) |
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); |
TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
|
if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) |
|
break; |
|
} |
|
if (pp1 == NULL) |
|
TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); |
|
else |
|
TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); |
|
if (__predict_true(!cold)) |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* Insert this into the list of pools using this allocator. */ |
/* Insert this into the list of pools using this allocator. */ |
|
if (__predict_true(!cold)) |
mutex_enter(&palloc->pa_lock); |
mutex_enter(&palloc->pa_lock); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
|
if (__predict_true(!cold)) |
mutex_exit(&palloc->pa_lock); |
mutex_exit(&palloc->pa_lock); |
} else { |
|
LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); |
|
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
|
} |
|
|
|
pool_reclaim_register(pp); |
pool_reclaim_register(pp); |
} |
} |
Line 875 pool_destroy(struct pool *pp) |
|
Line 902 pool_destroy(struct pool *pp) |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
while (pp->pr_refcnt != 0) |
while (pp->pr_refcnt != 0) |
cv_wait(&pool_busy, &pool_head_lock); |
cv_wait(&pool_busy, &pool_head_lock); |
LIST_REMOVE(pp, pr_poollist); |
TAILQ_REMOVE(&pool_head, pp, pr_poollist); |
if (drainpp == pp) |
if (drainpp == pp) |
drainpp = NULL; |
drainpp = NULL; |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
Line 1436 pool_prime_page(struct pool *pp, void *s |
|
Line 1463 pool_prime_page(struct pool *pp, void *s |
|
/* |
/* |
* Color this page. |
* Color this page. |
*/ |
*/ |
cp = (char *)cp + pp->pr_curcolor; |
ph->ph_off = pp->pr_curcolor; |
|
cp = (char *)cp + ph->ph_off; |
if ((pp->pr_curcolor += align) > pp->pr_maxcolor) |
if ((pp->pr_curcolor += align) > pp->pr_maxcolor) |
pp->pr_curcolor = 0; |
pp->pr_curcolor = 0; |
|
|
Line 1681 pool_drain_start(struct pool **ppp, uint |
|
Line 1709 pool_drain_start(struct pool **ppp, uint |
|
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
KASSERT(!LIST_EMPTY(&pool_head)); |
KASSERT(!TAILQ_EMPTY(&pool_head)); |
|
|
pp = NULL; |
pp = NULL; |
|
|
Line 1689 pool_drain_start(struct pool **ppp, uint |
|
Line 1717 pool_drain_start(struct pool **ppp, uint |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
do { |
do { |
if (drainpp == NULL) { |
if (drainpp == NULL) { |
drainpp = LIST_FIRST(&pool_head); |
drainpp = TAILQ_FIRST(&pool_head); |
} |
} |
if (drainpp != NULL) { |
if (drainpp != NULL) { |
pp = drainpp; |
pp = drainpp; |
drainpp = LIST_NEXT(pp, pr_poollist); |
drainpp = TAILQ_NEXT(pp, pr_poollist); |
} |
} |
/* |
/* |
* Skip completely idle pools. We depend on at least |
* Skip completely idle pools. We depend on at least |
Line 1749 pool_printall(const char *modif, void (* |
|
Line 1777 pool_printall(const char *modif, void (* |
|
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
pool_printit(pp, modif, pr); |
pool_printit(pp, modif, pr); |
} |
} |
} |
} |
Line 1866 pool_print1(struct pool *pp, const char |
|
Line 1894 pool_print1(struct pool *pp, const char |
|
|
|
#define PR_GROUPLIST(pcg) \ |
#define PR_GROUPLIST(pcg) \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
for (i = 0; i < PCG_NOBJECTS; i++) { \ |
for (i = 0; i < pcg->pcg_size; i++) { \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
if (pcg->pcg_objects[i].pcgo_pa != \ |
POOL_PADDR_INVALID) { \ |
POOL_PADDR_INVALID) { \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
(*pr)("\t\t\t%p, 0x%llx\n", \ |
Line 2039 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2067 pool_cache_bootstrap(pool_cache_t pc, si |
|
void *arg) |
void *arg) |
{ |
{ |
CPU_INFO_ITERATOR cii; |
CPU_INFO_ITERATOR cii; |
|
pool_cache_t pc1; |
struct cpu_info *ci; |
struct cpu_info *ci; |
struct pool *pp; |
struct pool *pp; |
|
|
Line 2078 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2107 pool_cache_bootstrap(pool_cache_t pc, si |
|
pc->pc_refcnt = 0; |
pc->pc_refcnt = 0; |
pc->pc_freecheck = NULL; |
pc->pc_freecheck = NULL; |
|
|
|
if ((flags & PR_LARGECACHE) != 0) { |
|
pc->pc_pcgsize = PCG_NOBJECTS_LARGE; |
|
} else { |
|
pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; |
|
} |
|
|
/* Allocate per-CPU caches. */ |
/* Allocate per-CPU caches. */ |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
pc->pc_ncpu = 0; |
pc->pc_ncpu = 0; |
if (ncpu == 0) { |
if (ncpu < 2) { |
/* XXX For sparc: boot CPU is not attached yet. */ |
/* XXX For sparc: boot CPU is not attached yet. */ |
pool_cache_cpu_init1(curcpu(), pc); |
pool_cache_cpu_init1(curcpu(), pc); |
} else { |
} else { |
Line 2089 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2124 pool_cache_bootstrap(pool_cache_t pc, si |
|
pool_cache_cpu_init1(ci, pc); |
pool_cache_cpu_init1(ci, pc); |
} |
} |
} |
} |
|
|
if (__predict_true(!cold)) { |
/* Add to list of all pools. */ |
mutex_enter(&pp->pr_lock); |
if (__predict_true(!cold)) |
pp->pr_cache = pc; |
|
mutex_exit(&pp->pr_lock); |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); |
TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { |
mutex_exit(&pool_head_lock); |
if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) |
} else { |
break; |
pp->pr_cache = pc; |
|
LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); |
|
} |
} |
|
if (pc1 == NULL) |
|
TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); |
|
else |
|
TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); |
|
if (__predict_true(!cold)) |
|
mutex_exit(&pool_head_lock); |
|
|
|
membar_sync(); |
|
pp->pr_cache = pc; |
} |
} |
|
|
/* |
/* |
Line 2111 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2151 pool_cache_bootstrap(pool_cache_t pc, si |
|
void |
void |
pool_cache_destroy(pool_cache_t pc) |
pool_cache_destroy(pool_cache_t pc) |
{ |
{ |
|
|
pool_cache_bootstrap_destroy(pc); |
|
pool_put(&cache_pool, pc); |
|
} |
|
|
|
/* |
|
* pool_cache_bootstrap_destroy: |
|
* |
|
* Kernel-private version of pool_cache_destroy(). |
|
* Destroy a pool cache initialized by pool_cache_bootstrap. |
|
*/ |
|
void |
|
pool_cache_bootstrap_destroy(pool_cache_t pc) |
|
{ |
|
struct pool *pp = &pc->pc_pool; |
struct pool *pp = &pc->pc_pool; |
pool_cache_cpu_t *cc; |
pool_cache_cpu_t *cc; |
pcg_t *pcg; |
pcg_t *pcg; |
Line 2134 pool_cache_bootstrap_destroy(pool_cache_ |
|
Line 2160 pool_cache_bootstrap_destroy(pool_cache_ |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
while (pc->pc_refcnt != 0) |
while (pc->pc_refcnt != 0) |
cv_wait(&pool_busy, &pool_head_lock); |
cv_wait(&pool_busy, &pool_head_lock); |
LIST_REMOVE(pc, pc_cachelist); |
TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* First, invalidate the entire cache. */ |
/* First, invalidate the entire cache. */ |
Line 2164 pool_cache_bootstrap_destroy(pool_cache_ |
|
Line 2190 pool_cache_bootstrap_destroy(pool_cache_ |
|
/* Finally, destroy it. */ |
/* Finally, destroy it. */ |
mutex_destroy(&pc->pc_lock); |
mutex_destroy(&pc->pc_lock); |
pool_destroy(pp); |
pool_destroy(pp); |
|
pool_put(&cache_pool, pc); |
} |
} |
|
|
/* |
/* |
Line 2224 pool_cache_cpu_init(struct cpu_info *ci) |
|
Line 2251 pool_cache_cpu_init(struct cpu_info *ci) |
|
pool_cache_t pc; |
pool_cache_t pc; |
|
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
LIST_FOREACH(pc, &pool_cache_head, pc_cachelist) { |
TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { |
pc->pc_refcnt++; |
pc->pc_refcnt++; |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
Line 2292 pool_cache_invalidate_groups(pool_cache_ |
|
Line 2319 pool_cache_invalidate_groups(pool_cache_ |
|
pool_cache_destruct_object1(pc, object); |
pool_cache_destruct_object1(pc, object); |
} |
} |
|
|
pool_put(&pcgpool, pcg); |
if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { |
|
pool_put(&pcg_large_pool, pcg); |
|
} else { |
|
KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); |
|
pool_put(&pcg_normal_pool, pcg); |
|
} |
} |
} |
} |
} |
|
|
Line 2432 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
Line 2464 pool_cache_get_slow(pool_cache_cpu_t *cc |
|
pc->pc_emptygroups = cur; |
pc->pc_emptygroups = cur; |
pc->pc_nempty++; |
pc->pc_nempty++; |
} |
} |
KASSERT(pcg->pcg_avail == PCG_NOBJECTS); |
KASSERT(pcg->pcg_avail == pcg->pcg_size); |
cc->cc_current = pcg; |
cc->cc_current = pcg; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_fullgroups = pcg->pcg_next; |
pc->pc_hits++; |
pc->pc_hits++; |
Line 2504 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2536 pool_cache_get_paddr(pool_cache_t pc, in |
|
if (pap != NULL) |
if (pap != NULL) |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
KASSERT(pcg->pcg_avail <= PCG_NOBJECTS); |
KASSERT(pcg->pcg_avail <= pcg->pcg_size); |
KASSERT(object != NULL); |
KASSERT(object != NULL); |
cc->cc_hits++; |
cc->cc_hits++; |
pool_cache_cpu_exit(cc, &s); |
pool_cache_cpu_exit(cc, &s); |
Line 2544 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2576 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
pcg_t *pcg, *cur; |
pcg_t *pcg, *cur; |
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
|
u_int nobj; |
|
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
cc->cc_misses++; |
cc->cc_misses++; |
Line 2576 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2609 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* group as cc_current and return. |
* group as cc_current and return. |
*/ |
*/ |
if ((cur = cc->cc_current) != NULL) { |
if ((cur = cc->cc_current) != NULL) { |
KASSERT(cur->pcg_avail == PCG_NOBJECTS); |
KASSERT(cur->pcg_avail == pcg->pcg_size); |
cur->pcg_next = pc->pc_fullgroups; |
cur->pcg_next = pc->pc_fullgroups; |
pc->pc_fullgroups = cur; |
pc->pc_fullgroups = cur; |
pc->pc_nfull++; |
pc->pc_nfull++; |
Line 2603 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2636 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* If we can't allocate a new group, just throw the |
* If we can't allocate a new group, just throw the |
* object away. |
* object away. |
*/ |
*/ |
pcg = pool_get(&pcgpool, PR_NOWAIT); |
nobj = pc->pc_pcgsize; |
|
if (nobj == PCG_NOBJECTS_LARGE) { |
|
pcg = pool_get(&pcg_large_pool, PR_NOWAIT); |
|
} else { |
|
pcg = pool_get(&pcg_normal_pool, PR_NOWAIT); |
|
} |
if (pcg == NULL) { |
if (pcg == NULL) { |
pool_cache_destruct_object(pc, object); |
pool_cache_destruct_object(pc, object); |
return NULL; |
return NULL; |
} |
} |
#ifdef DIAGNOSTIC |
|
memset(pcg, 0, sizeof(*pcg)); |
|
#else |
|
pcg->pcg_avail = 0; |
pcg->pcg_avail = 0; |
#endif |
pcg->pcg_size = nobj; |
|
|
/* |
/* |
* Add the empty group to the cache and try again. |
* Add the empty group to the cache and try again. |
Line 2645 pool_cache_put_paddr(pool_cache_t pc, vo |
|
Line 2680 pool_cache_put_paddr(pool_cache_t pc, vo |
|
do { |
do { |
/* If the current group isn't full, release it there. */ |
/* If the current group isn't full, release it there. */ |
pcg = cc->cc_current; |
pcg = cc->cc_current; |
if (pcg != NULL && pcg->pcg_avail < PCG_NOBJECTS) { |
if (pcg != NULL && pcg->pcg_avail < pcg->pcg_size) { |
KASSERT(pcg->pcg_objects[pcg->pcg_avail].pcgo_va |
|
== NULL); |
|
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
pcg->pcg_avail++; |
pcg->pcg_avail++; |
Line 2705 pool_cache_xcall(pool_cache_t pc) |
|
Line 2738 pool_cache_xcall(pool_cache_t pc) |
|
s = splvm(); |
s = splvm(); |
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
if (cur != NULL) { |
if (cur != NULL) { |
if (cur->pcg_avail == PCG_NOBJECTS) { |
if (cur->pcg_avail == cur->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (cur->pcg_avail == 0) { |
} else if (cur->pcg_avail == 0) { |
Line 2719 pool_cache_xcall(pool_cache_t pc) |
|
Line 2752 pool_cache_xcall(pool_cache_t pc) |
|
*list = cur; |
*list = cur; |
} |
} |
if (prev != NULL) { |
if (prev != NULL) { |
if (prev->pcg_avail == PCG_NOBJECTS) { |
if (prev->pcg_avail == prev->pcg_size) { |
list = &pc->pc_fullgroups; |
list = &pc->pc_fullgroups; |
pc->pc_nfull++; |
pc->pc_nfull++; |
} else if (prev->pcg_avail == 0) { |
} else if (prev->pcg_avail == 0) { |
Line 2755 void pool_page_free(struct pool *, void |
|
Line 2788 void pool_page_free(struct pool *, void |
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
struct pool_allocator pool_allocator_kmem_fullpage = { |
struct pool_allocator pool_allocator_kmem_fullpage = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
#else |
#else |
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
#endif |
#endif |
|
|
Line 2785 void pool_subpage_free(struct pool *, vo |
|
Line 2818 void pool_subpage_free(struct pool *, vo |
|
|
|
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
|
|
void *pool_subpage_alloc_nointr(struct pool *, int); |
void *pool_subpage_alloc_nointr(struct pool *, int); |
Line 2793 void pool_subpage_free_nointr(struct poo |
|
Line 2826 void pool_subpage_free_nointr(struct poo |
|
|
|
struct pool_allocator pool_allocator_nointr = { |
struct pool_allocator pool_allocator_nointr = { |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
.pa_backingmapptr = &kernel_map, |
.pa_backingmapptr = &kmem_map, |
}; |
}; |
#endif /* POOL_SUBPAGE */ |
#endif /* POOL_SUBPAGE */ |
|
|
Line 2831 pool_page_alloc(struct pool *pp, int fla |
|
Line 2864 pool_page_alloc(struct pool *pp, int fla |
|
{ |
{ |
bool waitok = (flags & PR_WAITOK) ? true : false; |
bool waitok = (flags & PR_WAITOK) ? true : false; |
|
|
return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); |
return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); |
} |
} |
|
|
void |
void |
pool_page_free(struct pool *pp, void *v) |
pool_page_free(struct pool *pp, void *v) |
{ |
{ |
|
|
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v); |
} |
} |
|
|
static void * |
static void * |
Line 2846 pool_page_alloc_meta(struct pool *pp, in |
|
Line 2879 pool_page_alloc_meta(struct pool *pp, in |
|
{ |
{ |
bool waitok = (flags & PR_WAITOK) ? true : false; |
bool waitok = (flags & PR_WAITOK) ? true : false; |
|
|
return ((void *) uvm_km_alloc_poolpage(kernel_map, waitok)); |
return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); |
} |
} |
|
|
static void |
static void |
pool_page_free_meta(struct pool *pp, void *v) |
pool_page_free_meta(struct pool *pp, void *v) |
{ |
{ |
|
|
uvm_km_free_poolpage(kernel_map, (vaddr_t) v); |
uvm_km_free_poolpage(kmem_map, (vaddr_t) v); |
} |
} |
|
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
Line 2899 pool_page_free_nointr(struct pool *pp, v |
|
Line 2932 pool_page_free_nointr(struct pool *pp, v |
|
|
|
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
} |
} |
|
|
|
#if defined(DDB) |
|
static bool |
|
pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) |
|
{ |
|
|
|
return (uintptr_t)ph->ph_page <= addr && |
|
addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
|
} |
|
|
|
static bool |
|
pool_in_item(struct pool *pp, void *item, uintptr_t addr) |
|
{ |
|
|
|
return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; |
|
} |
|
|
|
static bool |
|
pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) |
|
{ |
|
int i; |
|
|
|
if (pcg == NULL) { |
|
return false; |
|
} |
|
for (i = 0; i < pcg->pcg_avail; i++) { |
|
if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { |
|
return true; |
|
} |
|
} |
|
return false; |
|
} |
|
|
|
static bool |
|
pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) |
|
{ |
|
|
|
if ((pp->pr_roflags & PR_NOTOUCH) != 0) { |
|
unsigned int idx = pr_item_notouch_index(pp, ph, (void *)addr); |
|
pool_item_bitmap_t *bitmap = |
|
ph->ph_bitmap + (idx / BITMAP_SIZE); |
|
pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); |
|
|
|
return (*bitmap & mask) == 0; |
|
} else { |
|
struct pool_item *pi; |
|
|
|
LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { |
|
if (pool_in_item(pp, pi, addr)) { |
|
return false; |
|
} |
|
} |
|
return true; |
|
} |
|
} |
|
|
|
void |
|
pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
|
{ |
|
struct pool *pp; |
|
|
|
TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
|
struct pool_item_header *ph; |
|
uintptr_t item; |
|
bool allocated = true; |
|
bool incache = false; |
|
bool incpucache = false; |
|
char cpucachestr[32]; |
|
|
|
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
|
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
|
if (pool_in_page(pp, ph, addr)) { |
|
goto found; |
|
} |
|
} |
|
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
|
if (pool_in_page(pp, ph, addr)) { |
|
allocated = |
|
pool_allocated(pp, ph, addr); |
|
goto found; |
|
} |
|
} |
|
LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { |
|
if (pool_in_page(pp, ph, addr)) { |
|
allocated = false; |
|
goto found; |
|
} |
|
} |
|
continue; |
|
} else { |
|
ph = pr_find_pagehead_noalign(pp, (void *)addr); |
|
if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
|
continue; |
|
} |
|
allocated = pool_allocated(pp, ph, addr); |
|
} |
|
found: |
|
if (allocated && pp->pr_cache) { |
|
pool_cache_t pc = pp->pr_cache; |
|
struct pool_cache_group *pcg; |
|
int i; |
|
|
|
for (pcg = pc->pc_fullgroups; pcg != NULL; |
|
pcg = pcg->pcg_next) { |
|
if (pool_in_cg(pp, pcg, addr)) { |
|
incache = true; |
|
goto print; |
|
} |
|
} |
|
for (i = 0; i < MAXCPUS; i++) { |
|
pool_cache_cpu_t *cc; |
|
|
|
if ((cc = pc->pc_cpus[i]) == NULL) { |
|
continue; |
|
} |
|
if (pool_in_cg(pp, cc->cc_current, addr) || |
|
pool_in_cg(pp, cc->cc_previous, addr)) { |
|
struct cpu_info *ci = |
|
cpu_lookup_byindex(i); |
|
|
|
incpucache = true; |
|
snprintf(cpucachestr, |
|
sizeof(cpucachestr), |
|
"cached by CPU %u", |
|
(u_int)ci->ci_cpuid); |
|
goto print; |
|
} |
|
} |
|
} |
|
print: |
|
item = (uintptr_t)ph->ph_page + ph->ph_off; |
|
item = item + rounddown(addr - item, pp->pr_size); |
|
(*pr)("%p is %p+%zu in POOL '%s' (%s)\n", |
|
(void *)addr, item, (size_t)(addr - item), |
|
pp->pr_wchan, |
|
incpucache ? cpucachestr : |
|
incache ? "cached" : allocated ? "allocated" : "free"); |
|
} |
|
} |
|
#endif /* defined(DDB) */ |