version 1.137.2.5, 2007/12/26 21:39:42 |
version 1.143, 2007/12/22 03:28:48 |
Line 59 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 59 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include <sys/lockdebug.h> |
#include <sys/lockdebug.h> |
#include <sys/xcall.h> |
#include <sys/xcall.h> |
#include <sys/cpu.h> |
#include <sys/cpu.h> |
#include <sys/atomic.h> |
|
|
|
#include <uvm/uvm.h> |
#include <uvm/uvm.h> |
|
|
Line 77 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 76 __KERNEL_RCSID(0, "$NetBSD$"); |
|
*/ |
*/ |
|
|
/* List of all pools */ |
/* List of all pools */ |
TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head); |
|
|
|
/* List of all caches. */ |
|
LIST_HEAD(,pool_cache) pool_cache_head = |
|
LIST_HEAD_INITIALIZER(pool_cache_head); |
|
|
/* Private pool for page header structures */ |
/* Private pool for page header structures */ |
#define PHPOOL_MAX 8 |
#define PHPOOL_MAX 8 |
Line 183 static struct pool pcg_large_pool; |
|
Line 186 static struct pool pcg_large_pool; |
|
static struct pool cache_pool; |
static struct pool cache_pool; |
static struct pool cache_cpu_pool; |
static struct pool cache_cpu_pool; |
|
|
/* List of all caches. */ |
|
TAILQ_HEAD(,pool_cache) pool_cache_head = |
|
TAILQ_HEAD_INITIALIZER(pool_cache_head); |
|
|
|
int pool_cache_disable; |
|
|
|
|
|
static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, |
static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, |
void *, paddr_t); |
void *, paddr_t); |
static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, |
static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, |
|
|
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
const char *wchan, struct pool_allocator *palloc, int ipl) |
const char *wchan, struct pool_allocator *palloc, int ipl) |
{ |
{ |
|
#ifdef DEBUG |
struct pool *pp1; |
struct pool *pp1; |
|
#endif |
size_t trysize, phsize; |
size_t trysize, phsize; |
int off, slack; |
int off, slack; |
|
|
Line 639 pool_init(struct pool *pp, size_t size, |
|
Line 637 pool_init(struct pool *pp, size_t size, |
|
* Check that the pool hasn't already been initialised and |
* Check that the pool hasn't already been initialised and |
* added to the list of all pools. |
* added to the list of all pools. |
*/ |
*/ |
TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
LIST_FOREACH(pp1, &pool_head, pr_poollist) { |
if (pp == pp1) |
if (pp == pp1) |
panic("pool_init: pool %s already initialised", |
panic("pool_init: pool %s already initialised", |
wchan); |
wchan); |
Line 865 pool_init(struct pool *pp, size_t size, |
|
Line 863 pool_init(struct pool *pp, size_t size, |
|
"pcglarge", &pool_allocator_meta, IPL_VM); |
"pcglarge", &pool_allocator_meta, IPL_VM); |
} |
} |
|
|
/* Insert into the list of all pools. */ |
if (__predict_true(!cold)) { |
if (__predict_true(!cold)) |
/* Insert into the list of all pools. */ |
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); |
if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) |
|
break; |
|
} |
|
if (pp1 == NULL) |
|
TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); |
|
else |
|
TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); |
|
if (__predict_true(!cold)) |
|
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* Insert this into the list of pools using this allocator. */ |
/* Insert this into the list of pools using this allocator. */ |
if (__predict_true(!cold)) |
|
mutex_enter(&palloc->pa_lock); |
mutex_enter(&palloc->pa_lock); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
if (__predict_true(!cold)) |
|
mutex_exit(&palloc->pa_lock); |
mutex_exit(&palloc->pa_lock); |
|
} else { |
|
LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); |
|
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
|
} |
|
|
pool_reclaim_register(pp); |
pool_reclaim_register(pp); |
} |
} |
Line 902 pool_destroy(struct pool *pp) |
|
Line 894 pool_destroy(struct pool *pp) |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
while (pp->pr_refcnt != 0) |
while (pp->pr_refcnt != 0) |
cv_wait(&pool_busy, &pool_head_lock); |
cv_wait(&pool_busy, &pool_head_lock); |
TAILQ_REMOVE(&pool_head, pp, pr_poollist); |
LIST_REMOVE(pp, pr_poollist); |
if (drainpp == pp) |
if (drainpp == pp) |
drainpp = NULL; |
drainpp = NULL; |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
Line 1709 pool_drain_start(struct pool **ppp, uint |
|
Line 1701 pool_drain_start(struct pool **ppp, uint |
|
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
KASSERT(!TAILQ_EMPTY(&pool_head)); |
KASSERT(!LIST_EMPTY(&pool_head)); |
|
|
pp = NULL; |
pp = NULL; |
|
|
Line 1717 pool_drain_start(struct pool **ppp, uint |
|
Line 1709 pool_drain_start(struct pool **ppp, uint |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
do { |
do { |
if (drainpp == NULL) { |
if (drainpp == NULL) { |
drainpp = TAILQ_FIRST(&pool_head); |
drainpp = LIST_FIRST(&pool_head); |
} |
} |
if (drainpp != NULL) { |
if (drainpp != NULL) { |
pp = drainpp; |
pp = drainpp; |
drainpp = TAILQ_NEXT(pp, pr_poollist); |
drainpp = LIST_NEXT(pp, pr_poollist); |
} |
} |
/* |
/* |
* Skip completely idle pools. We depend on at least |
* Skip completely idle pools. We depend on at least |
Line 1777 pool_printall(const char *modif, void (* |
|
Line 1769 pool_printall(const char *modif, void (* |
|
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
pool_printit(pp, modif, pr); |
pool_printit(pp, modif, pr); |
} |
} |
} |
} |
Line 2067 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2059 pool_cache_bootstrap(pool_cache_t pc, si |
|
void *arg) |
void *arg) |
{ |
{ |
CPU_INFO_ITERATOR cii; |
CPU_INFO_ITERATOR cii; |
pool_cache_t pc1; |
|
struct cpu_info *ci; |
struct cpu_info *ci; |
struct pool *pp; |
struct pool *pp; |
|
|
Line 2124 pool_cache_bootstrap(pool_cache_t pc, si |
|
Line 2115 pool_cache_bootstrap(pool_cache_t pc, si |
|
pool_cache_cpu_init1(ci, pc); |
pool_cache_cpu_init1(ci, pc); |
} |
} |
} |
} |
|
|
/* Add to list of all pools. */ |
if (__predict_true(!cold)) { |
if (__predict_true(!cold)) |
mutex_enter(&pp->pr_lock); |
|
pp->pr_cache = pc; |
|
mutex_exit(&pp->pr_lock); |
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { |
LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); |
if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) |
|
break; |
|
} |
|
if (pc1 == NULL) |
|
TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); |
|
else |
|
TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); |
|
if (__predict_true(!cold)) |
|
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
} else { |
membar_sync(); |
pp->pr_cache = pc; |
pp->pr_cache = pc; |
LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); |
|
} |
} |
} |
|
|
/* |
/* |
Line 2160 pool_cache_destroy(pool_cache_t pc) |
|
Line 2146 pool_cache_destroy(pool_cache_t pc) |
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
while (pc->pc_refcnt != 0) |
while (pc->pc_refcnt != 0) |
cv_wait(&pool_busy, &pool_head_lock); |
cv_wait(&pool_busy, &pool_head_lock); |
TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); |
LIST_REMOVE(pc, pc_cachelist); |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
/* First, invalidate the entire cache. */ |
/* First, invalidate the entire cache. */ |
Line 2251 pool_cache_cpu_init(struct cpu_info *ci) |
|
Line 2237 pool_cache_cpu_init(struct cpu_info *ci) |
|
pool_cache_t pc; |
pool_cache_t pc; |
|
|
mutex_enter(&pool_head_lock); |
mutex_enter(&pool_head_lock); |
TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { |
LIST_FOREACH(pc, &pool_cache_head, pc_cachelist) { |
pc->pc_refcnt++; |
pc->pc_refcnt++; |
mutex_exit(&pool_head_lock); |
mutex_exit(&pool_head_lock); |
|
|
Line 2957 pool_in_cg(struct pool *pp, struct pool_ |
|
Line 2943 pool_in_cg(struct pool *pp, struct pool_ |
|
if (pcg == NULL) { |
if (pcg == NULL) { |
return false; |
return false; |
} |
} |
for (i = 0; i < pcg->pcg_avail; i++) { |
for (i = 0; i < pcg->pcg_size; i++) { |
if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { |
if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { |
return true; |
return true; |
} |
} |
Line 2993 pool_whatis(uintptr_t addr, void (*pr)(c |
|
Line 2979 pool_whatis(uintptr_t addr, void (*pr)(c |
|
{ |
{ |
struct pool *pp; |
struct pool *pp; |
|
|
TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
struct pool_item_header *ph; |
struct pool_item_header *ph; |
uintptr_t item; |
uintptr_t item; |
bool allocated = true; |
bool allocated = true; |