version 1.190.2.3, 2012/10/30 17:22:34 |
version 1.190.2.4, 2014/05/22 11:41:03 |
Line 67 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 67 __KERNEL_RCSID(0, "$NetBSD$"); |
|
* an internal pool of page headers (`phpool'). |
* an internal pool of page headers (`phpool'). |
*/ |
*/ |
|
|
/* List of all pools */ |
/* List of all pools. Non static as needed by 'vmstat -i' */ |
static TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
|
|
/* Private pool for page header structures */ |
/* Private pool for page header structures */ |
#define PHPOOL_MAX 8 |
#define PHPOOL_MAX 8 |
Line 203 static void *pool_allocator_alloc(struct |
|
Line 203 static void *pool_allocator_alloc(struct |
|
static void pool_allocator_free(struct pool *, void *); |
static void pool_allocator_free(struct pool *, void *); |
|
|
static void pool_print_pagelist(struct pool *, struct pool_pagelist *, |
static void pool_print_pagelist(struct pool *, struct pool_pagelist *, |
void (*)(const char *, ...)); |
void (*)(const char *, ...) __printflike(1, 2)); |
static void pool_print1(struct pool *, const char *, |
static void pool_print1(struct pool *, const char *, |
void (*)(const char *, ...)); |
void (*)(const char *, ...) __printflike(1, 2)); |
|
|
static int pool_chk_page(struct pool *, const char *, |
static int pool_chk_page(struct pool *, const char *, |
struct pool_item_header *); |
struct pool_item_header *); |
Line 561 pool_init(struct pool *pp, size_t size, |
|
Line 561 pool_init(struct pool *pp, size_t size, |
|
/* See the comment below about reserved bytes. */ |
/* See the comment below about reserved bytes. */ |
trysize = palloc->pa_pagesz - ((align - ioff) % align); |
trysize = palloc->pa_pagesz - ((align - ioff) % align); |
phsize = ALIGN(sizeof(struct pool_item_header)); |
phsize = ALIGN(sizeof(struct pool_item_header)); |
if ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 && |
if (pp->pr_roflags & PR_PHINPAGE || |
|
((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 && |
(pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) || |
(pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) || |
trysize / pp->pr_size == (trysize - phsize) / pp->pr_size)) { |
trysize / pp->pr_size == (trysize - phsize) / pp->pr_size))) { |
/* Use the end of the page for the page header */ |
/* Use the end of the page for the page header */ |
pp->pr_roflags |= PR_PHINPAGE; |
pp->pr_roflags |= PR_PHINPAGE; |
pp->pr_phoffset = off = palloc->pa_pagesz - phsize; |
pp->pr_phoffset = off = palloc->pa_pagesz - phsize; |
Line 2261 pool_cache_get_paddr(pool_cache_t pc, in |
|
Line 2262 pool_cache_get_paddr(pool_cache_t pc, in |
|
static bool __noinline |
static bool __noinline |
pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object) |
pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object) |
{ |
{ |
|
struct lwp *l = curlwp; |
pcg_t *pcg, *cur; |
pcg_t *pcg, *cur; |
uint64_t ncsw; |
uint64_t ncsw; |
pool_cache_t pc; |
pool_cache_t pc; |
Line 2271 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2273 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
pc = cc->cc_cache; |
pc = cc->cc_cache; |
pcg = NULL; |
pcg = NULL; |
cc->cc_misses++; |
cc->cc_misses++; |
|
ncsw = l->l_ncsw; |
|
|
/* |
/* |
* If there are no empty groups in the cache then allocate one |
* If there are no empty groups in the cache then allocate one |
Line 2280 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2283 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
if (__predict_true(!pool_cache_disable)) { |
if (__predict_true(!pool_cache_disable)) { |
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
} |
} |
|
/* |
|
* If pool_get() blocked, then our view of |
|
* the per-CPU data is invalid: retry. |
|
*/ |
|
if (__predict_false(l->l_ncsw != ncsw)) { |
|
if (pcg != NULL) { |
|
pool_put(pc->pc_pcgpool, pcg); |
|
} |
|
return true; |
|
} |
if (__predict_true(pcg != NULL)) { |
if (__predict_true(pcg != NULL)) { |
pcg->pcg_avail = 0; |
pcg->pcg_avail = 0; |
pcg->pcg_size = pc->pc_pcgsize; |
pcg->pcg_size = pc->pc_pcgsize; |
Line 2288 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2301 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
|
|
/* Lock the cache. */ |
/* Lock the cache. */ |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
ncsw = curlwp->l_ncsw; |
|
mutex_enter(&pc->pc_lock); |
mutex_enter(&pc->pc_lock); |
pc->pc_contended++; |
pc->pc_contended++; |
|
|
Line 2296 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
Line 2308 pool_cache_put_slow(pool_cache_cpu_t *cc |
|
* If we context switched while locking, then our view of |
* If we context switched while locking, then our view of |
* the per-CPU data is invalid: retry. |
* the per-CPU data is invalid: retry. |
*/ |
*/ |
if (__predict_false(curlwp->l_ncsw != ncsw)) { |
if (__predict_false(l->l_ncsw != ncsw)) { |
mutex_exit(&pc->pc_lock); |
mutex_exit(&pc->pc_lock); |
if (pcg != NULL) { |
if (pcg != NULL) { |
pool_put(pc->pc_pcgpool, pcg); |
pool_put(pc->pc_pcgpool, pcg); |