version 1.68, 2002/03/08 21:41:59 |
version 1.84, 2003/01/18 10:06:33 |
Line 94 struct pool_item_header { |
|
Line 94 struct pool_item_header { |
|
TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */ |
TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */ |
LIST_ENTRY(pool_item_header) |
LIST_ENTRY(pool_item_header) |
ph_hashlist; /* Off-page page headers */ |
ph_hashlist; /* Off-page page headers */ |
int ph_nmissing; /* # of chunks in use */ |
unsigned int ph_nmissing; /* # of chunks in use */ |
caddr_t ph_page; /* this page's address */ |
caddr_t ph_page; /* this page's address */ |
struct timeval ph_time; /* last referenced */ |
struct timeval ph_time; /* last referenced */ |
}; |
}; |
Line 102 TAILQ_HEAD(pool_pagelist,pool_item_heade |
|
Line 102 TAILQ_HEAD(pool_pagelist,pool_item_heade |
|
|
|
struct pool_item { |
struct pool_item { |
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
int pi_magic; |
u_int pi_magic; |
#endif |
#endif |
#define PI_MAGIC 0xdeadbeef |
#define PI_MAGIC 0xdeadbeefU |
/* Other entries use only this list entry */ |
/* Other entries use only this list entry */ |
TAILQ_ENTRY(pool_item) pi_list; |
TAILQ_ENTRY(pool_item) pi_list; |
}; |
}; |
Line 145 struct pool_item { |
|
Line 145 struct pool_item { |
|
/* The cache group pool. */ |
/* The cache group pool. */ |
static struct pool pcgpool; |
static struct pool pcgpool; |
|
|
/* The pool cache group. */ |
|
#define PCG_NOBJECTS 16 |
|
struct pool_cache_group { |
|
TAILQ_ENTRY(pool_cache_group) |
|
pcg_list; /* link in the pool cache's group list */ |
|
u_int pcg_avail; /* # available objects */ |
|
/* pointers to the objects */ |
|
void *pcg_objects[PCG_NOBJECTS]; |
|
}; |
|
|
|
static void pool_cache_reclaim(struct pool_cache *); |
static void pool_cache_reclaim(struct pool_cache *); |
|
|
static int pool_catchup(struct pool *); |
static int pool_catchup(struct pool *); |
Line 425 pool_init(struct pool *pp, size_t size, |
|
Line 415 pool_init(struct pool *pp, size_t size, |
|
if (size < sizeof(struct pool_item)) |
if (size < sizeof(struct pool_item)) |
size = sizeof(struct pool_item); |
size = sizeof(struct pool_item); |
|
|
size = ALIGN(size); |
size = roundup(size, align); |
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
if (size > palloc->pa_pagesz) |
if (size > palloc->pa_pagesz) |
panic("pool_init: pool item size (%lu) too large", |
panic("pool_init: pool item size (%lu) too large", |
Line 574 pool_destroy(struct pool *pp) |
|
Line 564 pool_destroy(struct pool *pp) |
|
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
if (pp->pr_nout != 0) { |
if (pp->pr_nout != 0) { |
pr_printlog(pp, NULL, printf); |
pr_printlog(pp, NULL, printf); |
panic("pool_destroy: pool busy: still out: %u\n", |
panic("pool_destroy: pool busy: still out: %u", |
pp->pr_nout); |
pp->pr_nout); |
} |
} |
#endif |
#endif |
|
|
/* Remove all pages */ |
/* Remove all pages */ |
if ((pp->pr_roflags & PR_STATIC) == 0) |
while ((ph = TAILQ_FIRST(&pp->pr_pagelist)) != NULL) |
while ((ph = TAILQ_FIRST(&pp->pr_pagelist)) != NULL) |
pr_rmpage(pp, ph, NULL); |
pr_rmpage(pp, ph, NULL); |
|
|
|
/* Remove from global pool list */ |
/* Remove from global pool list */ |
simple_lock(&pool_head_slock); |
simple_lock(&pool_head_slock); |
Line 645 pool_get(struct pool *pp, int flags) |
|
Line 634 pool_get(struct pool *pp, int flags) |
|
void *v; |
void *v; |
|
|
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
if (__predict_false((pp->pr_roflags & PR_STATIC) && |
if (__predict_false(curlwp == NULL && doing_shutdown == 0 && |
(flags & PR_MALLOCOK))) { |
|
pr_printlog(pp, NULL, printf); |
|
panic("pool_get: static"); |
|
} |
|
|
|
if (__predict_false(curproc == NULL && doing_shutdown == 0 && |
|
(flags & PR_WAITOK) != 0)) |
(flags & PR_WAITOK) != 0)) |
panic("pool_get: must have NOWAIT"); |
panic("pool_get: %s: must have NOWAIT", pp->pr_wchan); |
|
|
#ifdef LOCKDEBUG |
#ifdef LOCKDEBUG |
if (flags & PR_WAITOK) |
if (flags & PR_WAITOK) |
Line 732 pool_get(struct pool *pp, int flags) |
|
Line 715 pool_get(struct pool *pp, int flags) |
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
printf("pool_get: %s: curpage NULL, nitems %u\n", |
printf("pool_get: %s: curpage NULL, nitems %u\n", |
pp->pr_wchan, pp->pr_nitems); |
pp->pr_wchan, pp->pr_nitems); |
panic("pool_get: nitems inconsistent\n"); |
panic("pool_get: nitems inconsistent"); |
} |
} |
#endif |
#endif |
|
|
Line 802 pool_get(struct pool *pp, int flags) |
|
Line 785 pool_get(struct pool *pp, int flags) |
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
printf("pool_get: %s: items on itemlist, nitems %u\n", |
printf("pool_get: %s: items on itemlist, nitems %u\n", |
pp->pr_wchan, pp->pr_nitems); |
pp->pr_wchan, pp->pr_nitems); |
panic("pool_get: nitems inconsistent\n"); |
panic("pool_get: nitems inconsistent"); |
} |
} |
#endif |
#endif |
|
|
Line 933 pool_do_put(struct pool *pp, void *v) |
|
Line 916 pool_do_put(struct pool *pp, void *v) |
|
#endif |
#endif |
|
|
TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); |
TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); |
|
KDASSERT(ph->ph_nmissing != 0); |
ph->ph_nmissing--; |
ph->ph_nmissing--; |
pp->pr_nput++; |
pp->pr_nput++; |
pp->pr_nitems++; |
pp->pr_nitems++; |
Line 966 pool_do_put(struct pool *pp, void *v) |
|
Line 950 pool_do_put(struct pool *pp, void *v) |
|
*/ |
*/ |
if (ph->ph_nmissing == 0) { |
if (ph->ph_nmissing == 0) { |
pp->pr_nidle++; |
pp->pr_nidle++; |
if (pp->pr_npages > pp->pr_maxpages) { |
if (pp->pr_npages > pp->pr_maxpages || |
|
(pp->pr_alloc->pa_flags & PA_WANT) != 0) { |
pr_rmpage(pp, ph, NULL); |
pr_rmpage(pp, ph, NULL); |
} else { |
} else { |
TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); |
TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); |
Line 1052 pool_put(struct pool *pp, void *v) |
|
Line 1037 pool_put(struct pool *pp, void *v) |
|
int |
int |
pool_prime(struct pool *pp, int n) |
pool_prime(struct pool *pp, int n) |
{ |
{ |
struct pool_item_header *ph; |
struct pool_item_header *ph = NULL; |
caddr_t cp; |
caddr_t cp; |
int newpages, error = 0; |
int newpages; |
|
|
simple_lock(&pp->pr_slock); |
simple_lock(&pp->pr_slock); |
|
|
Line 1068 pool_prime(struct pool *pp, int n) |
|
Line 1053 pool_prime(struct pool *pp, int n) |
|
simple_lock(&pp->pr_slock); |
simple_lock(&pp->pr_slock); |
|
|
if (__predict_false(cp == NULL || ph == NULL)) { |
if (__predict_false(cp == NULL || ph == NULL)) { |
error = ENOMEM; |
|
if (cp != NULL) |
if (cp != NULL) |
pool_allocator_free(pp, cp); |
pool_allocator_free(pp, cp); |
break; |
break; |
Line 1142 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1126 pool_prime_page(struct pool *pp, caddr_t |
|
while (n--) { |
while (n--) { |
pi = (struct pool_item *)cp; |
pi = (struct pool_item *)cp; |
|
|
|
KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); |
|
|
/* Insert on page list */ |
/* Insert on page list */ |
TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); |
TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); |
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
Line 1166 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1152 pool_prime_page(struct pool *pp, caddr_t |
|
* |
* |
* Note 1, we never wait for memory here, we let the caller decide what to do. |
* Note 1, we never wait for memory here, we let the caller decide what to do. |
* |
* |
* Note 2, this doesn't work with static pools. |
* Note 2, we must be called with the pool already locked, and we return |
* |
|
* Note 3, we must be called with the pool already locked, and we return |
|
* with it locked. |
* with it locked. |
*/ |
*/ |
static int |
static int |
pool_catchup(struct pool *pp) |
pool_catchup(struct pool *pp) |
{ |
{ |
struct pool_item_header *ph; |
struct pool_item_header *ph = NULL; |
caddr_t cp; |
caddr_t cp; |
int error = 0; |
int error = 0; |
|
|
if (pp->pr_roflags & PR_STATIC) { |
|
/* |
|
* We dropped below the low water mark, and this is not a |
|
* good thing. Log a warning. |
|
* |
|
* XXX: rate-limit this? |
|
*/ |
|
printf("WARNING: static pool `%s' dropped below low water " |
|
"mark\n", pp->pr_wchan); |
|
return (0); |
|
} |
|
|
|
while (POOL_NEEDS_CATCHUP(pp)) { |
while (POOL_NEEDS_CATCHUP(pp)) { |
/* |
/* |
* Call the page back-end allocator for more memory. |
* Call the page back-end allocator for more memory. |
Line 1218 pool_catchup(struct pool *pp) |
|
Line 1190 pool_catchup(struct pool *pp) |
|
void |
void |
pool_setlowat(struct pool *pp, int n) |
pool_setlowat(struct pool *pp, int n) |
{ |
{ |
int error; |
|
|
|
simple_lock(&pp->pr_slock); |
simple_lock(&pp->pr_slock); |
|
|
Line 1228 pool_setlowat(struct pool *pp, int n) |
|
Line 1199 pool_setlowat(struct pool *pp, int n) |
|
: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
|
|
/* Make sure we're caught up with the newly-set low water mark. */ |
/* Make sure we're caught up with the newly-set low water mark. */ |
if (POOL_NEEDS_CATCHUP(pp) && (error = pool_catchup(pp) != 0)) { |
if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { |
/* |
/* |
* XXX: Should we log a warning? Should we set up a timeout |
* XXX: Should we log a warning? Should we set up a timeout |
* to try again in a second or so? The latter could break |
* to try again in a second or so? The latter could break |
Line 1291 pool_reclaim(struct pool *pp) |
|
Line 1262 pool_reclaim(struct pool *pp) |
|
struct pool_pagelist pq; |
struct pool_pagelist pq; |
int s; |
int s; |
|
|
if (pp->pr_roflags & PR_STATIC) |
|
return (0); |
|
|
|
if (pp->pr_drain_hook != NULL) { |
if (pp->pr_drain_hook != NULL) { |
/* |
/* |
* The drain hook must be called with the pool unlocked. |
* The drain hook must be called with the pool unlocked. |
Line 1457 pool_print1(struct pool *pp, const char |
|
Line 1425 pool_print1(struct pool *pp, const char |
|
print_pagelist = 1; |
print_pagelist = 1; |
if (c == 'c') |
if (c == 'c') |
print_cache = 1; |
print_cache = 1; |
modif++; |
|
} |
} |
|
|
(*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n", |
(*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n", |
Line 1977 pool_allocator_alloc(struct pool *org, i |
|
Line 1944 pool_allocator_alloc(struct pool *org, i |
|
TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); |
if (pp == org) |
if (pp == org) |
continue; |
continue; |
simple_unlock(&pa->pa_list); |
simple_unlock(&pa->pa_slock); |
freed = pool_reclaim(pp); |
freed = pool_reclaim(pp); |
simple_lock(&pa->pa_list); |
simple_lock(&pa->pa_slock); |
} while ((pp = TAILQ_FIRST(&pa->pa_list)) != start && |
} while ((pp = TAILQ_FIRST(&pa->pa_list)) != start && |
freed == 0); |
freed == 0); |
|
|
Line 2020 pool_allocator_free(struct pool *pp, voi |
|
Line 1987 pool_allocator_free(struct pool *pp, voi |
|
pp->pr_flags &= ~PR_WANTED; |
pp->pr_flags &= ~PR_WANTED; |
wakeup(pp); |
wakeup(pp); |
} |
} |
|
simple_unlock(&pp->pr_slock); |
} |
} |
pa->pa_flags &= ~PA_WANT; |
pa->pa_flags &= ~PA_WANT; |
simple_unlock(&pa->pa_slock); |
simple_unlock(&pa->pa_slock); |