src/sys/kern/subr_pool.c - diff

Return to subr_pool.c CVS log

Up to [cvs.NetBSD.org] / src / sys / kern

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/sys/kern/subr_pool.c between version 1.50 and 1.50.2.12

version 1.50, 2001/01/29 02:38:02

version 1.50.2.12, 2002/11/11 22:13:58

Line 37

* POSSIBILITY OF SUCH DAMAGE.

#include <sys/cdefs.h>

__KERNEL_RCSID(0, "$NetBSD$");

#include "opt_pool.h"

#include "opt_poollog.h"

#include "opt_lockdebug.h"

Line 70 TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD

Line 73 TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD

/* Private pool for page header structures */

static struct pool phpool;

#ifdef POOL_SUBPAGE

/* Pool of subpages for use by normal pools. */

static struct pool psppool;

#endif

/* # of seconds to retain page after last use */

int pool_inactive_time = 10;

Line 86 struct pool_item_header {

Line 94 struct pool_item_header {

TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */

LIST_ENTRY(pool_item_header)

ph_hashlist; /* Off-page page headers */

int ph_nmissing; /* # of chunks in use */

unsigned int ph_nmissing; /* # of chunks in use */

caddr_t ph_page; /* this page's address */

struct timeval ph_time; /* last referenced */

};

TAILQ_HEAD(pool_pagelist,pool_item_header);

struct pool_item {

#ifdef DIAGNOSTIC

int pi_magic;

u_int pi_magic;

#endif

#define PI_MAGIC 0xdeadbeef

#define PI_MAGIC 0xdeadbeefU

/* Other entries use only this list entry */

TAILQ_ENTRY(pool_item) pi_list;

};

#define PR_HASH_INDEX(pp,addr) \

(((u_long)(addr) >> (pp)->pr_pageshift) & (PR_HASHTABSIZE - 1))

(((u_long)(addr) >> (pp)->pr_alloc->pa_pageshift) & \

(PR_HASHTABSIZE - 1))

#define POOL_NEEDS_CATCHUP(pp) \

((pp)->pr_nitems < (pp)->pr_minitems)

* Pool cache management.

Line 132 struct pool_item {

Line 145 struct pool_item {

/* The cache group pool. */

static struct pool pcgpool;

/* The pool cache group. */

#define PCG_NOBJECTS 16

struct pool_cache_group {

TAILQ_ENTRY(pool_cache_group)

pcg_list; /* link in the pool cache's group list */

u_int pcg_avail; /* # available objects */

/* pointers to the objects */

void *pcg_objects[PCG_NOBJECTS];

};

static void pool_cache_reclaim(struct pool_cache *);

static int pool_catchup(struct pool *);

static int pool_prime_page(struct pool *, caddr_t, int);

static void pool_prime_page(struct pool *, caddr_t,

static void *pool_page_alloc(unsigned long, int, int);

struct pool_item_header *);

static void pool_page_free(void *, unsigned long, int);

void *pool_allocator_alloc(struct pool *, int);

void pool_allocator_free(struct pool *, void *);

static void pool_print1(struct pool *, const char *,

void (*)(const char *, ...));

* Pool log entry. An array of these is allocated in pool_create().

* Pool log entry. An array of these is allocated in pool_init().

struct pool_log {

const char *pl_file;

Line 171 struct pool_log {

Line 176 struct pool_log {

int pool_logsize = POOL_LOGSIZE;

#ifdef DIAGNOSTIC

#ifdef POOL_DIAGNOSTIC

static __inline void

pr_log(struct pool *pp, void *v, int action, const char *file, long line)

{

Line 268 pr_enter_check(struct pool *pp, void (*p

Line 273 pr_enter_check(struct pool *pp, void (*p

#define pr_enter(pp, file, line)

#define pr_leave(pp)

#define pr_enter_check(pp, pr)

#endif /* DIAGNOSTIC */

#endif /* POOL_DIAGNOSTIC */

* Return the pool page header based on page address.

Line 294 pr_find_pagehead(struct pool *pp, caddr_

Line 299 pr_find_pagehead(struct pool *pp, caddr_

* Remove a page from the pool.

static __inline void

pr_rmpage(struct pool *pp, struct pool_item_header *ph)

pr_rmpage(struct pool *pp, struct pool_item_header *ph,

struct pool_pagelist *pq)

{

int s;

* If the page was idle, decrement the idle page count.

Line 313 pr_rmpage(struct pool *pp, struct pool_i

Line 320 pr_rmpage(struct pool *pp, struct pool_i

pp->pr_nitems -= pp->pr_itemsperpage;

* Unlink a page from the pool and release it.

* Unlink a page from the pool and release it (or queue it for release).

TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);

(*pp->pr_free)(ph->ph_page, pp->pr_pagesz, pp->pr_mtype);

if (pq) {

TAILQ_INSERT_HEAD(pq, ph, ph_pagelist);

} else {

pool_allocator_free(pp, ph->ph_page);

if ((pp->pr_roflags & PR_PHINPAGE) == 0) {

LIST_REMOVE(ph, ph_hashlist);

s = splhigh();

pool_put(&phpool, ph);

splx(s);

}

pp->pr_npages--;

pp->pr_npagefree++;

if ((pp->pr_roflags & PR_PHINPAGE) == 0) {

int s;

LIST_REMOVE(ph, ph_hashlist);

s = splhigh();

pool_put(&phpool, ph);

splx(s);

}

if (pp->pr_curpage == ph) {

* Find a new non-empty page header, if any.

* Start search from the page head, to increase the

* chance for "high water" pages to be freed.

for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;

TAILQ_FOREACH(ph, &pp->pr_pagelist, ph_pagelist)

ph = TAILQ_NEXT(ph, ph_pagelist))

if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)

break;

Line 344 pr_rmpage(struct pool *pp, struct pool_i

Line 352 pr_rmpage(struct pool *pp, struct pool_i

}

* Allocate and initialize a pool.

struct pool *

pool_create(size_t size, u_int align, u_int ioff, int nitems,

const char *wchan, size_t pagesz,

void *(*alloc)(unsigned long, int, int),

void (*release)(void *, unsigned long, int),

int mtype)

{

struct pool *pp;

int flags;

pp = (struct pool *)malloc(sizeof(*pp), M_POOL, M_NOWAIT);

if (pp == NULL)

return (NULL);

flags = PR_FREEHEADER;

pool_init(pp, size, align, ioff, flags, wchan, pagesz,

alloc, release, mtype);

if (nitems != 0) {

if (pool_prime(pp, nitems, NULL) != 0) {

pool_destroy(pp);

return (NULL);

}

return (pp);

}

* Initialize the given pool resource structure.

* We export this routine to allow other kernel parts to declare

Line 382 pool_create(size_t size, u_int align, u_

Line 359 pool_create(size_t size, u_int align, u_

void

pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,

const char *wchan, size_t pagesz,

const char *wchan, struct pool_allocator *palloc)

void *(*alloc)(unsigned long, int, int),

void (*release)(void *, unsigned long, int),

int mtype)

{

int off, slack, i;

Line 397 pool_init(struct pool *pp, size_t size,

Line 371 pool_init(struct pool *pp, size_t size,

flags |= PR_LOGGING;

#endif

#ifdef POOL_SUBPAGE

* XXX We don't provide a real `nointr' back-end

* yet; all sub-pages come from a kmem back-end.

* maybe some day...

if (palloc == NULL) {

extern struct pool_allocator pool_allocator_kmem_subpage;

palloc = &pool_allocator_kmem_subpage;

}

* Check arguments and construct default values.

* We'll assume any user-specified back-end allocator

* will deal with sub-pages, or simply don't care.

if (!powerof2(pagesz))

#else

panic("pool_init: page size invalid (%lx)\n", (u_long)pagesz);

if (palloc == NULL)

palloc = &pool_allocator_kmem;

#endif /* POOL_SUBPAGE */

if ((palloc->pa_flags & PA_INITIALIZED) == 0) {

if (palloc->pa_pagesz == 0) {

#ifdef POOL_SUBPAGE

if (palloc == &pool_allocator_kmem)

palloc->pa_pagesz = PAGE_SIZE;

else

palloc->pa_pagesz = POOL_SUBPAGE;

#else

palloc->pa_pagesz = PAGE_SIZE;

#endif /* POOL_SUBPAGE */

}

if (alloc == NULL && release == NULL) {

TAILQ_INIT(&palloc->pa_list);

alloc = pool_page_alloc;

release = pool_page_free;

simple_lock_init(&palloc->pa_slock);

pagesz = PAGE_SIZE; /* Rounds to PAGE_SIZE anyhow. */

palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);

} else if ((alloc != NULL && release != NULL) == 0) {

palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;

/* If you specifiy one, must specify both. */

palloc->pa_flags |= PA_INITIALIZED;

panic("pool_init: must specify alloc and release together");

}

if (pagesz == 0)

pagesz = PAGE_SIZE;

if (align == 0)

align = ALIGN(1);

Line 421 pool_init(struct pool *pp, size_t size,

Line 415 pool_init(struct pool *pp, size_t size,

if (size < sizeof(struct pool_item))

size = sizeof(struct pool_item);

size = ALIGN(size);

size = roundup(size, align);

if (size > pagesz)

#ifdef DIAGNOSTIC

if (size > palloc->pa_pagesz)

panic("pool_init: pool item size (%lu) too large",

(u_long)size);

#endif

* Initialize the pool structure.

Line 441 pool_init(struct pool *pp, size_t size,

Line 437 pool_init(struct pool *pp, size_t size,

pp->pr_size = size;

pp->pr_align = align;

pp->pr_wchan = wchan;

pp->pr_mtype = mtype;

pp->pr_alloc = palloc;

pp->pr_alloc = alloc;

pp->pr_free = release;

pp->pr_pagesz = pagesz;

pp->pr_pagemask = ~(pagesz - 1);

pp->pr_pageshift = ffs(pagesz) - 1;

pp->pr_nitems = 0;

pp->pr_nout = 0;

pp->pr_hardlimit = UINT_MAX;

Line 455 pool_init(struct pool *pp, size_t size,

Line 446 pool_init(struct pool *pp, size_t size,

pp->pr_hardlimit_ratecap.tv_usec = 0;

pp->pr_hardlimit_warning_last.tv_sec = 0;

pp->pr_hardlimit_warning_last.tv_usec = 0;

pp->pr_drain_hook = NULL;

pp->pr_drain_hook_arg = NULL;

* Decide whether to put the page header off page to avoid

Line 463 pool_init(struct pool *pp, size_t size,

Line 456 pool_init(struct pool *pp, size_t size,

* with its header based on the page address.

* We use 1/16 of the page size as the threshold (XXX: tune)

if (pp->pr_size < pagesz/16) {

if (pp->pr_size < palloc->pa_pagesz/16) {

/* Use the end of the page for the page header */

pp->pr_roflags |= PR_PHINPAGE;

pp->pr_phoffset = off =

pp->pr_phoffset = off = palloc->pa_pagesz -

pagesz - ALIGN(sizeof(struct pool_item_header));

ALIGN(sizeof(struct pool_item_header));

} else {

/* The page header will be taken from our page header pool */

pp->pr_phoffset = 0;

off = pagesz;

off = palloc->pa_pagesz;

for (i = 0; i < PR_HASHTABSIZE; i++) {

LIST_INIT(&pp->pr_hashtab[i]);

}

Line 504 pool_init(struct pool *pp, size_t size,

Line 497 pool_init(struct pool *pp, size_t size,

pp->pr_hiwat = 0;

pp->pr_nidle = 0;

#ifdef POOL_DIAGNOSTIC

if (flags & PR_LOGGING) {

if (kmem_map == NULL ||

(pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log),

Line 512 pool_init(struct pool *pp, size_t size,

Line 506 pool_init(struct pool *pp, size_t size,

pp->pr_curlogentry = 0;

pp->pr_logsize = pool_logsize;

}

#endif

pp->pr_entered_file = NULL;

pp->pr_entered_line = 0;

Line 524 pool_init(struct pool *pp, size_t size,

Line 519 pool_init(struct pool *pp, size_t size,

* XXX LOCKING.

if (phpool.pr_size == 0) {

#ifdef POOL_SUBPAGE

pool_init(&phpool, sizeof(struct pool_item_header), 0, 0, 0,

"phpool", &pool_allocator_kmem);

pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0,

PR_RECURSIVE, "psppool", &pool_allocator_kmem);

#else

pool_init(&phpool, sizeof(struct pool_item_header), 0, 0,

0, "phpool", 0, 0, 0, 0);

0, "phpool", NULL);

#endif

pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0,

0, "pcgpool", 0, 0, 0, 0);

0, "pcgpool", NULL);

}

/* Insert into the list of all pools. */

simple_lock(&pool_head_slock);

TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);

simple_unlock(&pool_head_slock);

/* Insert this into the list of pools using this allocator. */

simple_lock(&palloc->pa_slock);

TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);

simple_unlock(&palloc->pa_slock);

}

Line 545 pool_destroy(struct pool *pp)

Line 552 pool_destroy(struct pool *pp)

struct pool_item_header *ph;

struct pool_cache *pc;

/* Locking order: pool_allocator -> pool */

simple_lock(&pp->pr_alloc->pa_slock);

TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);

simple_unlock(&pp->pr_alloc->pa_slock);

/* Destroy all caches for this pool. */

while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL)

pool_cache_destroy(pc);

Line 552 pool_destroy(struct pool *pp)

Line 564 pool_destroy(struct pool *pp)

#ifdef DIAGNOSTIC

if (pp->pr_nout != 0) {

pr_printlog(pp, NULL, printf);

panic("pool_destroy: pool busy: still out: %u\n",

panic("pool_destroy: pool busy: still out: %u",

pp->pr_nout);

}

#endif

/* Remove all pages */

if ((pp->pr_roflags & PR_STATIC) == 0)

while ((ph = TAILQ_FIRST(&pp->pr_pagelist)) != NULL)

while ((ph = pp->pr_pagelist.tqh_first) != NULL)

pr_rmpage(pp, ph, NULL);

pr_rmpage(pp, ph);

/* Remove from global pool list */

simple_lock(&pool_head_slock);

TAILQ_REMOVE(&pool_head, pp, pr_poollist);

/* XXX Only clear this if we were drainpp? */

if (drainpp == pp) {

drainpp = NULL;

}

simple_unlock(&pool_head_slock);

#ifdef POOL_DIAGNOSTIC

if ((pp->pr_roflags & PR_LOGGING) != 0)

free(pp->pr_log, M_TEMP);

#endif

}

if (pp->pr_roflags & PR_FREEHEADER)

void

free(pp, M_POOL);

pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg)

{

/* XXX no locking -- must be used just after pool_init() */

#ifdef DIAGNOSTIC

if (pp->pr_drain_hook != NULL)

panic("pool_set_drain_hook(%s): already set", pp->pr_wchan);

#endif

pp->pr_drain_hook = fn;

pp->pr_drain_hook_arg = arg;

}

static __inline struct pool_item_header *

pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags)

{

struct pool_item_header *ph;

int s;

LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0);

if ((pp->pr_roflags & PR_PHINPAGE) != 0)

ph = (struct pool_item_header *) (storage + pp->pr_phoffset);

else {

s = splhigh();

ph = pool_get(&phpool, flags);

splx(s);

}

return (ph);

}

* Grab an item from the pool; must be called at appropriate spl level

void *

#ifdef POOL_DIAGNOSTIC

_pool_get(struct pool *pp, int flags, const char *file, long line)

#else

pool_get(struct pool *pp, int flags)

#endif

{

void *v;

struct pool_item *pi;

struct pool_item_header *ph;

void *v;

#ifdef DIAGNOSTIC

if (__predict_false((pp->pr_roflags & PR_STATIC) &&

if (__predict_false(curlwp == NULL && doing_shutdown == 0 &&

(flags & PR_MALLOCOK))) {

pr_printlog(pp, NULL, printf);

panic("pool_get: static");

}

#endif

if (__predict_false(curproc == NULL && doing_shutdown == 0 &&

(flags & PR_WAITOK) != 0))

panic("pool_get: must have NOWAIT");

panic("pool_get: %s: must have NOWAIT", pp->pr_wchan);

#ifdef LOCKDEBUG

if (flags & PR_WAITOK)

simple_lock_only_held(NULL, "pool_get(PR_WAITOK)");

#endif

#endif /* DIAGNOSTIC */

simple_lock(&pp->pr_slock);

pr_enter(pp, file, line);

Line 616 _pool_get(struct pool *pp, int flags, co

Line 661 _pool_get(struct pool *pp, int flags, co

}

#endif

if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {

if (pp->pr_drain_hook != NULL) {

* Since the drain hook is going to free things

* back to the pool, unlock, call the hook, re-lock,

* and check the hardlimit condition again.

pr_leave(pp);

simple_unlock(&pp->pr_slock);

(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);

simple_lock(&pp->pr_slock);

pr_enter(pp, file, line);

if (pp->pr_nout < pp->pr_hardlimit)

goto startover;

}

if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {

* XXX: A warning isn't logged in this case. Should

Line 636 _pool_get(struct pool *pp, int flags, co

Line 696 _pool_get(struct pool *pp, int flags, co

&pp->pr_hardlimit_ratecap))

log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);

if (flags & PR_URGENT)

panic("pool_get: urgent");

pp->pr_nfail++;

pr_leave(pp);

Line 653 _pool_get(struct pool *pp, int flags, co

Line 710 _pool_get(struct pool *pp, int flags, co

* has no items in its bucket.

if ((ph = pp->pr_curpage) == NULL) {

void *v;

#ifdef DIAGNOSTIC

if (pp->pr_nitems != 0) {

simple_unlock(&pp->pr_slock);

printf("pool_get: %s: curpage NULL, nitems %u\n",

pp->pr_wchan, pp->pr_nitems);

panic("pool_get: nitems inconsistent\n");

panic("pool_get: nitems inconsistent");

}

#endif

Line 671 _pool_get(struct pool *pp, int flags, co

Line 726 _pool_get(struct pool *pp, int flags, co

pr_leave(pp);

simple_unlock(&pp->pr_slock);

v = (*pp->pr_alloc)(pp->pr_pagesz, flags, pp->pr_mtype);

v = pool_allocator_alloc(pp, flags);

if (__predict_true(v != NULL))

ph = pool_alloc_item_header(pp, v, flags);

simple_lock(&pp->pr_slock);

pr_enter(pp, file, line);

if (v == NULL) {

if (__predict_false(v == NULL || ph == NULL)) {

if (v != NULL)

pool_allocator_free(pp, v);

* We were unable to allocate a page, but

* We were unable to allocate a page or item

* we released the lock during allocation,

* header, but we released the lock during

* so perhaps items were freed back to the

* allocation, so perhaps items were freed

* pool. Check for this case.

* back to the pool. Check for this case.

if (pp->pr_curpage != NULL)

goto startover;

if (flags & PR_URGENT)

panic("pool_get: urgent");

if ((flags & PR_WAITOK) == 0) {

pp->pr_nfail++;

pr_leave(pp);

Line 698 _pool_get(struct pool *pp, int flags, co

Line 755 _pool_get(struct pool *pp, int flags, co

* Wait for items to be returned to this pool.

* XXX: we actually want to wait just until

* the page allocator has memory again. Depending

* on this pool's usage, we might get stuck here

* for a long time.

* XXX: maybe we should wake up once a second and

* try again?

pp->pr_flags |= PR_WANTED;

/* PA_WANTED is already set on the allocator. */

pr_leave(pp);

ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);

pr_enter(pp, file, line);

Line 714 _pool_get(struct pool *pp, int flags, co

Line 767 _pool_get(struct pool *pp, int flags, co

}

/* We have more memory; add it to the pool */

if (pool_prime_page(pp, v, flags & PR_WAITOK) != 0) {

pool_prime_page(pp, v, ph);

* Probably, we don't allowed to wait and

* couldn't allocate a page header.

(*pp->pr_free)(v, pp->pr_pagesz, pp->pr_mtype);

pp->pr_nfail++;

pr_leave(pp);

simple_unlock(&pp->pr_slock);

return (NULL);

}

pp->pr_npagealloc++;

/* Start the allocation process over. */

Line 742 _pool_get(struct pool *pp, int flags, co

Line 785 _pool_get(struct pool *pp, int flags, co

simple_unlock(&pp->pr_slock);

printf("pool_get: %s: items on itemlist, nitems %u\n",

pp->pr_wchan, pp->pr_nitems);

panic("pool_get: nitems inconsistent\n");

panic("pool_get: nitems inconsistent");

}

#endif

#ifdef POOL_DIAGNOSTIC

pr_log(pp, v, PRLOG_GET, file, line);

#endif

#ifdef DIAGNOSTIC

if (__predict_false(pi->pi_magic != PI_MAGIC)) {

Line 792 _pool_get(struct pool *pp, int flags, co

Line 838 _pool_get(struct pool *pp, int flags, co

TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);

TAILQ_INSERT_TAIL(&pp->pr_pagelist, ph, ph_pagelist);

for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;

TAILQ_FOREACH(ph, &pp->pr_pagelist, ph_pagelist)

ph = TAILQ_NEXT(ph, ph_pagelist))

if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)

break;

Line 806 _pool_get(struct pool *pp, int flags, co

Line 851 _pool_get(struct pool *pp, int flags, co

* If we have a low water mark and we are now below that low

* water mark, add more items to the pool.

if (pp->pr_nitems < pp->pr_minitems && pool_catchup(pp) != 0) {

if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {

* XXX: Should we log a warning? Should we set up a timeout

* to try again in a second or so? The latter could break

Line 823 _pool_get(struct pool *pp, int flags, co

Line 868 _pool_get(struct pool *pp, int flags, co

* Internal version of pool_put(). Pool is already locked/entered.

static void

pool_do_put(struct pool *pp, void *v, const char *file, long line)

pool_do_put(struct pool *pp, void *v)

{

struct pool_item *pi = v;

struct pool_item_header *ph;

caddr_t page;

int s;

page = (caddr_t)((u_long)v & pp->pr_pagemask);

LOCK_ASSERT(simple_lock_held(&pp->pr_slock));

page = (caddr_t)((u_long)v & pp->pr_alloc->pa_pagemask);

#ifdef DIAGNOSTIC

if (__predict_false(pp->pr_nout == 0)) {

Line 840 pool_do_put(struct pool *pp, void *v, co

Line 887 pool_do_put(struct pool *pp, void *v, co

}

#endif

pr_log(pp, v, PRLOG_PUT, file, line);

if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) {

pr_printlog(pp, NULL, printf);

panic("pool_put: %s: page header missing", pp->pr_wchan);

Line 871 pool_do_put(struct pool *pp, void *v, co

Line 916 pool_do_put(struct pool *pp, void *v, co

#endif

TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);

KDASSERT(ph->ph_nmissing != 0);

ph->ph_nmissing--;

pp->pr_nput++;

pp->pr_nitems++;

Line 904 pool_do_put(struct pool *pp, void *v, co

Line 950 pool_do_put(struct pool *pp, void *v, co

if (ph->ph_nmissing == 0) {

pp->pr_nidle++;

if (pp->pr_npages > pp->pr_maxpages) {

if (pp->pr_npages > pp->pr_maxpages ||

pr_rmpage(pp, ph);

(pp->pr_alloc->pa_flags & PA_WANT) != 0) {

pr_rmpage(pp, ph, NULL);

} else {

TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);

TAILQ_INSERT_TAIL(&pp->pr_pagelist, ph, ph_pagelist);

Line 928 pool_do_put(struct pool *pp, void *v, co

Line 975 pool_do_put(struct pool *pp, void *v, co

* page with the fewest available items, to minimize

* fragmentation?

for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;

TAILQ_FOREACH(ph, &pp->pr_pagelist, ph_pagelist)

ph = TAILQ_NEXT(ph, ph_pagelist))

if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)

break;

Line 952 pool_do_put(struct pool *pp, void *v, co

Line 998 pool_do_put(struct pool *pp, void *v, co

* Return resource to the pool; must be called at appropriate spl level

#ifdef POOL_DIAGNOSTIC

void

_pool_put(struct pool *pp, void *v, const char *file, long line)

{

Line 959 _pool_put(struct pool *pp, void *v, cons

Line 1006 _pool_put(struct pool *pp, void *v, cons

simple_lock(&pp->pr_slock);

pr_enter(pp, file, line);

pool_do_put(pp, v, file, line);

pr_log(pp, v, PRLOG_PUT, file, line);

pool_do_put(pp, v);

pr_leave(pp);

simple_unlock(&pp->pr_slock);

}

#undef pool_put

#endif /* POOL_DIAGNOSTIC */

void

pool_put(struct pool *pp, void *v)

{

simple_lock(&pp->pr_slock);

pool_do_put(pp, v);

simple_unlock(&pp->pr_slock);

}

#ifdef POOL_DIAGNOSTIC

#define pool_put(h, v) _pool_put((h), (v), __FILE__, __LINE__)

#endif

* Add N items to the pool.

int

pool_prime(struct pool *pp, int n, caddr_t storage)

pool_prime(struct pool *pp, int n)

{

struct pool_item_header *ph;

caddr_t cp;

int error, newnitems, newpages;

int newpages;

#ifdef DIAGNOSTIC

if (__predict_false(storage && !(pp->pr_roflags & PR_STATIC)))

panic("pool_prime: static");

/* !storage && static caught below */

#endif

simple_lock(&pp->pr_slock);

newnitems = pp->pr_minitems + n;

newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

newpages =

roundup(newnitems, pp->pr_itemsperpage) / pp->pr_itemsperpage

- pp->pr_minpages;

while (newpages-- > 0) {

if (pp->pr_roflags & PR_STATIC) {

simple_unlock(&pp->pr_slock);

cp = storage;

cp = pool_allocator_alloc(pp, PR_NOWAIT);

storage += pp->pr_pagesz;

if (__predict_true(cp != NULL))

} else {

ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);

simple_unlock(&pp->pr_slock);

simple_lock(&pp->pr_slock);

cp = (*pp->pr_alloc)(pp->pr_pagesz, 0, pp->pr_mtype);

simple_lock(&pp->pr_slock);

}

if (cp == NULL) {

if (__predict_false(cp == NULL || ph == NULL)) {

simple_unlock(&pp->pr_slock);

if (cp != NULL)

return (ENOMEM);

pool_allocator_free(pp, cp);

break;

}

if ((error = pool_prime_page(pp, cp, PR_NOWAIT)) != 0) {

pool_prime_page(pp, cp, ph);

if ((pp->pr_roflags & PR_STATIC) == 0)

(*pp->pr_free)(cp, pp->pr_pagesz,

pp->pr_mtype);

simple_unlock(&pp->pr_slock);

return (error);

}

pp->pr_npagealloc++;

pp->pr_minpages++;

}

pp->pr_minitems = newnitems;

if (pp->pr_minpages >= pp->pr_maxpages)

pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */

Line 1027 pool_prime(struct pool *pp, int n, caddr

Line 1075 pool_prime(struct pool *pp, int n, caddr

* Note, we must be called with the pool descriptor LOCKED.

static int

static void

pool_prime_page(struct pool *pp, caddr_t storage, int flags)

pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph)

{

struct pool_item *pi;

struct pool_item_header *ph;

caddr_t cp = storage;

unsigned int align = pp->pr_align;

unsigned int ioff = pp->pr_itemoffset;

int s, n;

int n;

if (((u_long)cp & (pp->pr_pagesz - 1)) != 0)

#ifdef DIAGNOSTIC

if (((u_long)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0)

panic("pool_prime_page: %s: unaligned page", pp->pr_wchan);

#endif

if ((pp->pr_roflags & PR_PHINPAGE) != 0) {

if ((pp->pr_roflags & PR_PHINPAGE) == 0)

ph = (struct pool_item_header *)(cp + pp->pr_phoffset);

} else {

s = splhigh();

ph = pool_get(&phpool, flags);

splx(s);

if (ph == NULL)

return (ENOMEM);

LIST_INSERT_HEAD(&pp->pr_hashtab[PR_HASH_INDEX(pp, cp)],

ph, ph_hashlist);

}

* Insert page header.

Line 1085 pool_prime_page(struct pool *pp, caddr_t

Line 1126 pool_prime_page(struct pool *pp, caddr_t

while (n--) {

pi = (struct pool_item *)cp;

KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0);

/* Insert on page list */

TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list);

#ifdef DIAGNOSTIC

Line 1101 pool_prime_page(struct pool *pp, caddr_t

Line 1144 pool_prime_page(struct pool *pp, caddr_t

if (++pp->pr_npages > pp->pr_hiwat)

pp->pr_hiwat = pp->pr_npages;

return (0);

}

* Like pool_prime(), except this is used by pool_get() when nitems

* Used by pool_get() when nitems drops below the low water mark. This

* drops below the low water mark. This is used to catch up nitmes

* is used to catch up nitmes with the low water mark.

* with the low water mark.

* Note 1, we never wait for memory here, we let the caller decide what to do.

* Note 2, this doesn't work with static pools.

* Note 2, we must be called with the pool already locked, and we return

* Note 3, we must be called with the pool already locked, and we return

* with it locked.

static int

pool_catchup(struct pool *pp)

{

struct pool_item_header *ph;

caddr_t cp;

int error = 0;

if (pp->pr_roflags & PR_STATIC) {

while (POOL_NEEDS_CATCHUP(pp)) {

* We dropped below the low water mark, and this is not a

* good thing. Log a warning.

* XXX: rate-limit this?

printf("WARNING: static pool `%s' dropped below low water "

"mark\n", pp->pr_wchan);

return (0);

}

while (pp->pr_nitems < pp->pr_minitems) {

* Call the page back-end allocator for more memory.

Line 1143 pool_catchup(struct pool *pp)

Line 1170 pool_catchup(struct pool *pp)

* the pool descriptor?

simple_unlock(&pp->pr_slock);

cp = (*pp->pr_alloc)(pp->pr_pagesz, 0, pp->pr_mtype);

cp = pool_allocator_alloc(pp, PR_NOWAIT);

if (__predict_true(cp != NULL))

ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);

simple_lock(&pp->pr_slock);

if (__predict_false(cp == NULL)) {

if (__predict_false(cp == NULL || ph == NULL)) {

if (cp != NULL)

pool_allocator_free(pp, cp);

error = ENOMEM;

break;

}

if ((error = pool_prime_page(pp, cp, PR_NOWAIT)) != 0) {

pool_prime_page(pp, cp, ph);

(*pp->pr_free)(cp, pp->pr_pagesz, pp->pr_mtype);

break;

}

pp->pr_npagealloc++;

}

Line 1162 pool_catchup(struct pool *pp)

Line 1190 pool_catchup(struct pool *pp)

void

pool_setlowat(struct pool *pp, int n)

{

int error;

simple_lock(&pp->pr_slock);

Line 1172 pool_setlowat(struct pool *pp, int n)

Line 1199 pool_setlowat(struct pool *pp, int n)

: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

/* Make sure we're caught up with the newly-set low water mark. */

if ((pp->pr_nitems < pp->pr_minitems) &&

if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {

(error = pool_catchup(pp)) != 0) {

* XXX: Should we log a warning? Should we set up a timeout

* to try again in a second or so? The latter could break

Line 1221 pool_sethardlimit(struct pool *pp, int n

Line 1247 pool_sethardlimit(struct pool *pp, int n

}

* Default page allocator.

static void *

pool_page_alloc(unsigned long sz, int flags, int mtype)

{

boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;

return ((void *)uvm_km_alloc_poolpage(waitok));

}

static void

pool_page_free(void *v, unsigned long sz, int mtype)

{

uvm_km_free_poolpage((vaddr_t)v);

}

* Alternate pool page allocator for pools that know they will

* never be accessed in interrupt context.

void *

pool_page_alloc_nointr(unsigned long sz, int flags, int mtype)

{

boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;

return ((void *)uvm_km_alloc_poolpage1(kernel_map, uvm.kernel_object,

waitok));

}

void

pool_page_free_nointr(void *v, unsigned long sz, int mtype)

{

uvm_km_free_poolpage1(kernel_map, (vaddr_t)v);

}

* Release all complete pages that have not been used recently.

void

int

#ifdef POOL_DIAGNOSTIC

_pool_reclaim(struct pool *pp, const char *file, long line)

#else

pool_reclaim(struct pool *pp)

#endif

{

struct pool_item_header *ph, *phnext;

struct pool_cache *pc;

struct timeval curtime;

struct pool_pagelist pq;

int s;

if (pp->pr_roflags & PR_STATIC)

if (pp->pr_drain_hook != NULL) {

return;

* The drain hook must be called with the pool unlocked.

(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT);

}

if (simple_lock_try(&pp->pr_slock) == 0)

return;

return (0);

pr_enter(pp, file, line);

TAILQ_INIT(&pq);

* Reclaim items from the pool's caches.

for (pc = TAILQ_FIRST(&pp->pr_cachelist); pc != NULL;

TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist)

pc = TAILQ_NEXT(pc, pc_poollist))

pool_cache_reclaim(pc);

s = splclock();

Line 1309 _pool_reclaim(struct pool *pp, const cha

Line 1306 _pool_reclaim(struct pool *pp, const cha

pp->pr_minitems)

break;

pr_rmpage(pp, ph);

pr_rmpage(pp, ph, &pq);

}

pr_leave(pp);

simple_unlock(&pp->pr_slock);

}

if (TAILQ_EMPTY(&pq))

return (0);

while ((ph = TAILQ_FIRST(&pq)) != NULL) {

TAILQ_REMOVE(&pq, ph, ph_pagelist);

pool_allocator_free(pp, ph->ph_page);

if (pp->pr_roflags & PR_PHINPAGE) {

continue;

}

LIST_REMOVE(ph, ph_hashlist);

s = splhigh();

pool_put(&phpool, ph);

splx(s);

}

return (1);

}

* Drain pools, one at a time.

Line 1329 pool_drain(void *arg)

Line 1341 pool_drain(void *arg)

struct pool *pp;

int s;

pp = NULL;

s = splvm();

simple_lock(&pool_head_slock);

if (drainpp == NULL) {

if (drainpp == NULL && (drainpp = TAILQ_FIRST(&pool_head)) == NULL)

drainpp = TAILQ_FIRST(&pool_head);

goto out;

}

if (drainpp) {

pp = drainpp;

drainpp = TAILQ_NEXT(pp, pr_poollist);

}

pool_reclaim(pp);

out:

simple_unlock(&pool_head_slock);

pool_reclaim(pp);

splx(s);

}

* Diagnostic helpers.

Line 1415 pool_print1(struct pool *pp, const char

Line 1425 pool_print1(struct pool *pp, const char

print_pagelist = 1;

if (c == 'c')

print_cache = 1;

modif++;

}

(*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n",

pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,

pp->pr_roflags);

(*pr)("\tpagesz %u, mtype %d\n", pp->pr_pagesz, pp->pr_mtype);

(*pr)("\talloc %p\n", pp->pr_alloc);

(*pr)("\talloc %p, release %p\n", pp->pr_alloc, pp->pr_free);

(*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",

pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);

(*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",

Line 1444 pool_print1(struct pool *pp, const char

Line 1452 pool_print1(struct pool *pp, const char

(u_long)ph->ph_time.tv_sec,

(u_long)ph->ph_time.tv_usec);

#ifdef DIAGNOSTIC

for (pi = TAILQ_FIRST(&ph->ph_itemlist); pi != NULL;

TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) {

pi = TAILQ_NEXT(pi, pi_list)) {

if (pi->pi_magic != PI_MAGIC) {

(*pr)("\t\t\titem %p, magic 0x%x\n",

pi, pi->pi_magic);

Line 1474 pool_print1(struct pool *pp, const char

Line 1481 pool_print1(struct pool *pp, const char

if (print_cache == 0)

goto skip_cache;

for (pc = TAILQ_FIRST(&pp->pr_cachelist); pc != NULL;

TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) {

pc = TAILQ_NEXT(pc, pc_poollist)) {

(*pr)("\tcache %p: allocfrom %p freeto %p\n", pc,

pc->pc_allocfrom, pc->pc_freeto);

(*pr)("\t hits %lu misses %lu ngroups %lu nitems %lu\n",

pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems);

for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;

TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {

pcg = TAILQ_NEXT(pcg, pcg_list)) {

(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);

for (i = 0; i < PCG_NOBJECTS; i++)

(*pr)("\t\t\t%p\n", pcg->pcg_objects[i]);

Line 1501 pool_chk(struct pool *pp, const char *la

Line 1506 pool_chk(struct pool *pp, const char *la

simple_lock(&pp->pr_slock);

for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;

TAILQ_FOREACH(ph, &pp->pr_pagelist, ph_pagelist) {

ph = TAILQ_NEXT(ph, ph_pagelist)) {

struct pool_item *pi;

int n;

caddr_t page;

page = (caddr_t)((u_long)ph & pp->pr_pagemask);

page = (caddr_t)((u_long)ph & pp->pr_alloc->pa_pagemask);

if (page != ph->ph_page &&

(pp->pr_roflags & PR_PHINPAGE) != 0) {

if (label != NULL)

Line 1537 pool_chk(struct pool *pp, const char *la

Line 1540 pool_chk(struct pool *pp, const char *la

panic("pool");

}

#endif

page = (caddr_t)((u_long)pi & pp->pr_pagemask);

page =

(caddr_t)((u_long)pi & pp->pr_alloc->pa_pagemask);

if (page == ph->ph_page)

continue;

Line 1653 pool_cache_get(struct pool_cache *pc, in

Line 1657 pool_cache_get(struct pool_cache *pc, in

struct pool_cache_group *pcg;

void *object;

#ifdef LOCKDEBUG

if (flags & PR_WAITOK)

simple_lock_only_held(NULL, "pool_cache_get(PR_WAITOK)");

#endif

simple_lock(&pc->pc_slock);

if ((pcg = pc->pc_allocfrom) == NULL) {

for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;

TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {

pcg = TAILQ_NEXT(pcg, pcg_list)) {

if (pcg->pcg_avail != 0) {

pc->pc_allocfrom = pcg;

goto have_group;

Line 1704 void

Line 1712 void

pool_cache_put(struct pool_cache *pc, void *object)

{

struct pool_cache_group *pcg;

int s;

simple_lock(&pc->pc_slock);

if ((pcg = pc->pc_freeto) == NULL) {

for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;

TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {

pcg = TAILQ_NEXT(pcg, pcg_list)) {

if (pcg->pcg_avail != PCG_NOBJECTS) {

pc->pc_freeto = pcg;

goto have_group;

Line 1721 pool_cache_put(struct pool_cache *pc, vo

Line 1729 pool_cache_put(struct pool_cache *pc, vo

* allocate one.

simple_unlock(&pc->pc_slock);

s = splvm();

pcg = pool_get(&pcgpool, PR_NOWAIT);

splx(s);

if (pcg != NULL) {

memset(pcg, 0, sizeof(*pcg));

simple_lock(&pc->pc_slock);

Line 1736 pool_cache_put(struct pool_cache *pc, vo

Line 1746 pool_cache_put(struct pool_cache *pc, vo

* Unable to allocate a cache group; destruct the object

* and free it back to the pool.

if (pc->pc_dtor != NULL)

pool_cache_destruct_object(pc, object);

(*pc->pc_dtor)(pc->pc_arg, object);

pool_put(pc->pc_pool, object);

return;

}

Line 1753 pool_cache_put(struct pool_cache *pc, vo

Line 1761 pool_cache_put(struct pool_cache *pc, vo

}

* pool_cache_destruct_object:

* Force destruction of an object and its release back into

* the pool.

void

pool_cache_destruct_object(struct pool_cache *pc, void *object)

{

if (pc->pc_dtor != NULL)

(*pc->pc_dtor)(pc->pc_arg, object);

pool_put(pc->pc_pool, object);

}

* pool_cache_do_invalidate:

* This internal function implements pool_cache_invalidate() and

Line 1760 pool_cache_put(struct pool_cache *pc, vo

Line 1783 pool_cache_put(struct pool_cache *pc, vo

static void

pool_cache_do_invalidate(struct pool_cache *pc, int free_groups,

void (*putit)(struct pool *, void *, const char *, long))

void (*putit)(struct pool *, void *))

{

struct pool_cache_group *pcg, *npcg;

void *object;

int s;

for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;

pcg = npcg) {

Line 1775 pool_cache_do_invalidate(struct pool_cac

Line 1799 pool_cache_do_invalidate(struct pool_cac

pc->pc_allocfrom = NULL;

if (pc->pc_dtor != NULL)

(*pc->pc_dtor)(pc->pc_arg, object);

(*putit)(pc->pc_pool, object, __FILE__, __LINE__);

(*putit)(pc->pc_pool, object);

}

if (free_groups) {

pc->pc_ngroups--;

TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list);

if (pc->pc_freeto == pcg)

pc->pc_freeto = NULL;

s = splvm();

pool_put(&pcgpool, pcg);

splx(s);

}

Line 1798 pool_cache_invalidate(struct pool_cache

Line 1824 pool_cache_invalidate(struct pool_cache

{

simple_lock(&pc->pc_slock);

pool_cache_do_invalidate(pc, 0, _pool_put);

pool_cache_do_invalidate(pc, 0, pool_put);

simple_unlock(&pc->pc_slock);

}

Line 1815 pool_cache_reclaim(struct pool_cache *pc

Line 1841 pool_cache_reclaim(struct pool_cache *pc

pool_cache_do_invalidate(pc, 1, pool_do_put);

simple_unlock(&pc->pc_slock);

}

* Pool backend allocators.

* Each pool has a backend allocator that handles allocation, deallocation,

* and any additional draining that might be needed.

* We provide two standard allocators:

* pool_allocator_kmem - the default when no allocator is specified

* pool_allocator_nointr - used for pools that will not be accessed

* in interrupt context.

void *pool_page_alloc(struct pool *, int);

void pool_page_free(struct pool *, void *);

struct pool_allocator pool_allocator_kmem = {

pool_page_alloc, pool_page_free, 0,

};

void *pool_page_alloc_nointr(struct pool *, int);

void pool_page_free_nointr(struct pool *, void *);

struct pool_allocator pool_allocator_nointr = {

pool_page_alloc_nointr, pool_page_free_nointr, 0,

};

#ifdef POOL_SUBPAGE

void *pool_subpage_alloc(struct pool *, int);

void pool_subpage_free(struct pool *, void *);

struct pool_allocator pool_allocator_kmem_subpage = {

pool_subpage_alloc, pool_subpage_free, 0,

};

#endif /* POOL_SUBPAGE */

* We have at least three different resources for the same allocation and

* each resource can be depleted. First, we have the ready elements in the

* pool. Then we have the resource (typically a vm_map) for this allocator.

* Finally, we have physical memory. Waiting for any of these can be

* unnecessary when any other is freed, but the kernel doesn't support

* sleeping on multiple wait channels, so we have to employ another strategy.

* The caller sleeps on the pool (so that it can be awakened when an item

* is returned to the pool), but we set PA_WANT on the allocator. When a

* page is returned to the allocator and PA_WANT is set, pool_allocator_free

* will wake up all sleeping pools belonging to this allocator.

* XXX Thundering herd.

void *

pool_allocator_alloc(struct pool *org, int flags)

{

struct pool_allocator *pa = org->pr_alloc;

struct pool *pp, *start;

int s, freed;

void *res;

do {

if ((res = (*pa->pa_alloc)(org, flags)) != NULL)

return (res);

if ((flags & PR_WAITOK) == 0) {

* We only run the drain hookhere if PR_NOWAIT.

* In other cases, the hook will be run in

* pool_reclaim().

if (org->pr_drain_hook != NULL) {

(*org->pr_drain_hook)(org->pr_drain_hook_arg,

flags);

if ((res = (*pa->pa_alloc)(org, flags)) != NULL)

return (res);

}

break;

}

* Drain all pools, except "org", that use this

* allocator. We do this to reclaim VA space.

* pa_alloc is responsible for waiting for

* physical memory.

* XXX We risk looping forever if start if someone

* calls pool_destroy on "start". But there is no

* other way to have potentially sleeping pool_reclaim,

* non-sleeping locks on pool_allocator, and some

* stirring of drained pools in the allocator.

* XXX Maybe we should use pool_head_slock for locking

* the allocators?

freed = 0;

s = splvm();

simple_lock(&pa->pa_slock);

pp = start = TAILQ_FIRST(&pa->pa_list);

do {

TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list);

TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list);

if (pp == org)

continue;

simple_unlock(&pa->pa_slock);

freed = pool_reclaim(pp);

simple_lock(&pa->pa_slock);

} while ((pp = TAILQ_FIRST(&pa->pa_list)) != start &&

freed == 0);

if (freed == 0) {

* We set PA_WANT here, the caller will most likely

* sleep waiting for pages (if not, this won't hurt

* that much), and there is no way to set this in

* the caller without violating locking order.

pa->pa_flags |= PA_WANT;

}

simple_unlock(&pa->pa_slock);

splx(s);

} while (freed);

return (NULL);

}

void

pool_allocator_free(struct pool *pp, void *v)

{

struct pool_allocator *pa = pp->pr_alloc;

int s;

(*pa->pa_free)(pp, v);

s = splvm();

simple_lock(&pa->pa_slock);

if ((pa->pa_flags & PA_WANT) == 0) {

simple_unlock(&pa->pa_slock);

splx(s);

return;

}

TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) {

simple_lock(&pp->pr_slock);

if ((pp->pr_flags & PR_WANTED) != 0) {

pp->pr_flags &= ~PR_WANTED;

wakeup(pp);

}

simple_unlock(&pp->pr_slock);

}

pa->pa_flags &= ~PA_WANT;

simple_unlock(&pa->pa_slock);

splx(s);

}

void *

pool_page_alloc(struct pool *pp, int flags)

{

boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;

return ((void *) uvm_km_alloc_poolpage(waitok));

}

void

pool_page_free(struct pool *pp, void *v)

{

uvm_km_free_poolpage((vaddr_t) v);

}

#ifdef POOL_SUBPAGE

/* Sub-page allocator, for machines with large hardware pages. */

void *

pool_subpage_alloc(struct pool *pp, int flags)

{

return (pool_get(&psppool, flags));

}

void

pool_subpage_free(struct pool *pp, void *v)

{

pool_put(&psppool, v);

}

/* We don't provide a real nointr allocator. Maybe later. */

void *

pool_page_alloc_nointr(struct pool *pp, int flags)

{

return (pool_subpage_alloc(pp, flags));

}

void

pool_page_free_nointr(struct pool *pp, void *v)

{

pool_subpage_free(pp, v);

}

#else

void *

pool_page_alloc_nointr(struct pool *pp, int flags)

{

boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;

return ((void *) uvm_km_alloc_poolpage1(kernel_map,

uvm.kernel_object, waitok));

}

void

pool_page_free_nointr(struct pool *pp, void *v)

{

uvm_km_free_poolpage1(kernel_map, (vaddr_t) v);

}

#endif /* POOL_SUBPAGE */

CVSweb <webmaster@jp.NetBSD.org>