src/sys/kern/subr_pool.c - diff

Return to subr_pool.c CVS log

Up to [cvs.NetBSD.org] / src / sys / kern

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/sys/kern/subr_pool.c between version 1.128.2.1 and 1.128.2.2

version 1.128.2.1, 2007/03/13 16:51:56

version 1.128.2.2, 2007/03/13 17:50:58

Line 1

/* $NetBSD$ */

/*-

* This code is derived from software contributed to The NetBSD Foundation

Line 101 int pool_inactive_time = 10;

/* Next candidate for drainage (see pool_drain()) */

static struct pool *drainpp;

/* This spin lock protects both pool_head and drainpp. */

/* This lock protects both pool_head and drainpp. */

struct simplelock pool_head_slock = SIMPLELOCK_INITIALIZER;

static kmutex_t pool_head_lock;

typedef uint8_t pool_item_freelist_t;

Line 422 static void

pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq)

{

struct pool_item_header *ph;

int s;

while ((ph = LIST_FIRST(pq)) != NULL) {

LIST_REMOVE(ph, ph_pagelist);

pool_allocator_free(pp, ph->ph_page);

if ((pp->pr_roflags & PR_PHINPAGE) == 0) {

if ((pp->pr_roflags & PR_PHINPAGE) == 0)

s = splvm();

pool_put(pp->pr_phpool, ph);

splx(s);

}

Line 443 pr_rmpage(struct pool *pp, struct pool_i

Line 439 pr_rmpage(struct pool *pp, struct pool_i

struct pool_pagelist *pq)

{

LOCK_ASSERT(simple_lock_held(&pp->pr_slock));

KASSERT(mutex_owned(&pp->pr_lock));

* If the page was idle, decrement the idle page count.

Line 557 pool_subsystem_init(void)

Line 553 pool_subsystem_init(void)

__link_set_decl(pools, struct link_pool_init);

struct link_pool_init * const *pi;

mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE);

__link_set_foreach(pi, pools)

pool_init((*pi)->pp, (*pi)->size, (*pi)->align,

(*pi)->align_offset, (*pi)->flags, (*pi)->wchan,

Line 584 pool_init(struct pool *pp, size_t size,

Line 582 pool_init(struct pool *pp, size_t size,

struct pool *pp1;

#endif

size_t trysize, phsize;

int off, slack, s;

int off, slack;

KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >=

PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1));

Line 625 pool_init(struct pool *pp, size_t size,

Line 623 pool_init(struct pool *pp, size_t size,

TAILQ_INIT(&palloc->pa_list);

simple_lock_init(&palloc->pa_slock);

mutex_init(&palloc->pa_lock, MUTEX_DRIVER, IPL_VM);

palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);

palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;

Line 769 pool_init(struct pool *pp, size_t size,

Line 767 pool_init(struct pool *pp, size_t size,

pp->pr_entered_file = NULL;

pp->pr_entered_line = 0;

simple_lock_init(&pp->pr_slock);

mutex_init(&pp->pr_lock, MUTEX_DRIVER, ipl);

cv_init(&pp->pr_cv, wchan);

pp->pr_ipl = ipl;

if (strcmp(wchan, "kmem-52") == 0) {

printf("kmem-52 initted, mutex @ %p\n", &pp->pr_lock);

printf("=> %x %x %x %x\n",

((uint32_t *)&pp->pr_lock)[0],

((uint32_t *)&pp->pr_lock)[1],

((uint32_t *)&pp->pr_lock)[2],

((uint32_t *)&pp->pr_lock)[3]);

}

* Initialize private page header pool and cache magazine pool if we

Line 802 pool_init(struct pool *pp, size_t size,

Line 811 pool_init(struct pool *pp, size_t size,

0, "pcgpool", &pool_allocator_meta, IPL_VM);

}

/* Insert into the list of all pools. */

if (__predict_true(!cold)) {

simple_lock(&pool_head_slock);

/* Insert into the list of all pools. */

LIST_INSERT_HEAD(&pool_head, pp, pr_poollist);

mutex_enter(&pool_head_lock);

simple_unlock(&pool_head_slock);

LIST_INSERT_HEAD(&pool_head, pp, pr_poollist);

mutex_exit(&pool_head_lock);

/* Insert this into the list of pools using this allocator. */

s = splvm();

/* Insert this into the list of pools using this allocator. */

simple_lock(&palloc->pa_slock);

mutex_enter(&palloc->pa_lock);

TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);

simple_unlock(&palloc->pa_slock);

mutex_exit(&palloc->pa_lock);

splx(s);

} else {

LIST_INSERT_HEAD(&pool_head, pp, pr_poollist);

TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);

}

pool_reclaim_register(pp);

}

Line 824 pool_destroy(struct pool *pp)

Line 837 pool_destroy(struct pool *pp)

{

struct pool_pagelist pq;

struct pool_item_header *ph;

int s;

/* Remove from global pool list */

simple_lock(&pool_head_slock);

mutex_enter(&pool_head_lock);

LIST_REMOVE(pp, pr_poollist);

if (drainpp == pp)

drainpp = NULL;

simple_unlock(&pool_head_slock);

mutex_exit(&pool_head_lock);

/* Remove this pool from its allocator's list of pools. */

pool_reclaim_unregister(pp);

s = splvm();

mutex_enter(&pp->pr_alloc->pa_lock);

simple_lock(&pp->pr_alloc->pa_slock);

TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);

simple_unlock(&pp->pr_alloc->pa_slock);

mutex_exit(&pp->pr_alloc->pa_lock);

splx(s);

s = splvm();

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

KASSERT(LIST_EMPTY(&pp->pr_cachelist));

Line 862 pool_destroy(struct pool *pp)

Line 871 pool_destroy(struct pool *pp)

while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)

pr_rmpage(pp, ph, &pq);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

splx(s);

pr_pagelist_free(pp, &pq);

Line 871 pool_destroy(struct pool *pp)

Line 879 pool_destroy(struct pool *pp)

if ((pp->pr_roflags & PR_LOGGING) != 0)

free(pp->pr_log, M_TEMP);

#endif

cv_destroy(&pp->pr_cv);

mutex_destroy(&pp->pr_lock);

}

void

Line 890 static struct pool_item_header *

Line 901 static struct pool_item_header *

pool_alloc_item_header(struct pool *pp, void *storage, int flags)

{

struct pool_item_header *ph;

int s;

LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0);

if ((pp->pr_roflags & PR_PHINPAGE) != 0)

ph = (struct pool_item_header *) ((char *)storage + pp->pr_phoffset);

else {

else

s = splvm();

ph = pool_get(pp->pr_phpool, flags);

splx(s);

}

return (ph);

}

Line 933 pool_get(struct pool *pp, int flags)

Line 938 pool_get(struct pool *pp, int flags)

ASSERT_SLEEPABLE(NULL, "pool_get(PR_WAITOK)");

#endif

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pr_enter(pp, file, line);

startover:

Line 945 pool_get(struct pool *pp, int flags)

Line 950 pool_get(struct pool *pp, int flags)

#ifdef DIAGNOSTIC

if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

panic("pool_get: %s: crossed hard limit", pp->pr_wchan);

}

#endif

Line 957 pool_get(struct pool *pp, int flags)

Line 962 pool_get(struct pool *pp, int flags)

* and check the hardlimit condition again.

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pr_enter(pp, file, line);

if (pp->pr_nout < pp->pr_hardlimit)

goto startover;

Line 972 pool_get(struct pool *pp, int flags)

Line 977 pool_get(struct pool *pp, int flags)

pp->pr_flags |= PR_WANTED;

pr_leave(pp);

ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);

cv_wait(&pp->pr_cv, &pp->pr_lock);

pr_enter(pp, file, line);

goto startover;

}

Line 988 pool_get(struct pool *pp, int flags)

Line 993 pool_get(struct pool *pp, int flags)

pp->pr_nfail++;

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

return (NULL);

}

Line 1003 pool_get(struct pool *pp, int flags)

Line 1008 pool_get(struct pool *pp, int flags)

#ifdef DIAGNOSTIC

if (pp->pr_nitems != 0) {

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

printf("pool_get: %s: curpage NULL, nitems %u\n",

pp->pr_wchan, pp->pr_nitems);

panic("pool_get: nitems inconsistent");

Line 1030 pool_get(struct pool *pp, int flags)

Line 1035 pool_get(struct pool *pp, int flags)

pp->pr_nfail++;

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

return (NULL);

}

Line 1041 pool_get(struct pool *pp, int flags)

Line 1046 pool_get(struct pool *pp, int flags)

#ifdef DIAGNOSTIC

if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) {

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

panic("pool_get: %s: page empty", pp->pr_wchan);

}

#endif

Line 1053 pool_get(struct pool *pp, int flags)

Line 1058 pool_get(struct pool *pp, int flags)

v = pi = LIST_FIRST(&ph->ph_itemlist);

if (__predict_false(v == NULL)) {

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

panic("pool_get: %s: page empty", pp->pr_wchan);

}

#ifdef DIAGNOSTIC

if (__predict_false(pp->pr_nitems == 0)) {

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

printf("pool_get: %s: items on itemlist, nitems %u\n",

pp->pr_wchan, pp->pr_nitems);

panic("pool_get: nitems inconsistent");

Line 1106 pool_get(struct pool *pp, int flags)

Line 1111 pool_get(struct pool *pp, int flags)

if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 &&

!LIST_EMPTY(&ph->ph_itemlist))) {

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

panic("pool_get: %s: nmissing inconsistent",

pp->pr_wchan);

}

Line 1135 pool_get(struct pool *pp, int flags)

Line 1140 pool_get(struct pool *pp, int flags)

}

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

KASSERT((((vaddr_t)v + pp->pr_itemoffset) & (pp->pr_align - 1)) == 0);

FREECHECK_OUT(&pp->pr_freecheck, v);

return (v);

Line 1150 pool_do_put(struct pool *pp, void *v, st

Line 1155 pool_do_put(struct pool *pp, void *v, st

struct pool_item *pi = v;

struct pool_item_header *ph;

LOCK_ASSERT(simple_lock_held(&pp->pr_slock));

KASSERT(mutex_owned(&pp->pr_lock));

FREECHECK_IN(&pp->pr_freecheck, v);

#ifdef DIAGNOSTIC

Line 1270 _pool_put(struct pool *pp, void *v, cons

Line 1275 _pool_put(struct pool *pp, void *v, cons

LIST_INIT(&pq);

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pr_enter(pp, file, line);

pr_log(pp, v, PRLOG_PUT, file, line);

Line 1278 _pool_put(struct pool *pp, void *v, cons

Line 1283 _pool_put(struct pool *pp, void *v, cons

pool_do_put(pp, v, &pq);

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

pr_pagelist_free(pp, &pq);

}

Line 1292 pool_put(struct pool *pp, void *v)

Line 1297 pool_put(struct pool *pp, void *v)

LIST_INIT(&pq);

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pool_do_put(pp, v, &pq);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

pr_pagelist_free(pp, &pq);

}

Line 1317 pool_grow(struct pool *pp, int flags)

Line 1322 pool_grow(struct pool *pp, int flags)

struct pool_item_header *ph = NULL;

char *cp;

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

cp = pool_allocator_alloc(pp, flags);

if (__predict_true(cp != NULL)) {

ph = pool_alloc_item_header(pp, cp, flags);

Line 1326 pool_grow(struct pool *pp, int flags)

Line 1331 pool_grow(struct pool *pp, int flags)

if (cp != NULL) {

pool_allocator_free(pp, cp);

}

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

return ENOMEM;

}

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pool_prime_page(pp, cp, ph);

pp->pr_npagealloc++;

return 0;

Line 1345 pool_prime(struct pool *pp, int n)

Line 1350 pool_prime(struct pool *pp, int n)

int newpages;

int error = 0;

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

Line 1360 pool_prime(struct pool *pp, int n)

Line 1365 pool_prime(struct pool *pp, int n)

if (pp->pr_minpages >= pp->pr_maxpages)

pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

return error;

}

Line 1378 pool_prime_page(struct pool *pp, void *s

Line 1383 pool_prime_page(struct pool *pp, void *s

const unsigned int ioff = pp->pr_itemoffset;

int n;

LOCK_ASSERT(simple_lock_held(&pp->pr_slock));

KASSERT(mutex_owned(&pp->pr_lock));

#ifdef DIAGNOSTIC

if ((pp->pr_roflags & PR_NOALIGN) == 0 &&

Line 1493 void

Line 1498 void

pool_setlowat(struct pool *pp, int n)

{

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pp->pr_minitems = n;

pp->pr_minpages = (n == 0)

Line 1509 pool_setlowat(struct pool *pp, int n)

Line 1514 pool_setlowat(struct pool *pp, int n)

}

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

}

void

pool_sethiwat(struct pool *pp, int n)

{

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pp->pr_maxpages = (n == 0)

? 0

: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

}

void

pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap)

{

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

pp->pr_hardlimit = n;

pp->pr_hardlimit_warning = warnmess;

Line 1545 pool_sethardlimit(struct pool *pp, int n

Line 1550 pool_sethardlimit(struct pool *pp, int n

? 0

: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

}

Line 1571 pool_reclaim(struct pool *pp)

Line 1576 pool_reclaim(struct pool *pp)

(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT);

}

if (simple_lock_try(&pp->pr_slock) == 0)

if (mutex_tryenter(&pp->pr_lock) == 0)

return (0);

pr_enter(pp, file, line);

Line 1611 pool_reclaim(struct pool *pp)

Line 1616 pool_reclaim(struct pool *pp)

}

pr_leave(pp);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

if (LIST_EMPTY(&pq) && LIST_EMPTY(&pcgl))

return 0;

Line 1632 pool_drain(void *arg)

Line 1637 pool_drain(void *arg)

int s;

pp = NULL;

s = splvm();

s = splvm(); /* XXX why? */

simple_lock(&pool_head_slock);

mutex_enter(&pool_head_lock);

if (drainpp == NULL) {

drainpp = LIST_FIRST(&pool_head);

}

Line 1641 pool_drain(void *arg)

Line 1646 pool_drain(void *arg)

pp = drainpp;

drainpp = LIST_NEXT(pp, pr_poollist);

}

simple_unlock(&pool_head_slock);

mutex_exit(&pool_head_lock);

if (pp)

pool_reclaim(pp);

splx(s);

Line 1653 pool_drain(void *arg)

Line 1658 pool_drain(void *arg)

void

pool_print(struct pool *pp, const char *modif)

{

int s;

s = splvm();

if (mutex_tryenter(&pp->pr_lock) == 0) {

if (simple_lock_try(&pp->pr_slock) == 0) {

printf("pool %s is locked; try again later\n",

pp->pr_wchan);

splx(s);

return;

}

pool_print1(pp, modif, printf);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

splx(s);

}

void

Line 1672 pool_printall(const char *modif, void (*

Line 1673 pool_printall(const char *modif, void (*

{

struct pool *pp;

if (simple_lock_try(&pool_head_slock) == 0) {

if (mutex_tryenter(&pool_head_lock) == 0) {

(*pr)("WARNING: pool_head_slock is locked\n");

} else {

simple_unlock(&pool_head_slock);

mutex_exit(&pool_head_lock);

}

LIST_FOREACH(pp, &pool_head, pr_poollist) {

Line 1697 pool_printit(struct pool *pp, const char

Line 1698 pool_printit(struct pool *pp, const char

* other processors should be paused. We can skip locking

* the pool in this case.

* We do a simple_lock_try() just to print the lock

* We do a mutex_tryenter() just to print the lock

* status, however.

if (simple_lock_try(&pp->pr_slock) == 0)

if (mutex_tryenter(&pp->pr_lock) == 0)

(*pr)("WARNING: pool %s is locked\n", pp->pr_wchan);

else

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

pool_print1(pp, modif, pr);

}

Line 1903 pool_chk(struct pool *pp, const char *la

Line 1904 pool_chk(struct pool *pp, const char *la

struct pool_item_header *ph;

int r = 0;

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {

r = pool_chk_page(pp, label, ph);

if (r) {

Line 1924 pool_chk(struct pool *pp, const char *la

Line 1925 pool_chk(struct pool *pp, const char *la

}

out:

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

return (r);

}

Line 1946 pool_cache_init(struct pool_cache *pc, s

Line 1947 pool_cache_init(struct pool_cache *pc, s

LIST_INIT(&pc->pc_emptygroups);

LIST_INIT(&pc->pc_fullgroups);

LIST_INIT(&pc->pc_partgroups);

simple_lock_init(&pc->pc_slock);

mutex_init(&pc->pc_lock, MUTEX_DRIVER, pp->pr_ipl);

pc->pc_pool = pp;

Line 1961 pool_cache_init(struct pool_cache *pc, s

Line 1962 pool_cache_init(struct pool_cache *pc, s

pc->pc_nitems = 0;

simple_lock(&pp->pr_slock);

if (__predict_true(!cold)) {

LIST_INSERT_HEAD(&pp->pr_cachelist, pc, pc_poollist);

mutex_enter(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

LIST_INSERT_HEAD(&pp->pr_cachelist, pc, pc_poollist);

mutex_exit(&pp->pr_lock);

} else

LIST_INSERT_HEAD(&pp->pr_cachelist, pc, pc_poollist);

}

Line 1980 pool_cache_destroy(struct pool_cache *pc

Line 1984 pool_cache_destroy(struct pool_cache *pc

pool_cache_invalidate(pc);

/* ...and remove it from the pool's cache list. */

simple_lock(&pp->pr_slock);

mutex_enter(&pp->pr_lock);

LIST_REMOVE(pc, pc_poollist);

simple_unlock(&pp->pr_slock);

mutex_exit(&pp->pr_lock);

mutex_destroy(&pc->pc_lock);

}

static inline void *

Line 2021 static void

Line 2027 static void

pcg_grouplist_free(struct pool_cache_grouplist *pcgl)

{

struct pool_cache_group *pcg;

int s;

s = splvm();

while ((pcg = LIST_FIRST(pcgl)) != NULL) {

LIST_REMOVE(pcg, pcg_list);

pool_put(&pcgpool, pcg);

}

splx(s);

}

Line 2048 pool_cache_get_paddr(struct pool_cache *

Line 2051 pool_cache_get_paddr(struct pool_cache *

ASSERT_SLEEPABLE(NULL, "pool_cache_get(PR_WAITOK)");

#endif

simple_lock(&pc->pc_slock);

mutex_enter(&pc->pc_lock);

pcg = LIST_FIRST(&pc->pc_partgroups);

if (pcg == NULL) {

Line 2067 pool_cache_get_paddr(struct pool_cache *

Line 2070 pool_cache_get_paddr(struct pool_cache *

* when the object is freed back to the cache.

pc->pc_misses++;

simple_unlock(&pc->pc_slock);

mutex_exit(&pc->pc_lock);

object = pool_get(pc->pc_pool, flags);

if (object != NULL && pc->pc_ctor != NULL) {

if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) {

Line 2097 pool_cache_get_paddr(struct pool_cache *

Line 2100 pool_cache_get_paddr(struct pool_cache *

LIST_REMOVE(pcg, pcg_list);

LIST_INSERT_HEAD(&pc->pc_emptygroups, pcg, pcg_list);

}

simple_unlock(&pc->pc_slock);

mutex_exit(&pc->pc_lock);

KASSERT((((vaddr_t)object + pc->pc_pool->pr_itemoffset) &

(pc->pc_pool->pr_align - 1)) == 0);

Line 2115 void

Line 2118 void

pool_cache_put_paddr(struct pool_cache *pc, void *object, paddr_t pa)

{

struct pool_cache_group *pcg;

int s;

FREECHECK_IN(&pc->pc_freecheck, object);

Line 2123 pool_cache_put_paddr(struct pool_cache *

Line 2125 pool_cache_put_paddr(struct pool_cache *

goto destruct;

}

simple_lock(&pc->pc_slock);

mutex_enter(&pc->pc_lock);

pcg = LIST_FIRST(&pc->pc_partgroups);

if (pcg == NULL) {

Line 2139 pool_cache_put_paddr(struct pool_cache *

Line 2141 pool_cache_put_paddr(struct pool_cache *

* No empty groups to free the object to. Attempt to

* allocate one.

simple_unlock(&pc->pc_slock);

mutex_exit(&pc->pc_lock);

s = splvm();

pcg = pool_get(&pcgpool, PR_NOWAIT);

splx(s);

if (pcg == NULL) {

destruct:

Line 2154 destruct:

return;

}

memset(pcg, 0, sizeof(*pcg));

simple_lock(&pc->pc_slock);

mutex_enter(&pc->pc_lock);

pc->pc_ngroups++;

LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list);

}

Line 2166 destruct:

LIST_REMOVE(pcg, pcg_list);

LIST_INSERT_HEAD(&pc->pc_fullgroups, pcg, pcg_list);

}

simple_unlock(&pc->pc_slock);

mutex_exit(&pc->pc_lock);

}

Line 2212 pool_do_cache_invalidate(struct pool_cac

struct pool_cache_grouplist *pcgl)

{

LOCK_ASSERT(simple_lock_held(&pc->pc_slock));

KASSERT(mutex_owned(&pc->pc_lock));

LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock));

KASSERT(mutex_owned(&pc->pc_pool->pr_lock));

pool_do_cache_invalidate_grouplist(&pc->pc_fullgroups, pc, pq, pcgl);

pool_do_cache_invalidate_grouplist(&pc->pc_partgroups, pc, pq, pcgl);

Line 2238 pool_cache_invalidate(struct pool_cache

LIST_INIT(&pq);

LIST_INIT(&pcgl);

simple_lock(&pc->pc_slock);

mutex_enter(&pc->pc_lock);

simple_lock(&pc->pc_pool->pr_slock);

mutex_enter(&pc->pc_pool->pr_lock);

pool_do_cache_invalidate(pc, &pq, &pcgl);

simple_unlock(&pc->pc_pool->pr_slock);

mutex_exit(&pc->pc_pool->pr_lock);

simple_unlock(&pc->pc_slock);

mutex_exit(&pc->pc_lock);

pr_pagelist_free(pc->pc_pool, &pq);

pcg_grouplist_free(&pcgl);

Line 2266 pool_cache_reclaim(struct pool_cache *pc

* to use trylock. If we can't lock the pool_cache, it's not really

* a big deal here.

if (simple_lock_try(&pc->pc_slock) == 0)

if (mutex_tryenter(&pc->pc_lock) == 0)

return;

pool_do_cache_invalidate(pc, pq, pcgl);

simple_unlock(&pc->pc_slock);

mutex_exit(&pc->pc_lock);

}

Line 2341 pool_allocator_alloc(struct pool *pp, in

struct pool_allocator *pa = pp->pr_alloc;

void *res;

LOCK_ASSERT(!simple_lock_held(&pp->pr_slock));

res = (*pa->pa_alloc)(pp, flags);

if (res == NULL && (flags & PR_WAITOK) == 0) {

Line 2363 pool_allocator_free(struct pool *pp, voi

Line 2361 pool_allocator_free(struct pool *pp, voi

{

struct pool_allocator *pa = pp->pr_alloc;

LOCK_ASSERT(!simple_lock_held(&pp->pr_slock));

(*pa->pa_free)(pp, v);

}

Line 2403 pool_page_free_meta(struct pool *pp, voi

Line 2399 pool_page_free_meta(struct pool *pp, voi

void *

pool_subpage_alloc(struct pool *pp, int flags)

{

void *v;

return pool_get(&psppool, flags);

int s;

s = splvm();

v = pool_get(&psppool, flags);

splx(s);

return v;

}

void

pool_subpage_free(struct pool *pp, void *v)

{

int s;

s = splvm();

pool_put(&psppool, v);

splx(s);

}

/* We don't provide a real nointr allocator. Maybe later. */

CVSweb <webmaster@jp.NetBSD.org>