src/sys/kern/subr_pool.c - diff

Return to subr_pool.c CVS log

Up to [cvs.NetBSD.org] / src / sys / kern

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/sys/kern/subr_pool.c between version 1.128.2.13 and 1.129

version 1.128.2.13, 2007/11/01 21:10:14

version 1.129, 2007/03/12 18:18:34

Line 1

/* $NetBSD$ */

/*-

* This code is derived from software contributed to The NetBSD Foundation

* by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace

* Simulation Facility, NASA Ames Research Center, and by Andrew Doran.

* Simulation Facility, NASA Ames Research Center.

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

Line 54 __KERNEL_RCSID(0, "$NetBSD$");

#include <sys/pool.h>

#include <sys/syslog.h>

#include <sys/debug.h>

#include <sys/lockdebug.h>

#include <sys/xcall.h>

#include <uvm/uvm.h>

Line 75 __KERNEL_RCSID(0, "$NetBSD$");

Line 73 __KERNEL_RCSID(0, "$NetBSD$");

/* List of all pools */

LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head);

/* List of all caches. */

LIST_HEAD(,pool_cache) pool_cache_head =

LIST_HEAD_INITIALIZER(pool_cache_head);

/* Private pool for page header structures */

#define PHPOOL_MAX 8

static struct pool phpool[PHPOOL_MAX];

Line 96 static void *pool_page_alloc_meta(struct

Line 90 static void *pool_page_alloc_meta(struct

static void pool_page_free_meta(struct pool *, void *);

/* allocator for pool metadata */

struct pool_allocator pool_allocator_meta = {

static struct pool_allocator pool_allocator_meta = {

pool_page_alloc_meta, pool_page_free_meta,

.pa_backingmapptr = &kmem_map,

};

Line 107 int pool_inactive_time = 10;

Line 101 int pool_inactive_time = 10;

/* Next candidate for drainage (see pool_drain()) */

static struct pool *drainpp;

/* This lock protects both pool_head and drainpp. */

/* This spin lock protects both pool_head and drainpp. */

static kmutex_t pool_head_lock;

struct simplelock pool_head_slock = SIMPLELOCK_INITIALIZER;

static kcondvar_t pool_busy;

typedef uint8_t pool_item_freelist_t;

Line 149 struct pool_item {

Line 142 struct pool_item {

#ifdef DIAGNOSTIC

u_int pi_magic;

#endif

#define PI_MAGIC 0xdeaddeadU

#define PI_MAGIC 0xdeadbeefU

/* Other entries use only this list entry */

LIST_ENTRY(pool_item) pi_list;

};

Line 165 struct pool_item {

Line 158 struct pool_item {

* needless object construction/destruction; it is deferred until absolutely

* necessary.

* Caches are grouped into cache groups. Each cache group references up

* Caches are grouped into cache groups. Each cache group references

* to PCG_NUMOBJECTS constructed objects. When a cache allocates an

* up to 16 constructed objects. When a cache allocates an object

* object from the pool, it calls the object's constructor and places it

* from the pool, it calls the object's constructor and places it into

* into a cache group. When a cache group frees an object back to the

* a cache group. When a cache group frees an object back to the pool,

* pool, it first calls the object's destructor. This allows the object

* it first calls the object's destructor. This allows the object to

* to persist in constructed form while freed to the cache.

* persist in constructed form while freed to the cache.

* The pool references each cache, so that when a pool is drained by the

* Multiple caches may exist for each pool. This allows a single

* pagedaemon, it can drain each individual cache as well. Each time a

* object type to have multiple constructed forms. The pool references

* cache is drained, the most idle cache group is freed to the pool in

* each cache, so that when a pool is drained by the pagedaemon, it can

* its entirety.

* drain each individual cache as well. Each time a cache is drained,

* the most idle cache group is freed to the pool in its entirety.

* Pool caches are layed on top of pools. By layering them, we can avoid

* the complexity of cache management for pools which would not benefit

* from it.

/* The cache group pool. */

static struct pool pcgpool;

static struct pool cache_pool;

static struct pool cache_cpu_pool;

static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *,

static void pool_cache_reclaim(struct pool_cache *, struct pool_pagelist *,

void *, paddr_t);

struct pool_cache_grouplist *);

static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *,

static void pcg_grouplist_free(struct pool_cache_grouplist *);

void **, paddr_t *, int);

static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t);

static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *);

static void pool_cache_xcall(pool_cache_t);

static int pool_catchup(struct pool *);

static void pool_prime_page(struct pool *, void *,

Line 433 static void

Line 422 static void

pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq)

{

struct pool_item_header *ph;

int s;

while ((ph = LIST_FIRST(pq)) != NULL) {

LIST_REMOVE(ph, ph_pagelist);

pool_allocator_free(pp, ph->ph_page);

if ((pp->pr_roflags & PR_PHINPAGE) == 0)

if ((pp->pr_roflags & PR_PHINPAGE) == 0) {

s = splvm();

pool_put(pp->pr_phpool, ph);

splx(s);

}

Line 450 pr_rmpage(struct pool *pp, struct pool_i

Line 443 pr_rmpage(struct pool *pp, struct pool_i

struct pool_pagelist *pq)

{

KASSERT(mutex_owned(&pp->pr_lock));

LOCK_ASSERT(simple_lock_held(&pp->pr_slock));

* If the page was idle, decrement the idle page count.

Line 564 pool_subsystem_init(void)

Line 557 pool_subsystem_init(void)

__link_set_decl(pools, struct link_pool_init);

struct link_pool_init * const *pi;

mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE);

cv_init(&pool_busy, "poolbusy");

__link_set_foreach(pi, pools)

pool_init((*pi)->pp, (*pi)->size, (*pi)->align,

(*pi)->align_offset, (*pi)->flags, (*pi)->wchan,

Line 578 pool_subsystem_init(void)

Line 568 pool_subsystem_init(void)

SLIST_REMOVE_HEAD(&pa_deferinitq, pa_q);

pa_reclaim_register(pa);

}

pool_init(&cache_pool, sizeof(struct pool_cache), CACHE_LINE_SIZE,

0, 0, "pcache", &pool_allocator_nointr, IPL_NONE);

pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), CACHE_LINE_SIZE,

0, 0, "pcachecpu", &pool_allocator_nointr, IPL_NONE);

}

Line 600 pool_init(struct pool *pp, size_t size,

Line 584 pool_init(struct pool *pp, size_t size,

struct pool *pp1;

#endif

size_t trysize, phsize;

int off, slack;

int off, slack, s;

KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >=

PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1));

Line 641 pool_init(struct pool *pp, size_t size,

Line 625 pool_init(struct pool *pp, size_t size,

TAILQ_INIT(&palloc->pa_list);

mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM);

simple_lock_init(&palloc->pa_slock);

palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);

palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;

Line 669 pool_init(struct pool *pp, size_t size,

Line 653 pool_init(struct pool *pp, size_t size,

LIST_INIT(&pp->pr_emptypages);

LIST_INIT(&pp->pr_fullpages);

LIST_INIT(&pp->pr_partpages);

pp->pr_cache = NULL;

LIST_INIT(&pp->pr_cachelist);

pp->pr_curpage = NULL;

pp->pr_npages = 0;

pp->pr_minitems = 0;

Line 770 pool_init(struct pool *pp, size_t size,

Line 754 pool_init(struct pool *pp, size_t size,

pp->pr_npagefree = 0;

pp->pr_hiwat = 0;

pp->pr_nidle = 0;

pp->pr_refcnt = 0;

#ifdef POOL_DIAGNOSTIC

if (flags & PR_LOGGING) {

Line 786 pool_init(struct pool *pp, size_t size,

Line 769 pool_init(struct pool *pp, size_t size,

pp->pr_entered_file = NULL;

pp->pr_entered_line = 0;

mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl);

simple_lock_init(&pp->pr_slock);

cv_init(&pp->pr_cv, wchan);

pp->pr_ipl = ipl;

* Initialize private page header pool and cache magazine pool if we

Line 817 pool_init(struct pool *pp, size_t size,

Line 798 pool_init(struct pool *pp, size_t size,

pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0,

PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM);

#endif

pool_init(&pcgpool, sizeof(pcg_t), CACHE_LINE_SIZE, 0, 0,

pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0,

"cachegrp", &pool_allocator_meta, IPL_VM);

0, "pcgpool", &pool_allocator_meta, IPL_VM);

}

if (__predict_true(!cold)) {

/* Insert into the list of all pools. */

simple_lock(&pool_head_slock);

mutex_enter(&pool_head_lock);

LIST_INSERT_HEAD(&pool_head, pp, pr_poollist);

simple_unlock(&pool_head_slock);

mutex_exit(&pool_head_lock);

/* Insert this into the list of pools using this allocator. */

mutex_enter(&palloc->pa_lock);

TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);

mutex_exit(&palloc->pa_lock);

} else {

LIST_INSERT_HEAD(&pool_head, pp, pr_poollist);

TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);

}

/* Insert this into the list of pools using this allocator. */

s = splvm();

simple_lock(&palloc->pa_slock);

TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);

simple_unlock(&palloc->pa_slock);

splx(s);

pool_reclaim_register(pp);

}

Line 847 pool_destroy(struct pool *pp)

Line 824 pool_destroy(struct pool *pp)

{

struct pool_pagelist pq;

struct pool_item_header *ph;

int s;

/* Remove from global pool list */

mutex_enter(&pool_head_lock);

simple_lock(&pool_head_slock);

while (pp->pr_refcnt != 0)

cv_wait(&pool_busy, &pool_head_lock);

LIST_REMOVE(pp, pr_poollist);

if (drainpp == pp)

drainpp = NULL;

mutex_exit(&pool_head_lock);

simple_unlock(&pool_head_slock);

/* Remove this pool from its allocator's list of pools. */

pool_reclaim_unregister(pp);

mutex_enter(&pp->pr_alloc->pa_lock);

s = splvm();

simple_lock(&pp->pr_alloc->pa_slock);

TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);

mutex_exit(&pp->pr_alloc->pa_lock);

simple_unlock(&pp->pr_alloc->pa_slock);

splx(s);

mutex_enter(&pp->pr_lock);

s = splvm();

simple_lock(&pp->pr_slock);

KASSERT(pp->pr_cache == NULL);

KASSERT(LIST_EMPTY(&pp->pr_cachelist));

#ifdef DIAGNOSTIC

if (pp->pr_nout != 0) {

Line 883 pool_destroy(struct pool *pp)

Line 862 pool_destroy(struct pool *pp)

while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)

pr_rmpage(pp, ph, &pq);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

splx(s);

pr_pagelist_free(pp, &pq);

Line 891 pool_destroy(struct pool *pp)

Line 871 pool_destroy(struct pool *pp)

if ((pp->pr_roflags & PR_LOGGING) != 0)

free(pp->pr_log, M_TEMP);

#endif

cv_destroy(&pp->pr_cv);

mutex_destroy(&pp->pr_lock);

}

void

Line 913 static struct pool_item_header *

Line 890 static struct pool_item_header *

pool_alloc_item_header(struct pool *pp, void *storage, int flags)

{

struct pool_item_header *ph;

int s;

LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0);

if ((pp->pr_roflags & PR_PHINPAGE) != 0)

ph = (struct pool_item_header *) ((char *)storage + pp->pr_phoffset);

else

else {

s = splvm();

ph = pool_get(pp->pr_phpool, flags);

splx(s);

}

return (ph);

}

* Grab an item from the pool.

* Grab an item from the pool; must be called at appropriate spl level

void *

#ifdef POOL_DIAGNOSTIC

Line 950 pool_get(struct pool *pp, int flags)

Line 933 pool_get(struct pool *pp, int flags)

ASSERT_SLEEPABLE(NULL, "pool_get(PR_WAITOK)");

#endif

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pr_enter(pp, file, line);

startover:

Line 962 pool_get(struct pool *pp, int flags)

Line 945 pool_get(struct pool *pp, int flags)

#ifdef DIAGNOSTIC

if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

panic("pool_get: %s: crossed hard limit", pp->pr_wchan);

}

#endif

Line 974 pool_get(struct pool *pp, int flags)

Line 957 pool_get(struct pool *pp, int flags)

* and check the hardlimit condition again.

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pr_enter(pp, file, line);

if (pp->pr_nout < pp->pr_hardlimit)

goto startover;

Line 989 pool_get(struct pool *pp, int flags)

Line 972 pool_get(struct pool *pp, int flags)

pp->pr_flags |= PR_WANTED;

pr_leave(pp);

cv_wait(&pp->pr_cv, &pp->pr_lock);

ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);

pr_enter(pp, file, line);

goto startover;

}

Line 1005 pool_get(struct pool *pp, int flags)

Line 988 pool_get(struct pool *pp, int flags)

pp->pr_nfail++;

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

return (NULL);

}

Line 1020 pool_get(struct pool *pp, int flags)

Line 1003 pool_get(struct pool *pp, int flags)

#ifdef DIAGNOSTIC

if (pp->pr_nitems != 0) {

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

printf("pool_get: %s: curpage NULL, nitems %u\n",

pp->pr_wchan, pp->pr_nitems);

panic("pool_get: nitems inconsistent");

Line 1047 pool_get(struct pool *pp, int flags)

Line 1030 pool_get(struct pool *pp, int flags)

pp->pr_nfail++;

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

return (NULL);

}

Line 1058 pool_get(struct pool *pp, int flags)

Line 1041 pool_get(struct pool *pp, int flags)

#ifdef DIAGNOSTIC

if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) {

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

panic("pool_get: %s: page empty", pp->pr_wchan);

}

#endif

Line 1070 pool_get(struct pool *pp, int flags)

Line 1053 pool_get(struct pool *pp, int flags)

v = pi = LIST_FIRST(&ph->ph_itemlist);

if (__predict_false(v == NULL)) {

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

panic("pool_get: %s: page empty", pp->pr_wchan);

}

#ifdef DIAGNOSTIC

if (__predict_false(pp->pr_nitems == 0)) {

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

printf("pool_get: %s: items on itemlist, nitems %u\n",

pp->pr_wchan, pp->pr_nitems);

panic("pool_get: nitems inconsistent");

Line 1123 pool_get(struct pool *pp, int flags)

Line 1106 pool_get(struct pool *pp, int flags)

if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 &&

!LIST_EMPTY(&ph->ph_itemlist))) {

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

panic("pool_get: %s: nmissing inconsistent",

pp->pr_wchan);

}

Line 1152 pool_get(struct pool *pp, int flags)

Line 1135 pool_get(struct pool *pp, int flags)

}

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

KASSERT((((vaddr_t)v + pp->pr_itemoffset) & (pp->pr_align - 1)) == 0);

FREECHECK_OUT(&pp->pr_freecheck, v);

return (v);

Line 1167 pool_do_put(struct pool *pp, void *v, st

Line 1150 pool_do_put(struct pool *pp, void *v, st

struct pool_item *pi = v;

struct pool_item_header *ph;

KASSERT(mutex_owned(&pp->pr_lock));

LOCK_ASSERT(simple_lock_held(&pp->pr_slock));

FREECHECK_IN(&pp->pr_freecheck, v);

LOCKDEBUG_MEM_CHECK(v, pp->pr_size);

#ifdef DIAGNOSTIC

if (__predict_false(pp->pr_nout == 0)) {

Line 1184 pool_do_put(struct pool *pp, void *v, st

Line 1166 pool_do_put(struct pool *pp, void *v, st

panic("pool_put: %s: page header missing", pp->pr_wchan);

}

#ifdef LOCKDEBUG

* Check if we're freeing a locked simple lock.

simple_lock_freecheck(pi, (char *)pi + pp->pr_size);

#endif

* Return to item list.

Line 1219 pool_do_put(struct pool *pp, void *v, st

Line 1208 pool_do_put(struct pool *pp, void *v, st

pp->pr_flags &= ~PR_WANTED;

if (ph->ph_nmissing == 0)

pp->pr_nidle++;

cv_broadcast(&pp->pr_cv);

wakeup((void *)pp);

return;

}

Line 1271 pool_do_put(struct pool *pp, void *v, st

Line 1260 pool_do_put(struct pool *pp, void *v, st

}

* Return resource to the pool.

* Return resource to the pool; must be called at appropriate spl level

#ifdef POOL_DIAGNOSTIC

void

Line 1281 _pool_put(struct pool *pp, void *v, cons

Line 1270 _pool_put(struct pool *pp, void *v, cons

LIST_INIT(&pq);

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pr_enter(pp, file, line);

pr_log(pp, v, PRLOG_PUT, file, line);

Line 1289 _pool_put(struct pool *pp, void *v, cons

Line 1278 _pool_put(struct pool *pp, void *v, cons

pool_do_put(pp, v, &pq);

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

pr_pagelist_free(pp, &pq);

}

Line 1303 pool_put(struct pool *pp, void *v)

Line 1292 pool_put(struct pool *pp, void *v)

LIST_INIT(&pq);

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pool_do_put(pp, v, &pq);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

pr_pagelist_free(pp, &pq);

}

Line 1328 pool_grow(struct pool *pp, int flags)

Line 1317 pool_grow(struct pool *pp, int flags)

struct pool_item_header *ph = NULL;

char *cp;

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

cp = pool_allocator_alloc(pp, flags);

if (__predict_true(cp != NULL)) {

ph = pool_alloc_item_header(pp, cp, flags);

Line 1337 pool_grow(struct pool *pp, int flags)

Line 1326 pool_grow(struct pool *pp, int flags)

if (cp != NULL) {

pool_allocator_free(pp, cp);

}

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

return ENOMEM;

}

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pool_prime_page(pp, cp, ph);

pp->pr_npagealloc++;

return 0;

Line 1356 pool_prime(struct pool *pp, int n)

Line 1345 pool_prime(struct pool *pp, int n)

int newpages;

int error = 0;

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

Line 1371 pool_prime(struct pool *pp, int n)

Line 1360 pool_prime(struct pool *pp, int n)

if (pp->pr_minpages >= pp->pr_maxpages)

pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

return error;

}

Line 1389 pool_prime_page(struct pool *pp, void *s

Line 1378 pool_prime_page(struct pool *pp, void *s

const unsigned int ioff = pp->pr_itemoffset;

int n;

KASSERT(mutex_owned(&pp->pr_lock));

LOCK_ASSERT(simple_lock_held(&pp->pr_slock));

#ifdef DIAGNOSTIC

if ((pp->pr_roflags & PR_NOALIGN) == 0 &&

Line 1504 void

Line 1493 void

pool_setlowat(struct pool *pp, int n)

{

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pp->pr_minitems = n;

pp->pr_minpages = (n == 0)

Line 1520 pool_setlowat(struct pool *pp, int n)

Line 1509 pool_setlowat(struct pool *pp, int n)

}

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

}

void

pool_sethiwat(struct pool *pp, int n)

{

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pp->pr_maxpages = (n == 0)

? 0

: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

}

void

pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap)

{

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pp->pr_hardlimit = n;

pp->pr_hardlimit_warning = warnmess;

Line 1556 pool_sethardlimit(struct pool *pp, int n

Line 1545 pool_sethardlimit(struct pool *pp, int n

? 0

: roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

}

Line 1570 pool_reclaim(struct pool *pp)

Line 1559 pool_reclaim(struct pool *pp)

#endif

{

struct pool_item_header *ph, *phnext;

struct pool_cache *pc;

struct pool_pagelist pq;

struct pool_cache_grouplist pcgl;

struct timeval curtime, diff;

bool klock;

int rv;

if (pp->pr_drain_hook != NULL) {

Line 1582 pool_reclaim(struct pool *pp)

Line 1571 pool_reclaim(struct pool *pp)

(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT);

}

if (simple_lock_try(&pp->pr_slock) == 0)

* XXXSMP Because mutexes at IPL_SOFTXXX are still spinlocks,

* and we are called from the pagedaemon without kernel_lock.

* Does not apply to IPL_SOFTBIO.

switch (pp->pr_ipl) {

case IPL_SOFTNET:

case IPL_SOFTCLOCK:

case IPL_SOFTSERIAL:

KERNEL_LOCK(1, NULL);

klock = true;

break;

default:

klock = false;

break;

}

/* Reclaim items from the pool's cache (if any). */

if (pp->pr_cache != NULL)

pool_cache_invalidate(pp->pr_cache);

if (mutex_tryenter(&pp->pr_lock) == 0) {

if (klock) {

KERNEL_UNLOCK_ONE(NULL);

}

return (0);

}

pr_enter(pp, file, line);

LIST_INIT(&pq);

LIST_INIT(&pcgl);

* Reclaim items from the pool's caches.

LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist)

pool_cache_reclaim(pc, &pq, &pcgl);

getmicrotime(&curtime);

Line 1640 pool_reclaim(struct pool *pp)

Line 1611 pool_reclaim(struct pool *pp)

}

pr_leave(pp);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

if (LIST_EMPTY(&pq) && LIST_EMPTY(&pcgl))

if (LIST_EMPTY(&pq))

return 0;

rv = 0;

else {

pr_pagelist_free(pp, &pq);

rv = 1;

}

if (klock) {

KERNEL_UNLOCK_ONE(NULL);

}

return (rv);

pr_pagelist_free(pp, &pq);

pcg_grouplist_free(&pcgl);

return (1);

}

* Drain pools, one at a time. This is a two stage process;

* Drain pools, one at a time.

* drain_start kicks off a cross call to drain CPU-level caches

* if the pool has an associated pool_cache. drain_end waits

* for those cross calls to finish, and then drains the cache

* (if any) and pool.

* Note, must never be called from interrupt context.

* Note, we must never be called from an interrupt context.

void

pool_drain_start(struct pool **ppp, uint64_t *wp)

pool_drain(void *arg)

{

struct pool *pp;

int s;

KASSERT(!LIST_EMPTY(&pool_head));

pp = NULL;

s = splvm();

/* Find next pool to drain, and add a reference. */

simple_lock(&pool_head_slock);

mutex_enter(&pool_head_lock);

if (drainpp == NULL) {

do {

drainpp = LIST_FIRST(&pool_head);

if (drainpp == NULL) {

}

drainpp = LIST_FIRST(&pool_head);

if (drainpp) {

}

pp = drainpp;

if (drainpp != NULL) {

drainpp = LIST_NEXT(pp, pr_poollist);

pp = drainpp;

}

drainpp = LIST_NEXT(pp, pr_poollist);

simple_unlock(&pool_head_slock);

}

if (pp)

pool_reclaim(pp);

* Skip completely idle pools. We depend on at least

splx(s);

* one pool in the system being active.

} while (pp == NULL || pp->pr_npages == 0);

pp->pr_refcnt++;

mutex_exit(&pool_head_lock);

/* If there is a pool_cache, drain CPU level caches. */

*ppp = pp;

if (pp->pr_cache != NULL) {

*wp = xc_broadcast(0, (xcfunc_t)pool_cache_xcall,

pp->pr_cache, NULL);

}

void

pool_drain_end(struct pool *pp, uint64_t where)

{

if (pp == NULL)

return;

KASSERT(pp->pr_refcnt > 0);

/* Wait for remote draining to complete. */

if (pp->pr_cache != NULL)

xc_wait(where);

/* Drain the cache (if any) and pool.. */

pool_reclaim(pp);

/* Finally, unlock the pool. */

mutex_enter(&pool_head_lock);

pp->pr_refcnt--;

cv_broadcast(&pool_busy);

mutex_exit(&pool_head_lock);

}

Line 1729 pool_drain_end(struct pool *pp, uint64_t

Line 1653 pool_drain_end(struct pool *pp, uint64_t

void

pool_print(struct pool *pp, const char *modif)

{

int s;

s = splvm();

if (simple_lock_try(&pp->pr_slock) == 0) {

printf("pool %s is locked; try again later\n",

pp->pr_wchan);

splx(s);

return;

}

pool_print1(pp, modif, printf);

simple_unlock(&pp->pr_slock);

splx(s);

}

void

Line 1738 pool_printall(const char *modif, void (*

Line 1672 pool_printall(const char *modif, void (*

{

struct pool *pp;

if (simple_lock_try(&pool_head_slock) == 0) {

(*pr)("WARNING: pool_head_slock is locked\n");

} else {

simple_unlock(&pool_head_slock);

}

LIST_FOREACH(pp, &pool_head, pr_poollist) {

pool_printit(pp, modif, pr);

}

Line 1752 pool_printit(struct pool *pp, const char

Line 1692 pool_printit(struct pool *pp, const char

return;

}

* Called from DDB; interrupts should be blocked, and all

* other processors should be paused. We can skip locking

* the pool in this case.

* We do a simple_lock_try() just to print the lock

* status, however.

if (simple_lock_try(&pp->pr_slock) == 0)

(*pr)("WARNING: pool %s is locked\n", pp->pr_wchan);

else

simple_unlock(&pp->pr_slock);

pool_print1(pp, modif, pr);

}

Line 1786 static void

Line 1740 static void

pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...))

{

struct pool_item_header *ph;

pool_cache_t pc;

struct pool_cache *pc;

pcg_t *pcg;

struct pool_cache_group *pcg;

pool_cache_cpu_t *cc;

uint64_t cpuhit, cpumiss;

int i, print_log = 0, print_pagelist = 0, print_cache = 0;

char c;

Line 1802 pool_print1(struct pool *pp, const char

Line 1754 pool_print1(struct pool *pp, const char

print_cache = 1;

}

if ((pc = pp->pr_cache) != NULL) {

(*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n",

(*pr)("POOL CACHE");

} else {

(*pr)("POOL");

}

(*pr)(" %s: size %u, align %u, ioff %u, roflags 0x%08x\n",

pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,

pp->pr_roflags);

(*pr)("\talloc %p\n", pp->pr_alloc);

Line 1817 pool_print1(struct pool *pp, const char

Line 1763 pool_print1(struct pool *pp, const char

(*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",

pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);

(*pr)("\tnget %lu, nfail %lu, nput %lu\n",

(*pr)("\n\tnget %lu, nfail %lu, nput %lu\n",

pp->pr_nget, pp->pr_nfail, pp->pr_nput);

(*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",

pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);

Line 1852 pool_print1(struct pool *pp, const char

Line 1798 pool_print1(struct pool *pp, const char

}

skip_log:

if (print_cache == 0)

goto skip_cache;

#define PR_GROUPLIST(pcg) \

(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \

Line 1868 pool_print1(struct pool *pp, const char

Line 1816 pool_print1(struct pool *pp, const char

} \

}

if (pc != NULL) {

LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) {

cpuhit = 0;

(*pr)("\tcache %p\n", pc);

cpumiss = 0;

(*pr)("\t hits %lu misses %lu ngroups %lu nitems %lu\n",

for (i = 0; i < MAXCPUS; i++) {

pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems);

if ((cc = pc->pc_cpus[i]) == NULL)

(*pr)("\t full groups:\n");

continue;

LIST_FOREACH(pcg, &pc->pc_fullgroups, pcg_list) {

cpuhit += cc->cc_hits;

PR_GROUPLIST(pcg);

cpumiss += cc->cc_misses;

}

(*pr)("\t partial groups:\n");

(*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss);

LIST_FOREACH(pcg, &pc->pc_partgroups, pcg_list) {

(*pr)("\tcache layer hits %llu misses %llu\n",

PR_GROUPLIST(pcg);

pc->pc_hits, pc->pc_misses);

}

(*pr)("\tcache layer entry uncontended %llu contended %llu\n",

(*pr)("\t empty groups:\n");

pc->pc_hits + pc->pc_misses - pc->pc_contended,

LIST_FOREACH(pcg, &pc->pc_emptygroups, pcg_list) {

pc->pc_contended);

PR_GROUPLIST(pcg);

(*pr)("\tcache layer empty groups %u full groups %u\n",

pc->pc_nempty, pc->pc_nfull);

if (print_cache) {

(*pr)("\tfull cache groups:\n");

for (pcg = pc->pc_fullgroups; pcg != NULL;

pcg = pcg->pcg_next) {

PR_GROUPLIST(pcg);

}

(*pr)("\tempty cache groups:\n");

for (pcg = pc->pc_emptygroups; pcg != NULL;

pcg = pcg->pcg_next) {

PR_GROUPLIST(pcg);

}

#undef PR_GROUPLIST

skip_cache:

pr_enter_check(pp, pr);

}

Line 1967 pool_chk(struct pool *pp, const char *la

Line 1903 pool_chk(struct pool *pp, const char *la

struct pool_item_header *ph;

int r = 0;

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {

r = pool_chk_page(pp, label, ph);

if (r) {

Line 1988 pool_chk(struct pool *pp, const char *la

Line 1924 pool_chk(struct pool *pp, const char *la

}

out:

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

return (r);

}

Line 1996 out:

Line 1932 out:

* pool_cache_init:

* Initialize a pool cache.

pool_cache_t

pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags,

const char *wchan, struct pool_allocator *palloc, int ipl,

int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg)

{

pool_cache_t pc;

pc = pool_get(&cache_pool, PR_WAITOK);

if (pc == NULL)

return NULL;

pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan,

palloc, ipl, ctor, dtor, arg);

return pc;

}

* pool_cache_bootstrap:

* Kernel-private version of pool_cache_init(). The caller

* NOTE: If the pool must be protected from interrupts, we expect

* provides initial storage.

* to be called at the appropriate interrupt priority level.

void

pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align,

pool_cache_init(struct pool_cache *pc, struct pool *pp,

u_int align_offset, u_int flags, const char *wchan,

int (*ctor)(void *, void *, int),

struct pool_allocator *palloc, int ipl,

void (*dtor)(void *, void *),

int (*ctor)(void *, void *, int), void (*dtor)(void *, void *),

void *arg)

{

CPU_INFO_ITERATOR cii;

struct cpu_info *ci;

struct pool *pp;

pp = &pc->pc_pool;

LIST_INIT(&pc->pc_emptygroups);

if (palloc == NULL && ipl == IPL_NONE)

LIST_INIT(&pc->pc_fullgroups);

palloc = &pool_allocator_nointr;

LIST_INIT(&pc->pc_partgroups);

pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl);

simple_lock_init(&pc->pc_slock);

mutex_init(&pc->pc_lock, MUTEX_DEFAULT, pp->pr_ipl);

pc->pc_pool = pp;

if (ctor == NULL) {

ctor = (int (*)(void *, void *, int))nullop;

}

if (dtor == NULL) {

dtor = (void (*)(void *, void *))nullop;

}

pc->pc_emptygroups = NULL;

pc->pc_fullgroups = NULL;

pc->pc_partgroups = NULL;

pc->pc_ctor = ctor;

pc->pc_dtor = dtor;

pc->pc_arg = arg;

pc->pc_hits = 0;

pc->pc_misses = 0;

pc->pc_nempty = 0;

pc->pc_npart = 0;

pc->pc_ngroups = 0;

pc->pc_nfull = 0;

pc->pc_contended = 0;

pc->pc_nitems = 0;

pc->pc_refcnt = 0;

simple_lock(&pp->pr_slock);

/* Allocate per-CPU caches. */

LIST_INSERT_HEAD(&pp->pr_cachelist, pc, pc_poollist);

memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus));

simple_unlock(&pp->pr_slock);

pc->pc_ncpu = 0;

for (CPU_INFO_FOREACH(cii, ci)) {

pool_cache_cpu_init1(ci, pc);

}

if (__predict_true(!cold)) {

mutex_enter(&pp->pr_lock);

pp->pr_cache = pc;

mutex_exit(&pp->pr_lock);

mutex_enter(&pool_head_lock);

LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist);

mutex_exit(&pool_head_lock);

} else {

pp->pr_cache = pc;

LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist);

}

Line 2085 pool_cache_bootstrap(pool_cache_t pc, si

Line 1972 pool_cache_bootstrap(pool_cache_t pc, si

* Destroy a pool cache.

void

pool_cache_destroy(pool_cache_t pc)

pool_cache_destroy(struct pool_cache *pc)

{

struct pool *pp = &pc->pc_pool;

struct pool *pp = pc->pc_pool;

pool_cache_cpu_t *cc;

pcg_t *pcg;

int i;

/* Remove it from the global list. */

mutex_enter(&pool_head_lock);

while (pc->pc_refcnt != 0)

cv_wait(&pool_busy, &pool_head_lock);

LIST_REMOVE(pc, pc_cachelist);

mutex_exit(&pool_head_lock);

/* First, invalidate the entire cache. */

pool_cache_invalidate(pc);

/* Disassociate it from the pool. */

/* ...and remove it from the pool's cache list. */

mutex_enter(&pp->pr_lock);

simple_lock(&pp->pr_slock);

pp->pr_cache = NULL;

LIST_REMOVE(pc, pc_poollist);

mutex_exit(&pp->pr_lock);

simple_unlock(&pp->pr_slock);

/* Destroy per-CPU data */

for (i = 0; i < MAXCPUS; i++) {

if ((cc = pc->pc_cpus[i]) == NULL)

continue;

if ((pcg = cc->cc_current) != NULL) {

pcg->pcg_next = NULL;

pool_cache_invalidate_groups(pc, pcg);

}

if ((pcg = cc->cc_previous) != NULL) {

pcg->pcg_next = NULL;

pool_cache_invalidate_groups(pc, pcg);

}

if (cc != &pc->pc_cpu0)

pool_put(&cache_cpu_pool, cc);

}

/* Finally, destroy it. */

mutex_destroy(&pc->pc_lock);

pool_destroy(pp);

pool_put(&cache_pool, pc);

}

static inline void *

* pool_cache_cpu_init1:

pcg_get(struct pool_cache_group *pcg, paddr_t *pap)

* Called for each pool_cache whenever a new CPU is attached.

static void

pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc)

{

pool_cache_cpu_t *cc;

void *object;

u_int idx;

KASSERT(((uintptr_t)pc->pc_cpus & (CACHE_LINE_SIZE - 1)) == 0);

if ((cc = pc->pc_cpus[ci->ci_index]) != NULL) {

KASSERT(cc->cc_cpu = ci);

return;

}

* The first CPU is 'free'. This needs to be the case for

* bootstrap - we may not be able to allocate yet.

if (pc->pc_ncpu == 0) {

cc = &pc->pc_cpu0;

pc->pc_ncpu = 1;

} else {

mutex_enter(&pc->pc_lock);

pc->pc_ncpu++;

mutex_exit(&pc->pc_lock);

cc = pool_get(&cache_cpu_pool, PR_WAITOK);

}

cc->cc_ipl = pc->pc_pool.pr_ipl;

KASSERT(pcg->pcg_avail <= PCG_NOBJECTS);

cc->cc_iplcookie = makeiplcookie(cc->cc_ipl);

KASSERT(pcg->pcg_avail != 0);

cc->cc_cache = pc;

idx = --pcg->pcg_avail;

cc->cc_cpu = ci;

cc->cc_hits = 0;

KASSERT(pcg->pcg_objects[idx].pcgo_va != NULL);

cc->cc_misses = 0;

object = pcg->pcg_objects[idx].pcgo_va;

cc->cc_current = NULL;

if (pap != NULL)

cc->cc_previous = NULL;

*pap = pcg->pcg_objects[idx].pcgo_pa;

pcg->pcg_objects[idx].pcgo_va = NULL;

pc->pc_cpus[ci->ci_index] = cc;

return (object);

}

static inline void

* pool_cache_cpu_init:

pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa)

* Called whenever a new CPU is attached.

void

pool_cache_cpu_init(struct cpu_info *ci)

{

pool_cache_t pc;

mutex_enter(&pool_head_lock);

LIST_FOREACH(pc, &pool_cache_head, pc_cachelist) {

pc->pc_refcnt++;

mutex_exit(&pool_head_lock);

pool_cache_cpu_init1(ci, pc);

mutex_enter(&pool_head_lock);

pc->pc_refcnt--;

cv_broadcast(&pool_busy);

}

mutex_exit(&pool_head_lock);

}

* pool_cache_reclaim:

* Reclaim memory from a pool cache.

bool

pool_cache_reclaim(pool_cache_t pc)

{

u_int idx;

return pool_reclaim(&pc->pc_pool);

KASSERT(pcg->pcg_avail < PCG_NOBJECTS);

}

idx = pcg->pcg_avail++;

KASSERT(pcg->pcg_objects[idx].pcgo_va == NULL);

* pool_cache_destruct_object:

pcg->pcg_objects[idx].pcgo_va = object;

pcg->pcg_objects[idx].pcgo_pa = pa;

* Force destruction of an object and its release back into

* the pool.

void

pool_cache_destruct_object(pool_cache_t pc, void *object)

{

(*pc->pc_dtor)(pc->pc_arg, object);

pool_put(&pc->pc_pool, object);

}

* pool_cache_invalidate_groups:

* Invalidate a chain of groups and destruct all objects.

static void

pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg)

pcg_grouplist_free(struct pool_cache_grouplist *pcgl)

{

void *object;

struct pool_cache_group *pcg;

pcg_t *next;

int s;

int i;

for (; pcg != NULL; pcg = next) {

next = pcg->pcg_next;

for (i = 0; i < pcg->pcg_avail; i++) {

object = pcg->pcg_objects[i].pcgo_va;

pool_cache_destruct_object(pc, object);

}

s = splvm();

while ((pcg = LIST_FIRST(pcgl)) != NULL) {

LIST_REMOVE(pcg, pcg_list);

pool_put(&pcgpool, pcg);

}

splx(s);

}

* pool_cache_invalidate:

* pool_cache_get{,_paddr}:

* Invalidate a pool cache (destruct and release all of the

* Get an object from a pool cache (optionally returning

* cached objects). Does not reclaim objects from the pool.

* the physical address of the object).

void

void *

pool_cache_invalidate(pool_cache_t pc)

pool_cache_get_paddr(struct pool_cache *pc, int flags, paddr_t *pap)

{

pcg_t *full, *empty, *part;

mutex_enter(&pc->pc_lock);

full = pc->pc_fullgroups;

empty = pc->pc_emptygroups;

part = pc->pc_partgroups;

pc->pc_fullgroups = NULL;

pc->pc_emptygroups = NULL;

pc->pc_partgroups = NULL;

pc->pc_nfull = 0;

pc->pc_nempty = 0;

pc->pc_npart = 0;

mutex_exit(&pc->pc_lock);

pool_cache_invalidate_groups(pc, full);

pool_cache_invalidate_groups(pc, empty);

pool_cache_invalidate_groups(pc, part);

}

void

pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg)

{

pool_set_drain_hook(&pc->pc_pool, fn, arg);

}

void

pool_cache_setlowat(pool_cache_t pc, int n)

{

pool_setlowat(&pc->pc_pool, n);

}

void

pool_cache_sethiwat(pool_cache_t pc, int n)

{

pool_sethiwat(&pc->pc_pool, n);

}

void

pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap)

{

pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap);

}

static inline pool_cache_cpu_t *

pool_cache_cpu_enter(pool_cache_t pc, int *s)

{

pool_cache_cpu_t *cc;

struct cpu_info *ci;

* Prevent other users of the cache from accessing our

* CPU-local data. To avoid touching shared state, we

* pull the neccessary information from CPU local data.

ci = curcpu();

KASSERT(ci->ci_data.cpu_index < MAXCPUS);

cc = pc->pc_cpus[ci->ci_data.cpu_index];

KASSERT(cc->cc_cache == pc);

if (cc->cc_ipl == IPL_NONE) {

crit_enter();

} else {

*s = splraiseipl(cc->cc_iplcookie);

}

/* Moved to another CPU before disabling preemption? */

if (__predict_false(ci != curcpu())) {

ci = curcpu();

cc = pc->pc_cpus[ci->ci_data.cpu_index];

}

#ifdef DIAGNOSTIC

KASSERT(cc->cc_cpu == ci);

KASSERT(((uintptr_t)cc & (CACHE_LINE_SIZE - 1)) == 0);

#endif

return cc;

}

static inline void

pool_cache_cpu_exit(pool_cache_cpu_t *cc, int *s)

{

struct pool_cache_group *pcg;

void *object;

/* No longer need exclusive access to the per-CPU data. */

#ifdef LOCKDEBUG

if (cc->cc_ipl == IPL_NONE) {

if (flags & PR_WAITOK)

crit_exit();

ASSERT_SLEEPABLE(NULL, "pool_cache_get(PR_WAITOK)");

} else {

splx(*s);

}

#if __GNUC_PREREQ__(3, 0)

__attribute ((noinline))

#endif

pool_cache_cpu_t *

pool_cache_get_slow(pool_cache_cpu_t *cc, int *s, void **objectp,

paddr_t *pap, int flags)

{

pcg_t *pcg, *cur;

uint64_t ncsw;

pool_cache_t pc;

void *object;

pc = cc->cc_cache;

simple_lock(&pc->pc_slock);

cc->cc_misses++;

pcg = LIST_FIRST(&pc->pc_partgroups);

* Nothing was available locally. Try and grab a group

if (pcg == NULL) {

* from the cache.

pcg = LIST_FIRST(&pc->pc_fullgroups);

if (pcg != NULL) {

if (!mutex_tryenter(&pc->pc_lock)) {

LIST_REMOVE(pcg, pcg_list);

ncsw = curlwp->l_ncsw;

LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list);

mutex_enter(&pc->pc_lock);

pc->pc_contended++;

* If we context switched while locking, then

* our view of the per-CPU data is invalid:

* retry.

if (curlwp->l_ncsw != ncsw) {

mutex_exit(&pc->pc_lock);

pool_cache_cpu_exit(cc, s);

return pool_cache_cpu_enter(pc, s);

}

if (pcg == NULL) {

if ((pcg = pc->pc_fullgroups) != NULL) {

* If there's a full group, release our empty

* No groups with any available objects. Allocate

* group back to the cache. Install the full

* a new object, construct it, and return it to

* group as cc_current and return.

* the caller. We will allocate a group, if necessary,

* when the object is freed back to the cache.

if ((cur = cc->cc_current) != NULL) {

pc->pc_misses++;

KASSERT(cur->pcg_avail == 0);

simple_unlock(&pc->pc_slock);

cur->pcg_next = pc->pc_emptygroups;

object = pool_get(pc->pc_pool, flags);

pc->pc_emptygroups = cur;

if (object != NULL && pc->pc_ctor != NULL) {

pc->pc_nempty++;

if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) {

pool_put(pc->pc_pool, object);

return (NULL);

}

KASSERT((((vaddr_t)object + pc->pc_pool->pr_itemoffset) &

(pc->pc_pool->pr_align - 1)) == 0);

if (object != NULL && pap != NULL) {

#ifdef POOL_VTOPHYS

*pap = POOL_VTOPHYS(object);

#else

*pap = POOL_PADDR_INVALID;

#endif

}

KASSERT(pcg->pcg_avail == PCG_NOBJECTS);

cc->cc_current = pcg;

pc->pc_fullgroups = pcg->pcg_next;

pc->pc_hits++;

pc->pc_nfull--;

mutex_exit(&pc->pc_lock);

return cc;

}

* Nothing available locally or in cache. Take the slow

* path: fetch a new object from the pool and construct

* it.

pc->pc_misses++;

mutex_exit(&pc->pc_lock);

pool_cache_cpu_exit(cc, s);

object = pool_get(&pc->pc_pool, flags);

*objectp = object;

if (object == NULL)

return NULL;

if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) {

FREECHECK_OUT(&pc->pc_freecheck, object);

pool_put(&pc->pc_pool, object);

return (object);

*objectp = NULL;

return NULL;

}

KASSERT((((vaddr_t)object + pc->pc_pool.pr_itemoffset) &

pc->pc_hits++;

(pc->pc_pool.pr_align - 1)) == 0);

pc->pc_nitems--;

object = pcg_get(pcg, pap);

if (pap != NULL) {

if (pcg->pcg_avail == 0) {

#ifdef POOL_VTOPHYS

LIST_REMOVE(pcg, pcg_list);

*pap = POOL_VTOPHYS(object);

LIST_INSERT_HEAD(&pc->pc_emptygroups, pcg, pcg_list);

#else

*pap = POOL_PADDR_INVALID;

#endif

}

simple_unlock(&pc->pc_slock);

KASSERT((((vaddr_t)object + pc->pc_pool->pr_itemoffset) &

(pc->pc_pool->pr_align - 1)) == 0);

FREECHECK_OUT(&pc->pc_freecheck, object);

return NULL;

return (object);

}

* pool_cache_get{,_paddr}:

* pool_cache_put{,_paddr}:

* Get an object from a pool cache (optionally returning

* Put an object back to the pool cache (optionally caching the

* the physical address of the object).

* physical address of the object).

void *

void

pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap)

pool_cache_put_paddr(struct pool_cache *pc, void *object, paddr_t pa)

{

pool_cache_cpu_t *cc;

struct pool_cache_group *pcg;

pcg_t *pcg;

void *object;

int s;

#ifdef LOCKDEBUG

FREECHECK_IN(&pc->pc_freecheck, object);

if (flags & PR_WAITOK)

ASSERT_SLEEPABLE(NULL, "pool_cache_get(PR_WAITOK)");

if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) {

#endif

goto destruct;

}

cc = pool_cache_cpu_enter(pc, &s);

simple_lock(&pc->pc_slock);

do {

/* Try and allocate an object from the current group. */

pcg = LIST_FIRST(&pc->pc_partgroups);

pcg = cc->cc_current;

if (pcg == NULL) {

if (pcg != NULL && pcg->pcg_avail > 0) {

pcg = LIST_FIRST(&pc->pc_emptygroups);

object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va;

if (pcg != NULL) {

if (pap != NULL)

LIST_REMOVE(pcg, pcg_list);

*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa;

LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list);

pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL;

KASSERT(pcg->pcg_avail <= PCG_NOBJECTS);

KASSERT(object != NULL);

cc->cc_hits++;

pool_cache_cpu_exit(cc, &s);

FREECHECK_OUT(&pc->pc_freecheck, object);

return object;

}

if (pcg == NULL) {

* That failed. If the previous group isn't empty, swap

* No empty groups to free the object to. Attempt to

* it with the current group and allocate from there.

* allocate one.

pcg = cc->cc_previous;

simple_unlock(&pc->pc_slock);

if (pcg != NULL && pcg->pcg_avail > 0) {

s = splvm();

cc->cc_previous = cc->cc_current;

pcg = pool_get(&pcgpool, PR_NOWAIT);

cc->cc_current = pcg;

splx(s);

continue;

if (pcg == NULL) {

destruct:

* Unable to allocate a cache group; destruct the object

* and free it back to the pool.

pool_cache_destruct_object(pc, object);

return;

}

memset(pcg, 0, sizeof(*pcg));

simple_lock(&pc->pc_slock);

pc->pc_ngroups++;

LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list);

}

pc->pc_nitems++;

* Can't allocate from either group: try the slow path.

pcg_put(pcg, object, pa);

* If get_slow() allocated an object for us, or if

* no more objects are available, it will return NULL.

* Otherwise, we need to retry.

cc = pool_cache_get_slow(cc, &s, &object, pap, flags);

} while (cc != NULL);

return object;

if (pcg->pcg_avail == PCG_NOBJECTS) {

LIST_REMOVE(pcg, pcg_list);

LIST_INSERT_HEAD(&pc->pc_fullgroups, pcg, pcg_list);

}

simple_unlock(&pc->pc_slock);

}

#if __GNUC_PREREQ__(3, 0)

__attribute ((noinline))

* pool_cache_destruct_object:

#endif

pool_cache_cpu_t *

* Force destruction of an object and its release back into

pool_cache_put_slow(pool_cache_cpu_t *cc, int *s, void *object, paddr_t pa)

* the pool.

void

pool_cache_destruct_object(struct pool_cache *pc, void *object)

{

pcg_t *pcg, *cur;

uint64_t ncsw;

pool_cache_t pc;

pc = cc->cc_cache;

cc->cc_misses++;

if (pc->pc_dtor != NULL)

* No free slots locally. Try to grab an empty, unused

(*pc->pc_dtor)(pc->pc_arg, object);

* group from the cache.

pool_put(pc->pc_pool, object);

}

if (!mutex_tryenter(&pc->pc_lock)) {

ncsw = curlwp->l_ncsw;

mutex_enter(&pc->pc_lock);

pc->pc_contended++;

static void

* If we context switched while locking, then

pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl,

* our view of the per-CPU data is invalid:

struct pool_cache *pc, struct pool_pagelist *pq,

* retry.

struct pool_cache_grouplist *pcgdl)

{

if (curlwp->l_ncsw != ncsw) {

struct pool_cache_group *pcg, *npcg;

mutex_exit(&pc->pc_lock);

void *object;

pool_cache_cpu_exit(cc, s);

return pool_cache_cpu_enter(pc, s);

}

if ((pcg = pc->pc_emptygroups) != NULL) {

for (pcg = LIST_FIRST(pcgsl); pcg != NULL; pcg = npcg) {

npcg = LIST_NEXT(pcg, pcg_list);

* If there's a empty group, release our full

while (pcg->pcg_avail != 0) {

* group back to the cache. Install the empty

pc->pc_nitems--;

* group as cc_current and return.

object = pcg_get(pcg, NULL);

if (pc->pc_dtor != NULL)

if ((cur = cc->cc_current) != NULL) {

(*pc->pc_dtor)(pc->pc_arg, object);

KASSERT(cur->pcg_avail == PCG_NOBJECTS);

pool_do_put(pc->pc_pool, object, pq);

cur->pcg_next = pc->pc_fullgroups;

}

pc->pc_fullgroups = cur;

pc->pc_ngroups--;

pc->pc_nfull++;

LIST_REMOVE(pcg, pcg_list);

}

LIST_INSERT_HEAD(pcgdl, pcg, pcg_list);

KASSERT(pcg->pcg_avail == 0);

cc->cc_current = pcg;

pc->pc_emptygroups = pcg->pcg_next;

pc->pc_hits++;

pc->pc_nempty--;

mutex_exit(&pc->pc_lock);

return cc;

}

static void

* Nothing available locally or in cache. Take the

pool_do_cache_invalidate(struct pool_cache *pc, struct pool_pagelist *pq,

* slow path and try to allocate a new group that we

struct pool_cache_grouplist *pcgl)

* can release to.

{

pc->pc_misses++;

mutex_exit(&pc->pc_lock);

pool_cache_cpu_exit(cc, s);

LOCK_ASSERT(simple_lock_held(&pc->pc_slock));

* If we can't allocate a new group, just throw the

LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock));

* object away.

pcg = pool_get(&pcgpool, PR_NOWAIT);

if (pcg == NULL) {

pool_cache_destruct_object(pc, object);

return NULL;

}

#ifdef DIAGNOSTIC

memset(pcg, 0, sizeof(*pcg));

#else

pcg->pcg_avail = 0;

#endif

pool_do_cache_invalidate_grouplist(&pc->pc_fullgroups, pc, pq, pcgl);

* Add the empty group to the cache and try again.

pool_do_cache_invalidate_grouplist(&pc->pc_partgroups, pc, pq, pcgl);

mutex_enter(&pc->pc_lock);

pcg->pcg_next = pc->pc_emptygroups;

pc->pc_emptygroups = pcg;

pc->pc_nempty++;

mutex_exit(&pc->pc_lock);

return pool_cache_cpu_enter(pc, s);

KASSERT(LIST_EMPTY(&pc->pc_partgroups));

}

KASSERT(LIST_EMPTY(&pc->pc_fullgroups));

KASSERT(pc->pc_nitems == 0);

}

* pool_cache_put{,_paddr}:

* pool_cache_invalidate:

* Put an object back to the pool cache (optionally caching the

* Invalidate a pool cache (destruct and release all of the

* physical address of the object).

* cached objects).

void

pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa)

pool_cache_invalidate(struct pool_cache *pc)

{

pool_cache_cpu_t *cc;

struct pool_pagelist pq;

pcg_t *pcg;

struct pool_cache_grouplist pcgl;

int s;

FREECHECK_IN(&pc->pc_freecheck, object);

LIST_INIT(&pq);

LIST_INIT(&pcgl);

cc = pool_cache_cpu_enter(pc, &s);

simple_lock(&pc->pc_slock);

do {

simple_lock(&pc->pc_pool->pr_slock);

/* If the current group isn't full, release it there. */

pcg = cc->cc_current;

if (pcg != NULL && pcg->pcg_avail < PCG_NOBJECTS) {

KASSERT(pcg->pcg_objects[pcg->pcg_avail].pcgo_va

== NULL);

pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object;

pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa;

pcg->pcg_avail++;

cc->cc_hits++;

pool_cache_cpu_exit(cc, &s);

return;

}

pool_do_cache_invalidate(pc, &pq, &pcgl);

* That failed. If the previous group is empty, swap

* it with the current group and try again.

pcg = cc->cc_previous;

if (pcg != NULL && pcg->pcg_avail == 0) {

cc->cc_previous = cc->cc_current;

cc->cc_current = pcg;

continue;

}

simple_unlock(&pc->pc_pool->pr_slock);

* Can't free to either group: try the slow path.

simple_unlock(&pc->pc_slock);

* If put_slow() releases the object for us, it

* will return NULL. Otherwise we need to retry.

pr_pagelist_free(pc->pc_pool, &pq);

pcg_grouplist_free(&pcgl);

cc = pool_cache_put_slow(cc, &s, object, pa);

} while (cc != NULL);

}

* pool_cache_xcall:

* pool_cache_reclaim:

* Transfer objects from the per-CPU cache to the global cache.

* Reclaim a pool cache for pool_reclaim().

* Run within a cross-call thread.

static void

pool_cache_xcall(pool_cache_t pc)

pool_cache_reclaim(struct pool_cache *pc, struct pool_pagelist *pq,

struct pool_cache_grouplist *pcgl)

{

pool_cache_cpu_t *cc;

pcg_t *prev, *cur, **list;

int s = 0; /* XXXgcc */

* We're locking in the wrong order (normally pool_cache -> pool,

* but the pool is already locked when we get here), so we have

cc = pool_cache_cpu_enter(pc, &s);

* to use trylock. If we can't lock the pool_cache, it's not really

cur = cc->cc_current;

* a big deal here.

cc->cc_current = NULL;

prev = cc->cc_previous;

cc->cc_previous = NULL;

pool_cache_cpu_exit(cc, &s);

* XXXSMP Go to splvm to prevent kernel_lock from being taken,

* because locks at IPL_SOFTXXX are still spinlocks. Does not

* apply to IPL_SOFTBIO. Cross-call threads do not take the

* kernel_lock.

s = splvm();

if (simple_lock_try(&pc->pc_slock) == 0)

mutex_enter(&pc->pc_lock);

return;

if (cur != NULL) {

if (cur->pcg_avail == PCG_NOBJECTS) {

pool_do_cache_invalidate(pc, pq, pcgl);

list = &pc->pc_fullgroups;

pc->pc_nfull++;

simple_unlock(&pc->pc_slock);

} else if (cur->pcg_avail == 0) {

list = &pc->pc_emptygroups;

pc->pc_nempty++;

} else {

list = &pc->pc_partgroups;

pc->pc_npart++;

}

cur->pcg_next = *list;

*list = cur;

}

if (prev != NULL) {

if (prev->pcg_avail == PCG_NOBJECTS) {

list = &pc->pc_fullgroups;

pc->pc_nfull++;

} else if (prev->pcg_avail == 0) {

list = &pc->pc_emptygroups;

pc->pc_nempty++;

} else {

list = &pc->pc_partgroups;

pc->pc_npart++;

}

prev->pcg_next = *list;

*list = prev;

}

mutex_exit(&pc->pc_lock);

splx(s);

}

Line 2768 pool_allocator_alloc(struct pool *pp, in

Line 2341 pool_allocator_alloc(struct pool *pp, in

struct pool_allocator *pa = pp->pr_alloc;

void *res;

LOCK_ASSERT(!simple_lock_held(&pp->pr_slock));

res = (*pa->pa_alloc)(pp, flags);

if (res == NULL && (flags & PR_WAITOK) == 0) {

Line 2788 pool_allocator_free(struct pool *pp, voi

Line 2363 pool_allocator_free(struct pool *pp, voi

{

struct pool_allocator *pa = pp->pr_alloc;

LOCK_ASSERT(!simple_lock_held(&pp->pr_slock));

(*pa->pa_free)(pp, v);

}

Line 2826 pool_page_free_meta(struct pool *pp, voi

Line 2403 pool_page_free_meta(struct pool *pp, voi

void *

pool_subpage_alloc(struct pool *pp, int flags)

{

return pool_get(&psppool, flags);

void *v;

int s;

s = splvm();

v = pool_get(&psppool, flags);

splx(s);

return v;

}

void

pool_subpage_free(struct pool *pp, void *v)

{

int s;

s = splvm();

pool_put(&psppool, v);

splx(s);

}

/* We don't provide a real nointr allocator. Maybe later. */

CVSweb <webmaster@jp.NetBSD.org>