version 1.206, 2016/02/05 03:04:52 |
version 1.207, 2017/03/14 03:13:50 |
Line 382 pr_rmpage(struct pool *pp, struct pool_i |
|
Line 382 pr_rmpage(struct pool *pp, struct pool_i |
|
* If the page was idle, decrement the idle page count. |
* If the page was idle, decrement the idle page count. |
*/ |
*/ |
if (ph->ph_nmissing == 0) { |
if (ph->ph_nmissing == 0) { |
#ifdef DIAGNOSTIC |
KASSERT(pp->pr_nidle != 0); |
if (pp->pr_nidle == 0) |
KASSERTMSG((pp->pr_nitems >= pp->pr_itemsperpage), |
panic("pr_rmpage: nidle inconsistent"); |
"nitems=%u < itemsperpage=%u", |
if (pp->pr_nitems < pp->pr_itemsperpage) |
pp->pr_nitems, pp->pr_itemsperpage); |
panic("pr_rmpage: nitems inconsistent"); |
|
#endif |
|
pp->pr_nidle--; |
pp->pr_nidle--; |
} |
} |
|
|
Line 525 pool_init(struct pool *pp, size_t size, |
|
Line 523 pool_init(struct pool *pp, size_t size, |
|
prsize = sizeof(struct pool_item); |
prsize = sizeof(struct pool_item); |
|
|
prsize = roundup(prsize, align); |
prsize = roundup(prsize, align); |
#ifdef DIAGNOSTIC |
KASSERTMSG((prsize <= palloc->pa_pagesz), |
if (prsize > palloc->pa_pagesz) |
"pool_init: pool item size (%zu) larger than page size (%u)", |
panic("pool_init: pool item size (%zu) too large", prsize); |
prsize, palloc->pa_pagesz); |
#endif |
|
|
|
/* |
/* |
* Initialize the pool structure. |
* Initialize the pool structure. |
Line 698 pool_destroy(struct pool *pp) |
|
Line 695 pool_destroy(struct pool *pp) |
|
mutex_enter(&pp->pr_lock); |
mutex_enter(&pp->pr_lock); |
|
|
KASSERT(pp->pr_cache == NULL); |
KASSERT(pp->pr_cache == NULL); |
|
KASSERTMSG((pp->pr_nout == 0), |
#ifdef DIAGNOSTIC |
"pool_destroy: pool busy: still out: %u", pp->pr_nout); |
if (pp->pr_nout != 0) { |
|
panic("pool_destroy: pool busy: still out: %u", |
|
pp->pr_nout); |
|
} |
|
#endif |
|
|
|
KASSERT(LIST_EMPTY(&pp->pr_fullpages)); |
KASSERT(LIST_EMPTY(&pp->pr_fullpages)); |
KASSERT(LIST_EMPTY(&pp->pr_partpages)); |
KASSERT(LIST_EMPTY(&pp->pr_partpages)); |
|
|
Line 726 pool_set_drain_hook(struct pool *pp, voi |
|
Line 717 pool_set_drain_hook(struct pool *pp, voi |
|
{ |
{ |
|
|
/* XXX no locking -- must be used just after pool_init() */ |
/* XXX no locking -- must be used just after pool_init() */ |
#ifdef DIAGNOSTIC |
KASSERTMSG((pp->pr_drain_hook == NULL), |
if (pp->pr_drain_hook != NULL) |
"pool_set_drain_hook(%s): already set", pp->pr_wchan); |
panic("pool_set_drain_hook(%s): already set", pp->pr_wchan); |
|
#endif |
|
pp->pr_drain_hook = fn; |
pp->pr_drain_hook = fn; |
pp->pr_drain_hook_arg = arg; |
pp->pr_drain_hook_arg = arg; |
} |
} |
Line 757 pool_get(struct pool *pp, int flags) |
|
Line 746 pool_get(struct pool *pp, int flags) |
|
struct pool_item_header *ph; |
struct pool_item_header *ph; |
void *v; |
void *v; |
|
|
#ifdef DIAGNOSTIC |
KASSERTMSG((pp->pr_itemsperpage != 0), |
if (pp->pr_itemsperpage == 0) |
"pool_get: pool '%s': pr_itemsperpage is zero, " |
panic("pool_get: pool '%s': pr_itemsperpage is zero, " |
"pool not initialized?", pp->pr_wchan); |
"pool not initialized?", pp->pr_wchan); |
KASSERTMSG((!(cpu_intr_p() || cpu_softintr_p()) |
if ((cpu_intr_p() || cpu_softintr_p()) && pp->pr_ipl == IPL_NONE && |
|| pp->pr_ipl != IPL_NONE || cold || panicstr != NULL), |
!cold && panicstr == NULL) |
"pool '%s' is IPL_NONE, but called from interrupt context", |
panic("pool '%s' is IPL_NONE, but called from " |
pp->pr_wchan); |
"interrupt context\n", pp->pr_wchan); |
|
#endif |
|
if (flags & PR_WAITOK) { |
if (flags & PR_WAITOK) { |
ASSERT_SLEEPABLE(); |
ASSERT_SLEEPABLE(); |
} |
} |
Line 777 pool_get(struct pool *pp, int flags) |
|
Line 764 pool_get(struct pool *pp, int flags) |
|
* and we can wait, then wait until an item has been returned to |
* and we can wait, then wait until an item has been returned to |
* the pool. |
* the pool. |
*/ |
*/ |
#ifdef DIAGNOSTIC |
KASSERTMSG((pp->pr_nout <= pp->pr_hardlimit), |
if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) { |
"pool_get: %s: crossed hard limit", pp->pr_wchan); |
mutex_exit(&pp->pr_lock); |
|
panic("pool_get: %s: crossed hard limit", pp->pr_wchan); |
|
} |
|
#endif |
|
if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { |
if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { |
if (pp->pr_drain_hook != NULL) { |
if (pp->pr_drain_hook != NULL) { |
/* |
/* |
Line 830 pool_get(struct pool *pp, int flags) |
|
Line 813 pool_get(struct pool *pp, int flags) |
|
if ((ph = pp->pr_curpage) == NULL) { |
if ((ph = pp->pr_curpage) == NULL) { |
int error; |
int error; |
|
|
#ifdef DIAGNOSTIC |
KASSERTMSG((pp->pr_nitems == 0), |
if (pp->pr_nitems != 0) { |
"pool_get: nitems inconsistent" |
mutex_exit(&pp->pr_lock); |
": %s: curpage NULL, nitems %u", |
printf("pool_get: %s: curpage NULL, nitems %u\n", |
pp->pr_wchan, pp->pr_nitems); |
pp->pr_wchan, pp->pr_nitems); |
|
panic("pool_get: nitems inconsistent"); |
|
} |
|
#endif |
|
|
|
/* |
/* |
* Call the back-end page allocator for more memory. |
* Call the back-end page allocator for more memory. |
Line 864 pool_get(struct pool *pp, int flags) |
|
Line 843 pool_get(struct pool *pp, int flags) |
|
goto startover; |
goto startover; |
} |
} |
if (pp->pr_roflags & PR_NOTOUCH) { |
if (pp->pr_roflags & PR_NOTOUCH) { |
#ifdef DIAGNOSTIC |
KASSERTMSG((ph->ph_nmissing < pp->pr_itemsperpage), |
if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) { |
"pool_get: %s: page empty", pp->pr_wchan); |
mutex_exit(&pp->pr_lock); |
|
panic("pool_get: %s: page empty", pp->pr_wchan); |
|
} |
|
#endif |
|
v = pr_item_notouch_get(pp, ph); |
v = pr_item_notouch_get(pp, ph); |
} else { |
} else { |
v = pi = LIST_FIRST(&ph->ph_itemlist); |
v = pi = LIST_FIRST(&ph->ph_itemlist); |
Line 877 pool_get(struct pool *pp, int flags) |
|
Line 852 pool_get(struct pool *pp, int flags) |
|
mutex_exit(&pp->pr_lock); |
mutex_exit(&pp->pr_lock); |
panic("pool_get: %s: page empty", pp->pr_wchan); |
panic("pool_get: %s: page empty", pp->pr_wchan); |
} |
} |
#ifdef DIAGNOSTIC |
KASSERTMSG((pp->pr_nitems > 0), |
if (__predict_false(pp->pr_nitems == 0)) { |
"pool_get: nitems inconsistent" |
mutex_exit(&pp->pr_lock); |
": %s: items on itemlist, nitems %u", |
printf("pool_get: %s: items on itemlist, nitems %u\n", |
pp->pr_wchan, pp->pr_nitems); |
pp->pr_wchan, pp->pr_nitems); |
KASSERTMSG((pi->pi_magic == PI_MAGIC), |
panic("pool_get: nitems inconsistent"); |
"pool_get(%s): free list modified: " |
} |
"magic=%x; page %p; item addr %p", |
#endif |
pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); |
|
|
#ifdef DIAGNOSTIC |
|
if (__predict_false(pi->pi_magic != PI_MAGIC)) { |
|
panic("pool_get(%s): free list modified: " |
|
"magic=%x; page %p; item addr %p\n", |
|
pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); |
|
} |
|
#endif |
|
|
|
/* |
/* |
* Remove from item list. |
* Remove from item list. |
Line 902 pool_get(struct pool *pp, int flags) |
|
Line 869 pool_get(struct pool *pp, int flags) |
|
pp->pr_nitems--; |
pp->pr_nitems--; |
pp->pr_nout++; |
pp->pr_nout++; |
if (ph->ph_nmissing == 0) { |
if (ph->ph_nmissing == 0) { |
#ifdef DIAGNOSTIC |
KASSERT(pp->pr_nidle > 0); |
if (__predict_false(pp->pr_nidle == 0)) |
|
panic("pool_get: nidle inconsistent"); |
|
#endif |
|
pp->pr_nidle--; |
pp->pr_nidle--; |
|
|
/* |
/* |
Line 917 pool_get(struct pool *pp, int flags) |
|
Line 881 pool_get(struct pool *pp, int flags) |
|
} |
} |
ph->ph_nmissing++; |
ph->ph_nmissing++; |
if (ph->ph_nmissing == pp->pr_itemsperpage) { |
if (ph->ph_nmissing == pp->pr_itemsperpage) { |
#ifdef DIAGNOSTIC |
KASSERTMSG(((pp->pr_roflags & PR_NOTOUCH) || |
if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 && |
LIST_EMPTY(&ph->ph_itemlist)), |
!LIST_EMPTY(&ph->ph_itemlist))) { |
"pool_get: %s: nmissing inconsistent", pp->pr_wchan); |
mutex_exit(&pp->pr_lock); |
|
panic("pool_get: %s: nmissing inconsistent", |
|
pp->pr_wchan); |
|
} |
|
#endif |
|
/* |
/* |
* This page is now full. Move it to the full list |
* This page is now full. Move it to the full list |
* and select a new current page. |
* and select a new current page. |
Line 969 pool_do_put(struct pool *pp, void *v, st |
|
Line 928 pool_do_put(struct pool *pp, void *v, st |
|
FREECHECK_IN(&pp->pr_freecheck, v); |
FREECHECK_IN(&pp->pr_freecheck, v); |
LOCKDEBUG_MEM_CHECK(v, pp->pr_size); |
LOCKDEBUG_MEM_CHECK(v, pp->pr_size); |
|
|
#ifdef DIAGNOSTIC |
KASSERTMSG((pp->pr_nout > 0), |
if (__predict_false(pp->pr_nout == 0)) { |
"pool_put: pool %s: putting with none out", pp->pr_wchan); |
printf("pool %s: putting with none out\n", |
|
pp->pr_wchan); |
|
panic("pool_put"); |
|
} |
|
#endif |
|
|
|
if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { |
if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { |
panic("pool_put: %s: page header missing", pp->pr_wchan); |
panic("pool_put: %s: page header missing", pp->pr_wchan); |
Line 1156 pool_prime_page(struct pool *pp, void *s |
|
Line 1110 pool_prime_page(struct pool *pp, void *s |
|
int n; |
int n; |
|
|
KASSERT(mutex_owned(&pp->pr_lock)); |
KASSERT(mutex_owned(&pp->pr_lock)); |
|
KASSERTMSG(((pp->pr_roflags & PR_NOALIGN) || |
#ifdef DIAGNOSTIC |
(((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) == 0)), |
if ((pp->pr_roflags & PR_NOALIGN) == 0 && |
"pool_prime_page: %s: unaligned page: %p", pp->pr_wchan, cp); |
((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0) |
|
panic("pool_prime_page: %s: unaligned page", pp->pr_wchan); |
|
#endif |
|
|
|
/* |
/* |
* Insert page header. |
* Insert page header. |
Line 1487 pool_print_pagelist(struct pool *pp, str |
|
Line 1438 pool_print_pagelist(struct pool *pp, str |
|
void (*pr)(const char *, ...)) |
void (*pr)(const char *, ...)) |
{ |
{ |
struct pool_item_header *ph; |
struct pool_item_header *ph; |
#ifdef DIAGNOSTIC |
struct pool_item *pi __diagused; |
struct pool_item *pi; |
|
#endif |
|
|
|
LIST_FOREACH(ph, pl, ph_pagelist) { |
LIST_FOREACH(ph, pl, ph_pagelist) { |
(*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n", |
(*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n", |