version 1.4, 1998/07/24 20:19:23 |
version 1.16, 1998/12/16 04:28:23 |
|
|
* documentation and/or other materials provided with the distribution. |
* documentation and/or other materials provided with the distribution. |
* 3. All advertising materials mentioning features or use of this software |
* 3. All advertising materials mentioning features or use of this software |
* must display the following acknowledgement: |
* must display the following acknowledgement: |
* This product includes software developed by the NetBSD |
* This product includes software developed by the NetBSD |
* Foundation, Inc. and its contributors. |
* Foundation, Inc. and its contributors. |
* 4. Neither the name of The NetBSD Foundation nor the names of its |
* 4. Neither the name of The NetBSD Foundation nor the names of its |
* contributors may be used to endorse or promote products derived |
* contributors may be used to endorse or promote products derived |
* from this software without specific prior written permission. |
* from this software without specific prior written permission. |
|
|
*/ |
*/ |
|
|
/* List of all pools */ |
/* List of all pools */ |
static TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
|
|
/* Private pool for page header structures */ |
/* Private pool for page header structures */ |
static struct pool phpool; |
static struct pool phpool; |
Line 109 static void pr_rmpage __P((struct pool * |
|
Line 109 static void pr_rmpage __P((struct pool * |
|
static int pool_prime_page __P((struct pool *, caddr_t)); |
static int pool_prime_page __P((struct pool *, caddr_t)); |
static void *pool_page_alloc __P((unsigned long, int, int)); |
static void *pool_page_alloc __P((unsigned long, int, int)); |
static void pool_page_free __P((void *, unsigned long, int)); |
static void pool_page_free __P((void *, unsigned long, int)); |
int pool_chk __P((struct pool *, char *)); |
|
|
|
|
|
#ifdef POOL_DIAGNOSTIC |
#ifdef POOL_DIAGNOSTIC |
Line 226 pr_rmpage(pp, ph) |
|
Line 225 pr_rmpage(pp, ph) |
|
{ |
{ |
|
|
/* |
/* |
|
* If the page was idle, decrement the idle page count. |
|
*/ |
|
if (ph->ph_nmissing == 0) { |
|
#ifdef DIAGNOSTIC |
|
if (pp->pr_nidle == 0) |
|
panic("pr_rmpage: nidle inconsistent"); |
|
#endif |
|
pp->pr_nidle--; |
|
} |
|
|
|
/* |
* Unlink a page from the pool and release it. |
* Unlink a page from the pool and release it. |
*/ |
*/ |
TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); |
TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); |
Line 313 pool_init(pp, size, align, ioff, flags, |
|
Line 323 pool_init(pp, size, align, ioff, flags, |
|
void (*release) __P((void *, unsigned long, int)); |
void (*release) __P((void *, unsigned long, int)); |
int mtype; |
int mtype; |
{ |
{ |
int off, slack; |
int off, slack, i; |
|
|
/* |
/* |
* Check arguments and construct default values. |
* Check arguments and construct default values. |
Line 336 pool_init(pp, size, align, ioff, flags, |
|
Line 346 pool_init(pp, size, align, ioff, flags, |
|
if (align == 0) |
if (align == 0) |
align = ALIGN(1); |
align = ALIGN(1); |
|
|
|
if (size < sizeof(struct pool_item)) |
|
size = sizeof(struct pool_item); |
|
|
/* |
/* |
* Initialize the pool structure. |
* Initialize the pool structure. |
*/ |
*/ |
Line 373 pool_init(pp, size, align, ioff, flags, |
|
Line 386 pool_init(pp, size, align, ioff, flags, |
|
/* The page header will be taken from our page header pool */ |
/* The page header will be taken from our page header pool */ |
pp->pr_phoffset = 0; |
pp->pr_phoffset = 0; |
off = pagesz; |
off = pagesz; |
bzero(pp->pr_hashtab, sizeof(pp->pr_hashtab)); |
for (i = 0; i < PR_HASHTABSIZE; i++) { |
|
LIST_INIT(&pp->pr_hashtab[i]); |
|
} |
} |
} |
|
|
/* |
/* |
Line 400 pool_init(pp, size, align, ioff, flags, |
|
Line 415 pool_init(pp, size, align, ioff, flags, |
|
pp->pr_npagealloc = 0; |
pp->pr_npagealloc = 0; |
pp->pr_npagefree = 0; |
pp->pr_npagefree = 0; |
pp->pr_hiwat = 0; |
pp->pr_hiwat = 0; |
|
pp->pr_nidle = 0; |
|
|
#ifdef POOL_DIAGNOSTIC |
#ifdef POOL_DIAGNOSTIC |
if ((flags & PR_LOGGING) != 0) { |
if ((flags & PR_LOGGING) != 0) { |
Line 413 pool_init(pp, size, align, ioff, flags, |
|
Line 429 pool_init(pp, size, align, ioff, flags, |
|
#endif |
#endif |
|
|
simple_lock_init(&pp->pr_lock); |
simple_lock_init(&pp->pr_lock); |
|
lockinit(&pp->pr_resourcelock, PSWP, wchan, 0, 0); |
|
|
/* |
/* |
* Initialize private page header pool if we haven't done so yet. |
* Initialize private page header pool if we haven't done so yet. |
Line 499 pool_get(pp, flags) |
|
Line 516 pool_get(pp, flags) |
|
* never points at a page header which has PR_PHINPAGE set and |
* never points at a page header which has PR_PHINPAGE set and |
* has no items in its bucket. |
* has no items in its bucket. |
*/ |
*/ |
again: |
while ((ph = pp->pr_curpage) == NULL) { |
if ((ph = pp->pr_curpage) == NULL) { |
void *v; |
void *v = (*pp->pr_alloc)(pp->pr_pagesz, flags, pp->pr_mtype); |
int lkflags = LK_EXCLUSIVE | LK_INTERLOCK | |
|
((flags & PR_WAITOK) == 0 ? LK_NOWAIT : 0); |
|
|
|
/* Get long-term lock on pool */ |
|
if (lockmgr(&pp->pr_resourcelock, lkflags, &pp->pr_lock) != 0) |
|
return (NULL); |
|
|
|
/* Check if pool became non-empty while we slept */ |
|
if ((ph = pp->pr_curpage) != NULL) |
|
goto again; |
|
|
|
/* Call the page back-end allocator for more memory */ |
|
v = (*pp->pr_alloc)(pp->pr_pagesz, flags, pp->pr_mtype); |
if (v == NULL) { |
if (v == NULL) { |
if (flags & PR_URGENT) |
if (flags & PR_URGENT) |
panic("pool_get: urgent"); |
panic("pool_get: urgent"); |
if ((flags & PR_WAITOK) == 0) { |
if ((flags & PR_WAITOK) == 0) { |
pp->pr_nfail++; |
pp->pr_nfail++; |
simple_unlock(&pp->pr_lock); |
lockmgr(&pp->pr_resourcelock, LK_RELEASE, NULL); |
return (NULL); |
return (NULL); |
} |
} |
|
|
|
/* |
|
* Wait for items to be returned to this pool. |
|
* XXX: we actually want to wait just until |
|
* the page allocator has memory again. Depending |
|
* on this pool's usage, we might get stuck here |
|
* for a long time. |
|
*/ |
pp->pr_flags |= PR_WANTED; |
pp->pr_flags |= PR_WANTED; |
simple_unlock(&pp->pr_lock); |
lockmgr(&pp->pr_resourcelock, LK_RELEASE, NULL); |
tsleep((caddr_t)pp, PSWP, pp->pr_wchan, 0); |
tsleep((caddr_t)pp, PSWP, pp->pr_wchan, 0); |
simple_lock(&pp->pr_lock); |
simple_lock(&pp->pr_lock); |
} else { |
continue; |
pp->pr_npagealloc++; |
|
pool_prime_page(pp, v); |
|
} |
} |
|
|
goto again; |
/* We have more memory; add it to the pool */ |
|
pp->pr_npagealloc++; |
|
pool_prime_page(pp, v); |
|
|
|
again: |
|
/* Re-acquire pool interlock */ |
|
simple_lock(&pp->pr_lock); |
|
lockmgr(&pp->pr_resourcelock, LK_RELEASE, NULL); |
} |
} |
|
|
if ((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL) |
if ((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL) |
|
|
* Remove from item list. |
* Remove from item list. |
*/ |
*/ |
TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); |
TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); |
|
if (ph->ph_nmissing == 0) { |
|
#ifdef DIAGNOSTIC |
|
if (pp->pr_nidle == 0) |
|
panic("pool_get: nidle inconsistent"); |
|
#endif |
|
pp->pr_nidle--; |
|
} |
ph->ph_nmissing++; |
ph->ph_nmissing++; |
if (TAILQ_FIRST(&ph->ph_itemlist) == NULL) { |
if (TAILQ_FIRST(&ph->ph_itemlist) == NULL) { |
/* |
/* |
|
|
|
|
if (pp->pr_flags & PR_WANTED) { |
if (pp->pr_flags & PR_WANTED) { |
pp->pr_flags &= ~PR_WANTED; |
pp->pr_flags &= ~PR_WANTED; |
|
if (ph->ph_nmissing == 0) |
|
pp->pr_nidle++; |
wakeup((caddr_t)pp); |
wakeup((caddr_t)pp); |
simple_unlock(&pp->pr_lock); |
simple_unlock(&pp->pr_lock); |
return; |
return; |
|
|
* If this page has just become un-empty, move it the head. |
* If this page has just become un-empty, move it the head. |
*/ |
*/ |
if (ph->ph_nmissing == 0) { |
if (ph->ph_nmissing == 0) { |
|
pp->pr_nidle++; |
if (pp->pr_npages > pp->pr_maxpages) { |
if (pp->pr_npages > pp->pr_maxpages) { |
#if 0 |
#if 0 |
timeout(pool_drain, 0, pool_inactive_time*hz); |
timeout(pool_drain, 0, pool_inactive_time*hz); |
Line 665 pool_prime(pp, n, storage) |
|
Line 716 pool_prime(pp, n, storage) |
|
/* !storage && static caught below */ |
/* !storage && static caught below */ |
#endif |
#endif |
|
|
|
(void)lockmgr(&pp->pr_resourcelock, LK_EXCLUSIVE, NULL); |
newnitems = pp->pr_minitems + n; |
newnitems = pp->pr_minitems + n; |
newpages = |
newpages = |
roundup(pp->pr_itemsperpage,newnitems) / pp->pr_itemsperpage |
roundup(pp->pr_itemsperpage,newnitems) / pp->pr_itemsperpage |
- pp->pr_minpages; |
- pp->pr_minpages; |
|
|
simple_lock(&pp->pr_lock); |
|
while (newpages-- > 0) { |
while (newpages-- > 0) { |
|
|
if (pp->pr_flags & PR_STATIC) { |
if (pp->pr_flags & PR_STATIC) { |
Line 681 pool_prime(pp, n, storage) |
|
Line 732 pool_prime(pp, n, storage) |
|
} |
} |
|
|
if (cp == NULL) { |
if (cp == NULL) { |
simple_unlock(&pp->pr_lock); |
(void)lockmgr(&pp->pr_resourcelock, LK_RELEASE, NULL); |
return (ENOMEM); |
return (ENOMEM); |
} |
} |
|
|
Line 694 pool_prime(pp, n, storage) |
|
Line 745 pool_prime(pp, n, storage) |
|
if (pp->pr_minpages >= pp->pr_maxpages) |
if (pp->pr_minpages >= pp->pr_maxpages) |
pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ |
pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ |
|
|
simple_unlock(&pp->pr_lock); |
(void)lockmgr(&pp->pr_resourcelock, LK_RELEASE, NULL); |
return (0); |
return (0); |
} |
} |
|
|
Line 713 pool_prime_page(pp, storage) |
|
Line 764 pool_prime_page(pp, storage) |
|
unsigned int ioff = pp->pr_itemoffset; |
unsigned int ioff = pp->pr_itemoffset; |
int n; |
int n; |
|
|
|
simple_lock(&pp->pr_lock); |
|
|
if ((pp->pr_flags & PR_PHINPAGE) != 0) { |
if ((pp->pr_flags & PR_PHINPAGE) != 0) { |
ph = (struct pool_item_header *)(cp + pp->pr_phoffset); |
ph = (struct pool_item_header *)(cp + pp->pr_phoffset); |
} else { |
} else { |
Line 730 pool_prime_page(pp, storage) |
|
Line 783 pool_prime_page(pp, storage) |
|
ph->ph_nmissing = 0; |
ph->ph_nmissing = 0; |
ph->ph_time.tv_sec = ph->ph_time.tv_usec = 0; |
ph->ph_time.tv_sec = ph->ph_time.tv_usec = 0; |
|
|
|
pp->pr_nidle++; |
|
|
/* |
/* |
* Color this page. |
* Color this page. |
*/ |
*/ |
Line 768 pool_prime_page(pp, storage) |
|
Line 823 pool_prime_page(pp, storage) |
|
if (++pp->pr_npages > pp->pr_hiwat) |
if (++pp->pr_npages > pp->pr_hiwat) |
pp->pr_hiwat = pp->pr_npages; |
pp->pr_hiwat = pp->pr_npages; |
|
|
|
simple_unlock(&pp->pr_lock); |
return (0); |
return (0); |
} |
} |
|
|
Line 776 pool_setlowat(pp, n) |
|
Line 832 pool_setlowat(pp, n) |
|
pool_handle_t pp; |
pool_handle_t pp; |
int n; |
int n; |
{ |
{ |
|
|
|
(void)lockmgr(&pp->pr_resourcelock, LK_EXCLUSIVE, NULL); |
pp->pr_minitems = n; |
pp->pr_minitems = n; |
if (n == 0) { |
pp->pr_minpages = (n == 0) |
pp->pr_minpages = 0; |
? 0 |
return; |
: roundup(pp->pr_itemsperpage,n) / pp->pr_itemsperpage; |
} |
(void)lockmgr(&pp->pr_resourcelock, LK_RELEASE, NULL); |
pp->pr_minpages = |
|
roundup(pp->pr_itemsperpage,n) / pp->pr_itemsperpage; |
|
} |
} |
|
|
void |
void |
Line 790 pool_sethiwat(pp, n) |
|
Line 846 pool_sethiwat(pp, n) |
|
pool_handle_t pp; |
pool_handle_t pp; |
int n; |
int n; |
{ |
{ |
if (n == 0) { |
|
pp->pr_maxpages = 0; |
(void)lockmgr(&pp->pr_resourcelock, LK_EXCLUSIVE, NULL); |
return; |
pp->pr_maxpages = (n == 0) |
} |
? 0 |
pp->pr_maxpages = |
: roundup(pp->pr_itemsperpage,n) / pp->pr_itemsperpage; |
roundup(pp->pr_itemsperpage,n) / pp->pr_itemsperpage; |
(void)lockmgr(&pp->pr_resourcelock, LK_RELEASE, NULL); |
} |
} |
|
|
|
|
Line 808 pool_page_alloc(sz, flags, mtype) |
|
Line 864 pool_page_alloc(sz, flags, mtype) |
|
int flags; |
int flags; |
int mtype; |
int mtype; |
{ |
{ |
|
boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; |
|
|
#if defined(UVM) |
#if defined(UVM) |
return ((void *)uvm_km_alloc_poolpage()); |
return ((void *)uvm_km_alloc_poolpage(waitok)); |
#else |
#else |
return ((void *)kmem_alloc_poolpage()); |
return ((void *)kmem_alloc_poolpage(waitok)); |
#endif |
#endif |
} |
} |
|
|
Line 824 pool_page_free(v, sz, mtype) |
|
Line 881 pool_page_free(v, sz, mtype) |
|
{ |
{ |
|
|
#if defined(UVM) |
#if defined(UVM) |
uvm_km_free_poolpage((vm_offset_t)v); |
uvm_km_free_poolpage((vaddr_t)v); |
#else |
#else |
kmem_free_poolpage((vm_offset_t)v); |
kmem_free_poolpage((vaddr_t)v); |
#endif |
#endif |
} |
} |
|
|
/* |
/* |
|
* Alternate pool page allocator for pools that know they will |
|
* never be accessed in interrupt context. |
|
*/ |
|
void * |
|
pool_page_alloc_nointr(sz, flags, mtype) |
|
unsigned long sz; |
|
int flags; |
|
int mtype; |
|
{ |
|
#if defined(UVM) |
|
boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; |
|
|
|
/* |
|
* With UVM, we can use the kernel_map. |
|
*/ |
|
return ((void *)uvm_km_alloc_poolpage1(kernel_map, uvm.kernel_object, |
|
waitok)); |
|
#else |
|
/* |
|
* Can't do anything so cool with Mach VM. |
|
*/ |
|
return (pool_page_alloc(sz, flags, mtype)); |
|
#endif |
|
} |
|
|
|
void |
|
pool_page_free_nointr(v, sz, mtype) |
|
void *v; |
|
unsigned long sz; |
|
int mtype; |
|
{ |
|
|
|
#if defined(UVM) |
|
uvm_km_free_poolpage1(kernel_map, (vaddr_t)v); |
|
#else |
|
pool_page_free(v, sz, mtype); |
|
#endif |
|
} |
|
|
|
|
|
/* |
* Release all complete pages that have not been used recently. |
* Release all complete pages that have not been used recently. |
*/ |
*/ |
void |
void |
Line 905 pool_print(pp, label) |
|
Line 1003 pool_print(pp, label) |
|
printf("%s: ", label); |
printf("%s: ", label); |
|
|
printf("pool %s: nalloc %lu nfree %lu npagealloc %lu npagefree %lu\n" |
printf("pool %s: nalloc %lu nfree %lu npagealloc %lu npagefree %lu\n" |
" npages %u minitems %u itemsperpage %u itemoffset %u\n", |
" npages %u minitems %u itemsperpage %u itemoffset %u\n" |
|
" nidle %lu\n", |
pp->pr_wchan, |
pp->pr_wchan, |
pp->pr_nget, |
pp->pr_nget, |
pp->pr_nput, |
pp->pr_nput, |
Line 914 pool_print(pp, label) |
|
Line 1013 pool_print(pp, label) |
|
pp->pr_npages, |
pp->pr_npages, |
pp->pr_minitems, |
pp->pr_minitems, |
pp->pr_itemsperpage, |
pp->pr_itemsperpage, |
pp->pr_itemoffset); |
pp->pr_itemoffset, |
|
pp->pr_nidle); |
} |
} |
|
|
int |
int |
Line 935 pool_chk(pp, label) |
|
Line 1035 pool_chk(pp, label) |
|
caddr_t page; |
caddr_t page; |
|
|
page = (caddr_t)((u_long)ph & pp->pr_pagemask); |
page = (caddr_t)((u_long)ph & pp->pr_pagemask); |
if (page != ph->ph_page) { |
if (page != ph->ph_page && (pp->pr_flags & PR_PHINPAGE) != 0) { |
if (label != NULL) |
if (label != NULL) |
printf("%s: ", label); |
printf("%s: ", label); |
printf("pool(%s): page inconsistency: page %p;" |
printf("pool(%p:%s): page inconsistency: page %p;" |
" at page head addr %p (p %p)\n", |
" at page head addr %p (p %p)\n", pp, |
pp->pr_wchan, ph->ph_page, |
pp->pr_wchan, ph->ph_page, |
ph, page); |
ph, page); |
r++; |
r++; |
Line 968 pool_chk(pp, label) |
|
Line 1068 pool_chk(pp, label) |
|
|
|
if (label != NULL) |
if (label != NULL) |
printf("%s: ", label); |
printf("%s: ", label); |
printf("pool(%s): page inconsistency: page %p;" |
printf("pool(%p:%s): page inconsistency: page %p;" |
" item ordinal %d; addr %p (p %p)\n", |
" item ordinal %d; addr %p (p %p)\n", pp, |
pp->pr_wchan, ph->ph_page, |
pp->pr_wchan, ph->ph_page, |
n, pi, page); |
n, pi, page); |
r++; |
r++; |