version 1.101.2.3, 2007/02/26 09:11:15 |
version 1.133, 2007/10/11 19:53:37 |
Line 112 struct pool_item_header { |
|
Line 112 struct pool_item_header { |
|
ph_pagelist; /* pool page list */ |
ph_pagelist; /* pool page list */ |
SPLAY_ENTRY(pool_item_header) |
SPLAY_ENTRY(pool_item_header) |
ph_node; /* Off-page page headers */ |
ph_node; /* Off-page page headers */ |
caddr_t ph_page; /* this page's address */ |
void * ph_page; /* this page's address */ |
struct timeval ph_time; /* last referenced */ |
struct timeval ph_time; /* last referenced */ |
union { |
union { |
/* !PR_NOTOUCH */ |
/* !PR_NOTOUCH */ |
Line 184 static void pool_cache_reclaim(struct po |
|
Line 184 static void pool_cache_reclaim(struct po |
|
static void pcg_grouplist_free(struct pool_cache_grouplist *); |
static void pcg_grouplist_free(struct pool_cache_grouplist *); |
|
|
static int pool_catchup(struct pool *); |
static int pool_catchup(struct pool *); |
static void pool_prime_page(struct pool *, caddr_t, |
static void pool_prime_page(struct pool *, void *, |
struct pool_item_header *); |
struct pool_item_header *); |
static void pool_update_curpage(struct pool *); |
static void pool_update_curpage(struct pool *); |
|
|
Line 326 pr_item_notouch_index(const struct pool |
|
Line 326 pr_item_notouch_index(const struct pool |
|
int idx; |
int idx; |
|
|
KASSERT(pp->pr_roflags & PR_NOTOUCH); |
KASSERT(pp->pr_roflags & PR_NOTOUCH); |
idx = (cp - ph->ph_page - ph->ph_off) / pp->pr_size; |
idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size; |
KASSERT(idx < pp->pr_itemsperpage); |
KASSERT(idx < pp->pr_itemsperpage); |
return idx; |
return idx; |
} |
} |
Line 359 pr_item_notouch_get(const struct pool *p |
|
Line 359 pr_item_notouch_get(const struct pool *p |
|
ph->ph_firstfree = freelist[idx]; |
ph->ph_firstfree = freelist[idx]; |
freelist[idx] = PR_INDEX_USED; |
freelist[idx] = PR_INDEX_USED; |
|
|
return ph->ph_page + ph->ph_off + idx * pp->pr_size; |
return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size; |
} |
} |
|
|
static inline int |
static inline int |
Line 391 pr_find_pagehead(struct pool *pp, void * |
|
Line 391 pr_find_pagehead(struct pool *pp, void * |
|
struct pool_item_header *ph, tmp; |
struct pool_item_header *ph, tmp; |
|
|
if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
tmp.ph_page = (caddr_t)(uintptr_t)v; |
tmp.ph_page = (void *)(uintptr_t)v; |
ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
if (ph == NULL) { |
if (ph == NULL) { |
ph = SPLAY_ROOT(&pp->pr_phtree); |
ph = SPLAY_ROOT(&pp->pr_phtree); |
Line 401 pr_find_pagehead(struct pool *pp, void * |
|
Line 401 pr_find_pagehead(struct pool *pp, void * |
|
KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); |
KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); |
} |
} |
} else { |
} else { |
caddr_t page = |
void *page = |
(caddr_t)((uintptr_t)v & pp->pr_alloc->pa_pagemask); |
(void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask); |
|
|
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
ph = (void *)(page + pp->pr_phoffset); |
ph = (struct pool_item_header *)((char *)page + pp->pr_phoffset); |
} else { |
} else { |
tmp.ph_page = page; |
tmp.ph_page = page; |
ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
Line 413 pr_find_pagehead(struct pool *pp, void * |
|
Line 413 pr_find_pagehead(struct pool *pp, void * |
|
} |
} |
|
|
KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || |
KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || |
(ph->ph_page <= (char *)v && |
((char *)ph->ph_page <= (char *)v && |
(char *)v < ph->ph_page + pp->pr_alloc->pa_pagesz)); |
(char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz)); |
return ph; |
return ph; |
} |
} |
|
|
Line 560 pool_subsystem_init(void) |
|
Line 560 pool_subsystem_init(void) |
|
__link_set_foreach(pi, pools) |
__link_set_foreach(pi, pools) |
pool_init((*pi)->pp, (*pi)->size, (*pi)->align, |
pool_init((*pi)->pp, (*pi)->size, (*pi)->align, |
(*pi)->align_offset, (*pi)->flags, (*pi)->wchan, |
(*pi)->align_offset, (*pi)->flags, (*pi)->wchan, |
(*pi)->palloc); |
(*pi)->palloc, (*pi)->ipl); |
|
|
while ((pa = SLIST_FIRST(&pa_deferinitq)) != NULL) { |
while ((pa = SLIST_FIRST(&pa_deferinitq)) != NULL) { |
KASSERT(pa->pa_backingmapptr != NULL); |
KASSERT(pa->pa_backingmapptr != NULL); |
Line 578 pool_subsystem_init(void) |
|
Line 578 pool_subsystem_init(void) |
|
*/ |
*/ |
void |
void |
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
const char *wchan, struct pool_allocator *palloc) |
const char *wchan, struct pool_allocator *palloc, int ipl) |
{ |
{ |
#ifdef DEBUG |
#ifdef DEBUG |
struct pool *pp1; |
struct pool *pp1; |
Line 792 pool_init(struct pool *pp, size_t size, |
|
Line 792 pool_init(struct pool *pp, size_t size, |
|
+ nelem * sizeof(pool_item_freelist_t); |
+ nelem * sizeof(pool_item_freelist_t); |
} |
} |
pool_init(&phpool[idx], sz, 0, 0, 0, |
pool_init(&phpool[idx], sz, 0, 0, 0, |
phpool_names[idx], &pool_allocator_meta); |
phpool_names[idx], &pool_allocator_meta, IPL_VM); |
} |
} |
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
PR_RECURSIVE, "psppool", &pool_allocator_meta); |
PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); |
#endif |
#endif |
pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0, |
pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0, |
0, "pcgpool", &pool_allocator_meta); |
0, "pcgpool", &pool_allocator_meta, IPL_VM); |
} |
} |
|
|
/* Insert into the list of all pools. */ |
/* Insert into the list of all pools. */ |
Line 887 pool_set_drain_hook(struct pool *pp, voi |
|
Line 887 pool_set_drain_hook(struct pool *pp, voi |
|
} |
} |
|
|
static struct pool_item_header * |
static struct pool_item_header * |
pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags) |
pool_alloc_item_header(struct pool *pp, void *storage, int flags) |
{ |
{ |
struct pool_item_header *ph; |
struct pool_item_header *ph; |
int s; |
int s; |
Line 895 pool_alloc_item_header(struct pool *pp, |
|
Line 895 pool_alloc_item_header(struct pool *pp, |
|
LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0); |
LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0); |
|
|
if ((pp->pr_roflags & PR_PHINPAGE) != 0) |
if ((pp->pr_roflags & PR_PHINPAGE) != 0) |
ph = (struct pool_item_header *) (storage + pp->pr_phoffset); |
ph = (struct pool_item_header *) ((char *)storage + pp->pr_phoffset); |
else { |
else { |
s = splvm(); |
s = splvm(); |
ph = pool_get(pp->pr_phpool, flags); |
ph = pool_get(pp->pr_phpool, flags); |
Line 1166 pool_do_put(struct pool *pp, void *v, st |
|
Line 1166 pool_do_put(struct pool *pp, void *v, st |
|
panic("pool_put: %s: page header missing", pp->pr_wchan); |
panic("pool_put: %s: page header missing", pp->pr_wchan); |
} |
} |
|
|
#ifdef LOCKDEBUG |
|
/* |
|
* Check if we're freeing a locked simple lock. |
|
*/ |
|
simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size); |
|
#endif |
|
|
|
/* |
/* |
* Return to item list. |
* Return to item list. |
*/ |
*/ |
Line 1208 pool_do_put(struct pool *pp, void *v, st |
|
Line 1201 pool_do_put(struct pool *pp, void *v, st |
|
pp->pr_flags &= ~PR_WANTED; |
pp->pr_flags &= ~PR_WANTED; |
if (ph->ph_nmissing == 0) |
if (ph->ph_nmissing == 0) |
pp->pr_nidle++; |
pp->pr_nidle++; |
wakeup((caddr_t)pp); |
wakeup((void *)pp); |
return; |
return; |
} |
} |
|
|
Line 1370 pool_prime(struct pool *pp, int n) |
|
Line 1363 pool_prime(struct pool *pp, int n) |
|
* Note, we must be called with the pool descriptor LOCKED. |
* Note, we must be called with the pool descriptor LOCKED. |
*/ |
*/ |
static void |
static void |
pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph) |
pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph) |
{ |
{ |
struct pool_item *pi; |
struct pool_item *pi; |
caddr_t cp = storage; |
void *cp = storage; |
const unsigned int align = pp->pr_align; |
const unsigned int align = pp->pr_align; |
const unsigned int ioff = pp->pr_itemoffset; |
const unsigned int ioff = pp->pr_itemoffset; |
int n; |
int n; |
Line 1402 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1395 pool_prime_page(struct pool *pp, caddr_t |
|
/* |
/* |
* Color this page. |
* Color this page. |
*/ |
*/ |
cp = (caddr_t)(cp + pp->pr_curcolor); |
cp = (char *)cp + pp->pr_curcolor; |
if ((pp->pr_curcolor += align) > pp->pr_maxcolor) |
if ((pp->pr_curcolor += align) > pp->pr_maxcolor) |
pp->pr_curcolor = 0; |
pp->pr_curcolor = 0; |
|
|
Line 1410 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1403 pool_prime_page(struct pool *pp, caddr_t |
|
* Adjust storage to apply aligment to `pr_itemoffset' in each item. |
* Adjust storage to apply aligment to `pr_itemoffset' in each item. |
*/ |
*/ |
if (ioff != 0) |
if (ioff != 0) |
cp = (caddr_t)(cp + (align - ioff)); |
cp = (char *)cp + align - ioff; |
|
|
KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); |
KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); |
|
|
Line 1424 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1417 pool_prime_page(struct pool *pp, caddr_t |
|
pool_item_freelist_t *freelist = PR_FREELIST(ph); |
pool_item_freelist_t *freelist = PR_FREELIST(ph); |
int i; |
int i; |
|
|
ph->ph_off = cp - storage; |
ph->ph_off = (char *)cp - (char *)storage; |
ph->ph_firstfree = 0; |
ph->ph_firstfree = 0; |
for (i = 0; i < n - 1; i++) |
for (i = 0; i < n - 1; i++) |
freelist[i] = i + 1; |
freelist[i] = i + 1; |
Line 1440 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1433 pool_prime_page(struct pool *pp, caddr_t |
|
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
pi->pi_magic = PI_MAGIC; |
pi->pi_magic = PI_MAGIC; |
#endif |
#endif |
cp = (caddr_t)(cp + pp->pr_size); |
cp = (char *)cp + pp->pr_size; |
|
|
KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); |
KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); |
} |
} |
Line 1624 pool_reclaim(struct pool *pp) |
|
Line 1617 pool_reclaim(struct pool *pp) |
|
* Drain pools, one at a time. |
* Drain pools, one at a time. |
* |
* |
* Note, we must never be called from an interrupt context. |
* Note, we must never be called from an interrupt context. |
|
* |
|
* XXX Pool can disappear while draining. |
*/ |
*/ |
void |
void |
pool_drain(void *arg) |
pool_drain(void *arg) |
|
|
pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) |
pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) |
{ |
{ |
struct pool_item *pi; |
struct pool_item *pi; |
caddr_t page; |
void *page; |
int n; |
int n; |
|
|
if ((pp->pr_roflags & PR_NOALIGN) == 0) { |
if ((pp->pr_roflags & PR_NOALIGN) == 0) { |
page = (caddr_t)((uintptr_t)ph & pp->pr_alloc->pa_pagemask); |
page = (void *)((uintptr_t)ph & pp->pr_alloc->pa_pagemask); |
if (page != ph->ph_page && |
if (page != ph->ph_page && |
(pp->pr_roflags & PR_PHINPAGE) != 0) { |
(pp->pr_roflags & PR_PHINPAGE) != 0) { |
if (label != NULL) |
if (label != NULL) |
Line 1881 pool_chk_page(struct pool *pp, const cha |
|
Line 1876 pool_chk_page(struct pool *pp, const cha |
|
if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
continue; |
continue; |
} |
} |
page = (caddr_t)((uintptr_t)pi & pp->pr_alloc->pa_pagemask); |
page = (void *)((uintptr_t)pi & pp->pr_alloc->pa_pagemask); |
if (page == ph->ph_page) |
if (page == ph->ph_page) |
continue; |
continue; |
|
|
Line 2184 pool_cache_destruct_object(struct pool_c |
|
Line 2179 pool_cache_destruct_object(struct pool_c |
|
pool_put(pc->pc_pool, object); |
pool_put(pc->pc_pool, object); |
} |
} |
|
|
|
/* |
|
* pool_do_cache_invalidate_grouplist: |
|
* |
|
* Invalidate a single grouplist and destruct all objects. |
|
* XXX This is too expensive. We should swap the list then |
|
* unlock. |
|
*/ |
static void |
static void |
pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl, |
pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl, |
struct pool_cache *pc, struct pool_pagelist *pq, |
struct pool_cache *pc, struct pool_pagelist *pq, |
struct pool_cache_grouplist *pcgdl) |
struct pool_cache_grouplist *pcgdl) |
{ |
{ |
struct pool_cache_group *pcg, *npcg; |
struct pool_cache_group *pcg; |
void *object; |
void *object; |
|
|
for (pcg = LIST_FIRST(pcgsl); pcg != NULL; pcg = npcg) { |
LOCK_ASSERT(simple_lock_held(&pc->pc_slock)); |
npcg = LIST_NEXT(pcg, pcg_list); |
LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock)); |
|
|
|
while ((pcg = LIST_FIRST(pcgsl)) != NULL) { |
|
pc->pc_ngroups--; |
|
LIST_REMOVE(pcg, pcg_list); |
|
LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); |
|
pc->pc_nitems -= pcg->pcg_avail; |
|
simple_unlock(&pc->pc_pool->pr_slock); |
|
simple_unlock(&pc->pc_slock); |
|
|
while (pcg->pcg_avail != 0) { |
while (pcg->pcg_avail != 0) { |
pc->pc_nitems--; |
|
object = pcg_get(pcg, NULL); |
object = pcg_get(pcg, NULL); |
if (pc->pc_dtor != NULL) |
if (pc->pc_dtor != NULL) |
(*pc->pc_dtor)(pc->pc_arg, object); |
(*pc->pc_dtor)(pc->pc_arg, object); |
|
simple_lock(&pc->pc_pool->pr_slock); |
pool_do_put(pc->pc_pool, object, pq); |
pool_do_put(pc->pc_pool, object, pq); |
|
simple_unlock(&pc->pc_pool->pr_slock); |
} |
} |
pc->pc_ngroups--; |
|
LIST_REMOVE(pcg, pcg_list); |
simple_lock(&pc->pc_slock); |
LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); |
simple_lock(&pc->pc_pool->pr_slock); |
} |
} |
} |
} |
|
|
Line 2341 pool_allocator_alloc(struct pool *pp, in |
|
Line 2353 pool_allocator_alloc(struct pool *pp, in |
|
struct pool_allocator *pa = pp->pr_alloc; |
struct pool_allocator *pa = pp->pr_alloc; |
void *res; |
void *res; |
|
|
LOCK_ASSERT(!simple_lock_held(&pp->pr_slock)); |
|
|
|
res = (*pa->pa_alloc)(pp, flags); |
res = (*pa->pa_alloc)(pp, flags); |
if (res == NULL && (flags & PR_WAITOK) == 0) { |
if (res == NULL && (flags & PR_WAITOK) == 0) { |
/* |
/* |
Line 2363 pool_allocator_free(struct pool *pp, voi |
|
Line 2373 pool_allocator_free(struct pool *pp, voi |
|
{ |
{ |
struct pool_allocator *pa = pp->pr_alloc; |
struct pool_allocator *pa = pp->pr_alloc; |
|
|
LOCK_ASSERT(!simple_lock_held(&pp->pr_slock)); |
|
|
|
(*pa->pa_free)(pp, v); |
(*pa->pa_free)(pp, v); |
} |
} |
|
|