version 1.109, 2005/12/20 15:52:14 |
version 1.112.2.2, 2006/04/11 11:55:47 |
Line 183 static void pool_prime_page(struct pool |
|
Line 183 static void pool_prime_page(struct pool |
|
struct pool_item_header *); |
struct pool_item_header *); |
static void pool_update_curpage(struct pool *); |
static void pool_update_curpage(struct pool *); |
|
|
|
static int pool_grow(struct pool *, int); |
void *pool_allocator_alloc(struct pool *, int); |
void *pool_allocator_alloc(struct pool *, int); |
void pool_allocator_free(struct pool *, void *); |
void pool_allocator_free(struct pool *, void *); |
|
|
Line 214 struct pool_log { |
|
Line 215 struct pool_log { |
|
|
|
int pool_logsize = POOL_LOGSIZE; |
int pool_logsize = POOL_LOGSIZE; |
|
|
static __inline void |
static inline void |
pr_log(struct pool *pp, void *v, int action, const char *file, long line) |
pr_log(struct pool *pp, void *v, int action, const char *file, long line) |
{ |
{ |
int n = pp->pr_curlogentry; |
int n = pp->pr_curlogentry; |
Line 267 pr_printlog(struct pool *pp, struct pool |
|
Line 268 pr_printlog(struct pool *pp, struct pool |
|
} |
} |
} |
} |
|
|
static __inline void |
static inline void |
pr_enter(struct pool *pp, const char *file, long line) |
pr_enter(struct pool *pp, const char *file, long line) |
{ |
{ |
|
|
Line 283 pr_enter(struct pool *pp, const char *fi |
|
Line 284 pr_enter(struct pool *pp, const char *fi |
|
pp->pr_entered_line = line; |
pp->pr_entered_line = line; |
} |
} |
|
|
static __inline void |
static inline void |
pr_leave(struct pool *pp) |
pr_leave(struct pool *pp) |
{ |
{ |
|
|
Line 296 pr_leave(struct pool *pp) |
|
Line 297 pr_leave(struct pool *pp) |
|
pp->pr_entered_line = 0; |
pp->pr_entered_line = 0; |
} |
} |
|
|
static __inline void |
static inline void |
pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) |
pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) |
{ |
{ |
|
|
Line 312 pr_enter_check(struct pool *pp, void (*p |
|
Line 313 pr_enter_check(struct pool *pp, void (*p |
|
#define pr_enter_check(pp, pr) |
#define pr_enter_check(pp, pr) |
#endif /* POOL_DIAGNOSTIC */ |
#endif /* POOL_DIAGNOSTIC */ |
|
|
static __inline int |
static inline int |
pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, |
pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, |
const void *v) |
const void *v) |
{ |
{ |
Line 331 pr_item_notouch_index(const struct pool |
|
Line 332 pr_item_notouch_index(const struct pool |
|
#define PR_INDEX_USED ((pool_item_freelist_t)-1) |
#define PR_INDEX_USED ((pool_item_freelist_t)-1) |
#define PR_INDEX_EOL ((pool_item_freelist_t)-2) |
#define PR_INDEX_EOL ((pool_item_freelist_t)-2) |
|
|
static __inline void |
static inline void |
pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, |
pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, |
void *obj) |
void *obj) |
{ |
{ |
Line 343 pr_item_notouch_put(const struct pool *p |
|
Line 344 pr_item_notouch_put(const struct pool *p |
|
ph->ph_firstfree = idx; |
ph->ph_firstfree = idx; |
} |
} |
|
|
static __inline void * |
static inline void * |
pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) |
pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) |
{ |
{ |
int idx = ph->ph_firstfree; |
int idx = ph->ph_firstfree; |
Line 356 pr_item_notouch_get(const struct pool *p |
|
Line 357 pr_item_notouch_get(const struct pool *p |
|
return ph->ph_page + ph->ph_off + idx * pp->pr_size; |
return ph->ph_page + ph->ph_off + idx * pp->pr_size; |
} |
} |
|
|
static __inline int |
static inline int |
phtree_compare(struct pool_item_header *a, struct pool_item_header *b) |
phtree_compare(struct pool_item_header *a, struct pool_item_header *b) |
{ |
{ |
if (a->ph_page < b->ph_page) |
if (a->ph_page < b->ph_page) |
Line 373 SPLAY_GENERATE(phtree, pool_item_header, |
|
Line 374 SPLAY_GENERATE(phtree, pool_item_header, |
|
/* |
/* |
* Return the pool page header based on page address. |
* Return the pool page header based on page address. |
*/ |
*/ |
static __inline struct pool_item_header * |
static inline struct pool_item_header * |
pr_find_pagehead(struct pool *pp, caddr_t page) |
pr_find_pagehead(struct pool *pp, caddr_t page) |
{ |
{ |
struct pool_item_header *ph, tmp; |
struct pool_item_header *ph, tmp; |
Line 406 pr_pagelist_free(struct pool *pp, struct |
|
Line 407 pr_pagelist_free(struct pool *pp, struct |
|
/* |
/* |
* Remove a page from the pool. |
* Remove a page from the pool. |
*/ |
*/ |
static __inline void |
static inline void |
pr_rmpage(struct pool *pp, struct pool_item_header *ph, |
pr_rmpage(struct pool *pp, struct pool_item_header *ph, |
struct pool_pagelist *pq) |
struct pool_pagelist *pq) |
{ |
{ |
Line 482 pool_init(struct pool *pp, size_t size, |
|
Line 483 pool_init(struct pool *pp, size_t size, |
|
flags |= PR_LOGGING; |
flags |= PR_LOGGING; |
#endif |
#endif |
|
|
#ifdef POOL_SUBPAGE |
|
/* |
|
* XXX We don't provide a real `nointr' back-end |
|
* yet; all sub-pages come from a kmem back-end. |
|
* maybe some day... |
|
*/ |
|
if (palloc == NULL) { |
|
extern struct pool_allocator pool_allocator_kmem_subpage; |
|
palloc = &pool_allocator_kmem_subpage; |
|
} |
|
/* |
|
* We'll assume any user-specified back-end allocator |
|
* will deal with sub-pages, or simply don't care. |
|
*/ |
|
#else |
|
if (palloc == NULL) |
if (palloc == NULL) |
palloc = &pool_allocator_kmem; |
palloc = &pool_allocator_kmem; |
|
#ifdef POOL_SUBPAGE |
|
if (size > palloc->pa_pagesz) { |
|
if (palloc == &pool_allocator_kmem) |
|
palloc = &pool_allocator_kmem_fullpage; |
|
else if (palloc == &pool_allocator_nointr) |
|
palloc = &pool_allocator_nointr_fullpage; |
|
} |
#endif /* POOL_SUBPAGE */ |
#endif /* POOL_SUBPAGE */ |
if ((palloc->pa_flags & PA_INITIALIZED) == 0) { |
if ((palloc->pa_flags & PA_INITIALIZED) == 0) { |
if (palloc->pa_pagesz == 0) { |
if (palloc->pa_pagesz == 0) |
#ifdef POOL_SUBPAGE |
|
if (palloc == &pool_allocator_kmem) |
|
palloc->pa_pagesz = PAGE_SIZE; |
|
else |
|
palloc->pa_pagesz = POOL_SUBPAGE; |
|
#else |
|
palloc->pa_pagesz = PAGE_SIZE; |
palloc->pa_pagesz = PAGE_SIZE; |
#endif /* POOL_SUBPAGE */ |
|
} |
|
|
|
TAILQ_INIT(&palloc->pa_list); |
TAILQ_INIT(&palloc->pa_list); |
|
|
Line 883 pool_get(struct pool *pp, int flags) |
|
Line 868 pool_get(struct pool *pp, int flags) |
|
* has no items in its bucket. |
* has no items in its bucket. |
*/ |
*/ |
if ((ph = pp->pr_curpage) == NULL) { |
if ((ph = pp->pr_curpage) == NULL) { |
|
int error; |
|
|
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
if (pp->pr_nitems != 0) { |
if (pp->pr_nitems != 0) { |
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
Line 898 pool_get(struct pool *pp, int flags) |
|
Line 885 pool_get(struct pool *pp, int flags) |
|
* may block. |
* may block. |
*/ |
*/ |
pr_leave(pp); |
pr_leave(pp); |
simple_unlock(&pp->pr_slock); |
error = pool_grow(pp, flags); |
v = pool_allocator_alloc(pp, flags); |
pr_enter(pp, file, line); |
if (__predict_true(v != NULL)) |
if (error != 0) { |
ph = pool_alloc_item_header(pp, v, flags); |
|
|
|
if (__predict_false(v == NULL || ph == NULL)) { |
|
if (v != NULL) |
|
pool_allocator_free(pp, v); |
|
|
|
simple_lock(&pp->pr_slock); |
|
pr_enter(pp, file, line); |
|
|
|
/* |
/* |
* We were unable to allocate a page or item |
* We were unable to allocate a page or item |
* header, but we released the lock during |
* header, but we released the lock during |
Line 937 pool_get(struct pool *pp, int flags) |
|
Line 915 pool_get(struct pool *pp, int flags) |
|
pr_leave(pp); |
pr_leave(pp); |
ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); |
ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); |
pr_enter(pp, file, line); |
pr_enter(pp, file, line); |
goto startover; |
|
} |
} |
|
|
/* We have more memory; add it to the pool */ |
|
simple_lock(&pp->pr_slock); |
|
pr_enter(pp, file, line); |
|
pool_prime_page(pp, v, ph); |
|
pp->pr_npagealloc++; |
|
|
|
/* Start the allocation process over. */ |
/* Start the allocation process over. */ |
goto startover; |
goto startover; |
} |
} |
Line 1033 pool_get(struct pool *pp, int flags) |
|
Line 1004 pool_get(struct pool *pp, int flags) |
|
} |
} |
|
|
pp->pr_nget++; |
pp->pr_nget++; |
|
pr_leave(pp); |
|
|
/* |
/* |
* If we have a low water mark and we are now below that low |
* If we have a low water mark and we are now below that low |
Line 1046 pool_get(struct pool *pp, int flags) |
|
Line 1018 pool_get(struct pool *pp, int flags) |
|
*/ |
*/ |
} |
} |
|
|
pr_leave(pp); |
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
return (v); |
return (v); |
} |
} |
Line 1220 pool_put(struct pool *pp, void *v) |
|
Line 1191 pool_put(struct pool *pp, void *v) |
|
#endif |
#endif |
|
|
/* |
/* |
|
* pool_grow: grow a pool by a page. |
|
* |
|
* => called with pool locked. |
|
* => unlock and relock the pool. |
|
* => return with pool locked. |
|
*/ |
|
|
|
static int |
|
pool_grow(struct pool *pp, int flags) |
|
{ |
|
struct pool_item_header *ph = NULL; |
|
char *cp; |
|
|
|
simple_unlock(&pp->pr_slock); |
|
cp = pool_allocator_alloc(pp, flags); |
|
if (__predict_true(cp != NULL)) { |
|
ph = pool_alloc_item_header(pp, cp, flags); |
|
} |
|
if (__predict_false(cp == NULL || ph == NULL)) { |
|
if (cp != NULL) { |
|
pool_allocator_free(pp, cp); |
|
} |
|
simple_lock(&pp->pr_slock); |
|
return ENOMEM; |
|
} |
|
|
|
simple_lock(&pp->pr_slock); |
|
pool_prime_page(pp, cp, ph); |
|
pp->pr_npagealloc++; |
|
return 0; |
|
} |
|
|
|
/* |
* Add N items to the pool. |
* Add N items to the pool. |
*/ |
*/ |
int |
int |
pool_prime(struct pool *pp, int n) |
pool_prime(struct pool *pp, int n) |
{ |
{ |
struct pool_item_header *ph = NULL; |
|
caddr_t cp; |
|
int newpages; |
int newpages; |
|
int error = 0; |
|
|
simple_lock(&pp->pr_slock); |
simple_lock(&pp->pr_slock); |
|
|
newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
|
|
while (newpages-- > 0) { |
while (newpages-- > 0) { |
simple_unlock(&pp->pr_slock); |
error = pool_grow(pp, PR_NOWAIT); |
cp = pool_allocator_alloc(pp, PR_NOWAIT); |
if (error) { |
if (__predict_true(cp != NULL)) |
|
ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); |
|
|
|
if (__predict_false(cp == NULL || ph == NULL)) { |
|
if (cp != NULL) |
|
pool_allocator_free(pp, cp); |
|
simple_lock(&pp->pr_slock); |
|
break; |
break; |
} |
} |
|
|
simple_lock(&pp->pr_slock); |
|
pool_prime_page(pp, cp, ph); |
|
pp->pr_npagealloc++; |
|
pp->pr_minpages++; |
pp->pr_minpages++; |
} |
} |
|
|
Line 1256 pool_prime(struct pool *pp, int n) |
|
Line 1248 pool_prime(struct pool *pp, int n) |
|
pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ |
pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ |
|
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
return (0); |
return error; |
} |
} |
|
|
/* |
/* |
Line 1361 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1353 pool_prime_page(struct pool *pp, caddr_t |
|
static int |
static int |
pool_catchup(struct pool *pp) |
pool_catchup(struct pool *pp) |
{ |
{ |
struct pool_item_header *ph = NULL; |
|
caddr_t cp; |
|
int error = 0; |
int error = 0; |
|
|
while (POOL_NEEDS_CATCHUP(pp)) { |
while (POOL_NEEDS_CATCHUP(pp)) { |
/* |
error = pool_grow(pp, PR_NOWAIT); |
* Call the page back-end allocator for more memory. |
if (error) { |
* |
|
* XXX: We never wait, so should we bother unlocking |
|
* the pool descriptor? |
|
*/ |
|
simple_unlock(&pp->pr_slock); |
|
cp = pool_allocator_alloc(pp, PR_NOWAIT); |
|
if (__predict_true(cp != NULL)) |
|
ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); |
|
if (__predict_false(cp == NULL || ph == NULL)) { |
|
if (cp != NULL) |
|
pool_allocator_free(pp, cp); |
|
error = ENOMEM; |
|
simple_lock(&pp->pr_slock); |
|
break; |
break; |
} |
} |
simple_lock(&pp->pr_slock); |
|
pool_prime_page(pp, cp, ph); |
|
pp->pr_npagealloc++; |
|
} |
} |
|
return error; |
return (error); |
|
} |
} |
|
|
static void |
static void |
Line 1894 pool_cache_destroy(struct pool_cache *pc |
|
Line 1867 pool_cache_destroy(struct pool_cache *pc |
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
} |
} |
|
|
static __inline void * |
static inline void * |
pcg_get(struct pool_cache_group *pcg, paddr_t *pap) |
pcg_get(struct pool_cache_group *pcg, paddr_t *pap) |
{ |
{ |
void *object; |
void *object; |
Line 1913 pcg_get(struct pool_cache_group *pcg, pa |
|
Line 1886 pcg_get(struct pool_cache_group *pcg, pa |
|
return (object); |
return (object); |
} |
} |
|
|
static __inline void |
static inline void |
pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) |
pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) |
{ |
{ |
u_int idx; |
u_int idx; |
Line 2190 pool_cache_reclaim(struct pool_cache *pc |
|
Line 2163 pool_cache_reclaim(struct pool_cache *pc |
|
void *pool_page_alloc(struct pool *, int); |
void *pool_page_alloc(struct pool *, int); |
void pool_page_free(struct pool *, void *); |
void pool_page_free(struct pool *, void *); |
|
|
|
#ifdef POOL_SUBPAGE |
|
struct pool_allocator pool_allocator_kmem_fullpage = { |
|
pool_page_alloc, pool_page_free, 0, |
|
}; |
|
#else |
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
}; |
}; |
|
#endif |
|
|
void *pool_page_alloc_nointr(struct pool *, int); |
void *pool_page_alloc_nointr(struct pool *, int); |
void pool_page_free_nointr(struct pool *, void *); |
void pool_page_free_nointr(struct pool *, void *); |
|
|
|
#ifdef POOL_SUBPAGE |
|
struct pool_allocator pool_allocator_nointr_fullpage = { |
|
pool_page_alloc_nointr, pool_page_free_nointr, 0, |
|
}; |
|
#else |
struct pool_allocator pool_allocator_nointr = { |
struct pool_allocator pool_allocator_nointr = { |
pool_page_alloc_nointr, pool_page_free_nointr, 0, |
pool_page_alloc_nointr, pool_page_free_nointr, 0, |
}; |
}; |
|
#endif |
|
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
void *pool_subpage_alloc(struct pool *, int); |
void *pool_subpage_alloc(struct pool *, int); |
void pool_subpage_free(struct pool *, void *); |
void pool_subpage_free(struct pool *, void *); |
|
|
struct pool_allocator pool_allocator_kmem_subpage = { |
struct pool_allocator pool_allocator_kmem = { |
pool_subpage_alloc, pool_subpage_free, 0, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
|
}; |
|
|
|
void *pool_subpage_alloc_nointr(struct pool *, int); |
|
void pool_subpage_free_nointr(struct pool *, void *); |
|
|
|
struct pool_allocator pool_allocator_nointr = { |
|
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
}; |
}; |
#endif /* POOL_SUBPAGE */ |
#endif /* POOL_SUBPAGE */ |
|
|
Line 2382 pool_subpage_free(struct pool *pp, void |
|
Line 2374 pool_subpage_free(struct pool *pp, void |
|
|
|
/* We don't provide a real nointr allocator. Maybe later. */ |
/* We don't provide a real nointr allocator. Maybe later. */ |
void * |
void * |
pool_page_alloc_nointr(struct pool *pp, int flags) |
pool_subpage_alloc_nointr(struct pool *pp, int flags) |
{ |
{ |
|
|
return (pool_subpage_alloc(pp, flags)); |
return (pool_subpage_alloc(pp, flags)); |
} |
} |
|
|
void |
void |
pool_page_free_nointr(struct pool *pp, void *v) |
pool_subpage_free_nointr(struct pool *pp, void *v) |
{ |
{ |
|
|
pool_subpage_free(pp, v); |
pool_subpage_free(pp, v); |
} |
} |
#else |
#endif /* POOL_SUBPAGE */ |
void * |
void * |
pool_page_alloc_nointr(struct pool *pp, int flags) |
pool_page_alloc_nointr(struct pool *pp, int flags) |
{ |
{ |
Line 2409 pool_page_free_nointr(struct pool *pp, v |
|
Line 2401 pool_page_free_nointr(struct pool *pp, v |
|
|
|
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
} |
} |
#endif /* POOL_SUBPAGE */ |
|