version 1.87.2.6, 2005/11/10 14:09:45 |
version 1.111.4.2, 2006/04/22 11:39:59 |
Line 183 static void pool_prime_page(struct pool |
|
Line 183 static void pool_prime_page(struct pool |
|
struct pool_item_header *); |
struct pool_item_header *); |
static void pool_update_curpage(struct pool *); |
static void pool_update_curpage(struct pool *); |
|
|
|
static int pool_grow(struct pool *, int); |
void *pool_allocator_alloc(struct pool *, int); |
void *pool_allocator_alloc(struct pool *, int); |
void pool_allocator_free(struct pool *, void *); |
void pool_allocator_free(struct pool *, void *); |
|
|
Line 214 struct pool_log { |
|
Line 215 struct pool_log { |
|
|
|
int pool_logsize = POOL_LOGSIZE; |
int pool_logsize = POOL_LOGSIZE; |
|
|
static __inline void |
static inline void |
pr_log(struct pool *pp, void *v, int action, const char *file, long line) |
pr_log(struct pool *pp, void *v, int action, const char *file, long line) |
{ |
{ |
int n = pp->pr_curlogentry; |
int n = pp->pr_curlogentry; |
Line 267 pr_printlog(struct pool *pp, struct pool |
|
Line 268 pr_printlog(struct pool *pp, struct pool |
|
} |
} |
} |
} |
|
|
static __inline void |
static inline void |
pr_enter(struct pool *pp, const char *file, long line) |
pr_enter(struct pool *pp, const char *file, long line) |
{ |
{ |
|
|
Line 283 pr_enter(struct pool *pp, const char *fi |
|
Line 284 pr_enter(struct pool *pp, const char *fi |
|
pp->pr_entered_line = line; |
pp->pr_entered_line = line; |
} |
} |
|
|
static __inline void |
static inline void |
pr_leave(struct pool *pp) |
pr_leave(struct pool *pp) |
{ |
{ |
|
|
Line 296 pr_leave(struct pool *pp) |
|
Line 297 pr_leave(struct pool *pp) |
|
pp->pr_entered_line = 0; |
pp->pr_entered_line = 0; |
} |
} |
|
|
static __inline void |
static inline void |
pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) |
pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) |
{ |
{ |
|
|
Line 312 pr_enter_check(struct pool *pp, void (*p |
|
Line 313 pr_enter_check(struct pool *pp, void (*p |
|
#define pr_enter_check(pp, pr) |
#define pr_enter_check(pp, pr) |
#endif /* POOL_DIAGNOSTIC */ |
#endif /* POOL_DIAGNOSTIC */ |
|
|
static __inline int |
static inline int |
pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, |
pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, |
const void *v) |
const void *v) |
{ |
{ |
Line 331 pr_item_notouch_index(const struct pool |
|
Line 332 pr_item_notouch_index(const struct pool |
|
#define PR_INDEX_USED ((pool_item_freelist_t)-1) |
#define PR_INDEX_USED ((pool_item_freelist_t)-1) |
#define PR_INDEX_EOL ((pool_item_freelist_t)-2) |
#define PR_INDEX_EOL ((pool_item_freelist_t)-2) |
|
|
static __inline void |
static inline void |
pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, |
pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, |
void *obj) |
void *obj) |
{ |
{ |
Line 343 pr_item_notouch_put(const struct pool *p |
|
Line 344 pr_item_notouch_put(const struct pool *p |
|
ph->ph_firstfree = idx; |
ph->ph_firstfree = idx; |
} |
} |
|
|
static __inline void * |
static inline void * |
pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) |
pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) |
{ |
{ |
int idx = ph->ph_firstfree; |
int idx = ph->ph_firstfree; |
Line 356 pr_item_notouch_get(const struct pool *p |
|
Line 357 pr_item_notouch_get(const struct pool *p |
|
return ph->ph_page + ph->ph_off + idx * pp->pr_size; |
return ph->ph_page + ph->ph_off + idx * pp->pr_size; |
} |
} |
|
|
static __inline int |
static inline int |
phtree_compare(struct pool_item_header *a, struct pool_item_header *b) |
phtree_compare(struct pool_item_header *a, struct pool_item_header *b) |
{ |
{ |
if (a->ph_page < b->ph_page) |
if (a->ph_page < b->ph_page) |
Line 373 SPLAY_GENERATE(phtree, pool_item_header, |
|
Line 374 SPLAY_GENERATE(phtree, pool_item_header, |
|
/* |
/* |
* Return the pool page header based on page address. |
* Return the pool page header based on page address. |
*/ |
*/ |
static __inline struct pool_item_header * |
static inline struct pool_item_header * |
pr_find_pagehead(struct pool *pp, caddr_t page) |
pr_find_pagehead(struct pool *pp, caddr_t page) |
{ |
{ |
struct pool_item_header *ph, tmp; |
struct pool_item_header *ph, tmp; |
Line 406 pr_pagelist_free(struct pool *pp, struct |
|
Line 407 pr_pagelist_free(struct pool *pp, struct |
|
/* |
/* |
* Remove a page from the pool. |
* Remove a page from the pool. |
*/ |
*/ |
static __inline void |
static inline void |
pr_rmpage(struct pool *pp, struct pool_item_header *ph, |
pr_rmpage(struct pool *pp, struct pool_item_header *ph, |
struct pool_pagelist *pq) |
struct pool_pagelist *pq) |
{ |
{ |
|
|
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
const char *wchan, struct pool_allocator *palloc) |
const char *wchan, struct pool_allocator *palloc) |
{ |
{ |
int off, slack; |
#ifdef DEBUG |
|
struct pool *pp1; |
|
#endif |
size_t trysize, phsize; |
size_t trysize, phsize; |
int s; |
int off, slack, s; |
|
|
KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >= |
KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >= |
PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1)); |
PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1)); |
|
|
|
#ifdef DEBUG |
|
/* |
|
* Check that the pool hasn't already been initialised and |
|
* added to the list of all pools. |
|
*/ |
|
LIST_FOREACH(pp1, &pool_head, pr_poollist) { |
|
if (pp == pp1) |
|
panic("pool_init: pool %s already initialised", |
|
wchan); |
|
} |
|
#endif |
|
|
#ifdef POOL_DIAGNOSTIC |
#ifdef POOL_DIAGNOSTIC |
/* |
/* |
* Always log if POOL_DIAGNOSTIC is defined. |
* Always log if POOL_DIAGNOSTIC is defined. |
Line 482 pool_init(struct pool *pp, size_t size, |
|
Line 497 pool_init(struct pool *pp, size_t size, |
|
flags |= PR_LOGGING; |
flags |= PR_LOGGING; |
#endif |
#endif |
|
|
#ifdef POOL_SUBPAGE |
|
/* |
|
* XXX We don't provide a real `nointr' back-end |
|
* yet; all sub-pages come from a kmem back-end. |
|
* maybe some day... |
|
*/ |
|
if (palloc == NULL) { |
|
extern struct pool_allocator pool_allocator_kmem_subpage; |
|
palloc = &pool_allocator_kmem_subpage; |
|
} |
|
/* |
|
* We'll assume any user-specified back-end allocator |
|
* will deal with sub-pages, or simply don't care. |
|
*/ |
|
#else |
|
if (palloc == NULL) |
if (palloc == NULL) |
palloc = &pool_allocator_kmem; |
palloc = &pool_allocator_kmem; |
|
#ifdef POOL_SUBPAGE |
|
if (size > palloc->pa_pagesz) { |
|
if (palloc == &pool_allocator_kmem) |
|
palloc = &pool_allocator_kmem_fullpage; |
|
else if (palloc == &pool_allocator_nointr) |
|
palloc = &pool_allocator_nointr_fullpage; |
|
} |
#endif /* POOL_SUBPAGE */ |
#endif /* POOL_SUBPAGE */ |
if ((palloc->pa_flags & PA_INITIALIZED) == 0) { |
if ((palloc->pa_flags & PA_INITIALIZED) == 0) { |
if (palloc->pa_pagesz == 0) { |
if (palloc->pa_pagesz == 0) |
#ifdef POOL_SUBPAGE |
|
if (palloc == &pool_allocator_kmem) |
|
palloc->pa_pagesz = PAGE_SIZE; |
|
else |
|
palloc->pa_pagesz = POOL_SUBPAGE; |
|
#else |
|
palloc->pa_pagesz = PAGE_SIZE; |
palloc->pa_pagesz = PAGE_SIZE; |
#endif /* POOL_SUBPAGE */ |
|
} |
|
|
|
TAILQ_INIT(&palloc->pa_list); |
TAILQ_INIT(&palloc->pa_list); |
|
|
Line 883 pool_get(struct pool *pp, int flags) |
|
Line 882 pool_get(struct pool *pp, int flags) |
|
* has no items in its bucket. |
* has no items in its bucket. |
*/ |
*/ |
if ((ph = pp->pr_curpage) == NULL) { |
if ((ph = pp->pr_curpage) == NULL) { |
|
int error; |
|
|
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
if (pp->pr_nitems != 0) { |
if (pp->pr_nitems != 0) { |
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
Line 898 pool_get(struct pool *pp, int flags) |
|
Line 899 pool_get(struct pool *pp, int flags) |
|
* may block. |
* may block. |
*/ |
*/ |
pr_leave(pp); |
pr_leave(pp); |
simple_unlock(&pp->pr_slock); |
error = pool_grow(pp, flags); |
v = pool_allocator_alloc(pp, flags); |
pr_enter(pp, file, line); |
if (__predict_true(v != NULL)) |
if (error != 0) { |
ph = pool_alloc_item_header(pp, v, flags); |
|
|
|
if (__predict_false(v == NULL || ph == NULL)) { |
|
if (v != NULL) |
|
pool_allocator_free(pp, v); |
|
|
|
simple_lock(&pp->pr_slock); |
|
pr_enter(pp, file, line); |
|
|
|
/* |
/* |
* We were unable to allocate a page or item |
* We were unable to allocate a page or item |
* header, but we released the lock during |
* header, but we released the lock during |
Line 929 pool_get(struct pool *pp, int flags) |
|
Line 921 pool_get(struct pool *pp, int flags) |
|
/* |
/* |
* Wait for items to be returned to this pool. |
* Wait for items to be returned to this pool. |
* |
* |
* XXX: maybe we should wake up once a second and |
* wake up once a second and try again, |
* try again? |
* as the check in pool_cache_put_paddr() is racy. |
*/ |
*/ |
pp->pr_flags |= PR_WANTED; |
pp->pr_flags |= PR_WANTED; |
/* PA_WANTED is already set on the allocator. */ |
/* PA_WANTED is already set on the allocator. */ |
pr_leave(pp); |
pr_leave(pp); |
ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock); |
ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); |
pr_enter(pp, file, line); |
pr_enter(pp, file, line); |
goto startover; |
|
} |
} |
|
|
/* We have more memory; add it to the pool */ |
|
simple_lock(&pp->pr_slock); |
|
pr_enter(pp, file, line); |
|
pool_prime_page(pp, v, ph); |
|
pp->pr_npagealloc++; |
|
|
|
/* Start the allocation process over. */ |
/* Start the allocation process over. */ |
goto startover; |
goto startover; |
} |
} |
Line 1033 pool_get(struct pool *pp, int flags) |
|
Line 1018 pool_get(struct pool *pp, int flags) |
|
} |
} |
|
|
pp->pr_nget++; |
pp->pr_nget++; |
|
pr_leave(pp); |
|
|
/* |
/* |
* If we have a low water mark and we are now below that low |
* If we have a low water mark and we are now below that low |
Line 1046 pool_get(struct pool *pp, int flags) |
|
Line 1032 pool_get(struct pool *pp, int flags) |
|
*/ |
*/ |
} |
} |
|
|
pr_leave(pp); |
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
return (v); |
return (v); |
} |
} |
Line 1060 pool_do_put(struct pool *pp, void *v, st |
|
Line 1045 pool_do_put(struct pool *pp, void *v, st |
|
struct pool_item *pi = v; |
struct pool_item *pi = v; |
struct pool_item_header *ph; |
struct pool_item_header *ph; |
caddr_t page; |
caddr_t page; |
int s; |
|
|
|
LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); |
LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); |
SCHED_ASSERT_UNLOCKED(); |
SCHED_ASSERT_UNLOCKED(); |
Line 1155 pool_do_put(struct pool *pp, void *v, st |
|
Line 1139 pool_do_put(struct pool *pp, void *v, st |
|
* be reclaimed by the pagedaemon. This minimizes |
* be reclaimed by the pagedaemon. This minimizes |
* ping-pong'ing for memory. |
* ping-pong'ing for memory. |
*/ |
*/ |
s = splclock(); |
getmicrotime(&ph->ph_time); |
ph->ph_time = mono_time; |
|
splx(s); |
|
} |
} |
pool_update_curpage(pp); |
pool_update_curpage(pp); |
} |
} |
Line 1220 pool_put(struct pool *pp, void *v) |
|
Line 1202 pool_put(struct pool *pp, void *v) |
|
#endif |
#endif |
|
|
/* |
/* |
|
* pool_grow: grow a pool by a page. |
|
* |
|
* => called with pool locked. |
|
* => unlock and relock the pool. |
|
* => return with pool locked. |
|
*/ |
|
|
|
static int |
|
pool_grow(struct pool *pp, int flags) |
|
{ |
|
struct pool_item_header *ph = NULL; |
|
char *cp; |
|
|
|
simple_unlock(&pp->pr_slock); |
|
cp = pool_allocator_alloc(pp, flags); |
|
if (__predict_true(cp != NULL)) { |
|
ph = pool_alloc_item_header(pp, cp, flags); |
|
} |
|
if (__predict_false(cp == NULL || ph == NULL)) { |
|
if (cp != NULL) { |
|
pool_allocator_free(pp, cp); |
|
} |
|
simple_lock(&pp->pr_slock); |
|
return ENOMEM; |
|
} |
|
|
|
simple_lock(&pp->pr_slock); |
|
pool_prime_page(pp, cp, ph); |
|
pp->pr_npagealloc++; |
|
return 0; |
|
} |
|
|
|
/* |
* Add N items to the pool. |
* Add N items to the pool. |
*/ |
*/ |
int |
int |
pool_prime(struct pool *pp, int n) |
pool_prime(struct pool *pp, int n) |
{ |
{ |
struct pool_item_header *ph = NULL; |
|
caddr_t cp; |
|
int newpages; |
int newpages; |
|
int error = 0; |
|
|
simple_lock(&pp->pr_slock); |
simple_lock(&pp->pr_slock); |
|
|
newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
|
|
while (newpages-- > 0) { |
while (newpages-- > 0) { |
simple_unlock(&pp->pr_slock); |
error = pool_grow(pp, PR_NOWAIT); |
cp = pool_allocator_alloc(pp, PR_NOWAIT); |
if (error) { |
if (__predict_true(cp != NULL)) |
|
ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); |
|
|
|
if (__predict_false(cp == NULL || ph == NULL)) { |
|
if (cp != NULL) |
|
pool_allocator_free(pp, cp); |
|
simple_lock(&pp->pr_slock); |
|
break; |
break; |
} |
} |
|
|
simple_lock(&pp->pr_slock); |
|
pool_prime_page(pp, cp, ph); |
|
pp->pr_npagealloc++; |
|
pp->pr_minpages++; |
pp->pr_minpages++; |
} |
} |
|
|
Line 1256 pool_prime(struct pool *pp, int n) |
|
Line 1259 pool_prime(struct pool *pp, int n) |
|
pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ |
pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ |
|
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
return (0); |
return error; |
} |
} |
|
|
/* |
/* |
Line 1272 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1275 pool_prime_page(struct pool *pp, caddr_t |
|
unsigned int align = pp->pr_align; |
unsigned int align = pp->pr_align; |
unsigned int ioff = pp->pr_itemoffset; |
unsigned int ioff = pp->pr_itemoffset; |
int n; |
int n; |
int s; |
|
|
|
LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); |
LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); |
|
|
Line 1288 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1290 pool_prime_page(struct pool *pp, caddr_t |
|
LIST_INIT(&ph->ph_itemlist); |
LIST_INIT(&ph->ph_itemlist); |
ph->ph_page = storage; |
ph->ph_page = storage; |
ph->ph_nmissing = 0; |
ph->ph_nmissing = 0; |
s = splclock(); |
getmicrotime(&ph->ph_time); |
ph->ph_time = mono_time; |
|
splx(s); |
|
if ((pp->pr_roflags & PR_PHINPAGE) == 0) |
if ((pp->pr_roflags & PR_PHINPAGE) == 0) |
SPLAY_INSERT(phtree, &pp->pr_phtree, ph); |
SPLAY_INSERT(phtree, &pp->pr_phtree, ph); |
|
|
Line 1361 pool_prime_page(struct pool *pp, caddr_t |
|
Line 1361 pool_prime_page(struct pool *pp, caddr_t |
|
static int |
static int |
pool_catchup(struct pool *pp) |
pool_catchup(struct pool *pp) |
{ |
{ |
struct pool_item_header *ph = NULL; |
|
caddr_t cp; |
|
int error = 0; |
int error = 0; |
|
|
while (POOL_NEEDS_CATCHUP(pp)) { |
while (POOL_NEEDS_CATCHUP(pp)) { |
/* |
error = pool_grow(pp, PR_NOWAIT); |
* Call the page back-end allocator for more memory. |
if (error) { |
* |
|
* XXX: We never wait, so should we bother unlocking |
|
* the pool descriptor? |
|
*/ |
|
simple_unlock(&pp->pr_slock); |
|
cp = pool_allocator_alloc(pp, PR_NOWAIT); |
|
if (__predict_true(cp != NULL)) |
|
ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); |
|
if (__predict_false(cp == NULL || ph == NULL)) { |
|
if (cp != NULL) |
|
pool_allocator_free(pp, cp); |
|
error = ENOMEM; |
|
simple_lock(&pp->pr_slock); |
|
break; |
break; |
} |
} |
simple_lock(&pp->pr_slock); |
|
pool_prime_page(pp, cp, ph); |
|
pp->pr_npagealloc++; |
|
} |
} |
|
return error; |
return (error); |
|
} |
} |
|
|
static void |
static void |
Line 1475 pool_reclaim(struct pool *pp) |
|
Line 1456 pool_reclaim(struct pool *pp) |
|
struct pool_pagelist pq; |
struct pool_pagelist pq; |
struct pool_cache_grouplist pcgl; |
struct pool_cache_grouplist pcgl; |
struct timeval curtime, diff; |
struct timeval curtime, diff; |
int s; |
|
|
|
if (pp->pr_drain_hook != NULL) { |
if (pp->pr_drain_hook != NULL) { |
/* |
/* |
Line 1497 pool_reclaim(struct pool *pp) |
|
Line 1477 pool_reclaim(struct pool *pp) |
|
LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) |
LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) |
pool_cache_reclaim(pc, &pq, &pcgl); |
pool_cache_reclaim(pc, &pq, &pcgl); |
|
|
s = splclock(); |
getmicrotime(&curtime); |
curtime = mono_time; |
|
splx(s); |
|
|
|
for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { |
for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { |
phnext = LIST_NEXT(ph, ph_pagelist); |
phnext = LIST_NEXT(ph, ph_pagelist); |
Line 1556 pool_drain(void *arg) |
|
Line 1534 pool_drain(void *arg) |
|
drainpp = LIST_NEXT(pp, pr_poollist); |
drainpp = LIST_NEXT(pp, pr_poollist); |
} |
} |
simple_unlock(&pool_head_slock); |
simple_unlock(&pool_head_slock); |
pool_reclaim(pp); |
if (pp) |
|
pool_reclaim(pp); |
splx(s); |
splx(s); |
} |
} |
|
|
Line 1581 pool_print(struct pool *pp, const char * |
|
Line 1560 pool_print(struct pool *pp, const char * |
|
} |
} |
|
|
void |
void |
|
pool_printall(const char *modif, void (*pr)(const char *, ...)) |
|
{ |
|
struct pool *pp; |
|
|
|
if (simple_lock_try(&pool_head_slock) == 0) { |
|
(*pr)("WARNING: pool_head_slock is locked\n"); |
|
} else { |
|
simple_unlock(&pool_head_slock); |
|
} |
|
|
|
LIST_FOREACH(pp, &pool_head, pr_poollist) { |
|
pool_printit(pp, modif, pr); |
|
} |
|
} |
|
|
|
void |
pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) |
pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) |
{ |
{ |
|
|
Line 1878 pool_cache_destroy(struct pool_cache *pc |
|
Line 1873 pool_cache_destroy(struct pool_cache *pc |
|
simple_unlock(&pp->pr_slock); |
simple_unlock(&pp->pr_slock); |
} |
} |
|
|
static __inline void * |
static inline void * |
pcg_get(struct pool_cache_group *pcg, paddr_t *pap) |
pcg_get(struct pool_cache_group *pcg, paddr_t *pap) |
{ |
{ |
void *object; |
void *object; |
Line 1897 pcg_get(struct pool_cache_group *pcg, pa |
|
Line 1892 pcg_get(struct pool_cache_group *pcg, pa |
|
return (object); |
return (object); |
} |
} |
|
|
static __inline void |
static inline void |
pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) |
pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) |
{ |
{ |
u_int idx; |
u_int idx; |
Line 2003 pool_cache_put_paddr(struct pool_cache * |
|
Line 1998 pool_cache_put_paddr(struct pool_cache * |
|
struct pool_cache_group *pcg; |
struct pool_cache_group *pcg; |
int s; |
int s; |
|
|
|
if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) { |
|
goto destruct; |
|
} |
|
|
simple_lock(&pc->pc_slock); |
simple_lock(&pc->pc_slock); |
|
|
pcg = LIST_FIRST(&pc->pc_partgroups); |
pcg = LIST_FIRST(&pc->pc_partgroups); |
Line 2024 pool_cache_put_paddr(struct pool_cache * |
|
Line 2023 pool_cache_put_paddr(struct pool_cache * |
|
pcg = pool_get(&pcgpool, PR_NOWAIT); |
pcg = pool_get(&pcgpool, PR_NOWAIT); |
splx(s); |
splx(s); |
if (pcg == NULL) { |
if (pcg == NULL) { |
|
destruct: |
|
|
/* |
/* |
* Unable to allocate a cache group; destruct the object |
* Unable to allocate a cache group; destruct the object |
Line 2169 pool_cache_reclaim(struct pool_cache *pc |
|
Line 2169 pool_cache_reclaim(struct pool_cache *pc |
|
void *pool_page_alloc(struct pool *, int); |
void *pool_page_alloc(struct pool *, int); |
void pool_page_free(struct pool *, void *); |
void pool_page_free(struct pool *, void *); |
|
|
|
#ifdef POOL_SUBPAGE |
|
struct pool_allocator pool_allocator_kmem_fullpage = { |
|
pool_page_alloc, pool_page_free, 0, |
|
}; |
|
#else |
struct pool_allocator pool_allocator_kmem = { |
struct pool_allocator pool_allocator_kmem = { |
pool_page_alloc, pool_page_free, 0, |
pool_page_alloc, pool_page_free, 0, |
}; |
}; |
|
#endif |
|
|
void *pool_page_alloc_nointr(struct pool *, int); |
void *pool_page_alloc_nointr(struct pool *, int); |
void pool_page_free_nointr(struct pool *, void *); |
void pool_page_free_nointr(struct pool *, void *); |
|
|
|
#ifdef POOL_SUBPAGE |
|
struct pool_allocator pool_allocator_nointr_fullpage = { |
|
pool_page_alloc_nointr, pool_page_free_nointr, 0, |
|
}; |
|
#else |
struct pool_allocator pool_allocator_nointr = { |
struct pool_allocator pool_allocator_nointr = { |
pool_page_alloc_nointr, pool_page_free_nointr, 0, |
pool_page_alloc_nointr, pool_page_free_nointr, 0, |
}; |
}; |
|
#endif |
|
|
#ifdef POOL_SUBPAGE |
#ifdef POOL_SUBPAGE |
void *pool_subpage_alloc(struct pool *, int); |
void *pool_subpage_alloc(struct pool *, int); |
void pool_subpage_free(struct pool *, void *); |
void pool_subpage_free(struct pool *, void *); |
|
|
struct pool_allocator pool_allocator_kmem_subpage = { |
struct pool_allocator pool_allocator_kmem = { |
pool_subpage_alloc, pool_subpage_free, 0, |
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
|
}; |
|
|
|
void *pool_subpage_alloc_nointr(struct pool *, int); |
|
void pool_subpage_free_nointr(struct pool *, void *); |
|
|
|
struct pool_allocator pool_allocator_nointr = { |
|
pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, |
}; |
}; |
#endif /* POOL_SUBPAGE */ |
#endif /* POOL_SUBPAGE */ |
|
|
Line 2233 pool_allocator_alloc(struct pool *org, i |
|
Line 2252 pool_allocator_alloc(struct pool *org, i |
|
} |
} |
|
|
/* |
/* |
* Drain all pools, except "org", that use this |
* Drain all pools, that use this allocator. |
* allocator. We do this to reclaim VA space. |
* We do this to reclaim VA space. |
* pa_alloc is responsible for waiting for |
* pa_alloc is responsible for waiting for |
* physical memory. |
* physical memory. |
* |
* |
Line 2255 pool_allocator_alloc(struct pool *org, i |
|
Line 2274 pool_allocator_alloc(struct pool *org, i |
|
do { |
do { |
TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list); |
TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); |
TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); |
if (pp == org) |
|
continue; |
|
simple_unlock(&pa->pa_slock); |
simple_unlock(&pa->pa_slock); |
freed = pool_reclaim(pp); |
freed = pool_reclaim(pp); |
simple_lock(&pa->pa_slock); |
simple_lock(&pa->pa_slock); |
Line 2363 pool_subpage_free(struct pool *pp, void |
|
Line 2380 pool_subpage_free(struct pool *pp, void |
|
|
|
/* We don't provide a real nointr allocator. Maybe later. */ |
/* We don't provide a real nointr allocator. Maybe later. */ |
void * |
void * |
pool_page_alloc_nointr(struct pool *pp, int flags) |
pool_subpage_alloc_nointr(struct pool *pp, int flags) |
{ |
{ |
|
|
return (pool_subpage_alloc(pp, flags)); |
return (pool_subpage_alloc(pp, flags)); |
} |
} |
|
|
void |
void |
pool_page_free_nointr(struct pool *pp, void *v) |
pool_subpage_free_nointr(struct pool *pp, void *v) |
{ |
{ |
|
|
pool_subpage_free(pp, v); |
pool_subpage_free(pp, v); |
} |
} |
#else |
#endif /* POOL_SUBPAGE */ |
void * |
void * |
pool_page_alloc_nointr(struct pool *pp, int flags) |
pool_page_alloc_nointr(struct pool *pp, int flags) |
{ |
{ |
Line 2390 pool_page_free_nointr(struct pool *pp, v |
|
Line 2407 pool_page_free_nointr(struct pool *pp, v |
|
|
|
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); |
} |
} |
#endif /* POOL_SUBPAGE */ |
|