Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.108 retrieving revision 1.112.2.2 diff -u -p -r1.108 -r1.112.2.2 --- src/sys/kern/subr_pool.c 2005/12/01 13:21:05 1.108 +++ src/sys/kern/subr_pool.c 2006/04/11 11:55:47 1.112.2.2 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.108 2005/12/01 13:21:05 yamt Exp $ */ +/* $NetBSD: subr_pool.c,v 1.112.2.2 2006/04/11 11:55:47 yamt Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.108 2005/12/01 13:21:05 yamt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.112.2.2 2006/04/11 11:55:47 yamt Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -183,6 +183,7 @@ static void pool_prime_page(struct pool struct pool_item_header *); static void pool_update_curpage(struct pool *); +static int pool_grow(struct pool *, int); void *pool_allocator_alloc(struct pool *, int); void pool_allocator_free(struct pool *, void *); @@ -214,7 +215,7 @@ struct pool_log { int pool_logsize = POOL_LOGSIZE; -static __inline void +static inline void pr_log(struct pool *pp, void *v, int action, const char *file, long line) { int n = pp->pr_curlogentry; @@ -267,7 +268,7 @@ pr_printlog(struct pool *pp, struct pool } } -static __inline void +static inline void pr_enter(struct pool *pp, const char *file, long line) { @@ -283,7 +284,7 @@ pr_enter(struct pool *pp, const char *fi pp->pr_entered_line = line; } -static __inline void +static inline void pr_leave(struct pool *pp) { @@ -296,7 +297,7 @@ pr_leave(struct pool *pp) pp->pr_entered_line = 0; } -static __inline void +static inline void pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) { @@ -312,7 +313,7 @@ pr_enter_check(struct pool *pp, void (*p #define pr_enter_check(pp, pr) #endif /* POOL_DIAGNOSTIC */ -static __inline int +static inline int pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, const void *v) { @@ -331,7 +332,7 @@ pr_item_notouch_index(const struct pool #define PR_INDEX_USED ((pool_item_freelist_t)-1) #define PR_INDEX_EOL ((pool_item_freelist_t)-2) -static __inline void +static inline void pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, void *obj) { @@ -343,7 +344,7 @@ pr_item_notouch_put(const struct pool *p ph->ph_firstfree = idx; } -static __inline void * +static inline void * pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) { int idx = ph->ph_firstfree; @@ -356,7 +357,7 @@ pr_item_notouch_get(const struct pool *p return ph->ph_page + ph->ph_off + idx * pp->pr_size; } -static __inline int +static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { if (a->ph_page < b->ph_page) @@ -373,7 +374,7 @@ SPLAY_GENERATE(phtree, pool_item_header, /* * Return the pool page header based on page address. */ -static __inline struct pool_item_header * +static inline struct pool_item_header * pr_find_pagehead(struct pool *pp, caddr_t page) { struct pool_item_header *ph, tmp; @@ -406,7 +407,7 @@ pr_pagelist_free(struct pool *pp, struct /* * Remove a page from the pool. */ -static __inline void +static inline void pr_rmpage(struct pool *pp, struct pool_item_header *ph, struct pool_pagelist *pq) { @@ -482,35 +483,19 @@ pool_init(struct pool *pp, size_t size, flags |= PR_LOGGING; #endif -#ifdef POOL_SUBPAGE - /* - * XXX We don't provide a real `nointr' back-end - * yet; all sub-pages come from a kmem back-end. - * maybe some day... - */ - if (palloc == NULL) { - extern struct pool_allocator pool_allocator_kmem_subpage; - palloc = &pool_allocator_kmem_subpage; - } - /* - * We'll assume any user-specified back-end allocator - * will deal with sub-pages, or simply don't care. - */ -#else if (palloc == NULL) palloc = &pool_allocator_kmem; +#ifdef POOL_SUBPAGE + if (size > palloc->pa_pagesz) { + if (palloc == &pool_allocator_kmem) + palloc = &pool_allocator_kmem_fullpage; + else if (palloc == &pool_allocator_nointr) + palloc = &pool_allocator_nointr_fullpage; + } #endif /* POOL_SUBPAGE */ if ((palloc->pa_flags & PA_INITIALIZED) == 0) { - if (palloc->pa_pagesz == 0) { -#ifdef POOL_SUBPAGE - if (palloc == &pool_allocator_kmem) - palloc->pa_pagesz = PAGE_SIZE; - else - palloc->pa_pagesz = POOL_SUBPAGE; -#else + if (palloc->pa_pagesz == 0) palloc->pa_pagesz = PAGE_SIZE; -#endif /* POOL_SUBPAGE */ - } TAILQ_INIT(&palloc->pa_list); @@ -883,6 +868,8 @@ pool_get(struct pool *pp, int flags) * has no items in its bucket. */ if ((ph = pp->pr_curpage) == NULL) { + int error; + #ifdef DIAGNOSTIC if (pp->pr_nitems != 0) { simple_unlock(&pp->pr_slock); @@ -898,18 +885,9 @@ pool_get(struct pool *pp, int flags) * may block. */ pr_leave(pp); - simple_unlock(&pp->pr_slock); - v = pool_allocator_alloc(pp, flags); - if (__predict_true(v != NULL)) - ph = pool_alloc_item_header(pp, v, flags); - - if (__predict_false(v == NULL || ph == NULL)) { - if (v != NULL) - pool_allocator_free(pp, v); - - simple_lock(&pp->pr_slock); - pr_enter(pp, file, line); - + error = pool_grow(pp, flags); + pr_enter(pp, file, line); + if (error != 0) { /* * We were unable to allocate a page or item * header, but we released the lock during @@ -929,23 +907,16 @@ pool_get(struct pool *pp, int flags) /* * Wait for items to be returned to this pool. * - * XXX: maybe we should wake up once a second and - * try again? + * wake up once a second and try again, + * as the check in pool_cache_put_paddr() is racy. */ pp->pr_flags |= PR_WANTED; /* PA_WANTED is already set on the allocator. */ pr_leave(pp); - ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock); + ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); pr_enter(pp, file, line); - goto startover; } - /* We have more memory; add it to the pool */ - simple_lock(&pp->pr_slock); - pr_enter(pp, file, line); - pool_prime_page(pp, v, ph); - pp->pr_npagealloc++; - /* Start the allocation process over. */ goto startover; } @@ -1033,6 +1004,7 @@ pool_get(struct pool *pp, int flags) } pp->pr_nget++; + pr_leave(pp); /* * If we have a low water mark and we are now below that low @@ -1046,7 +1018,6 @@ pool_get(struct pool *pp, int flags) */ } - pr_leave(pp); simple_unlock(&pp->pr_slock); return (v); } @@ -1220,35 +1191,56 @@ pool_put(struct pool *pp, void *v) #endif /* + * pool_grow: grow a pool by a page. + * + * => called with pool locked. + * => unlock and relock the pool. + * => return with pool locked. + */ + +static int +pool_grow(struct pool *pp, int flags) +{ + struct pool_item_header *ph = NULL; + char *cp; + + simple_unlock(&pp->pr_slock); + cp = pool_allocator_alloc(pp, flags); + if (__predict_true(cp != NULL)) { + ph = pool_alloc_item_header(pp, cp, flags); + } + if (__predict_false(cp == NULL || ph == NULL)) { + if (cp != NULL) { + pool_allocator_free(pp, cp); + } + simple_lock(&pp->pr_slock); + return ENOMEM; + } + + simple_lock(&pp->pr_slock); + pool_prime_page(pp, cp, ph); + pp->pr_npagealloc++; + return 0; +} + +/* * Add N items to the pool. */ int pool_prime(struct pool *pp, int n) { - struct pool_item_header *ph = NULL; - caddr_t cp; int newpages; + int error = 0; simple_lock(&pp->pr_slock); newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; while (newpages-- > 0) { - simple_unlock(&pp->pr_slock); - cp = pool_allocator_alloc(pp, PR_NOWAIT); - if (__predict_true(cp != NULL)) - ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); - - if (__predict_false(cp == NULL || ph == NULL)) { - if (cp != NULL) - pool_allocator_free(pp, cp); - simple_lock(&pp->pr_slock); + error = pool_grow(pp, PR_NOWAIT); + if (error) { break; } - - simple_lock(&pp->pr_slock); - pool_prime_page(pp, cp, ph); - pp->pr_npagealloc++; pp->pr_minpages++; } @@ -1256,7 +1248,7 @@ pool_prime(struct pool *pp, int n) pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ simple_unlock(&pp->pr_slock); - return (0); + return error; } /* @@ -1361,34 +1353,15 @@ pool_prime_page(struct pool *pp, caddr_t static int pool_catchup(struct pool *pp) { - struct pool_item_header *ph = NULL; - caddr_t cp; int error = 0; while (POOL_NEEDS_CATCHUP(pp)) { - /* - * Call the page back-end allocator for more memory. - * - * XXX: We never wait, so should we bother unlocking - * the pool descriptor? - */ - simple_unlock(&pp->pr_slock); - cp = pool_allocator_alloc(pp, PR_NOWAIT); - if (__predict_true(cp != NULL)) - ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); - if (__predict_false(cp == NULL || ph == NULL)) { - if (cp != NULL) - pool_allocator_free(pp, cp); - error = ENOMEM; - simple_lock(&pp->pr_slock); + error = pool_grow(pp, PR_NOWAIT); + if (error) { break; } - simple_lock(&pp->pr_slock); - pool_prime_page(pp, cp, ph); - pp->pr_npagealloc++; } - - return (error); + return error; } static void @@ -1894,7 +1867,7 @@ pool_cache_destroy(struct pool_cache *pc simple_unlock(&pp->pr_slock); } -static __inline void * +static inline void * pcg_get(struct pool_cache_group *pcg, paddr_t *pap) { void *object; @@ -1913,7 +1886,7 @@ pcg_get(struct pool_cache_group *pcg, pa return (object); } -static __inline void +static inline void pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) { u_int idx; @@ -2019,6 +1992,10 @@ pool_cache_put_paddr(struct pool_cache * struct pool_cache_group *pcg; int s; + if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) { + goto destruct; + } + simple_lock(&pc->pc_slock); pcg = LIST_FIRST(&pc->pc_partgroups); @@ -2040,6 +2017,7 @@ pool_cache_put_paddr(struct pool_cache * pcg = pool_get(&pcgpool, PR_NOWAIT); splx(s); if (pcg == NULL) { +destruct: /* * Unable to allocate a cache group; destruct the object @@ -2185,23 +2163,42 @@ pool_cache_reclaim(struct pool_cache *pc void *pool_page_alloc(struct pool *, int); void pool_page_free(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_kmem_fullpage = { + pool_page_alloc, pool_page_free, 0, +}; +#else struct pool_allocator pool_allocator_kmem = { pool_page_alloc, pool_page_free, 0, }; +#endif void *pool_page_alloc_nointr(struct pool *, int); void pool_page_free_nointr(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_nointr_fullpage = { + pool_page_alloc_nointr, pool_page_free_nointr, 0, +}; +#else struct pool_allocator pool_allocator_nointr = { pool_page_alloc_nointr, pool_page_free_nointr, 0, }; +#endif #ifdef POOL_SUBPAGE void *pool_subpage_alloc(struct pool *, int); void pool_subpage_free(struct pool *, void *); -struct pool_allocator pool_allocator_kmem_subpage = { - pool_subpage_alloc, pool_subpage_free, 0, +struct pool_allocator pool_allocator_kmem = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, +}; + +void *pool_subpage_alloc_nointr(struct pool *, int); +void pool_subpage_free_nointr(struct pool *, void *); + +struct pool_allocator pool_allocator_nointr = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, }; #endif /* POOL_SUBPAGE */ @@ -2249,8 +2246,8 @@ pool_allocator_alloc(struct pool *org, i } /* - * Drain all pools, except "org", that use this - * allocator. We do this to reclaim VA space. + * Drain all pools, that use this allocator. + * We do this to reclaim VA space. * pa_alloc is responsible for waiting for * physical memory. * @@ -2271,8 +2268,6 @@ pool_allocator_alloc(struct pool *org, i do { TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list); TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); - if (pp == org) - continue; simple_unlock(&pa->pa_slock); freed = pool_reclaim(pp); simple_lock(&pa->pa_slock); @@ -2379,19 +2374,19 @@ pool_subpage_free(struct pool *pp, void /* We don't provide a real nointr allocator. Maybe later. */ void * -pool_page_alloc_nointr(struct pool *pp, int flags) +pool_subpage_alloc_nointr(struct pool *pp, int flags) { return (pool_subpage_alloc(pp, flags)); } void -pool_page_free_nointr(struct pool *pp, void *v) +pool_subpage_free_nointr(struct pool *pp, void *v) { pool_subpage_free(pp, v); } -#else +#endif /* POOL_SUBPAGE */ void * pool_page_alloc_nointr(struct pool *pp, int flags) { @@ -2406,4 +2401,3 @@ pool_page_free_nointr(struct pool *pp, v uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); } -#endif /* POOL_SUBPAGE */