Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.109 retrieving revision 1.116 diff -u -p -r1.109 -r1.116 --- src/sys/kern/subr_pool.c 2005/12/20 15:52:14 1.109 +++ src/sys/kern/subr_pool.c 2006/04/15 14:23:11 1.116 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.109 2005/12/20 15:52:14 christos Exp $ */ +/* $NetBSD: subr_pool.c,v 1.116 2006/04/15 14:23:11 simonb Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.109 2005/12/20 15:52:14 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.116 2006/04/15 14:23:11 simonb Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -183,6 +183,7 @@ static void pool_prime_page(struct pool struct pool_item_header *); static void pool_update_curpage(struct pool *); +static int pool_grow(struct pool *, int); void *pool_allocator_alloc(struct pool *, int); void pool_allocator_free(struct pool *, void *); @@ -214,7 +215,7 @@ struct pool_log { int pool_logsize = POOL_LOGSIZE; -static __inline void +static inline void pr_log(struct pool *pp, void *v, int action, const char *file, long line) { int n = pp->pr_curlogentry; @@ -267,7 +268,7 @@ pr_printlog(struct pool *pp, struct pool } } -static __inline void +static inline void pr_enter(struct pool *pp, const char *file, long line) { @@ -283,7 +284,7 @@ pr_enter(struct pool *pp, const char *fi pp->pr_entered_line = line; } -static __inline void +static inline void pr_leave(struct pool *pp) { @@ -296,7 +297,7 @@ pr_leave(struct pool *pp) pp->pr_entered_line = 0; } -static __inline void +static inline void pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) { @@ -312,7 +313,7 @@ pr_enter_check(struct pool *pp, void (*p #define pr_enter_check(pp, pr) #endif /* POOL_DIAGNOSTIC */ -static __inline int +static inline int pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, const void *v) { @@ -331,7 +332,7 @@ pr_item_notouch_index(const struct pool #define PR_INDEX_USED ((pool_item_freelist_t)-1) #define PR_INDEX_EOL ((pool_item_freelist_t)-2) -static __inline void +static inline void pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, void *obj) { @@ -343,7 +344,7 @@ pr_item_notouch_put(const struct pool *p ph->ph_firstfree = idx; } -static __inline void * +static inline void * pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) { int idx = ph->ph_firstfree; @@ -356,7 +357,7 @@ pr_item_notouch_get(const struct pool *p return ph->ph_page + ph->ph_off + idx * pp->pr_size; } -static __inline int +static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { if (a->ph_page < b->ph_page) @@ -373,7 +374,7 @@ SPLAY_GENERATE(phtree, pool_item_header, /* * Return the pool page header based on page address. */ -static __inline struct pool_item_header * +static inline struct pool_item_header * pr_find_pagehead(struct pool *pp, caddr_t page) { struct pool_item_header *ph, tmp; @@ -406,7 +407,7 @@ pr_pagelist_free(struct pool *pp, struct /* * Remove a page from the pool. */ -static __inline void +static inline void pr_rmpage(struct pool *pp, struct pool_item_header *ph, struct pool_pagelist *pq) { @@ -467,13 +468,27 @@ void pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, const char *wchan, struct pool_allocator *palloc) { - int off, slack; +#ifdef DEBUG + struct pool *pp1; +#endif size_t trysize, phsize; - int s; + int off, slack, s; KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >= PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1)); +#ifdef DEBUG + /* + * Check that the pool hasn't already been initialised and + * added to the list of all pools. + */ + LIST_FOREACH(pp1, &pool_head, pr_poollist) { + if (pp == pp1) + panic("pool_init: pool %s already initialised", + wchan); + } +#endif + #ifdef POOL_DIAGNOSTIC /* * Always log if POOL_DIAGNOSTIC is defined. @@ -482,35 +497,19 @@ pool_init(struct pool *pp, size_t size, flags |= PR_LOGGING; #endif -#ifdef POOL_SUBPAGE - /* - * XXX We don't provide a real `nointr' back-end - * yet; all sub-pages come from a kmem back-end. - * maybe some day... - */ - if (palloc == NULL) { - extern struct pool_allocator pool_allocator_kmem_subpage; - palloc = &pool_allocator_kmem_subpage; - } - /* - * We'll assume any user-specified back-end allocator - * will deal with sub-pages, or simply don't care. - */ -#else if (palloc == NULL) palloc = &pool_allocator_kmem; +#ifdef POOL_SUBPAGE + if (size > palloc->pa_pagesz) { + if (palloc == &pool_allocator_kmem) + palloc = &pool_allocator_kmem_fullpage; + else if (palloc == &pool_allocator_nointr) + palloc = &pool_allocator_nointr_fullpage; + } #endif /* POOL_SUBPAGE */ if ((palloc->pa_flags & PA_INITIALIZED) == 0) { - if (palloc->pa_pagesz == 0) { -#ifdef POOL_SUBPAGE - if (palloc == &pool_allocator_kmem) - palloc->pa_pagesz = PAGE_SIZE; - else - palloc->pa_pagesz = POOL_SUBPAGE; -#else + if (palloc->pa_pagesz == 0) palloc->pa_pagesz = PAGE_SIZE; -#endif /* POOL_SUBPAGE */ - } TAILQ_INIT(&palloc->pa_list); @@ -883,6 +882,8 @@ pool_get(struct pool *pp, int flags) * has no items in its bucket. */ if ((ph = pp->pr_curpage) == NULL) { + int error; + #ifdef DIAGNOSTIC if (pp->pr_nitems != 0) { simple_unlock(&pp->pr_slock); @@ -898,18 +899,9 @@ pool_get(struct pool *pp, int flags) * may block. */ pr_leave(pp); - simple_unlock(&pp->pr_slock); - v = pool_allocator_alloc(pp, flags); - if (__predict_true(v != NULL)) - ph = pool_alloc_item_header(pp, v, flags); - - if (__predict_false(v == NULL || ph == NULL)) { - if (v != NULL) - pool_allocator_free(pp, v); - - simple_lock(&pp->pr_slock); - pr_enter(pp, file, line); - + error = pool_grow(pp, flags); + pr_enter(pp, file, line); + if (error != 0) { /* * We were unable to allocate a page or item * header, but we released the lock during @@ -937,15 +929,8 @@ pool_get(struct pool *pp, int flags) pr_leave(pp); ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); pr_enter(pp, file, line); - goto startover; } - /* We have more memory; add it to the pool */ - simple_lock(&pp->pr_slock); - pr_enter(pp, file, line); - pool_prime_page(pp, v, ph); - pp->pr_npagealloc++; - /* Start the allocation process over. */ goto startover; } @@ -1033,6 +1018,7 @@ pool_get(struct pool *pp, int flags) } pp->pr_nget++; + pr_leave(pp); /* * If we have a low water mark and we are now below that low @@ -1046,7 +1032,6 @@ pool_get(struct pool *pp, int flags) */ } - pr_leave(pp); simple_unlock(&pp->pr_slock); return (v); } @@ -1220,35 +1205,56 @@ pool_put(struct pool *pp, void *v) #endif /* + * pool_grow: grow a pool by a page. + * + * => called with pool locked. + * => unlock and relock the pool. + * => return with pool locked. + */ + +static int +pool_grow(struct pool *pp, int flags) +{ + struct pool_item_header *ph = NULL; + char *cp; + + simple_unlock(&pp->pr_slock); + cp = pool_allocator_alloc(pp, flags); + if (__predict_true(cp != NULL)) { + ph = pool_alloc_item_header(pp, cp, flags); + } + if (__predict_false(cp == NULL || ph == NULL)) { + if (cp != NULL) { + pool_allocator_free(pp, cp); + } + simple_lock(&pp->pr_slock); + return ENOMEM; + } + + simple_lock(&pp->pr_slock); + pool_prime_page(pp, cp, ph); + pp->pr_npagealloc++; + return 0; +} + +/* * Add N items to the pool. */ int pool_prime(struct pool *pp, int n) { - struct pool_item_header *ph = NULL; - caddr_t cp; int newpages; + int error = 0; simple_lock(&pp->pr_slock); newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; while (newpages-- > 0) { - simple_unlock(&pp->pr_slock); - cp = pool_allocator_alloc(pp, PR_NOWAIT); - if (__predict_true(cp != NULL)) - ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); - - if (__predict_false(cp == NULL || ph == NULL)) { - if (cp != NULL) - pool_allocator_free(pp, cp); - simple_lock(&pp->pr_slock); + error = pool_grow(pp, PR_NOWAIT); + if (error) { break; } - - simple_lock(&pp->pr_slock); - pool_prime_page(pp, cp, ph); - pp->pr_npagealloc++; pp->pr_minpages++; } @@ -1256,7 +1262,7 @@ pool_prime(struct pool *pp, int n) pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ simple_unlock(&pp->pr_slock); - return (0); + return error; } /* @@ -1361,34 +1367,15 @@ pool_prime_page(struct pool *pp, caddr_t static int pool_catchup(struct pool *pp) { - struct pool_item_header *ph = NULL; - caddr_t cp; int error = 0; while (POOL_NEEDS_CATCHUP(pp)) { - /* - * Call the page back-end allocator for more memory. - * - * XXX: We never wait, so should we bother unlocking - * the pool descriptor? - */ - simple_unlock(&pp->pr_slock); - cp = pool_allocator_alloc(pp, PR_NOWAIT); - if (__predict_true(cp != NULL)) - ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); - if (__predict_false(cp == NULL || ph == NULL)) { - if (cp != NULL) - pool_allocator_free(pp, cp); - error = ENOMEM; - simple_lock(&pp->pr_slock); + error = pool_grow(pp, PR_NOWAIT); + if (error) { break; } - simple_lock(&pp->pr_slock); - pool_prime_page(pp, cp, ph); - pp->pr_npagealloc++; } - - return (error); + return error; } static void @@ -1556,7 +1543,8 @@ pool_drain(void *arg) drainpp = LIST_NEXT(pp, pr_poollist); } simple_unlock(&pool_head_slock); - pool_reclaim(pp); + if (pp) + pool_reclaim(pp); splx(s); } @@ -1894,7 +1882,7 @@ pool_cache_destroy(struct pool_cache *pc simple_unlock(&pp->pr_slock); } -static __inline void * +static inline void * pcg_get(struct pool_cache_group *pcg, paddr_t *pap) { void *object; @@ -1913,7 +1901,7 @@ pcg_get(struct pool_cache_group *pcg, pa return (object); } -static __inline void +static inline void pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) { u_int idx; @@ -2190,23 +2178,42 @@ pool_cache_reclaim(struct pool_cache *pc void *pool_page_alloc(struct pool *, int); void pool_page_free(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_kmem_fullpage = { + pool_page_alloc, pool_page_free, 0, +}; +#else struct pool_allocator pool_allocator_kmem = { pool_page_alloc, pool_page_free, 0, }; +#endif void *pool_page_alloc_nointr(struct pool *, int); void pool_page_free_nointr(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_nointr_fullpage = { + pool_page_alloc_nointr, pool_page_free_nointr, 0, +}; +#else struct pool_allocator pool_allocator_nointr = { pool_page_alloc_nointr, pool_page_free_nointr, 0, }; +#endif #ifdef POOL_SUBPAGE void *pool_subpage_alloc(struct pool *, int); void pool_subpage_free(struct pool *, void *); -struct pool_allocator pool_allocator_kmem_subpage = { - pool_subpage_alloc, pool_subpage_free, 0, +struct pool_allocator pool_allocator_kmem = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, +}; + +void *pool_subpage_alloc_nointr(struct pool *, int); +void pool_subpage_free_nointr(struct pool *, void *); + +struct pool_allocator pool_allocator_nointr = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, }; #endif /* POOL_SUBPAGE */ @@ -2382,19 +2389,19 @@ pool_subpage_free(struct pool *pp, void /* We don't provide a real nointr allocator. Maybe later. */ void * -pool_page_alloc_nointr(struct pool *pp, int flags) +pool_subpage_alloc_nointr(struct pool *pp, int flags) { return (pool_subpage_alloc(pp, flags)); } void -pool_page_free_nointr(struct pool *pp, void *v) +pool_subpage_free_nointr(struct pool *pp, void *v) { pool_subpage_free(pp, v); } -#else +#endif /* POOL_SUBPAGE */ void * pool_page_alloc_nointr(struct pool *pp, int flags) { @@ -2409,4 +2416,3 @@ pool_page_free_nointr(struct pool *pp, v uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); } -#endif /* POOL_SUBPAGE */