Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.87.2.4 retrieving revision 1.112.6.1 diff -u -p -r1.87.2.4 -r1.112.6.1 --- src/sys/kern/subr_pool.c 2005/01/17 19:32:25 1.87.2.4 +++ src/sys/kern/subr_pool.c 2006/03/28 09:42:26 1.112.6.1 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.87.2.4 2005/01/17 19:32:25 skrll Exp $ */ +/* $NetBSD: subr_pool.c,v 1.112.6.1 2006/03/28 09:42:26 tron Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.87.2.4 2005/01/17 19:32:25 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.112.6.1 2006/03/28 09:42:26 tron Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -70,7 +70,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, */ /* List of all pools */ -TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); +LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head); /* Private pool for page header structures */ #define PHPOOL_MAX 8 @@ -112,7 +112,7 @@ struct pool_item_header { union { /* !PR_NOTOUCH */ struct { - TAILQ_HEAD(, pool_item) + LIST_HEAD(, pool_item) phu_itemlist; /* chunk list for this page */ } phu_normal; /* PR_NOTOUCH */ @@ -139,7 +139,7 @@ struct pool_item { #endif #define PI_MAGIC 0xdeadbeefU /* Other entries use only this list entry */ - TAILQ_ENTRY(pool_item) pi_list; + LIST_ENTRY(pool_item) pi_list; }; #define POOL_NEEDS_CATCHUP(pp) \ @@ -174,13 +174,16 @@ struct pool_item { /* The cache group pool. */ static struct pool pcgpool; -static void pool_cache_reclaim(struct pool_cache *); +static void pool_cache_reclaim(struct pool_cache *, struct pool_pagelist *, + struct pool_cache_grouplist *); +static void pcg_grouplist_free(struct pool_cache_grouplist *); static int pool_catchup(struct pool *); static void pool_prime_page(struct pool *, caddr_t, struct pool_item_header *); static void pool_update_curpage(struct pool *); +static int pool_grow(struct pool *, int); void *pool_allocator_alloc(struct pool *, int); void pool_allocator_free(struct pool *, void *); @@ -212,7 +215,7 @@ struct pool_log { int pool_logsize = POOL_LOGSIZE; -static __inline void +static inline void pr_log(struct pool *pp, void *v, int action, const char *file, long line) { int n = pp->pr_curlogentry; @@ -265,7 +268,7 @@ pr_printlog(struct pool *pp, struct pool } } -static __inline void +static inline void pr_enter(struct pool *pp, const char *file, long line) { @@ -281,7 +284,7 @@ pr_enter(struct pool *pp, const char *fi pp->pr_entered_line = line; } -static __inline void +static inline void pr_leave(struct pool *pp) { @@ -294,7 +297,7 @@ pr_leave(struct pool *pp) pp->pr_entered_line = 0; } -static __inline void +static inline void pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) { @@ -310,7 +313,7 @@ pr_enter_check(struct pool *pp, void (*p #define pr_enter_check(pp, pr) #endif /* POOL_DIAGNOSTIC */ -static __inline int +static inline int pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, const void *v) { @@ -329,7 +332,7 @@ pr_item_notouch_index(const struct pool #define PR_INDEX_USED ((pool_item_freelist_t)-1) #define PR_INDEX_EOL ((pool_item_freelist_t)-2) -static __inline void +static inline void pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, void *obj) { @@ -341,7 +344,7 @@ pr_item_notouch_put(const struct pool *p ph->ph_firstfree = idx; } -static __inline void * +static inline void * pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) { int idx = ph->ph_firstfree; @@ -354,7 +357,7 @@ pr_item_notouch_get(const struct pool *p return ph->ph_page + ph->ph_off + idx * pp->pr_size; } -static __inline int +static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { if (a->ph_page < b->ph_page) @@ -371,7 +374,7 @@ SPLAY_GENERATE(phtree, pool_item_header, /* * Return the pool page header based on page address. */ -static __inline struct pool_item_header * +static inline struct pool_item_header * pr_find_pagehead(struct pool *pp, caddr_t page) { struct pool_item_header *ph, tmp; @@ -384,16 +387,32 @@ pr_find_pagehead(struct pool *pp, caddr_ return ph; } +static void +pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq) +{ + struct pool_item_header *ph; + int s; + + while ((ph = LIST_FIRST(pq)) != NULL) { + LIST_REMOVE(ph, ph_pagelist); + pool_allocator_free(pp, ph->ph_page); + if ((pp->pr_roflags & PR_PHINPAGE) == 0) { + s = splvm(); + pool_put(pp->pr_phpool, ph); + splx(s); + } + } +} + /* * Remove a page from the pool. */ -static __inline void +static inline void pr_rmpage(struct pool *pp, struct pool_item_header *ph, struct pool_pagelist *pq) { - int s; - LOCK_ASSERT(!simple_lock_held(&pp->pr_slock) || pq != NULL); + LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); /* * If the page was idle, decrement the idle page count. @@ -411,21 +430,13 @@ pr_rmpage(struct pool *pp, struct pool_i pp->pr_nitems -= pp->pr_itemsperpage; /* - * Unlink a page from the pool and release it (or queue it for release). + * Unlink the page from the pool and queue it for release. */ LIST_REMOVE(ph, ph_pagelist); if ((pp->pr_roflags & PR_PHINPAGE) == 0) SPLAY_REMOVE(phtree, &pp->pr_phtree, ph); - if (pq) { - LIST_INSERT_HEAD(pq, ph, ph_pagelist); - } else { - pool_allocator_free(pp, ph->ph_page); - if ((pp->pr_roflags & PR_PHINPAGE) == 0) { - s = splvm(); - pool_put(pp->pr_phpool, ph); - splx(s); - } - } + LIST_INSERT_HEAD(pq, ph, ph_pagelist); + pp->pr_npages--; pp->pr_npagefree++; @@ -472,35 +483,19 @@ pool_init(struct pool *pp, size_t size, flags |= PR_LOGGING; #endif -#ifdef POOL_SUBPAGE - /* - * XXX We don't provide a real `nointr' back-end - * yet; all sub-pages come from a kmem back-end. - * maybe some day... - */ - if (palloc == NULL) { - extern struct pool_allocator pool_allocator_kmem_subpage; - palloc = &pool_allocator_kmem_subpage; - } - /* - * We'll assume any user-specified back-end allocator - * will deal with sub-pages, or simply don't care. - */ -#else if (palloc == NULL) palloc = &pool_allocator_kmem; +#ifdef POOL_SUBPAGE + if (size > palloc->pa_pagesz) { + if (palloc == &pool_allocator_kmem) + palloc = &pool_allocator_kmem_fullpage; + else if (palloc == &pool_allocator_nointr) + palloc = &pool_allocator_nointr_fullpage; + } #endif /* POOL_SUBPAGE */ if ((palloc->pa_flags & PA_INITIALIZED) == 0) { - if (palloc->pa_pagesz == 0) { -#ifdef POOL_SUBPAGE - if (palloc == &pool_allocator_kmem) - palloc->pa_pagesz = PAGE_SIZE; - else - palloc->pa_pagesz = POOL_SUBPAGE; -#else + if (palloc->pa_pagesz == 0) palloc->pa_pagesz = PAGE_SIZE; -#endif /* POOL_SUBPAGE */ - } TAILQ_INIT(&palloc->pa_list); @@ -529,7 +524,7 @@ pool_init(struct pool *pp, size_t size, LIST_INIT(&pp->pr_emptypages); LIST_INIT(&pp->pr_fullpages); LIST_INIT(&pp->pr_partpages); - TAILQ_INIT(&pp->pr_cachelist); + LIST_INIT(&pp->pr_cachelist); pp->pr_curpage = NULL; pp->pr_npages = 0; pp->pr_minitems = 0; @@ -679,7 +674,7 @@ pool_init(struct pool *pp, size_t size, /* Insert into the list of all pools. */ simple_lock(&pool_head_slock); - TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); + LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); simple_unlock(&pool_head_slock); /* Insert this into the list of pools using this allocator. */ @@ -696,20 +691,28 @@ pool_init(struct pool *pp, size_t size, void pool_destroy(struct pool *pp) { + struct pool_pagelist pq; struct pool_item_header *ph; - struct pool_cache *pc; int s; - /* Locking order: pool_allocator -> pool */ + /* Remove from global pool list */ + simple_lock(&pool_head_slock); + LIST_REMOVE(pp, pr_poollist); + if (drainpp == pp) + drainpp = NULL; + simple_unlock(&pool_head_slock); + + /* Remove this pool from its allocator's list of pools. */ s = splvm(); simple_lock(&pp->pr_alloc->pa_slock); TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list); simple_unlock(&pp->pr_alloc->pa_slock); splx(s); - /* Destroy all caches for this pool. */ - while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL) - pool_cache_destroy(pc); + s = splvm(); + simple_lock(&pp->pr_slock); + + KASSERT(LIST_EMPTY(&pp->pr_cachelist)); #ifdef DIAGNOSTIC if (pp->pr_nout != 0) { @@ -719,19 +722,18 @@ pool_destroy(struct pool *pp) } #endif - /* Remove all pages */ - while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) - pr_rmpage(pp, ph, NULL); KASSERT(LIST_EMPTY(&pp->pr_fullpages)); KASSERT(LIST_EMPTY(&pp->pr_partpages)); - /* Remove from global pool list */ - simple_lock(&pool_head_slock); - TAILQ_REMOVE(&pool_head, pp, pr_poollist); - if (drainpp == pp) { - drainpp = NULL; - } - simple_unlock(&pool_head_slock); + /* Remove all pages */ + LIST_INIT(&pq); + while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) + pr_rmpage(pp, ph, &pq); + + simple_unlock(&pp->pr_slock); + splx(s); + + pr_pagelist_free(pp, &pq); #ifdef POOL_DIAGNOSTIC if ((pp->pr_roflags & PR_LOGGING) != 0) @@ -793,11 +795,12 @@ pool_get(struct pool *pp, int flags) (flags & PR_WAITOK) != 0)) panic("pool_get: %s: must have NOWAIT", pp->pr_wchan); +#endif /* DIAGNOSTIC */ #ifdef LOCKDEBUG if (flags & PR_WAITOK) simple_lock_only_held(NULL, "pool_get(PR_WAITOK)"); + SCHED_ASSERT_UNLOCKED(); #endif -#endif /* DIAGNOSTIC */ simple_lock(&pp->pr_slock); pr_enter(pp, file, line); @@ -865,6 +868,8 @@ pool_get(struct pool *pp, int flags) * has no items in its bucket. */ if ((ph = pp->pr_curpage) == NULL) { + int error; + #ifdef DIAGNOSTIC if (pp->pr_nitems != 0) { simple_unlock(&pp->pr_slock); @@ -880,18 +885,9 @@ pool_get(struct pool *pp, int flags) * may block. */ pr_leave(pp); - simple_unlock(&pp->pr_slock); - v = pool_allocator_alloc(pp, flags); - if (__predict_true(v != NULL)) - ph = pool_alloc_item_header(pp, v, flags); - - if (__predict_false(v == NULL || ph == NULL)) { - if (v != NULL) - pool_allocator_free(pp, v); - - simple_lock(&pp->pr_slock); - pr_enter(pp, file, line); - + error = pool_grow(pp, flags); + pr_enter(pp, file, line); + if (error != 0) { /* * We were unable to allocate a page or item * header, but we released the lock during @@ -911,23 +907,16 @@ pool_get(struct pool *pp, int flags) /* * Wait for items to be returned to this pool. * - * XXX: maybe we should wake up once a second and - * try again? + * wake up once a second and try again, + * as the check in pool_cache_put_paddr() is racy. */ pp->pr_flags |= PR_WANTED; /* PA_WANTED is already set on the allocator. */ pr_leave(pp); - ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock); + ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); pr_enter(pp, file, line); - goto startover; } - /* We have more memory; add it to the pool */ - simple_lock(&pp->pr_slock); - pr_enter(pp, file, line); - pool_prime_page(pp, v, ph); - pp->pr_npagealloc++; - /* Start the allocation process over. */ goto startover; } @@ -944,7 +933,7 @@ pool_get(struct pool *pp, int flags) pr_log(pp, v, PRLOG_GET, file, line); #endif } else { - v = pi = TAILQ_FIRST(&ph->ph_itemlist); + v = pi = LIST_FIRST(&ph->ph_itemlist); if (__predict_false(v == NULL)) { pr_leave(pp); simple_unlock(&pp->pr_slock); @@ -976,7 +965,7 @@ pool_get(struct pool *pp, int flags) /* * Remove from item list. */ - TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); + LIST_REMOVE(pi, pi_list); } pp->pr_nitems--; pp->pr_nout++; @@ -998,7 +987,7 @@ pool_get(struct pool *pp, int flags) if (ph->ph_nmissing == pp->pr_itemsperpage) { #ifdef DIAGNOSTIC if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 && - !TAILQ_EMPTY(&ph->ph_itemlist))) { + !LIST_EMPTY(&ph->ph_itemlist))) { pr_leave(pp); simple_unlock(&pp->pr_slock); panic("pool_get: %s: nmissing inconsistent", @@ -1015,6 +1004,7 @@ pool_get(struct pool *pp, int flags) } pp->pr_nget++; + pr_leave(pp); /* * If we have a low water mark and we are now below that low @@ -1028,7 +1018,6 @@ pool_get(struct pool *pp, int flags) */ } - pr_leave(pp); simple_unlock(&pp->pr_slock); return (v); } @@ -1037,7 +1026,7 @@ pool_get(struct pool *pp, int flags) * Internal version of pool_put(). Pool is already locked/entered. */ static void -pool_do_put(struct pool *pp, void *v) +pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq) { struct pool_item *pi = v; struct pool_item_header *ph; @@ -1045,6 +1034,7 @@ pool_do_put(struct pool *pp, void *v) int s; LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); + SCHED_ASSERT_UNLOCKED(); page = (caddr_t)((u_long)v & pp->pr_alloc->pa_pagemask); @@ -1087,7 +1077,7 @@ pool_do_put(struct pool *pp, void *v) } #endif - TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); + LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); } KDASSERT(ph->ph_nmissing != 0); ph->ph_nmissing--; @@ -1125,9 +1115,7 @@ pool_do_put(struct pool *pp, void *v) if (pp->pr_npages > pp->pr_minpages && (pp->pr_npages > pp->pr_maxpages || (pp->pr_alloc->pa_flags & PA_WANT) != 0)) { - simple_unlock(&pp->pr_slock); - pr_rmpage(pp, ph, NULL); - simple_lock(&pp->pr_slock); + pr_rmpage(pp, ph, pq); } else { LIST_REMOVE(ph, ph_pagelist); LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); @@ -1165,16 +1153,21 @@ pool_do_put(struct pool *pp, void *v) void _pool_put(struct pool *pp, void *v, const char *file, long line) { + struct pool_pagelist pq; + + LIST_INIT(&pq); simple_lock(&pp->pr_slock); pr_enter(pp, file, line); pr_log(pp, v, PRLOG_PUT, file, line); - pool_do_put(pp, v); + pool_do_put(pp, v, &pq); pr_leave(pp); simple_unlock(&pp->pr_slock); + + pr_pagelist_free(pp, &pq); } #undef pool_put #endif /* POOL_DIAGNOSTIC */ @@ -1182,12 +1175,15 @@ _pool_put(struct pool *pp, void *v, cons void pool_put(struct pool *pp, void *v) { + struct pool_pagelist pq; - simple_lock(&pp->pr_slock); - - pool_do_put(pp, v); + LIST_INIT(&pq); + simple_lock(&pp->pr_slock); + pool_do_put(pp, v, &pq); simple_unlock(&pp->pr_slock); + + pr_pagelist_free(pp, &pq); } #ifdef POOL_DIAGNOSTIC @@ -1195,35 +1191,57 @@ pool_put(struct pool *pp, void *v) #endif /* + * pool_grow: grow a pool by a page. + * + * => called with pool locked. + * => unlock and relock the pool. + * => return with pool locked. + */ + +static int +pool_grow(struct pool *pp, int flags) +{ + struct pool_item_header *ph = NULL; + char *cp; + + simple_unlock(&pp->pr_slock); + cp = pool_allocator_alloc(pp, flags); + if (__predict_true(cp != NULL)) { + ph = pool_alloc_item_header(pp, cp, flags); + } + if (__predict_false(cp == NULL || ph == NULL)) { + if (cp != NULL) { + pool_allocator_free(pp, cp); + } + simple_lock(&pp->pr_slock); + return ENOMEM; + } + + simple_lock(&pp->pr_slock); + pool_prime_page(pp, cp, ph); + pp->pr_npagealloc++; + pp->pr_minpages++; + return 0; +} + +/* * Add N items to the pool. */ int pool_prime(struct pool *pp, int n) { - struct pool_item_header *ph = NULL; - caddr_t cp; int newpages; + int error = 0; simple_lock(&pp->pr_slock); newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; while (newpages-- > 0) { - simple_unlock(&pp->pr_slock); - cp = pool_allocator_alloc(pp, PR_NOWAIT); - if (__predict_true(cp != NULL)) - ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); - - if (__predict_false(cp == NULL || ph == NULL)) { - if (cp != NULL) - pool_allocator_free(pp, cp); - simple_lock(&pp->pr_slock); + error = pool_grow(pp, PR_NOWAIT); + if (error) { break; } - - simple_lock(&pp->pr_slock); - pool_prime_page(pp, cp, ph); - pp->pr_npagealloc++; pp->pr_minpages++; } @@ -1231,7 +1249,7 @@ pool_prime(struct pool *pp, int n) pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ simple_unlock(&pp->pr_slock); - return (0); + return error; } /* @@ -1260,7 +1278,7 @@ pool_prime_page(struct pool *pp, caddr_t * Insert page header. */ LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); - TAILQ_INIT(&ph->ph_itemlist); + LIST_INIT(&ph->ph_itemlist); ph->ph_page = storage; ph->ph_nmissing = 0; s = splclock(); @@ -1306,7 +1324,7 @@ pool_prime_page(struct pool *pp, caddr_t KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); /* Insert on page list */ - TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); + LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); #ifdef DIAGNOSTIC pi->pi_magic = PI_MAGIC; #endif @@ -1336,34 +1354,15 @@ pool_prime_page(struct pool *pp, caddr_t static int pool_catchup(struct pool *pp) { - struct pool_item_header *ph = NULL; - caddr_t cp; int error = 0; while (POOL_NEEDS_CATCHUP(pp)) { - /* - * Call the page back-end allocator for more memory. - * - * XXX: We never wait, so should we bother unlocking - * the pool descriptor? - */ - simple_unlock(&pp->pr_slock); - cp = pool_allocator_alloc(pp, PR_NOWAIT); - if (__predict_true(cp != NULL)) - ph = pool_alloc_item_header(pp, cp, PR_NOWAIT); - if (__predict_false(cp == NULL || ph == NULL)) { - if (cp != NULL) - pool_allocator_free(pp, cp); - error = ENOMEM; - simple_lock(&pp->pr_slock); + error = pool_grow(pp, PR_NOWAIT); + if (error) { break; } - simple_lock(&pp->pr_slock); - pool_prime_page(pp, cp, ph); - pp->pr_npagealloc++; } - - return (error); + return error; } static void @@ -1447,9 +1446,9 @@ pool_reclaim(struct pool *pp) { struct pool_item_header *ph, *phnext; struct pool_cache *pc; - struct timeval curtime; struct pool_pagelist pq; - struct timeval diff; + struct pool_cache_grouplist pcgl; + struct timeval curtime, diff; int s; if (pp->pr_drain_hook != NULL) { @@ -1464,12 +1463,13 @@ pool_reclaim(struct pool *pp) pr_enter(pp, file, line); LIST_INIT(&pq); + LIST_INIT(&pcgl); /* * Reclaim items from the pool's caches. */ - TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) - pool_cache_reclaim(pc); + LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) + pool_cache_reclaim(pc, &pq, &pcgl); s = splclock(); curtime = mono_time; @@ -1500,20 +1500,11 @@ pool_reclaim(struct pool *pp) pr_leave(pp); simple_unlock(&pp->pr_slock); - if (LIST_EMPTY(&pq)) - return (0); - - while ((ph = LIST_FIRST(&pq)) != NULL) { - LIST_REMOVE(ph, ph_pagelist); - pool_allocator_free(pp, ph->ph_page); - if (pp->pr_roflags & PR_PHINPAGE) { - continue; - } - s = splvm(); - pool_put(pp->pr_phpool, ph); - splx(s); - } + if (LIST_EMPTY(&pq) && LIST_EMPTY(&pcgl)) + return 0; + pr_pagelist_free(pp, &pq); + pcg_grouplist_free(&pcgl); return (1); } @@ -1532,11 +1523,11 @@ pool_drain(void *arg) s = splvm(); simple_lock(&pool_head_slock); if (drainpp == NULL) { - drainpp = TAILQ_FIRST(&pool_head); + drainpp = LIST_FIRST(&pool_head); } if (drainpp) { pp = drainpp; - drainpp = TAILQ_NEXT(pp, pr_poollist); + drainpp = LIST_NEXT(pp, pr_poollist); } simple_unlock(&pool_head_slock); pool_reclaim(pp); @@ -1564,9 +1555,24 @@ pool_print(struct pool *pp, const char * } void +pool_printall(const char *modif, void (*pr)(const char *, ...)) +{ + struct pool *pp; + + if (simple_lock_try(&pool_head_slock) == 0) { + (*pr)("WARNING: pool_head_slock is locked\n"); + } else { + simple_unlock(&pool_head_slock); + } + + LIST_FOREACH(pp, &pool_head, pr_poollist) { + pool_printit(pp, modif, pr); + } +} + +void pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) { - int didlock = 0; if (pp == NULL) { (*pr)("Must specify a pool to print.\n"); @@ -1585,12 +1591,9 @@ pool_printit(struct pool *pp, const char if (simple_lock_try(&pp->pr_slock) == 0) (*pr)("WARNING: pool %s is locked\n", pp->pr_wchan); else - didlock = 1; + simple_unlock(&pp->pr_slock); pool_print1(pp, modif, pr); - - if (didlock) - simple_unlock(&pp->pr_slock); } static void @@ -1609,7 +1612,7 @@ pool_print_pagelist(struct pool *pp, str (u_long)ph->ph_time.tv_usec); #ifdef DIAGNOSTIC if (!(pp->pr_roflags & PR_NOTOUCH)) { - TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) { + LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { if (pi->pi_magic != PI_MAGIC) { (*pr)("\t\t\titem %p, magic 0x%x\n", pi, pi->pi_magic); @@ -1684,27 +1687,39 @@ pool_print1(struct pool *pp, const char if (print_cache == 0) goto skip_cache; - TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) { - (*pr)("\tcache %p: allocfrom %p freeto %p\n", pc, - pc->pc_allocfrom, pc->pc_freeto); +#define PR_GROUPLIST(pcg) \ + (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ + for (i = 0; i < PCG_NOBJECTS; i++) { \ + if (pcg->pcg_objects[i].pcgo_pa != \ + POOL_PADDR_INVALID) { \ + (*pr)("\t\t\t%p, 0x%llx\n", \ + pcg->pcg_objects[i].pcgo_va, \ + (unsigned long long) \ + pcg->pcg_objects[i].pcgo_pa); \ + } else { \ + (*pr)("\t\t\t%p\n", \ + pcg->pcg_objects[i].pcgo_va); \ + } \ + } + + LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) { + (*pr)("\tcache %p\n", pc); (*pr)("\t hits %lu misses %lu ngroups %lu nitems %lu\n", pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems); - TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) { - (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); - for (i = 0; i < PCG_NOBJECTS; i++) { - if (pcg->pcg_objects[i].pcgo_pa != - POOL_PADDR_INVALID) { - (*pr)("\t\t\t%p, 0x%llx\n", - pcg->pcg_objects[i].pcgo_va, - (unsigned long long) - pcg->pcg_objects[i].pcgo_pa); - } else { - (*pr)("\t\t\t%p\n", - pcg->pcg_objects[i].pcgo_va); - } - } + (*pr)("\t full groups:\n"); + LIST_FOREACH(pcg, &pc->pc_fullgroups, pcg_list) { + PR_GROUPLIST(pcg); + } + (*pr)("\t partial groups:\n"); + LIST_FOREACH(pcg, &pc->pc_partgroups, pcg_list) { + PR_GROUPLIST(pcg); + } + (*pr)("\t empty groups:\n"); + LIST_FOREACH(pcg, &pc->pc_emptygroups, pcg_list) { + PR_GROUPLIST(pcg); } } +#undef PR_GROUPLIST skip_cache: pr_enter_check(pp, pr); @@ -1732,9 +1747,9 @@ pool_chk_page(struct pool *pp, const cha if ((pp->pr_roflags & PR_NOTOUCH) != 0) return 0; - for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0; + for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0; pi != NULL; - pi = TAILQ_NEXT(pi,pi_list), n++) { + pi = LIST_NEXT(pi,pi_list), n++) { #ifdef DIAGNOSTIC if (pi->pi_magic != PI_MAGIC) { @@ -1811,11 +1826,11 @@ pool_cache_init(struct pool_cache *pc, s void *arg) { - TAILQ_INIT(&pc->pc_grouplist); + LIST_INIT(&pc->pc_emptygroups); + LIST_INIT(&pc->pc_fullgroups); + LIST_INIT(&pc->pc_partgroups); simple_lock_init(&pc->pc_slock); - pc->pc_allocfrom = NULL; - pc->pc_freeto = NULL; pc->pc_pool = pp; pc->pc_ctor = ctor; @@ -1830,7 +1845,7 @@ pool_cache_init(struct pool_cache *pc, s pc->pc_nitems = 0; simple_lock(&pp->pr_slock); - TAILQ_INSERT_TAIL(&pp->pr_cachelist, pc, pc_poollist); + LIST_INSERT_HEAD(&pp->pr_cachelist, pc, pc_poollist); simple_unlock(&pp->pr_slock); } @@ -1849,11 +1864,11 @@ pool_cache_destroy(struct pool_cache *pc /* ...and remove it from the pool's cache list. */ simple_lock(&pp->pr_slock); - TAILQ_REMOVE(&pp->pr_cachelist, pc, pc_poollist); + LIST_REMOVE(pc, pc_poollist); simple_unlock(&pp->pr_slock); } -static __inline void * +static inline void * pcg_get(struct pool_cache_group *pcg, paddr_t *pap) { void *object; @@ -1872,7 +1887,7 @@ pcg_get(struct pool_cache_group *pcg, pa return (object); } -static __inline void +static inline void pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) { u_int idx; @@ -1885,6 +1900,20 @@ pcg_put(struct pool_cache_group *pcg, vo pcg->pcg_objects[idx].pcgo_pa = pa; } +static void +pcg_grouplist_free(struct pool_cache_grouplist *pcgl) +{ + struct pool_cache_group *pcg; + int s; + + s = splvm(); + while ((pcg = LIST_FIRST(pcgl)) != NULL) { + LIST_REMOVE(pcg, pcg_list); + pool_put(&pcgpool, pcg); + } + splx(s); +} + /* * pool_cache_get{,_paddr}: * @@ -1904,13 +1933,15 @@ pool_cache_get_paddr(struct pool_cache * simple_lock(&pc->pc_slock); - if ((pcg = pc->pc_allocfrom) == NULL) { - TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) { - if (pcg->pcg_avail != 0) { - pc->pc_allocfrom = pcg; - goto have_group; - } + pcg = LIST_FIRST(&pc->pc_partgroups); + if (pcg == NULL) { + pcg = LIST_FIRST(&pc->pc_fullgroups); + if (pcg != NULL) { + LIST_REMOVE(pcg, pcg_list); + LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list); } + } + if (pcg == NULL) { /* * No groups with any available objects. Allocate @@ -1937,14 +1968,14 @@ pool_cache_get_paddr(struct pool_cache * return (object); } - have_group: pc->pc_hits++; pc->pc_nitems--; object = pcg_get(pcg, pap); - if (pcg->pcg_avail == 0) - pc->pc_allocfrom = NULL; - + if (pcg->pcg_avail == 0) { + LIST_REMOVE(pcg, pcg_list); + LIST_INSERT_HEAD(&pc->pc_emptygroups, pcg, pcg_list); + } simple_unlock(&pc->pc_slock); return (object); @@ -1962,15 +1993,21 @@ pool_cache_put_paddr(struct pool_cache * struct pool_cache_group *pcg; int s; + if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) { + goto destruct; + } + simple_lock(&pc->pc_slock); - if ((pcg = pc->pc_freeto) == NULL) { - TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) { - if (pcg->pcg_avail != PCG_NOBJECTS) { - pc->pc_freeto = pcg; - goto have_group; - } + pcg = LIST_FIRST(&pc->pc_partgroups); + if (pcg == NULL) { + pcg = LIST_FIRST(&pc->pc_emptygroups); + if (pcg != NULL) { + LIST_REMOVE(pcg, pcg_list); + LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list); } + } + if (pcg == NULL) { /* * No empty groups to free the object to. Attempt to @@ -1980,31 +2017,29 @@ pool_cache_put_paddr(struct pool_cache * s = splvm(); pcg = pool_get(&pcgpool, PR_NOWAIT); splx(s); - if (pcg != NULL) { - memset(pcg, 0, sizeof(*pcg)); - simple_lock(&pc->pc_slock); - pc->pc_ngroups++; - TAILQ_INSERT_TAIL(&pc->pc_grouplist, pcg, pcg_list); - if (pc->pc_freeto == NULL) - pc->pc_freeto = pcg; - goto have_group; - } + if (pcg == NULL) { +destruct: - /* - * Unable to allocate a cache group; destruct the object - * and free it back to the pool. - */ - pool_cache_destruct_object(pc, object); - return; + /* + * Unable to allocate a cache group; destruct the object + * and free it back to the pool. + */ + pool_cache_destruct_object(pc, object); + return; + } + memset(pcg, 0, sizeof(*pcg)); + simple_lock(&pc->pc_slock); + pc->pc_ngroups++; + LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list); } - have_group: pc->pc_nitems++; pcg_put(pcg, object, pa); - if (pcg->pcg_avail == PCG_NOBJECTS) - pc->pc_freeto = NULL; - + if (pcg->pcg_avail == PCG_NOBJECTS) { + LIST_REMOVE(pcg, pcg_list); + LIST_INSERT_HEAD(&pc->pc_fullgroups, pcg, pcg_list); + } simple_unlock(&pc->pc_slock); } @@ -2023,44 +2058,45 @@ pool_cache_destruct_object(struct pool_c pool_put(pc->pc_pool, object); } -/* - * pool_cache_do_invalidate: - * - * This internal function implements pool_cache_invalidate() and - * pool_cache_reclaim(). - */ static void -pool_cache_do_invalidate(struct pool_cache *pc, int free_groups, - void (*putit)(struct pool *, void *)) +pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl, + struct pool_cache *pc, struct pool_pagelist *pq, + struct pool_cache_grouplist *pcgdl) { struct pool_cache_group *pcg, *npcg; void *object; - int s; - for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL; - pcg = npcg) { - npcg = TAILQ_NEXT(pcg, pcg_list); + for (pcg = LIST_FIRST(pcgsl); pcg != NULL; pcg = npcg) { + npcg = LIST_NEXT(pcg, pcg_list); while (pcg->pcg_avail != 0) { pc->pc_nitems--; object = pcg_get(pcg, NULL); - if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg) - pc->pc_allocfrom = NULL; if (pc->pc_dtor != NULL) (*pc->pc_dtor)(pc->pc_arg, object); - (*putit)(pc->pc_pool, object); - } - if (free_groups) { - pc->pc_ngroups--; - TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list); - if (pc->pc_freeto == pcg) - pc->pc_freeto = NULL; - s = splvm(); - pool_put(&pcgpool, pcg); - splx(s); + pool_do_put(pc->pc_pool, object, pq); } + pc->pc_ngroups--; + LIST_REMOVE(pcg, pcg_list); + LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); } } +static void +pool_do_cache_invalidate(struct pool_cache *pc, struct pool_pagelist *pq, + struct pool_cache_grouplist *pcgl) +{ + + LOCK_ASSERT(simple_lock_held(&pc->pc_slock)); + LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock)); + + pool_do_cache_invalidate_grouplist(&pc->pc_fullgroups, pc, pq, pcgl); + pool_do_cache_invalidate_grouplist(&pc->pc_partgroups, pc, pq, pcgl); + + KASSERT(LIST_EMPTY(&pc->pc_partgroups)); + KASSERT(LIST_EMPTY(&pc->pc_fullgroups)); + KASSERT(pc->pc_nitems == 0); +} + /* * pool_cache_invalidate: * @@ -2070,10 +2106,22 @@ pool_cache_do_invalidate(struct pool_cac void pool_cache_invalidate(struct pool_cache *pc) { + struct pool_pagelist pq; + struct pool_cache_grouplist pcgl; + + LIST_INIT(&pq); + LIST_INIT(&pcgl); simple_lock(&pc->pc_slock); - pool_cache_do_invalidate(pc, 0, pool_put); + simple_lock(&pc->pc_pool->pr_slock); + + pool_do_cache_invalidate(pc, &pq, &pcgl); + + simple_unlock(&pc->pc_pool->pr_slock); simple_unlock(&pc->pc_slock); + + pr_pagelist_free(pc->pc_pool, &pq); + pcg_grouplist_free(&pcgl); } /* @@ -2082,11 +2130,21 @@ pool_cache_invalidate(struct pool_cache * Reclaim a pool cache for pool_reclaim(). */ static void -pool_cache_reclaim(struct pool_cache *pc) +pool_cache_reclaim(struct pool_cache *pc, struct pool_pagelist *pq, + struct pool_cache_grouplist *pcgl) { - simple_lock(&pc->pc_slock); - pool_cache_do_invalidate(pc, 1, pool_do_put); + /* + * We're locking in the wrong order (normally pool_cache -> pool, + * but the pool is already locked when we get here), so we have + * to use trylock. If we can't lock the pool_cache, it's not really + * a big deal here. + */ + if (simple_lock_try(&pc->pc_slock) == 0) + return; + + pool_do_cache_invalidate(pc, pq, pcgl); + simple_unlock(&pc->pc_slock); } @@ -2106,23 +2164,42 @@ pool_cache_reclaim(struct pool_cache *pc void *pool_page_alloc(struct pool *, int); void pool_page_free(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_kmem_fullpage = { + pool_page_alloc, pool_page_free, 0, +}; +#else struct pool_allocator pool_allocator_kmem = { pool_page_alloc, pool_page_free, 0, }; +#endif void *pool_page_alloc_nointr(struct pool *, int); void pool_page_free_nointr(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_nointr_fullpage = { + pool_page_alloc_nointr, pool_page_free_nointr, 0, +}; +#else struct pool_allocator pool_allocator_nointr = { pool_page_alloc_nointr, pool_page_free_nointr, 0, }; +#endif #ifdef POOL_SUBPAGE void *pool_subpage_alloc(struct pool *, int); void pool_subpage_free(struct pool *, void *); -struct pool_allocator pool_allocator_kmem_subpage = { - pool_subpage_alloc, pool_subpage_free, 0, +struct pool_allocator pool_allocator_kmem = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, +}; + +void *pool_subpage_alloc_nointr(struct pool *, int); +void pool_subpage_free_nointr(struct pool *, void *); + +struct pool_allocator pool_allocator_nointr = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, }; #endif /* POOL_SUBPAGE */ @@ -2170,8 +2247,8 @@ pool_allocator_alloc(struct pool *org, i } /* - * Drain all pools, except "org", that use this - * allocator. We do this to reclaim VA space. + * Drain all pools, that use this allocator. + * We do this to reclaim VA space. * pa_alloc is responsible for waiting for * physical memory. * @@ -2192,8 +2269,6 @@ pool_allocator_alloc(struct pool *org, i do { TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list); TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); - if (pp == org) - continue; simple_unlock(&pa->pa_slock); freed = pool_reclaim(pp); simple_lock(&pa->pa_slock); @@ -2251,7 +2326,7 @@ pool_page_alloc(struct pool *pp, int fla { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, NULL, waitok)); + return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); } void @@ -2266,14 +2341,14 @@ pool_page_alloc_meta(struct pool *pp, in { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage1(kmem_map, NULL, waitok)); + return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); } static void pool_page_free_meta(struct pool *pp, void *v) { - uvm_km_free_poolpage1(kmem_map, (vaddr_t) v); + uvm_km_free_poolpage(kmem_map, (vaddr_t) v); } #ifdef POOL_SUBPAGE @@ -2300,26 +2375,25 @@ pool_subpage_free(struct pool *pp, void /* We don't provide a real nointr allocator. Maybe later. */ void * -pool_page_alloc_nointr(struct pool *pp, int flags) +pool_subpage_alloc_nointr(struct pool *pp, int flags) { return (pool_subpage_alloc(pp, flags)); } void -pool_page_free_nointr(struct pool *pp, void *v) +pool_subpage_free_nointr(struct pool *pp, void *v) { pool_subpage_free(pp, v); } -#else +#endif /* POOL_SUBPAGE */ void * pool_page_alloc_nointr(struct pool *pp, int flags) { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, - uvm.kernel_object, waitok)); + return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); } void @@ -2328,4 +2402,3 @@ pool_page_free_nointr(struct pool *pp, v uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); } -#endif /* POOL_SUBPAGE */