Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.87.2.6 retrieving revision 1.98 diff -u -p -r1.87.2.6 -r1.98 --- src/sys/kern/subr_pool.c 2005/11/10 14:09:45 1.87.2.6 +++ src/sys/kern/subr_pool.c 2005/01/01 21:08:02 1.98 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.87.2.6 2005/11/10 14:09:45 skrll Exp $ */ +/* $NetBSD: subr_pool.c,v 1.98 2005/01/01 21:08:02 yamt Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.87.2.6 2005/11/10 14:09:45 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.98 2005/01/01 21:08:02 yamt Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -70,7 +70,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, */ /* List of all pools */ -LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head); +TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); /* Private pool for page header structures */ #define PHPOOL_MAX 8 @@ -99,8 +99,6 @@ static struct pool *drainpp; /* This spin lock protects both pool_head and drainpp. */ struct simplelock pool_head_slock = SIMPLELOCK_INITIALIZER; -typedef uint8_t pool_item_freelist_t; - struct pool_item_header { /* Page headers */ LIST_ENTRY(pool_item_header) @@ -112,19 +110,15 @@ struct pool_item_header { union { /* !PR_NOTOUCH */ struct { - LIST_HEAD(, pool_item) + TAILQ_HEAD(, pool_item) phu_itemlist; /* chunk list for this page */ } phu_normal; /* PR_NOTOUCH */ struct { uint16_t phu_off; /* start offset in page */ - pool_item_freelist_t + uint16_t phu_firstfree; /* first free item */ - /* - * XXX it might be better to use - * a simple bitmap and ffs(3) - */ } phu_notouch; } ph_u; uint16_t ph_nmissing; /* # of chunks in use */ @@ -139,7 +133,7 @@ struct pool_item { #endif #define PI_MAGIC 0xdeadbeefU /* Other entries use only this list entry */ - LIST_ENTRY(pool_item) pi_list; + TAILQ_ENTRY(pool_item) pi_list; }; #define POOL_NEEDS_CATCHUP(pp) \ @@ -174,9 +168,7 @@ struct pool_item { /* The cache group pool. */ static struct pool pcgpool; -static void pool_cache_reclaim(struct pool_cache *, struct pool_pagelist *, - struct pool_cache_grouplist *); -static void pcg_grouplist_free(struct pool_cache_grouplist *); +static void pool_cache_reclaim(struct pool_cache *); static int pool_catchup(struct pool *); static void pool_prime_page(struct pool *, caddr_t, @@ -325,18 +317,17 @@ pr_item_notouch_index(const struct pool return idx; } -#define PR_FREELIST_ALIGN(p) \ - roundup((uintptr_t)(p), sizeof(pool_item_freelist_t)) -#define PR_FREELIST(ph) ((pool_item_freelist_t *)PR_FREELIST_ALIGN((ph) + 1)) -#define PR_INDEX_USED ((pool_item_freelist_t)-1) -#define PR_INDEX_EOL ((pool_item_freelist_t)-2) +#define PR_FREELIST_ALIGN(p) roundup((uintptr_t)(p), sizeof(uint16_t)) +#define PR_FREELIST(ph) ((uint16_t *)PR_FREELIST_ALIGN((ph) + 1)) +#define PR_INDEX_USED ((uint16_t)-1) +#define PR_INDEX_EOL ((uint16_t)-2) static __inline void pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, void *obj) { int idx = pr_item_notouch_index(pp, ph, obj); - pool_item_freelist_t *freelist = PR_FREELIST(ph); + uint16_t *freelist = PR_FREELIST(ph); KASSERT(freelist[idx] == PR_INDEX_USED); freelist[idx] = ph->ph_firstfree; @@ -347,7 +338,7 @@ static __inline void * pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) { int idx = ph->ph_firstfree; - pool_item_freelist_t *freelist = PR_FREELIST(ph); + uint16_t *freelist = PR_FREELIST(ph); KASSERT(freelist[idx] != PR_INDEX_USED); ph->ph_firstfree = freelist[idx]; @@ -386,23 +377,6 @@ pr_find_pagehead(struct pool *pp, caddr_ return ph; } -static void -pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq) -{ - struct pool_item_header *ph; - int s; - - while ((ph = LIST_FIRST(pq)) != NULL) { - LIST_REMOVE(ph, ph_pagelist); - pool_allocator_free(pp, ph->ph_page); - if ((pp->pr_roflags & PR_PHINPAGE) == 0) { - s = splvm(); - pool_put(pp->pr_phpool, ph); - splx(s); - } - } -} - /* * Remove a page from the pool. */ @@ -410,8 +384,9 @@ static __inline void pr_rmpage(struct pool *pp, struct pool_item_header *ph, struct pool_pagelist *pq) { + int s; - LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); + LOCK_ASSERT(!simple_lock_held(&pp->pr_slock) || pq != NULL); /* * If the page was idle, decrement the idle page count. @@ -429,13 +404,21 @@ pr_rmpage(struct pool *pp, struct pool_i pp->pr_nitems -= pp->pr_itemsperpage; /* - * Unlink the page from the pool and queue it for release. + * Unlink a page from the pool and release it (or queue it for release). */ LIST_REMOVE(ph, ph_pagelist); if ((pp->pr_roflags & PR_PHINPAGE) == 0) SPLAY_REMOVE(phtree, &pp->pr_phtree, ph); - LIST_INSERT_HEAD(pq, ph, ph_pagelist); - + if (pq) { + LIST_INSERT_HEAD(pq, ph, ph_pagelist); + } else { + pool_allocator_free(pp, ph->ph_page); + if ((pp->pr_roflags & PR_PHINPAGE) == 0) { + s = splvm(); + pool_put(pp->pr_phpool, ph); + splx(s); + } + } pp->pr_npages--; pp->pr_npagefree++; @@ -471,9 +454,6 @@ pool_init(struct pool *pp, size_t size, size_t trysize, phsize; int s; - KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >= - PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1)); - #ifdef POOL_DIAGNOSTIC /* * Always log if POOL_DIAGNOSTIC is defined. @@ -539,7 +519,7 @@ pool_init(struct pool *pp, size_t size, LIST_INIT(&pp->pr_emptypages); LIST_INIT(&pp->pr_fullpages); LIST_INIT(&pp->pr_partpages); - LIST_INIT(&pp->pr_cachelist); + TAILQ_INIT(&pp->pr_cachelist); pp->pr_curpage = NULL; pp->pr_npages = 0; pp->pr_minitems = 0; @@ -674,7 +654,7 @@ pool_init(struct pool *pp, size_t size, sz = sizeof(struct pool_item_header); if (nelem) { sz = PR_FREELIST_ALIGN(sz) - + nelem * sizeof(pool_item_freelist_t); + + nelem * sizeof(uint16_t); } pool_init(&phpool[idx], sz, 0, 0, 0, phpool_names[idx], &pool_allocator_meta); @@ -689,7 +669,7 @@ pool_init(struct pool *pp, size_t size, /* Insert into the list of all pools. */ simple_lock(&pool_head_slock); - LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); + TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); simple_unlock(&pool_head_slock); /* Insert this into the list of pools using this allocator. */ @@ -706,28 +686,20 @@ pool_init(struct pool *pp, size_t size, void pool_destroy(struct pool *pp) { - struct pool_pagelist pq; struct pool_item_header *ph; + struct pool_cache *pc; int s; - /* Remove from global pool list */ - simple_lock(&pool_head_slock); - LIST_REMOVE(pp, pr_poollist); - if (drainpp == pp) - drainpp = NULL; - simple_unlock(&pool_head_slock); - - /* Remove this pool from its allocator's list of pools. */ + /* Locking order: pool_allocator -> pool */ s = splvm(); simple_lock(&pp->pr_alloc->pa_slock); TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list); simple_unlock(&pp->pr_alloc->pa_slock); splx(s); - s = splvm(); - simple_lock(&pp->pr_slock); - - KASSERT(LIST_EMPTY(&pp->pr_cachelist)); + /* Destroy all caches for this pool. */ + while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL) + pool_cache_destroy(pc); #ifdef DIAGNOSTIC if (pp->pr_nout != 0) { @@ -737,18 +709,19 @@ pool_destroy(struct pool *pp) } #endif - KASSERT(LIST_EMPTY(&pp->pr_fullpages)); - KASSERT(LIST_EMPTY(&pp->pr_partpages)); - /* Remove all pages */ - LIST_INIT(&pq); while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) - pr_rmpage(pp, ph, &pq); - - simple_unlock(&pp->pr_slock); - splx(s); + pr_rmpage(pp, ph, NULL); + KASSERT(LIST_EMPTY(&pp->pr_fullpages)); + KASSERT(LIST_EMPTY(&pp->pr_partpages)); - pr_pagelist_free(pp, &pq); + /* Remove from global pool list */ + simple_lock(&pool_head_slock); + TAILQ_REMOVE(&pool_head, pp, pr_poollist); + if (drainpp == pp) { + drainpp = NULL; + } + simple_unlock(&pool_head_slock); #ifdef POOL_DIAGNOSTIC if ((pp->pr_roflags & PR_LOGGING) != 0) @@ -810,12 +783,11 @@ pool_get(struct pool *pp, int flags) (flags & PR_WAITOK) != 0)) panic("pool_get: %s: must have NOWAIT", pp->pr_wchan); -#endif /* DIAGNOSTIC */ #ifdef LOCKDEBUG if (flags & PR_WAITOK) simple_lock_only_held(NULL, "pool_get(PR_WAITOK)"); - SCHED_ASSERT_UNLOCKED(); #endif +#endif /* DIAGNOSTIC */ simple_lock(&pp->pr_slock); pr_enter(pp, file, line); @@ -962,7 +934,7 @@ pool_get(struct pool *pp, int flags) pr_log(pp, v, PRLOG_GET, file, line); #endif } else { - v = pi = LIST_FIRST(&ph->ph_itemlist); + v = pi = TAILQ_FIRST(&ph->ph_itemlist); if (__predict_false(v == NULL)) { pr_leave(pp); simple_unlock(&pp->pr_slock); @@ -994,7 +966,7 @@ pool_get(struct pool *pp, int flags) /* * Remove from item list. */ - LIST_REMOVE(pi, pi_list); + TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); } pp->pr_nitems--; pp->pr_nout++; @@ -1016,7 +988,7 @@ pool_get(struct pool *pp, int flags) if (ph->ph_nmissing == pp->pr_itemsperpage) { #ifdef DIAGNOSTIC if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 && - !LIST_EMPTY(&ph->ph_itemlist))) { + !TAILQ_EMPTY(&ph->ph_itemlist))) { pr_leave(pp); simple_unlock(&pp->pr_slock); panic("pool_get: %s: nmissing inconsistent", @@ -1055,7 +1027,7 @@ pool_get(struct pool *pp, int flags) * Internal version of pool_put(). Pool is already locked/entered. */ static void -pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq) +pool_do_put(struct pool *pp, void *v) { struct pool_item *pi = v; struct pool_item_header *ph; @@ -1063,7 +1035,6 @@ pool_do_put(struct pool *pp, void *v, st int s; LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); - SCHED_ASSERT_UNLOCKED(); page = (caddr_t)((u_long)v & pp->pr_alloc->pa_pagemask); @@ -1106,7 +1077,7 @@ pool_do_put(struct pool *pp, void *v, st } #endif - LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); + TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); } KDASSERT(ph->ph_nmissing != 0); ph->ph_nmissing--; @@ -1144,7 +1115,9 @@ pool_do_put(struct pool *pp, void *v, st if (pp->pr_npages > pp->pr_minpages && (pp->pr_npages > pp->pr_maxpages || (pp->pr_alloc->pa_flags & PA_WANT) != 0)) { - pr_rmpage(pp, ph, pq); + simple_unlock(&pp->pr_slock); + pr_rmpage(pp, ph, NULL); + simple_lock(&pp->pr_slock); } else { LIST_REMOVE(ph, ph_pagelist); LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); @@ -1182,21 +1155,16 @@ pool_do_put(struct pool *pp, void *v, st void _pool_put(struct pool *pp, void *v, const char *file, long line) { - struct pool_pagelist pq; - - LIST_INIT(&pq); simple_lock(&pp->pr_slock); pr_enter(pp, file, line); pr_log(pp, v, PRLOG_PUT, file, line); - pool_do_put(pp, v, &pq); + pool_do_put(pp, v); pr_leave(pp); simple_unlock(&pp->pr_slock); - - pr_pagelist_free(pp, &pq); } #undef pool_put #endif /* POOL_DIAGNOSTIC */ @@ -1204,15 +1172,12 @@ _pool_put(struct pool *pp, void *v, cons void pool_put(struct pool *pp, void *v) { - struct pool_pagelist pq; - - LIST_INIT(&pq); simple_lock(&pp->pr_slock); - pool_do_put(pp, v, &pq); - simple_unlock(&pp->pr_slock); - pr_pagelist_free(pp, &pq); + pool_do_put(pp, v); + + simple_unlock(&pp->pr_slock); } #ifdef POOL_DIAGNOSTIC @@ -1285,7 +1250,7 @@ pool_prime_page(struct pool *pp, caddr_t * Insert page header. */ LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); - LIST_INIT(&ph->ph_itemlist); + TAILQ_INIT(&ph->ph_itemlist); ph->ph_page = storage; ph->ph_nmissing = 0; s = splclock(); @@ -1315,11 +1280,12 @@ pool_prime_page(struct pool *pp, caddr_t n = pp->pr_itemsperpage; pp->pr_nitems += n; + ph->ph_off = cp - storage; + if (pp->pr_roflags & PR_NOTOUCH) { - pool_item_freelist_t *freelist = PR_FREELIST(ph); + uint16_t *freelist = PR_FREELIST(ph); int i; - ph->ph_off = cp - storage; ph->ph_firstfree = 0; for (i = 0; i < n - 1; i++) freelist[i] = i + 1; @@ -1331,7 +1297,7 @@ pool_prime_page(struct pool *pp, caddr_t KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); /* Insert on page list */ - LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); + TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); #ifdef DIAGNOSTIC pi->pi_magic = PI_MAGIC; #endif @@ -1472,9 +1438,9 @@ pool_reclaim(struct pool *pp) { struct pool_item_header *ph, *phnext; struct pool_cache *pc; + struct timeval curtime; struct pool_pagelist pq; - struct pool_cache_grouplist pcgl; - struct timeval curtime, diff; + struct timeval diff; int s; if (pp->pr_drain_hook != NULL) { @@ -1489,13 +1455,12 @@ pool_reclaim(struct pool *pp) pr_enter(pp, file, line); LIST_INIT(&pq); - LIST_INIT(&pcgl); /* * Reclaim items from the pool's caches. */ - LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) - pool_cache_reclaim(pc, &pq, &pcgl); + TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) + pool_cache_reclaim(pc); s = splclock(); curtime = mono_time; @@ -1526,11 +1491,20 @@ pool_reclaim(struct pool *pp) pr_leave(pp); simple_unlock(&pp->pr_slock); - if (LIST_EMPTY(&pq) && LIST_EMPTY(&pcgl)) - return 0; + if (LIST_EMPTY(&pq)) + return (0); + + while ((ph = LIST_FIRST(&pq)) != NULL) { + LIST_REMOVE(ph, ph_pagelist); + pool_allocator_free(pp, ph->ph_page); + if (pp->pr_roflags & PR_PHINPAGE) { + continue; + } + s = splvm(); + pool_put(pp->pr_phpool, ph); + splx(s); + } - pr_pagelist_free(pp, &pq); - pcg_grouplist_free(&pcgl); return (1); } @@ -1549,11 +1523,11 @@ pool_drain(void *arg) s = splvm(); simple_lock(&pool_head_slock); if (drainpp == NULL) { - drainpp = LIST_FIRST(&pool_head); + drainpp = TAILQ_FIRST(&pool_head); } if (drainpp) { pp = drainpp; - drainpp = LIST_NEXT(pp, pr_poollist); + drainpp = TAILQ_NEXT(pp, pr_poollist); } simple_unlock(&pool_head_slock); pool_reclaim(pp); @@ -1583,6 +1557,7 @@ pool_print(struct pool *pp, const char * void pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) { + int didlock = 0; if (pp == NULL) { (*pr)("Must specify a pool to print.\n"); @@ -1601,9 +1576,12 @@ pool_printit(struct pool *pp, const char if (simple_lock_try(&pp->pr_slock) == 0) (*pr)("WARNING: pool %s is locked\n", pp->pr_wchan); else - simple_unlock(&pp->pr_slock); + didlock = 1; pool_print1(pp, modif, pr); + + if (didlock) + simple_unlock(&pp->pr_slock); } static void @@ -1622,7 +1600,7 @@ pool_print_pagelist(struct pool *pp, str (u_long)ph->ph_time.tv_usec); #ifdef DIAGNOSTIC if (!(pp->pr_roflags & PR_NOTOUCH)) { - LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { + TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) { if (pi->pi_magic != PI_MAGIC) { (*pr)("\t\t\titem %p, magic 0x%x\n", pi, pi->pi_magic); @@ -1697,39 +1675,27 @@ pool_print1(struct pool *pp, const char if (print_cache == 0) goto skip_cache; -#define PR_GROUPLIST(pcg) \ - (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ - for (i = 0; i < PCG_NOBJECTS; i++) { \ - if (pcg->pcg_objects[i].pcgo_pa != \ - POOL_PADDR_INVALID) { \ - (*pr)("\t\t\t%p, 0x%llx\n", \ - pcg->pcg_objects[i].pcgo_va, \ - (unsigned long long) \ - pcg->pcg_objects[i].pcgo_pa); \ - } else { \ - (*pr)("\t\t\t%p\n", \ - pcg->pcg_objects[i].pcgo_va); \ - } \ - } - - LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) { - (*pr)("\tcache %p\n", pc); + TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) { + (*pr)("\tcache %p: allocfrom %p freeto %p\n", pc, + pc->pc_allocfrom, pc->pc_freeto); (*pr)("\t hits %lu misses %lu ngroups %lu nitems %lu\n", pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems); - (*pr)("\t full groups:\n"); - LIST_FOREACH(pcg, &pc->pc_fullgroups, pcg_list) { - PR_GROUPLIST(pcg); - } - (*pr)("\t partial groups:\n"); - LIST_FOREACH(pcg, &pc->pc_partgroups, pcg_list) { - PR_GROUPLIST(pcg); - } - (*pr)("\t empty groups:\n"); - LIST_FOREACH(pcg, &pc->pc_emptygroups, pcg_list) { - PR_GROUPLIST(pcg); + TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) { + (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); + for (i = 0; i < PCG_NOBJECTS; i++) { + if (pcg->pcg_objects[i].pcgo_pa != + POOL_PADDR_INVALID) { + (*pr)("\t\t\t%p, 0x%llx\n", + pcg->pcg_objects[i].pcgo_va, + (unsigned long long) + pcg->pcg_objects[i].pcgo_pa); + } else { + (*pr)("\t\t\t%p\n", + pcg->pcg_objects[i].pcgo_va); + } + } } } -#undef PR_GROUPLIST skip_cache: pr_enter_check(pp, pr); @@ -1757,9 +1723,9 @@ pool_chk_page(struct pool *pp, const cha if ((pp->pr_roflags & PR_NOTOUCH) != 0) return 0; - for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0; + for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0; pi != NULL; - pi = LIST_NEXT(pi,pi_list), n++) { + pi = TAILQ_NEXT(pi,pi_list), n++) { #ifdef DIAGNOSTIC if (pi->pi_magic != PI_MAGIC) { @@ -1836,11 +1802,11 @@ pool_cache_init(struct pool_cache *pc, s void *arg) { - LIST_INIT(&pc->pc_emptygroups); - LIST_INIT(&pc->pc_fullgroups); - LIST_INIT(&pc->pc_partgroups); + TAILQ_INIT(&pc->pc_grouplist); simple_lock_init(&pc->pc_slock); + pc->pc_allocfrom = NULL; + pc->pc_freeto = NULL; pc->pc_pool = pp; pc->pc_ctor = ctor; @@ -1855,7 +1821,7 @@ pool_cache_init(struct pool_cache *pc, s pc->pc_nitems = 0; simple_lock(&pp->pr_slock); - LIST_INSERT_HEAD(&pp->pr_cachelist, pc, pc_poollist); + TAILQ_INSERT_TAIL(&pp->pr_cachelist, pc, pc_poollist); simple_unlock(&pp->pr_slock); } @@ -1874,7 +1840,7 @@ pool_cache_destroy(struct pool_cache *pc /* ...and remove it from the pool's cache list. */ simple_lock(&pp->pr_slock); - LIST_REMOVE(pc, pc_poollist); + TAILQ_REMOVE(&pp->pr_cachelist, pc, pc_poollist); simple_unlock(&pp->pr_slock); } @@ -1910,20 +1876,6 @@ pcg_put(struct pool_cache_group *pcg, vo pcg->pcg_objects[idx].pcgo_pa = pa; } -static void -pcg_grouplist_free(struct pool_cache_grouplist *pcgl) -{ - struct pool_cache_group *pcg; - int s; - - s = splvm(); - while ((pcg = LIST_FIRST(pcgl)) != NULL) { - LIST_REMOVE(pcg, pcg_list); - pool_put(&pcgpool, pcg); - } - splx(s); -} - /* * pool_cache_get{,_paddr}: * @@ -1943,15 +1895,13 @@ pool_cache_get_paddr(struct pool_cache * simple_lock(&pc->pc_slock); - pcg = LIST_FIRST(&pc->pc_partgroups); - if (pcg == NULL) { - pcg = LIST_FIRST(&pc->pc_fullgroups); - if (pcg != NULL) { - LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list); + if ((pcg = pc->pc_allocfrom) == NULL) { + TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) { + if (pcg->pcg_avail != 0) { + pc->pc_allocfrom = pcg; + goto have_group; + } } - } - if (pcg == NULL) { /* * No groups with any available objects. Allocate @@ -1978,14 +1928,14 @@ pool_cache_get_paddr(struct pool_cache * return (object); } + have_group: pc->pc_hits++; pc->pc_nitems--; object = pcg_get(pcg, pap); - if (pcg->pcg_avail == 0) { - LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(&pc->pc_emptygroups, pcg, pcg_list); - } + if (pcg->pcg_avail == 0) + pc->pc_allocfrom = NULL; + simple_unlock(&pc->pc_slock); return (object); @@ -2005,15 +1955,13 @@ pool_cache_put_paddr(struct pool_cache * simple_lock(&pc->pc_slock); - pcg = LIST_FIRST(&pc->pc_partgroups); - if (pcg == NULL) { - pcg = LIST_FIRST(&pc->pc_emptygroups); - if (pcg != NULL) { - LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list); + if ((pcg = pc->pc_freeto) == NULL) { + TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) { + if (pcg->pcg_avail != PCG_NOBJECTS) { + pc->pc_freeto = pcg; + goto have_group; + } } - } - if (pcg == NULL) { /* * No empty groups to free the object to. Attempt to @@ -2023,28 +1971,31 @@ pool_cache_put_paddr(struct pool_cache * s = splvm(); pcg = pool_get(&pcgpool, PR_NOWAIT); splx(s); - if (pcg == NULL) { - - /* - * Unable to allocate a cache group; destruct the object - * and free it back to the pool. - */ - pool_cache_destruct_object(pc, object); - return; + if (pcg != NULL) { + memset(pcg, 0, sizeof(*pcg)); + simple_lock(&pc->pc_slock); + pc->pc_ngroups++; + TAILQ_INSERT_TAIL(&pc->pc_grouplist, pcg, pcg_list); + if (pc->pc_freeto == NULL) + pc->pc_freeto = pcg; + goto have_group; } - memset(pcg, 0, sizeof(*pcg)); - simple_lock(&pc->pc_slock); - pc->pc_ngroups++; - LIST_INSERT_HEAD(&pc->pc_partgroups, pcg, pcg_list); + + /* + * Unable to allocate a cache group; destruct the object + * and free it back to the pool. + */ + pool_cache_destruct_object(pc, object); + return; } + have_group: pc->pc_nitems++; pcg_put(pcg, object, pa); - if (pcg->pcg_avail == PCG_NOBJECTS) { - LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(&pc->pc_fullgroups, pcg, pcg_list); - } + if (pcg->pcg_avail == PCG_NOBJECTS) + pc->pc_freeto = NULL; + simple_unlock(&pc->pc_slock); } @@ -2063,45 +2014,44 @@ pool_cache_destruct_object(struct pool_c pool_put(pc->pc_pool, object); } +/* + * pool_cache_do_invalidate: + * + * This internal function implements pool_cache_invalidate() and + * pool_cache_reclaim(). + */ static void -pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl, - struct pool_cache *pc, struct pool_pagelist *pq, - struct pool_cache_grouplist *pcgdl) +pool_cache_do_invalidate(struct pool_cache *pc, int free_groups, + void (*putit)(struct pool *, void *)) { struct pool_cache_group *pcg, *npcg; void *object; + int s; - for (pcg = LIST_FIRST(pcgsl); pcg != NULL; pcg = npcg) { - npcg = LIST_NEXT(pcg, pcg_list); + for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL; + pcg = npcg) { + npcg = TAILQ_NEXT(pcg, pcg_list); while (pcg->pcg_avail != 0) { pc->pc_nitems--; object = pcg_get(pcg, NULL); + if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg) + pc->pc_allocfrom = NULL; if (pc->pc_dtor != NULL) (*pc->pc_dtor)(pc->pc_arg, object); - pool_do_put(pc->pc_pool, object, pq); + (*putit)(pc->pc_pool, object); + } + if (free_groups) { + pc->pc_ngroups--; + TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list); + if (pc->pc_freeto == pcg) + pc->pc_freeto = NULL; + s = splvm(); + pool_put(&pcgpool, pcg); + splx(s); } - pc->pc_ngroups--; - LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); } } -static void -pool_do_cache_invalidate(struct pool_cache *pc, struct pool_pagelist *pq, - struct pool_cache_grouplist *pcgl) -{ - - LOCK_ASSERT(simple_lock_held(&pc->pc_slock)); - LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock)); - - pool_do_cache_invalidate_grouplist(&pc->pc_fullgroups, pc, pq, pcgl); - pool_do_cache_invalidate_grouplist(&pc->pc_partgroups, pc, pq, pcgl); - - KASSERT(LIST_EMPTY(&pc->pc_partgroups)); - KASSERT(LIST_EMPTY(&pc->pc_fullgroups)); - KASSERT(pc->pc_nitems == 0); -} - /* * pool_cache_invalidate: * @@ -2111,22 +2061,10 @@ pool_do_cache_invalidate(struct pool_cac void pool_cache_invalidate(struct pool_cache *pc) { - struct pool_pagelist pq; - struct pool_cache_grouplist pcgl; - - LIST_INIT(&pq); - LIST_INIT(&pcgl); simple_lock(&pc->pc_slock); - simple_lock(&pc->pc_pool->pr_slock); - - pool_do_cache_invalidate(pc, &pq, &pcgl); - - simple_unlock(&pc->pc_pool->pr_slock); + pool_cache_do_invalidate(pc, 0, pool_put); simple_unlock(&pc->pc_slock); - - pr_pagelist_free(pc->pc_pool, &pq); - pcg_grouplist_free(&pcgl); } /* @@ -2135,21 +2073,11 @@ pool_cache_invalidate(struct pool_cache * Reclaim a pool cache for pool_reclaim(). */ static void -pool_cache_reclaim(struct pool_cache *pc, struct pool_pagelist *pq, - struct pool_cache_grouplist *pcgl) +pool_cache_reclaim(struct pool_cache *pc) { - /* - * We're locking in the wrong order (normally pool_cache -> pool, - * but the pool is already locked when we get here), so we have - * to use trylock. If we can't lock the pool_cache, it's not really - * a big deal here. - */ - if (simple_lock_try(&pc->pc_slock) == 0) - return; - - pool_do_cache_invalidate(pc, pq, pcgl); - + simple_lock(&pc->pc_slock); + pool_cache_do_invalidate(pc, 1, pool_do_put); simple_unlock(&pc->pc_slock); } @@ -2314,7 +2242,7 @@ pool_page_alloc(struct pool *pp, int fla { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); + return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, NULL, waitok)); } void @@ -2329,14 +2257,14 @@ pool_page_alloc_meta(struct pool *pp, in { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); + return ((void *) uvm_km_alloc_poolpage1(kmem_map, NULL, waitok)); } static void pool_page_free_meta(struct pool *pp, void *v) { - uvm_km_free_poolpage(kmem_map, (vaddr_t) v); + uvm_km_free_poolpage1(kmem_map, (vaddr_t) v); } #ifdef POOL_SUBPAGE @@ -2381,7 +2309,8 @@ pool_page_alloc_nointr(struct pool *pp, { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); + return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, + uvm.kernel_object, waitok)); } void