Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.112.2.3 retrieving revision 1.129.12.2 diff -u -p -r1.112.2.3 -r1.129.12.2 --- src/sys/kern/subr_pool.c 2006/05/24 10:58:41 1.112.2.3 +++ src/sys/kern/subr_pool.c 2007/10/26 15:48:40 1.129.12.2 @@ -1,7 +1,7 @@ -/* $NetBSD: subr_pool.c,v 1.112.2.3 2006/05/24 10:58:41 yamt Exp $ */ +/* $NetBSD: subr_pool.c,v 1.129.12.2 2007/10/26 15:48:40 joerg Exp $ */ /*- - * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. + * Copyright (c) 1997, 1999, 2000, 2002 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.112.2.3 2006/05/24 10:58:41 yamt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.129.12.2 2007/10/26 15:48:40 joerg Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -53,6 +53,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, #include #include #include +#include #include @@ -82,12 +83,16 @@ static struct pool phpool[PHPOOL_MAX]; static struct pool psppool; #endif +static SLIST_HEAD(, pool_allocator) pa_deferinitq = + SLIST_HEAD_INITIALIZER(pa_deferinitq); + static void *pool_page_alloc_meta(struct pool *, int); static void pool_page_free_meta(struct pool *, void *); /* allocator for pool metadata */ static struct pool_allocator pool_allocator_meta = { - pool_page_alloc_meta, pool_page_free_meta + pool_page_alloc_meta, pool_page_free_meta, + .pa_backingmapptr = &kmem_map, }; /* # of seconds to retain page after last use */ @@ -107,7 +112,7 @@ struct pool_item_header { ph_pagelist; /* pool page list */ SPLAY_ENTRY(pool_item_header) ph_node; /* Off-page page headers */ - caddr_t ph_page; /* this page's address */ + void * ph_page; /* this page's address */ struct timeval ph_time; /* last referenced */ union { /* !PR_NOTOUCH */ @@ -179,13 +184,13 @@ static void pool_cache_reclaim(struct po static void pcg_grouplist_free(struct pool_cache_grouplist *); static int pool_catchup(struct pool *); -static void pool_prime_page(struct pool *, caddr_t, +static void pool_prime_page(struct pool *, void *, struct pool_item_header *); static void pool_update_curpage(struct pool *); static int pool_grow(struct pool *, int); -void *pool_allocator_alloc(struct pool *, int); -void pool_allocator_free(struct pool *, void *); +static void *pool_allocator_alloc(struct pool *, int); +static void pool_allocator_free(struct pool *, void *); static void pool_print_pagelist(struct pool *, struct pool_pagelist *, void (*)(const char *, ...)); @@ -321,7 +326,7 @@ pr_item_notouch_index(const struct pool int idx; KASSERT(pp->pr_roflags & PR_NOTOUCH); - idx = (cp - ph->ph_page - ph->ph_off) / pp->pr_size; + idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size; KASSERT(idx < pp->pr_itemsperpage); return idx; } @@ -354,16 +359,22 @@ pr_item_notouch_get(const struct pool *p ph->ph_firstfree = freelist[idx]; freelist[idx] = PR_INDEX_USED; - return ph->ph_page + ph->ph_off + idx * pp->pr_size; + return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size; } static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { + + /* + * we consider pool_item_header with smaller ph_page bigger. + * (this unnatural ordering is for the benefit of pr_find_pagehead.) + */ + if (a->ph_page < b->ph_page) - return (-1); - else if (a->ph_page > b->ph_page) return (1); + else if (a->ph_page > b->ph_page) + return (-1); else return (0); } @@ -372,18 +383,38 @@ SPLAY_PROTOTYPE(phtree, pool_item_header SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); /* - * Return the pool page header based on page address. + * Return the pool page header based on item address. */ static inline struct pool_item_header * -pr_find_pagehead(struct pool *pp, caddr_t page) +pr_find_pagehead(struct pool *pp, void *v) { struct pool_item_header *ph, tmp; - if ((pp->pr_roflags & PR_PHINPAGE) != 0) - return ((struct pool_item_header *)(page + pp->pr_phoffset)); + if ((pp->pr_roflags & PR_NOALIGN) != 0) { + tmp.ph_page = (void *)(uintptr_t)v; + ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); + if (ph == NULL) { + ph = SPLAY_ROOT(&pp->pr_phtree); + if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { + ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); + } + KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); + } + } else { + void *page = + (void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask); + + if ((pp->pr_roflags & PR_PHINPAGE) != 0) { + ph = (struct pool_item_header *)((char *)page + pp->pr_phoffset); + } else { + tmp.ph_page = page; + ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); + } + } - tmp.ph_page = page; - ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); + KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || + ((char *)ph->ph_page <= (char *)v && + (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz)); return ph; } @@ -443,19 +474,100 @@ pr_rmpage(struct pool *pp, struct pool_i pool_update_curpage(pp); } +static bool +pa_starved_p(struct pool_allocator *pa) +{ + + if (pa->pa_backingmap != NULL) { + return vm_map_starved_p(pa->pa_backingmap); + } + return false; +} + +static int +pool_reclaim_callback(struct callback_entry *ce, void *obj, void *arg) +{ + struct pool *pp = obj; + struct pool_allocator *pa = pp->pr_alloc; + + KASSERT(&pp->pr_reclaimerentry == ce); + pool_reclaim(pp); + if (!pa_starved_p(pa)) { + return CALLBACK_CHAIN_ABORT; + } + return CALLBACK_CHAIN_CONTINUE; +} + +static void +pool_reclaim_register(struct pool *pp) +{ + struct vm_map *map = pp->pr_alloc->pa_backingmap; + int s; + + if (map == NULL) { + return; + } + + s = splvm(); /* not necessary for INTRSAFE maps, but don't care. */ + callback_register(&vm_map_to_kernel(map)->vmk_reclaim_callback, + &pp->pr_reclaimerentry, pp, pool_reclaim_callback); + splx(s); +} + +static void +pool_reclaim_unregister(struct pool *pp) +{ + struct vm_map *map = pp->pr_alloc->pa_backingmap; + int s; + + if (map == NULL) { + return; + } + + s = splvm(); /* not necessary for INTRSAFE maps, but don't care. */ + callback_unregister(&vm_map_to_kernel(map)->vmk_reclaim_callback, + &pp->pr_reclaimerentry); + splx(s); +} + +static void +pa_reclaim_register(struct pool_allocator *pa) +{ + struct vm_map *map = *pa->pa_backingmapptr; + struct pool *pp; + + KASSERT(pa->pa_backingmap == NULL); + if (map == NULL) { + SLIST_INSERT_HEAD(&pa_deferinitq, pa, pa_q); + return; + } + pa->pa_backingmap = map; + TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) { + pool_reclaim_register(pp); + } +} + /* * Initialize all the pools listed in the "pools" link set. */ void -link_pool_init(void) +pool_subsystem_init(void) { + struct pool_allocator *pa; __link_set_decl(pools, struct link_pool_init); struct link_pool_init * const *pi; __link_set_foreach(pi, pools) pool_init((*pi)->pp, (*pi)->size, (*pi)->align, (*pi)->align_offset, (*pi)->flags, (*pi)->wchan, - (*pi)->palloc); + (*pi)->palloc, (*pi)->ipl); + + while ((pa = SLIST_FIRST(&pa_deferinitq)) != NULL) { + KASSERT(pa->pa_backingmapptr != NULL); + KASSERT(*pa->pa_backingmapptr != NULL); + SLIST_REMOVE_HEAD(&pa_deferinitq, pa_q); + pa_reclaim_register(pa); + } } /* @@ -466,7 +578,7 @@ link_pool_init(void) */ void pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, - const char *wchan, struct pool_allocator *palloc) + const char *wchan, struct pool_allocator *palloc, int ipl) { #ifdef DEBUG struct pool *pp1; @@ -516,20 +628,23 @@ pool_init(struct pool *pp, size_t size, simple_lock_init(&palloc->pa_slock); palloc->pa_pagemask = ~(palloc->pa_pagesz - 1); palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1; + + if (palloc->pa_backingmapptr != NULL) { + pa_reclaim_register(palloc); + } palloc->pa_flags |= PA_INITIALIZED; } if (align == 0) align = ALIGN(1); - if (size < sizeof(struct pool_item)) + if ((flags & PR_NOTOUCH) == 0 && size < sizeof(struct pool_item)) size = sizeof(struct pool_item); size = roundup(size, align); #ifdef DIAGNOSTIC if (size > palloc->pa_pagesz) - panic("pool_init: pool item size (%lu) too large", - (u_long)size); + panic("pool_init: pool item size (%zu) too large", size); #endif /* @@ -560,6 +675,7 @@ pool_init(struct pool *pp, size_t size, pp->pr_hardlimit_warning_last.tv_usec = 0; pp->pr_drain_hook = NULL; pp->pr_drain_hook_arg = NULL; + pp->pr_freecheck = NULL; /* * Decide whether to put the page header off page to avoid @@ -578,7 +694,7 @@ pool_init(struct pool *pp, size_t size, /* See the comment below about reserved bytes. */ trysize = palloc->pa_pagesz - ((align - ioff) % align); phsize = ALIGN(sizeof(struct pool_item_header)); - if ((pp->pr_roflags & PR_NOTOUCH) == 0 && + if ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 && (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) || trysize / pp->pr_size == (trysize - phsize) / pp->pr_size)) { /* Use the end of the page for the page header */ @@ -676,14 +792,14 @@ pool_init(struct pool *pp, size_t size, + nelem * sizeof(pool_item_freelist_t); } pool_init(&phpool[idx], sz, 0, 0, 0, - phpool_names[idx], &pool_allocator_meta); + phpool_names[idx], &pool_allocator_meta, IPL_VM); } #ifdef POOL_SUBPAGE pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, - PR_RECURSIVE, "psppool", &pool_allocator_meta); + PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM); #endif pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0, - 0, "pcgpool", &pool_allocator_meta); + 0, "pcgpool", &pool_allocator_meta, IPL_VM); } /* Insert into the list of all pools. */ @@ -697,6 +813,7 @@ pool_init(struct pool *pp, size_t size, TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); simple_unlock(&palloc->pa_slock); splx(s); + pool_reclaim_register(pp); } /* @@ -717,6 +834,7 @@ pool_destroy(struct pool *pp) simple_unlock(&pool_head_slock); /* Remove this pool from its allocator's list of pools. */ + pool_reclaim_unregister(pp); s = splvm(); simple_lock(&pp->pr_alloc->pa_slock); TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list); @@ -769,7 +887,7 @@ pool_set_drain_hook(struct pool *pp, voi } static struct pool_item_header * -pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags) +pool_alloc_item_header(struct pool *pp, void *storage, int flags) { struct pool_item_header *ph; int s; @@ -777,7 +895,7 @@ pool_alloc_item_header(struct pool *pp, LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0); if ((pp->pr_roflags & PR_PHINPAGE) != 0) - ph = (struct pool_item_header *) (storage + pp->pr_phoffset); + ph = (struct pool_item_header *) ((char *)storage + pp->pr_phoffset); else { s = splvm(); ph = pool_get(pp->pr_phpool, flags); @@ -812,8 +930,7 @@ pool_get(struct pool *pp, int flags) #endif /* DIAGNOSTIC */ #ifdef LOCKDEBUG if (flags & PR_WAITOK) - simple_lock_only_held(NULL, "pool_get(PR_WAITOK)"); - SCHED_ASSERT_UNLOCKED(); + ASSERT_SLEEPABLE(NULL, "pool_get(PR_WAITOK)"); #endif simple_lock(&pp->pr_slock); @@ -911,24 +1028,10 @@ pool_get(struct pool *pp, int flags) if (pp->pr_curpage != NULL) goto startover; - if ((flags & PR_WAITOK) == 0) { - pp->pr_nfail++; - pr_leave(pp); - simple_unlock(&pp->pr_slock); - return (NULL); - } - - /* - * Wait for items to be returned to this pool. - * - * wake up once a second and try again, - * as the check in pool_cache_put_paddr() is racy. - */ - pp->pr_flags |= PR_WANTED; - /* PA_WANTED is already set on the allocator. */ + pp->pr_nfail++; pr_leave(pp); - ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); - pr_enter(pp, file, line); + simple_unlock(&pp->pr_slock); + return (NULL); } /* Start the allocation process over. */ @@ -1033,6 +1136,8 @@ pool_get(struct pool *pp, int flags) } simple_unlock(&pp->pr_slock); + KASSERT((((vaddr_t)v + pp->pr_itemoffset) & (pp->pr_align - 1)) == 0); + FREECHECK_OUT(&pp->pr_freecheck, v); return (v); } @@ -1044,13 +1149,9 @@ pool_do_put(struct pool *pp, void *v, st { struct pool_item *pi = v; struct pool_item_header *ph; - caddr_t page; - int s; LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); - SCHED_ASSERT_UNLOCKED(); - - page = (caddr_t)((u_long)v & pp->pr_alloc->pa_pagemask); + FREECHECK_IN(&pp->pr_freecheck, v); #ifdef DIAGNOSTIC if (__predict_false(pp->pr_nout == 0)) { @@ -1060,18 +1161,11 @@ pool_do_put(struct pool *pp, void *v, st } #endif - if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) { + if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { pr_printlog(pp, NULL, printf); panic("pool_put: %s: page header missing", pp->pr_wchan); } -#ifdef LOCKDEBUG - /* - * Check if we're freeing a locked simple lock. - */ - simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size); -#endif - /* * Return to item list. */ @@ -1107,7 +1201,7 @@ pool_do_put(struct pool *pp, void *v, st pp->pr_flags &= ~PR_WANTED; if (ph->ph_nmissing == 0) pp->pr_nidle++; - wakeup((caddr_t)pp); + wakeup((void *)pp); return; } @@ -1128,7 +1222,7 @@ pool_do_put(struct pool *pp, void *v, st pp->pr_nidle++; if (pp->pr_npages > pp->pr_minpages && (pp->pr_npages > pp->pr_maxpages || - (pp->pr_alloc->pa_flags & PA_WANT) != 0)) { + pa_starved_p(pp->pr_alloc))) { pr_rmpage(pp, ph, pq); } else { LIST_REMOVE(ph, ph_pagelist); @@ -1140,9 +1234,7 @@ pool_do_put(struct pool *pp, void *v, st * be reclaimed by the pagedaemon. This minimizes * ping-pong'ing for memory. */ - s = splclock(); - ph->ph_time = mono_time; - splx(s); + getmicrotime(&ph->ph_time); } pool_update_curpage(pp); } @@ -1271,19 +1363,19 @@ pool_prime(struct pool *pp, int n) * Note, we must be called with the pool descriptor LOCKED. */ static void -pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph) +pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph) { struct pool_item *pi; - caddr_t cp = storage; - unsigned int align = pp->pr_align; - unsigned int ioff = pp->pr_itemoffset; + void *cp = storage; + const unsigned int align = pp->pr_align; + const unsigned int ioff = pp->pr_itemoffset; int n; - int s; LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); #ifdef DIAGNOSTIC - if (((u_long)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0) + if ((pp->pr_roflags & PR_NOALIGN) == 0 && + ((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0) panic("pool_prime_page: %s: unaligned page", pp->pr_wchan); #endif @@ -1294,9 +1386,7 @@ pool_prime_page(struct pool *pp, caddr_t LIST_INIT(&ph->ph_itemlist); ph->ph_page = storage; ph->ph_nmissing = 0; - s = splclock(); - ph->ph_time = mono_time; - splx(s); + getmicrotime(&ph->ph_time); if ((pp->pr_roflags & PR_PHINPAGE) == 0) SPLAY_INSERT(phtree, &pp->pr_phtree, ph); @@ -1305,7 +1395,7 @@ pool_prime_page(struct pool *pp, caddr_t /* * Color this page. */ - cp = (caddr_t)(cp + pp->pr_curcolor); + cp = (char *)cp + pp->pr_curcolor; if ((pp->pr_curcolor += align) > pp->pr_maxcolor) pp->pr_curcolor = 0; @@ -1313,7 +1403,9 @@ pool_prime_page(struct pool *pp, caddr_t * Adjust storage to apply aligment to `pr_itemoffset' in each item. */ if (ioff != 0) - cp = (caddr_t)(cp + (align - ioff)); + cp = (char *)cp + align - ioff; + + KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); /* * Insert remaining chunks on the bucket list. @@ -1325,7 +1417,7 @@ pool_prime_page(struct pool *pp, caddr_t pool_item_freelist_t *freelist = PR_FREELIST(ph); int i; - ph->ph_off = cp - storage; + ph->ph_off = (char *)cp - (char *)storage; ph->ph_firstfree = 0; for (i = 0; i < n - 1; i++) freelist[i] = i + 1; @@ -1341,7 +1433,9 @@ pool_prime_page(struct pool *pp, caddr_t #ifdef DIAGNOSTIC pi->pi_magic = PI_MAGIC; #endif - cp = (caddr_t)(cp + pp->pr_size); + cp = (char *)cp + pp->pr_size; + + KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); } } @@ -1462,7 +1556,6 @@ pool_reclaim(struct pool *pp) struct pool_pagelist pq; struct pool_cache_grouplist pcgl; struct timeval curtime, diff; - int s; if (pp->pr_drain_hook != NULL) { /* @@ -1484,9 +1577,7 @@ pool_reclaim(struct pool *pp) LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) pool_cache_reclaim(pc, &pq, &pcgl); - s = splclock(); - curtime = mono_time; - splx(s); + getmicrotime(&curtime); for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { phnext = LIST_NEXT(ph, ph_pagelist); @@ -1497,7 +1588,8 @@ pool_reclaim(struct pool *pp) KASSERT(ph->ph_nmissing == 0); timersub(&curtime, &ph->ph_time, &diff); - if (diff.tv_sec < pool_inactive_time) + if (diff.tv_sec < pool_inactive_time + && !pa_starved_p(pp->pr_alloc)) continue; /* @@ -1525,6 +1617,8 @@ pool_reclaim(struct pool *pp) * Drain pools, one at a time. * * Note, we must never be called from an interrupt context. + * + * XXX Pool can disappear while draining. */ void pool_drain(void *arg) @@ -1694,8 +1788,9 @@ pool_print1(struct pool *pp, const char (*pr)("\n"); if ((pp->pr_roflags & PR_LOGGING) == 0) (*pr)("\tno log\n"); - else + else { pr_printlog(pp, NULL, pr); + } skip_log: if (print_cache == 0) @@ -1743,19 +1838,21 @@ static int pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) { struct pool_item *pi; - caddr_t page; + void *page; int n; - page = (caddr_t)((u_long)ph & pp->pr_alloc->pa_pagemask); - if (page != ph->ph_page && - (pp->pr_roflags & PR_PHINPAGE) != 0) { - if (label != NULL) - printf("%s: ", label); - printf("pool(%p:%s): page inconsistency: page %p;" - " at page head addr %p (p %p)\n", pp, - pp->pr_wchan, ph->ph_page, - ph, page); - return 1; + if ((pp->pr_roflags & PR_NOALIGN) == 0) { + page = (void *)((uintptr_t)ph & pp->pr_alloc->pa_pagemask); + if (page != ph->ph_page && + (pp->pr_roflags & PR_PHINPAGE) != 0) { + if (label != NULL) + printf("%s: ", label); + printf("pool(%p:%s): page inconsistency: page %p;" + " at page head addr %p (p %p)\n", pp, + pp->pr_wchan, ph->ph_page, + ph, page); + return 1; + } } if ((pp->pr_roflags & PR_NOTOUCH) != 0) @@ -1770,15 +1867,16 @@ pool_chk_page(struct pool *pp, const cha if (label != NULL) printf("%s: ", label); printf("pool(%s): free list modified: magic=%x;" - " page %p; item ordinal %d;" - " addr %p (p %p)\n", + " page %p; item ordinal %d; addr %p\n", pp->pr_wchan, pi->pi_magic, ph->ph_page, - n, pi, page); + n, pi); panic("pool"); } #endif - page = - (caddr_t)((u_long)pi & pp->pr_alloc->pa_pagemask); + if ((pp->pr_roflags & PR_NOALIGN) != 0) { + continue; + } + page = (void *)((uintptr_t)pi & pp->pr_alloc->pa_pagemask); if (page == ph->ph_page) continue; @@ -1942,7 +2040,7 @@ pool_cache_get_paddr(struct pool_cache * #ifdef LOCKDEBUG if (flags & PR_WAITOK) - simple_lock_only_held(NULL, "pool_cache_get(PR_WAITOK)"); + ASSERT_SLEEPABLE(NULL, "pool_cache_get(PR_WAITOK)"); #endif simple_lock(&pc->pc_slock); @@ -1972,6 +2070,8 @@ pool_cache_get_paddr(struct pool_cache * return (NULL); } } + KASSERT((((vaddr_t)object + pc->pc_pool->pr_itemoffset) & + (pc->pc_pool->pr_align - 1)) == 0); if (object != NULL && pap != NULL) { #ifdef POOL_VTOPHYS *pap = POOL_VTOPHYS(object); @@ -1979,6 +2079,8 @@ pool_cache_get_paddr(struct pool_cache * *pap = POOL_PADDR_INVALID; #endif } + + FREECHECK_OUT(&pc->pc_freecheck, object); return (object); } @@ -1992,6 +2094,9 @@ pool_cache_get_paddr(struct pool_cache * } simple_unlock(&pc->pc_slock); + KASSERT((((vaddr_t)object + pc->pc_pool->pr_itemoffset) & + (pc->pc_pool->pr_align - 1)) == 0); + FREECHECK_OUT(&pc->pc_freecheck, object); return (object); } @@ -2007,6 +2112,8 @@ pool_cache_put_paddr(struct pool_cache * struct pool_cache_group *pcg; int s; + FREECHECK_IN(&pc->pc_freecheck, object); + if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) { goto destruct; } @@ -2072,26 +2179,43 @@ pool_cache_destruct_object(struct pool_c pool_put(pc->pc_pool, object); } +/* + * pool_do_cache_invalidate_grouplist: + * + * Invalidate a single grouplist and destruct all objects. + * XXX This is too expensive. We should swap the list then + * unlock. + */ static void pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl, struct pool_cache *pc, struct pool_pagelist *pq, struct pool_cache_grouplist *pcgdl) { - struct pool_cache_group *pcg, *npcg; + struct pool_cache_group *pcg; void *object; - for (pcg = LIST_FIRST(pcgsl); pcg != NULL; pcg = npcg) { - npcg = LIST_NEXT(pcg, pcg_list); + LOCK_ASSERT(simple_lock_held(&pc->pc_slock)); + LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock)); + + while ((pcg = LIST_FIRST(pcgsl)) != NULL) { + pc->pc_ngroups--; + LIST_REMOVE(pcg, pcg_list); + LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); + pc->pc_nitems -= pcg->pcg_avail; + simple_unlock(&pc->pc_pool->pr_slock); + simple_unlock(&pc->pc_slock); + while (pcg->pcg_avail != 0) { - pc->pc_nitems--; object = pcg_get(pcg, NULL); if (pc->pc_dtor != NULL) (*pc->pc_dtor)(pc->pc_arg, object); + simple_lock(&pc->pc_pool->pr_slock); pool_do_put(pc->pc_pool, object, pq); + simple_unlock(&pc->pc_pool->pr_slock); } - pc->pc_ngroups--; - LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); + + simple_lock(&pc->pc_slock); + simple_lock(&pc->pc_pool->pr_slock); } } @@ -2181,10 +2305,12 @@ void pool_page_free(struct pool *, void #ifdef POOL_SUBPAGE struct pool_allocator pool_allocator_kmem_fullpage = { pool_page_alloc, pool_page_free, 0, + .pa_backingmapptr = &kmem_map, }; #else struct pool_allocator pool_allocator_kmem = { pool_page_alloc, pool_page_free, 0, + .pa_backingmapptr = &kmem_map, }; #endif @@ -2194,10 +2320,12 @@ void pool_page_free_nointr(struct pool * #ifdef POOL_SUBPAGE struct pool_allocator pool_allocator_nointr_fullpage = { pool_page_alloc_nointr, pool_page_free_nointr, 0, + .pa_backingmapptr = &kernel_map, }; #else struct pool_allocator pool_allocator_nointr = { pool_page_alloc_nointr, pool_page_free_nointr, 0, + .pa_backingmapptr = &kernel_map, }; #endif @@ -2207,6 +2335,7 @@ void pool_subpage_free(struct pool *, vo struct pool_allocator pool_allocator_kmem = { pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, + .pa_backingmapptr = &kmem_map, }; void *pool_subpage_alloc_nointr(struct pool *, int); @@ -2214,131 +2343,43 @@ void pool_subpage_free_nointr(struct poo struct pool_allocator pool_allocator_nointr = { pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, + .pa_backingmapptr = &kmem_map, }; #endif /* POOL_SUBPAGE */ -/* - * We have at least three different resources for the same allocation and - * each resource can be depleted. First, we have the ready elements in the - * pool. Then we have the resource (typically a vm_map) for this allocator. - * Finally, we have physical memory. Waiting for any of these can be - * unnecessary when any other is freed, but the kernel doesn't support - * sleeping on multiple wait channels, so we have to employ another strategy. - * - * The caller sleeps on the pool (so that it can be awakened when an item - * is returned to the pool), but we set PA_WANT on the allocator. When a - * page is returned to the allocator and PA_WANT is set, pool_allocator_free - * will wake up all sleeping pools belonging to this allocator. - * - * XXX Thundering herd. - */ -void * -pool_allocator_alloc(struct pool *org, int flags) +static void * +pool_allocator_alloc(struct pool *pp, int flags) { - struct pool_allocator *pa = org->pr_alloc; - struct pool *pp, *start; - int s, freed; + struct pool_allocator *pa = pp->pr_alloc; void *res; - LOCK_ASSERT(!simple_lock_held(&org->pr_slock)); - - do { - if ((res = (*pa->pa_alloc)(org, flags)) != NULL) - return (res); - if ((flags & PR_WAITOK) == 0) { - /* - * We only run the drain hookhere if PR_NOWAIT. - * In other cases, the hook will be run in - * pool_reclaim(). - */ - if (org->pr_drain_hook != NULL) { - (*org->pr_drain_hook)(org->pr_drain_hook_arg, - flags); - if ((res = (*pa->pa_alloc)(org, flags)) != NULL) - return (res); - } - break; - } - + res = (*pa->pa_alloc)(pp, flags); + if (res == NULL && (flags & PR_WAITOK) == 0) { /* - * Drain all pools, that use this allocator. - * We do this to reclaim VA space. - * pa_alloc is responsible for waiting for - * physical memory. - * - * XXX We risk looping forever if start if someone - * calls pool_destroy on "start". But there is no - * other way to have potentially sleeping pool_reclaim, - * non-sleeping locks on pool_allocator, and some - * stirring of drained pools in the allocator. - * - * XXX Maybe we should use pool_head_slock for locking - * the allocators? + * We only run the drain hook here if PR_NOWAIT. + * In other cases, the hook will be run in + * pool_reclaim(). */ - freed = 0; - - s = splvm(); - simple_lock(&pa->pa_slock); - pp = start = TAILQ_FIRST(&pa->pa_list); - do { - TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list); - TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); - simple_unlock(&pa->pa_slock); - freed = pool_reclaim(pp); - simple_lock(&pa->pa_slock); - } while ((pp = TAILQ_FIRST(&pa->pa_list)) != start && - freed == 0); - - if (freed == 0) { - /* - * We set PA_WANT here, the caller will most likely - * sleep waiting for pages (if not, this won't hurt - * that much), and there is no way to set this in - * the caller without violating locking order. - */ - pa->pa_flags |= PA_WANT; + if (pp->pr_drain_hook != NULL) { + (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); + res = (*pa->pa_alloc)(pp, flags); } - simple_unlock(&pa->pa_slock); - splx(s); - } while (freed); - return (NULL); + } + return res; } -void +static void pool_allocator_free(struct pool *pp, void *v) { struct pool_allocator *pa = pp->pr_alloc; - int s; - - LOCK_ASSERT(!simple_lock_held(&pp->pr_slock)); (*pa->pa_free)(pp, v); - - s = splvm(); - simple_lock(&pa->pa_slock); - if ((pa->pa_flags & PA_WANT) == 0) { - simple_unlock(&pa->pa_slock); - splx(s); - return; - } - - TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) { - simple_lock(&pp->pr_slock); - if ((pp->pr_flags & PR_WANTED) != 0) { - pp->pr_flags &= ~PR_WANTED; - wakeup(pp); - } - simple_unlock(&pp->pr_slock); - } - pa->pa_flags &= ~PA_WANT; - simple_unlock(&pa->pa_slock); - splx(s); } void * pool_page_alloc(struct pool *pp, int flags) { - boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; + bool waitok = (flags & PR_WAITOK) ? true : false; return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); } @@ -2353,7 +2394,7 @@ pool_page_free(struct pool *pp, void *v) static void * pool_page_alloc_meta(struct pool *pp, int flags) { - boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; + bool waitok = (flags & PR_WAITOK) ? true : false; return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); } @@ -2405,7 +2446,7 @@ pool_subpage_free_nointr(struct pool *pp void * pool_page_alloc_nointr(struct pool *pp, int flags) { - boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; + bool waitok = (flags & PR_WAITOK) ? true : false; return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); }