Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.105 retrieving revision 1.110.2.2 diff -u -p -r1.105 -r1.110.2.2 --- src/sys/kern/subr_pool.c 2005/10/16 02:21:40 1.105 +++ src/sys/kern/subr_pool.c 2006/03/01 09:28:46 1.110.2.2 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.105 2005/10/16 02:21:40 christos Exp $ */ +/* $NetBSD: subr_pool.c,v 1.110.2.2 2006/03/01 09:28:46 yamt Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.105 2005/10/16 02:21:40 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.110.2.2 2006/03/01 09:28:46 yamt Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -214,7 +214,7 @@ struct pool_log { int pool_logsize = POOL_LOGSIZE; -static __inline void +static inline void pr_log(struct pool *pp, void *v, int action, const char *file, long line) { int n = pp->pr_curlogentry; @@ -267,7 +267,7 @@ pr_printlog(struct pool *pp, struct pool } } -static __inline void +static inline void pr_enter(struct pool *pp, const char *file, long line) { @@ -283,7 +283,7 @@ pr_enter(struct pool *pp, const char *fi pp->pr_entered_line = line; } -static __inline void +static inline void pr_leave(struct pool *pp) { @@ -296,7 +296,7 @@ pr_leave(struct pool *pp) pp->pr_entered_line = 0; } -static __inline void +static inline void pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) { @@ -312,7 +312,7 @@ pr_enter_check(struct pool *pp, void (*p #define pr_enter_check(pp, pr) #endif /* POOL_DIAGNOSTIC */ -static __inline int +static inline int pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, const void *v) { @@ -331,7 +331,7 @@ pr_item_notouch_index(const struct pool #define PR_INDEX_USED ((pool_item_freelist_t)-1) #define PR_INDEX_EOL ((pool_item_freelist_t)-2) -static __inline void +static inline void pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, void *obj) { @@ -343,7 +343,7 @@ pr_item_notouch_put(const struct pool *p ph->ph_firstfree = idx; } -static __inline void * +static inline void * pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) { int idx = ph->ph_firstfree; @@ -356,7 +356,7 @@ pr_item_notouch_get(const struct pool *p return ph->ph_page + ph->ph_off + idx * pp->pr_size; } -static __inline int +static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { if (a->ph_page < b->ph_page) @@ -373,7 +373,7 @@ SPLAY_GENERATE(phtree, pool_item_header, /* * Return the pool page header based on page address. */ -static __inline struct pool_item_header * +static inline struct pool_item_header * pr_find_pagehead(struct pool *pp, caddr_t page) { struct pool_item_header *ph, tmp; @@ -406,7 +406,7 @@ pr_pagelist_free(struct pool *pp, struct /* * Remove a page from the pool. */ -static __inline void +static inline void pr_rmpage(struct pool *pp, struct pool_item_header *ph, struct pool_pagelist *pq) { @@ -482,35 +482,19 @@ pool_init(struct pool *pp, size_t size, flags |= PR_LOGGING; #endif -#ifdef POOL_SUBPAGE - /* - * XXX We don't provide a real `nointr' back-end - * yet; all sub-pages come from a kmem back-end. - * maybe some day... - */ - if (palloc == NULL) { - extern struct pool_allocator pool_allocator_kmem_subpage; - palloc = &pool_allocator_kmem_subpage; - } - /* - * We'll assume any user-specified back-end allocator - * will deal with sub-pages, or simply don't care. - */ -#else if (palloc == NULL) palloc = &pool_allocator_kmem; +#ifdef POOL_SUBPAGE + if (size > palloc->pa_pagesz) { + if (palloc == &pool_allocator_kmem) + palloc = &pool_allocator_kmem_fullpage; + else if (palloc == &pool_allocator_nointr) + palloc = &pool_allocator_nointr_fullpage; + } #endif /* POOL_SUBPAGE */ if ((palloc->pa_flags & PA_INITIALIZED) == 0) { - if (palloc->pa_pagesz == 0) { -#ifdef POOL_SUBPAGE - if (palloc == &pool_allocator_kmem) - palloc->pa_pagesz = PAGE_SIZE; - else - palloc->pa_pagesz = POOL_SUBPAGE; -#else + if (palloc->pa_pagesz == 0) palloc->pa_pagesz = PAGE_SIZE; -#endif /* POOL_SUBPAGE */ - } TAILQ_INIT(&palloc->pa_list); @@ -929,13 +913,13 @@ pool_get(struct pool *pp, int flags) /* * Wait for items to be returned to this pool. * - * XXX: maybe we should wake up once a second and - * try again? + * wake up once a second and try again, + * as the check in pool_cache_put_paddr() is racy. */ pp->pr_flags |= PR_WANTED; /* PA_WANTED is already set on the allocator. */ pr_leave(pp); - ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock); + ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); pr_enter(pp, file, line); goto startover; } @@ -1033,6 +1017,7 @@ pool_get(struct pool *pp, int flags) } pp->pr_nget++; + pr_leave(pp); /* * If we have a low water mark and we are now below that low @@ -1046,7 +1031,6 @@ pool_get(struct pool *pp, int flags) */ } - pr_leave(pp); simple_unlock(&pp->pr_slock); return (v); } @@ -1581,9 +1565,24 @@ pool_print(struct pool *pp, const char * } void +pool_printall(const char *modif, void (*pr)(const char *, ...)) +{ + struct pool *pp; + + if (simple_lock_try(&pool_head_slock) == 0) { + (*pr)("WARNING: pool_head_slock is locked\n"); + } else { + simple_unlock(&pool_head_slock); + } + + LIST_FOREACH(pp, &pool_head, pr_poollist) { + pool_printit(pp, modif, pr); + } +} + +void pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) { - int didlock = 0; if (pp == NULL) { (*pr)("Must specify a pool to print.\n"); @@ -1602,12 +1601,9 @@ pool_printit(struct pool *pp, const char if (simple_lock_try(&pp->pr_slock) == 0) (*pr)("WARNING: pool %s is locked\n", pp->pr_wchan); else - didlock = 1; + simple_unlock(&pp->pr_slock); pool_print1(pp, modif, pr); - - if (didlock) - simple_unlock(&pp->pr_slock); } static void @@ -1882,7 +1878,7 @@ pool_cache_destroy(struct pool_cache *pc simple_unlock(&pp->pr_slock); } -static __inline void * +static inline void * pcg_get(struct pool_cache_group *pcg, paddr_t *pap) { void *object; @@ -1901,7 +1897,7 @@ pcg_get(struct pool_cache_group *pcg, pa return (object); } -static __inline void +static inline void pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) { u_int idx; @@ -2007,6 +2003,10 @@ pool_cache_put_paddr(struct pool_cache * struct pool_cache_group *pcg; int s; + if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) { + goto destruct; + } + simple_lock(&pc->pc_slock); pcg = LIST_FIRST(&pc->pc_partgroups); @@ -2028,6 +2028,7 @@ pool_cache_put_paddr(struct pool_cache * pcg = pool_get(&pcgpool, PR_NOWAIT); splx(s); if (pcg == NULL) { +destruct: /* * Unable to allocate a cache group; destruct the object @@ -2068,14 +2069,14 @@ pool_cache_destruct_object(struct pool_c } static void -pool_do_cache_invalidate_grouplist(struct pool_cache_group *pcg, +pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl, struct pool_cache *pc, struct pool_pagelist *pq, - struct pool_cache_grouplist *pcgl) + struct pool_cache_grouplist *pcgdl) { - struct pool_cache_group *npcg; + struct pool_cache_group *pcg, *npcg; void *object; - for (; pcg != NULL; pcg = npcg) { + for (pcg = LIST_FIRST(pcgsl); pcg != NULL; pcg = npcg) { npcg = LIST_NEXT(pcg, pcg_list); while (pcg->pcg_avail != 0) { pc->pc_nitems--; @@ -2086,7 +2087,7 @@ pool_do_cache_invalidate_grouplist(struc } pc->pc_ngroups--; LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(pcgl, pcg, pcg_list); + LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); } } @@ -2098,10 +2099,8 @@ pool_do_cache_invalidate(struct pool_cac LOCK_ASSERT(simple_lock_held(&pc->pc_slock)); LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock)); - pool_do_cache_invalidate_grouplist(LIST_FIRST(&pc->pc_fullgroups), - pc, pq, pcgl); - pool_do_cache_invalidate_grouplist(LIST_FIRST(&pc->pc_partgroups), - pc, pq, pcgl); + pool_do_cache_invalidate_grouplist(&pc->pc_fullgroups, pc, pq, pcgl); + pool_do_cache_invalidate_grouplist(&pc->pc_partgroups, pc, pq, pcgl); KASSERT(LIST_EMPTY(&pc->pc_partgroups)); KASSERT(LIST_EMPTY(&pc->pc_fullgroups)); @@ -2175,23 +2174,42 @@ pool_cache_reclaim(struct pool_cache *pc void *pool_page_alloc(struct pool *, int); void pool_page_free(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_kmem_fullpage = { + pool_page_alloc, pool_page_free, 0, +}; +#else struct pool_allocator pool_allocator_kmem = { pool_page_alloc, pool_page_free, 0, }; +#endif void *pool_page_alloc_nointr(struct pool *, int); void pool_page_free_nointr(struct pool *, void *); +#ifdef POOL_SUBPAGE +struct pool_allocator pool_allocator_nointr_fullpage = { + pool_page_alloc_nointr, pool_page_free_nointr, 0, +}; +#else struct pool_allocator pool_allocator_nointr = { pool_page_alloc_nointr, pool_page_free_nointr, 0, }; +#endif #ifdef POOL_SUBPAGE void *pool_subpage_alloc(struct pool *, int); void pool_subpage_free(struct pool *, void *); -struct pool_allocator pool_allocator_kmem_subpage = { - pool_subpage_alloc, pool_subpage_free, 0, +struct pool_allocator pool_allocator_kmem = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, +}; + +void *pool_subpage_alloc_nointr(struct pool *, int); +void pool_subpage_free_nointr(struct pool *, void *); + +struct pool_allocator pool_allocator_nointr = { + pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, }; #endif /* POOL_SUBPAGE */ @@ -2239,8 +2257,8 @@ pool_allocator_alloc(struct pool *org, i } /* - * Drain all pools, except "org", that use this - * allocator. We do this to reclaim VA space. + * Drain all pools, that use this allocator. + * We do this to reclaim VA space. * pa_alloc is responsible for waiting for * physical memory. * @@ -2261,8 +2279,6 @@ pool_allocator_alloc(struct pool *org, i do { TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list); TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); - if (pp == org) - continue; simple_unlock(&pa->pa_slock); freed = pool_reclaim(pp); simple_lock(&pa->pa_slock); @@ -2369,19 +2385,19 @@ pool_subpage_free(struct pool *pp, void /* We don't provide a real nointr allocator. Maybe later. */ void * -pool_page_alloc_nointr(struct pool *pp, int flags) +pool_subpage_alloc_nointr(struct pool *pp, int flags) { return (pool_subpage_alloc(pp, flags)); } void -pool_page_free_nointr(struct pool *pp, void *v) +pool_subpage_free_nointr(struct pool *pp, void *v) { pool_subpage_free(pp, v); } -#else +#endif /* POOL_SUBPAGE */ void * pool_page_alloc_nointr(struct pool *pp, int flags) { @@ -2396,4 +2412,3 @@ pool_page_free_nointr(struct pool *pp, v uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); } -#endif /* POOL_SUBPAGE */