Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.103 retrieving revision 1.110.2.1 diff -u -p -r1.103 -r1.110.2.1 --- src/sys/kern/subr_pool.c 2005/10/15 21:22:46 1.103 +++ src/sys/kern/subr_pool.c 2006/02/01 14:52:20 1.110.2.1 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.103 2005/10/15 21:22:46 chs Exp $ */ +/* $NetBSD: subr_pool.c,v 1.110.2.1 2006/02/01 14:52:20 yamt Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.103 2005/10/15 21:22:46 chs Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.110.2.1 2006/02/01 14:52:20 yamt Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -214,7 +214,7 @@ struct pool_log { int pool_logsize = POOL_LOGSIZE; -static __inline void +static inline void pr_log(struct pool *pp, void *v, int action, const char *file, long line) { int n = pp->pr_curlogentry; @@ -267,7 +267,7 @@ pr_printlog(struct pool *pp, struct pool } } -static __inline void +static inline void pr_enter(struct pool *pp, const char *file, long line) { @@ -283,7 +283,7 @@ pr_enter(struct pool *pp, const char *fi pp->pr_entered_line = line; } -static __inline void +static inline void pr_leave(struct pool *pp) { @@ -296,7 +296,7 @@ pr_leave(struct pool *pp) pp->pr_entered_line = 0; } -static __inline void +static inline void pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) { @@ -312,7 +312,7 @@ pr_enter_check(struct pool *pp, void (*p #define pr_enter_check(pp, pr) #endif /* POOL_DIAGNOSTIC */ -static __inline int +static inline int pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, const void *v) { @@ -331,7 +331,7 @@ pr_item_notouch_index(const struct pool #define PR_INDEX_USED ((pool_item_freelist_t)-1) #define PR_INDEX_EOL ((pool_item_freelist_t)-2) -static __inline void +static inline void pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, void *obj) { @@ -343,7 +343,7 @@ pr_item_notouch_put(const struct pool *p ph->ph_firstfree = idx; } -static __inline void * +static inline void * pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) { int idx = ph->ph_firstfree; @@ -356,7 +356,7 @@ pr_item_notouch_get(const struct pool *p return ph->ph_page + ph->ph_off + idx * pp->pr_size; } -static __inline int +static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { if (a->ph_page < b->ph_page) @@ -373,7 +373,7 @@ SPLAY_GENERATE(phtree, pool_item_header, /* * Return the pool page header based on page address. */ -static __inline struct pool_item_header * +static inline struct pool_item_header * pr_find_pagehead(struct pool *pp, caddr_t page) { struct pool_item_header *ph, tmp; @@ -406,7 +406,7 @@ pr_pagelist_free(struct pool *pp, struct /* * Remove a page from the pool. */ -static __inline void +static inline void pr_rmpage(struct pool *pp, struct pool_item_header *ph, struct pool_pagelist *pq) { @@ -929,13 +929,13 @@ pool_get(struct pool *pp, int flags) /* * Wait for items to be returned to this pool. * - * XXX: maybe we should wake up once a second and - * try again? + * wake up once a second and try again, + * as the check in pool_cache_put_paddr() is racy. */ pp->pr_flags |= PR_WANTED; /* PA_WANTED is already set on the allocator. */ pr_leave(pp); - ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock); + ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock); pr_enter(pp, file, line); goto startover; } @@ -1033,6 +1033,7 @@ pool_get(struct pool *pp, int flags) } pp->pr_nget++; + pr_leave(pp); /* * If we have a low water mark and we are now below that low @@ -1046,7 +1047,6 @@ pool_get(struct pool *pp, int flags) */ } - pr_leave(pp); simple_unlock(&pp->pr_slock); return (v); } @@ -1581,9 +1581,24 @@ pool_print(struct pool *pp, const char * } void +pool_printall(const char *modif, void (*pr)(const char *, ...)) +{ + struct pool *pp; + + if (simple_lock_try(&pool_head_slock) == 0) { + (*pr)("WARNING: pool_head_slock is locked\n"); + } else { + simple_unlock(&pool_head_slock); + } + + LIST_FOREACH(pp, &pool_head, pr_poollist) { + pool_printit(pp, modif, pr); + } +} + +void pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) { - int didlock = 0; if (pp == NULL) { (*pr)("Must specify a pool to print.\n"); @@ -1602,12 +1617,9 @@ pool_printit(struct pool *pp, const char if (simple_lock_try(&pp->pr_slock) == 0) (*pr)("WARNING: pool %s is locked\n", pp->pr_wchan); else - didlock = 1; + simple_unlock(&pp->pr_slock); pool_print1(pp, modif, pr); - - if (didlock) - simple_unlock(&pp->pr_slock); } static void @@ -1882,7 +1894,7 @@ pool_cache_destroy(struct pool_cache *pc simple_unlock(&pp->pr_slock); } -static __inline void * +static inline void * pcg_get(struct pool_cache_group *pcg, paddr_t *pap) { void *object; @@ -1901,7 +1913,7 @@ pcg_get(struct pool_cache_group *pcg, pa return (object); } -static __inline void +static inline void pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) { u_int idx; @@ -2007,6 +2019,10 @@ pool_cache_put_paddr(struct pool_cache * struct pool_cache_group *pcg; int s; + if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) { + goto destruct; + } + simple_lock(&pc->pc_slock); pcg = LIST_FIRST(&pc->pc_partgroups); @@ -2028,6 +2044,7 @@ pool_cache_put_paddr(struct pool_cache * pcg = pool_get(&pcgpool, PR_NOWAIT); splx(s); if (pcg == NULL) { +destruct: /* * Unable to allocate a cache group; destruct the object @@ -2068,18 +2085,14 @@ pool_cache_destruct_object(struct pool_c } static void -pool_do_cache_invalidate(struct pool_cache *pc, struct pool_pagelist *pq, - struct pool_cache_grouplist *pcgl) +pool_do_cache_invalidate_grouplist(struct pool_cache_grouplist *pcgsl, + struct pool_cache *pc, struct pool_pagelist *pq, + struct pool_cache_grouplist *pcgdl) { struct pool_cache_group *pcg, *npcg; void *object; - LOCK_ASSERT(simple_lock_held(&pc->pc_slock)); - LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock)); - - for (pcg = LIST_FIRST(&pc->pc_fullgroups); pcg != NULL; pcg = npcg) { - -loop: + for (pcg = LIST_FIRST(pcgsl); pcg != NULL; pcg = npcg) { npcg = LIST_NEXT(pcg, pcg_list); while (pcg->pcg_avail != 0) { pc->pc_nitems--; @@ -2090,12 +2103,20 @@ loop: } pc->pc_ngroups--; LIST_REMOVE(pcg, pcg_list); - LIST_INSERT_HEAD(pcgl, pcg, pcg_list); - } - pcg = LIST_FIRST(&pc->pc_partgroups); - if (pcg != NULL) { - goto loop; + LIST_INSERT_HEAD(pcgdl, pcg, pcg_list); } +} + +static void +pool_do_cache_invalidate(struct pool_cache *pc, struct pool_pagelist *pq, + struct pool_cache_grouplist *pcgl) +{ + + LOCK_ASSERT(simple_lock_held(&pc->pc_slock)); + LOCK_ASSERT(simple_lock_held(&pc->pc_pool->pr_slock)); + + pool_do_cache_invalidate_grouplist(&pc->pc_fullgroups, pc, pq, pcgl); + pool_do_cache_invalidate_grouplist(&pc->pc_partgroups, pc, pq, pcgl); KASSERT(LIST_EMPTY(&pc->pc_partgroups)); KASSERT(LIST_EMPTY(&pc->pc_fullgroups)); @@ -2233,8 +2254,8 @@ pool_allocator_alloc(struct pool *org, i } /* - * Drain all pools, except "org", that use this - * allocator. We do this to reclaim VA space. + * Drain all pools, that use this allocator. + * We do this to reclaim VA space. * pa_alloc is responsible for waiting for * physical memory. * @@ -2255,8 +2276,6 @@ pool_allocator_alloc(struct pool *org, i do { TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list); TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); - if (pp == org) - continue; simple_unlock(&pa->pa_slock); freed = pool_reclaim(pp); simple_lock(&pa->pa_slock);