Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

===================================================================
RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v
rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present.
retrieving revision 1.87.2.7
retrieving revision 1.112
diff -u -p -r1.87.2.7 -r1.112
--- src/sys/kern/subr_pool.c	2005/12/11 10:29:12	1.87.2.7
+++ src/sys/kern/subr_pool.c	2006/02/24 11:46:20	1.112
@@ -1,4 +1,4 @@
-/*	$NetBSD: subr_pool.c,v 1.87.2.7 2005/12/11 10:29:12 christos Exp $	*/
+/*	$NetBSD: subr_pool.c,v 1.112 2006/02/24 11:46:20 bjh21 Exp $	*/
 
 /*-
  * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.87.2.7 2005/12/11 10:29:12 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.112 2006/02/24 11:46:20 bjh21 Exp $");
 
 #include "opt_pool.h"
 #include "opt_poollog.h"
@@ -214,7 +214,7 @@ struct pool_log {
 
 int pool_logsize = POOL_LOGSIZE;
 
-static __inline void
+static inline void
 pr_log(struct pool *pp, void *v, int action, const char *file, long line)
 {
 	int n = pp->pr_curlogentry;
@@ -267,7 +267,7 @@ pr_printlog(struct pool *pp, struct pool
 	}
 }
 
-static __inline void
+static inline void
 pr_enter(struct pool *pp, const char *file, long line)
 {
 
@@ -283,7 +283,7 @@ pr_enter(struct pool *pp, const char *fi
 	pp->pr_entered_line = line;
 }
 
-static __inline void
+static inline void
 pr_leave(struct pool *pp)
 {
 
@@ -296,7 +296,7 @@ pr_leave(struct pool *pp)
 	pp->pr_entered_line = 0;
 }
 
-static __inline void
+static inline void
 pr_enter_check(struct pool *pp, void (*pr)(const char *, ...))
 {
 
@@ -312,7 +312,7 @@ pr_enter_check(struct pool *pp, void (*p
 #define	pr_enter_check(pp, pr)
 #endif /* POOL_DIAGNOSTIC */
 
-static __inline int
+static inline int
 pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph,
     const void *v)
 {
@@ -331,7 +331,7 @@ pr_item_notouch_index(const struct pool 
 #define	PR_INDEX_USED	((pool_item_freelist_t)-1)
 #define	PR_INDEX_EOL	((pool_item_freelist_t)-2)
 
-static __inline void
+static inline void
 pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph,
     void *obj)
 {
@@ -343,7 +343,7 @@ pr_item_notouch_put(const struct pool *p
 	ph->ph_firstfree = idx;
 }
 
-static __inline void *
+static inline void *
 pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph)
 {
 	int idx = ph->ph_firstfree;
@@ -356,7 +356,7 @@ pr_item_notouch_get(const struct pool *p
 	return ph->ph_page + ph->ph_off + idx * pp->pr_size;
 }
 
-static __inline int
+static inline int
 phtree_compare(struct pool_item_header *a, struct pool_item_header *b)
 {
 	if (a->ph_page < b->ph_page)
@@ -373,7 +373,7 @@ SPLAY_GENERATE(phtree, pool_item_header,
 /*
  * Return the pool page header based on page address.
  */
-static __inline struct pool_item_header *
+static inline struct pool_item_header *
 pr_find_pagehead(struct pool *pp, caddr_t page)
 {
 	struct pool_item_header *ph, tmp;
@@ -406,7 +406,7 @@ pr_pagelist_free(struct pool *pp, struct
 /*
  * Remove a page from the pool.
  */
-static __inline void
+static inline void
 pr_rmpage(struct pool *pp, struct pool_item_header *ph,
      struct pool_pagelist *pq)
 {
@@ -482,35 +482,19 @@ pool_init(struct pool *pp, size_t size, 
 		flags |= PR_LOGGING;
 #endif
 
-#ifdef POOL_SUBPAGE
-	/*
-	 * XXX We don't provide a real `nointr' back-end
-	 * yet; all sub-pages come from a kmem back-end.
-	 * maybe some day...
-	 */
-	if (palloc == NULL) {
-		extern struct pool_allocator pool_allocator_kmem_subpage;
-		palloc = &pool_allocator_kmem_subpage;
-	}
-	/*
-	 * We'll assume any user-specified back-end allocator
-	 * will deal with sub-pages, or simply don't care.
-	 */
-#else
 	if (palloc == NULL)
 		palloc = &pool_allocator_kmem;
+#ifdef POOL_SUBPAGE
+	if (size > palloc->pa_pagesz) {
+		if (palloc == &pool_allocator_kmem)
+			palloc = &pool_allocator_kmem_fullpage;
+		else if (palloc == &pool_allocator_nointr)
+			palloc = &pool_allocator_nointr_fullpage;
+	}		
 #endif /* POOL_SUBPAGE */
 	if ((palloc->pa_flags & PA_INITIALIZED) == 0) {
-		if (palloc->pa_pagesz == 0) {
-#ifdef POOL_SUBPAGE
-			if (palloc == &pool_allocator_kmem)
-				palloc->pa_pagesz = PAGE_SIZE;
-			else
-				palloc->pa_pagesz = POOL_SUBPAGE;
-#else
+		if (palloc->pa_pagesz == 0)
 			palloc->pa_pagesz = PAGE_SIZE;
-#endif /* POOL_SUBPAGE */
-		}
 
 		TAILQ_INIT(&palloc->pa_list);
 
@@ -929,13 +913,13 @@ pool_get(struct pool *pp, int flags)
 			/*
 			 * Wait for items to be returned to this pool.
 			 *
-			 * XXX: maybe we should wake up once a second and
-			 * try again?
+			 * wake up once a second and try again,
+			 * as the check in pool_cache_put_paddr() is racy.
 			 */
 			pp->pr_flags |= PR_WANTED;
 			/* PA_WANTED is already set on the allocator. */
 			pr_leave(pp);
-			ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);
+			ltsleep(pp, PSWP, pp->pr_wchan, hz, &pp->pr_slock);
 			pr_enter(pp, file, line);
 			goto startover;
 		}
@@ -1033,6 +1017,7 @@ pool_get(struct pool *pp, int flags)
 	}
 
 	pp->pr_nget++;
+	pr_leave(pp);
 
 	/*
 	 * If we have a low water mark and we are now below that low
@@ -1046,7 +1031,6 @@ pool_get(struct pool *pp, int flags)
 		 */
 	}
 
-	pr_leave(pp);
 	simple_unlock(&pp->pr_slock);
 	return (v);
 }
@@ -1894,7 +1878,7 @@ pool_cache_destroy(struct pool_cache *pc
 	simple_unlock(&pp->pr_slock);
 }
 
-static __inline void *
+static inline void *
 pcg_get(struct pool_cache_group *pcg, paddr_t *pap)
 {
 	void *object;
@@ -1913,7 +1897,7 @@ pcg_get(struct pool_cache_group *pcg, pa
 	return (object);
 }
 
-static __inline void
+static inline void
 pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa)
 {
 	u_int idx;
@@ -2019,6 +2003,10 @@ pool_cache_put_paddr(struct pool_cache *
 	struct pool_cache_group *pcg;
 	int s;
 
+	if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) {
+		goto destruct;
+	}
+
 	simple_lock(&pc->pc_slock);
 
 	pcg = LIST_FIRST(&pc->pc_partgroups);
@@ -2040,6 +2028,7 @@ pool_cache_put_paddr(struct pool_cache *
 		pcg = pool_get(&pcgpool, PR_NOWAIT);
 		splx(s);
 		if (pcg == NULL) {
+destruct:
 
 			/*
 			 * Unable to allocate a cache group; destruct the object
@@ -2185,23 +2174,42 @@ pool_cache_reclaim(struct pool_cache *pc
 void	*pool_page_alloc(struct pool *, int);
 void	pool_page_free(struct pool *, void *);
 
+#ifdef POOL_SUBPAGE
+struct pool_allocator pool_allocator_kmem_fullpage = {
+	pool_page_alloc, pool_page_free, 0,
+};
+#else
 struct pool_allocator pool_allocator_kmem = {
 	pool_page_alloc, pool_page_free, 0,
 };
+#endif
 
 void	*pool_page_alloc_nointr(struct pool *, int);
 void	pool_page_free_nointr(struct pool *, void *);
 
+#ifdef POOL_SUBPAGE
+struct pool_allocator pool_allocator_nointr_fullpage = {
+	pool_page_alloc_nointr, pool_page_free_nointr, 0,
+};
+#else
 struct pool_allocator pool_allocator_nointr = {
 	pool_page_alloc_nointr, pool_page_free_nointr, 0,
 };
+#endif
 
 #ifdef POOL_SUBPAGE
 void	*pool_subpage_alloc(struct pool *, int);
 void	pool_subpage_free(struct pool *, void *);
 
-struct pool_allocator pool_allocator_kmem_subpage = {
-	pool_subpage_alloc, pool_subpage_free, 0,
+struct pool_allocator pool_allocator_kmem = {
+	pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
+};
+
+void	*pool_subpage_alloc_nointr(struct pool *, int);
+void	pool_subpage_free_nointr(struct pool *, void *);
+
+struct pool_allocator pool_allocator_nointr = {
+	pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
 };
 #endif /* POOL_SUBPAGE */
 
@@ -2249,8 +2257,8 @@ pool_allocator_alloc(struct pool *org, i
 		}
 
 		/*
-		 * Drain all pools, except "org", that use this
-		 * allocator.  We do this to reclaim VA space.
+		 * Drain all pools, that use this allocator.
+		 * We do this to reclaim VA space.
 		 * pa_alloc is responsible for waiting for
 		 * physical memory.
 		 *
@@ -2271,8 +2279,6 @@ pool_allocator_alloc(struct pool *org, i
 		do {
 			TAILQ_REMOVE(&pa->pa_list, pp, pr_alloc_list);
 			TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list);
-			if (pp == org)
-				continue;
 			simple_unlock(&pa->pa_slock);
 			freed = pool_reclaim(pp);
 			simple_lock(&pa->pa_slock);
@@ -2379,19 +2385,19 @@ pool_subpage_free(struct pool *pp, void 
 
 /* We don't provide a real nointr allocator.  Maybe later. */
 void *
-pool_page_alloc_nointr(struct pool *pp, int flags)
+pool_subpage_alloc_nointr(struct pool *pp, int flags)
 {
 
 	return (pool_subpage_alloc(pp, flags));
 }
 
 void
-pool_page_free_nointr(struct pool *pp, void *v)
+pool_subpage_free_nointr(struct pool *pp, void *v)
 {
 
 	pool_subpage_free(pp, v);
 }
-#else
+#endif /* POOL_SUBPAGE */
 void *
 pool_page_alloc_nointr(struct pool *pp, int flags)
 {
@@ -2406,4 +2412,3 @@ pool_page_free_nointr(struct pool *pp, v
 
 	uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v);
 }
-#endif /* POOL_SUBPAGE */