Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

===================================================================
RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v
retrieving revision 1.99.8.2
retrieving revision 1.100
diff -u -p -r1.99.8.2 -r1.100
--- src/sys/kern/subr_pool.c	2006/03/10 13:19:42	1.99.8.2
+++ src/sys/kern/subr_pool.c	2005/04/01 11:59:37	1.100
@@ -1,4 +1,4 @@
-/*	$NetBSD: subr_pool.c,v 1.99.8.2 2006/03/10 13:19:42 tron Exp $	*/
+/*	$NetBSD: subr_pool.c,v 1.100 2005/04/01 11:59:37 yamt Exp $	*/
 
 /*-
  * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.99.8.2 2006/03/10 13:19:42 tron Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.100 2005/04/01 11:59:37 yamt Exp $");
 
 #include "opt_pool.h"
 #include "opt_poollog.h"
@@ -174,7 +174,7 @@ struct pool_item {
 /* The cache group pool. */
 static struct pool pcgpool;
 
-static void	pool_cache_reclaim(struct pool_cache *, struct pool_pagelist *);
+static void	pool_cache_reclaim(struct pool_cache *);
 
 static int	pool_catchup(struct pool *);
 static void	pool_prime_page(struct pool *, caddr_t,
@@ -384,23 +384,6 @@ pr_find_pagehead(struct pool *pp, caddr_
 	return ph;
 }
 
-static void
-pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq)
-{
-	struct pool_item_header *ph;
-	int s;
-
-	while ((ph = LIST_FIRST(pq)) != NULL) {
-		LIST_REMOVE(ph, ph_pagelist);
-		pool_allocator_free(pp, ph->ph_page);
-		if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
-			s = splvm();
-			pool_put(pp->pr_phpool, ph);
-			splx(s);
-		}
-	}
-}
-
 /*
  * Remove a page from the pool.
  */
@@ -408,8 +391,9 @@ static __inline void
 pr_rmpage(struct pool *pp, struct pool_item_header *ph,
      struct pool_pagelist *pq)
 {
+	int s;
 
-	LOCK_ASSERT(simple_lock_held(&pp->pr_slock));
+	LOCK_ASSERT(!simple_lock_held(&pp->pr_slock) || pq != NULL);
 
 	/*
 	 * If the page was idle, decrement the idle page count.
@@ -427,13 +411,21 @@ pr_rmpage(struct pool *pp, struct pool_i
 	pp->pr_nitems -= pp->pr_itemsperpage;
 
 	/*
-	 * Unlink the page from the pool and queue it for release.
+	 * Unlink a page from the pool and release it (or queue it for release).
 	 */
 	LIST_REMOVE(ph, ph_pagelist);
 	if ((pp->pr_roflags & PR_PHINPAGE) == 0)
 		SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
-	LIST_INSERT_HEAD(pq, ph, ph_pagelist);
-
+	if (pq) {
+		LIST_INSERT_HEAD(pq, ph, ph_pagelist);
+	} else {
+		pool_allocator_free(pp, ph->ph_page);
+		if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
+			s = splvm();
+			pool_put(pp->pr_phpool, ph);
+			splx(s);
+		}
+	}
 	pp->pr_npages--;
 	pp->pr_npagefree++;
 
@@ -480,19 +472,35 @@ pool_init(struct pool *pp, size_t size, 
 		flags |= PR_LOGGING;
 #endif
 
+#ifdef POOL_SUBPAGE
+	/*
+	 * XXX We don't provide a real `nointr' back-end
+	 * yet; all sub-pages come from a kmem back-end.
+	 * maybe some day...
+	 */
+	if (palloc == NULL) {
+		extern struct pool_allocator pool_allocator_kmem_subpage;
+		palloc = &pool_allocator_kmem_subpage;
+	}
+	/*
+	 * We'll assume any user-specified back-end allocator
+	 * will deal with sub-pages, or simply don't care.
+	 */
+#else
 	if (palloc == NULL)
 		palloc = &pool_allocator_kmem;
-#ifdef POOL_SUBPAGE
-	if (size > palloc->pa_pagesz) {
-		if (palloc == &pool_allocator_kmem)
-			palloc = &pool_allocator_kmem_fullpage;
-		else if (palloc == &pool_allocator_nointr)
-			palloc = &pool_allocator_nointr_fullpage;
-	}		
 #endif /* POOL_SUBPAGE */
 	if ((palloc->pa_flags & PA_INITIALIZED) == 0) {
-		if (palloc->pa_pagesz == 0)
+		if (palloc->pa_pagesz == 0) {
+#ifdef POOL_SUBPAGE
+			if (palloc == &pool_allocator_kmem)
+				palloc->pa_pagesz = PAGE_SIZE;
+			else
+				palloc->pa_pagesz = POOL_SUBPAGE;
+#else
 			palloc->pa_pagesz = PAGE_SIZE;
+#endif /* POOL_SUBPAGE */
+		}
 
 		TAILQ_INIT(&palloc->pa_list);
 
@@ -688,28 +696,20 @@ pool_init(struct pool *pp, size_t size, 
 void
 pool_destroy(struct pool *pp)
 {
-	struct pool_pagelist pq;
 	struct pool_item_header *ph;
+	struct pool_cache *pc;
 	int s;
 
-	/* Remove from global pool list */
-	simple_lock(&pool_head_slock);
-	TAILQ_REMOVE(&pool_head, pp, pr_poollist);
-	if (drainpp == pp)
-		drainpp = NULL;
-	simple_unlock(&pool_head_slock);
-
-	/* Remove this pool from its allocator's list of pools. */
+	/* Locking order: pool_allocator -> pool */
 	s = splvm();
 	simple_lock(&pp->pr_alloc->pa_slock);
 	TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);
 	simple_unlock(&pp->pr_alloc->pa_slock);
 	splx(s);
 
-	s = splvm();
-	simple_lock(&pp->pr_slock);
-
-	KASSERT(TAILQ_EMPTY(&pp->pr_cachelist));
+	/* Destroy all caches for this pool. */
+	while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL)
+		pool_cache_destroy(pc);
 
 #ifdef DIAGNOSTIC
 	if (pp->pr_nout != 0) {
@@ -719,18 +719,19 @@ pool_destroy(struct pool *pp)
 	}
 #endif
 
-	KASSERT(LIST_EMPTY(&pp->pr_fullpages));
-	KASSERT(LIST_EMPTY(&pp->pr_partpages));
-
 	/* Remove all pages */
-	LIST_INIT(&pq);
 	while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
-		pr_rmpage(pp, ph, &pq);
-
-	simple_unlock(&pp->pr_slock);
-	splx(s);
+		pr_rmpage(pp, ph, NULL);
+	KASSERT(LIST_EMPTY(&pp->pr_fullpages));
+	KASSERT(LIST_EMPTY(&pp->pr_partpages));
 
-	pr_pagelist_free(pp, &pq);
+	/* Remove from global pool list */
+	simple_lock(&pool_head_slock);
+	TAILQ_REMOVE(&pool_head, pp, pr_poollist);
+	if (drainpp == pp) {
+		drainpp = NULL;
+	}
+	simple_unlock(&pool_head_slock);
 
 #ifdef POOL_DIAGNOSTIC
 	if ((pp->pr_roflags & PR_LOGGING) != 0)
@@ -1036,7 +1037,7 @@ pool_get(struct pool *pp, int flags)
  * Internal version of pool_put().  Pool is already locked/entered.
  */
 static void
-pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq)
+pool_do_put(struct pool *pp, void *v)
 {
 	struct pool_item *pi = v;
 	struct pool_item_header *ph;
@@ -1124,7 +1125,9 @@ pool_do_put(struct pool *pp, void *v, st
 		if (pp->pr_npages > pp->pr_minpages &&
 		    (pp->pr_npages > pp->pr_maxpages ||
 		     (pp->pr_alloc->pa_flags & PA_WANT) != 0)) {
-			pr_rmpage(pp, ph, pq);
+			simple_unlock(&pp->pr_slock);
+			pr_rmpage(pp, ph, NULL);
+			simple_lock(&pp->pr_slock);
 		} else {
 			LIST_REMOVE(ph, ph_pagelist);
 			LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
@@ -1162,22 +1165,16 @@ pool_do_put(struct pool *pp, void *v, st
 void
 _pool_put(struct pool *pp, void *v, const char *file, long line)
 {
-	struct pool_pagelist pq;
-
-	LIST_INIT(&pq);
 
 	simple_lock(&pp->pr_slock);
 	pr_enter(pp, file, line);
 
 	pr_log(pp, v, PRLOG_PUT, file, line);
 
-	pool_do_put(pp, v, &pq);
+	pool_do_put(pp, v);
 
 	pr_leave(pp);
 	simple_unlock(&pp->pr_slock);
-
-	if (! LIST_EMPTY(&pq))
-		pr_pagelist_free(pp, &pq);
 }
 #undef pool_put
 #endif /* POOL_DIAGNOSTIC */
@@ -1185,16 +1182,12 @@ _pool_put(struct pool *pp, void *v, cons
 void
 pool_put(struct pool *pp, void *v)
 {
-	struct pool_pagelist pq;
-
-	LIST_INIT(&pq);
 
 	simple_lock(&pp->pr_slock);
-	pool_do_put(pp, v, &pq);
-	simple_unlock(&pp->pr_slock);
 
-	if (! LIST_EMPTY(&pq))
-		pr_pagelist_free(pp, &pq);
+	pool_do_put(pp, v);
+
+	simple_unlock(&pp->pr_slock);
 }
 
 #ifdef POOL_DIAGNOSTIC
@@ -1476,7 +1469,7 @@ pool_reclaim(struct pool *pp)
 	 * Reclaim items from the pool's caches.
 	 */
 	TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist)
-		pool_cache_reclaim(pc, &pq);
+		pool_cache_reclaim(pc);
 
 	s = splclock();
 	curtime = mono_time;
@@ -1510,7 +1503,17 @@ pool_reclaim(struct pool *pp)
 	if (LIST_EMPTY(&pq))
 		return (0);
 
-	pr_pagelist_free(pp, &pq);
+	while ((ph = LIST_FIRST(&pq)) != NULL) {
+		LIST_REMOVE(ph, ph_pagelist);
+		pool_allocator_free(pp, ph->ph_page);
+		if (pp->pr_roflags & PR_PHINPAGE) {
+			continue;
+		}
+		s = splvm();
+		pool_put(pp->pr_phpool, ph);
+		splx(s);
+	}
+
 	return (1);
 }
 
@@ -2021,22 +2024,18 @@ pool_cache_destruct_object(struct pool_c
 }
 
 /*
- * pool_cache_invalidate:
+ * pool_cache_do_invalidate:
  *
- *	Invalidate a pool cache (destruct and release all of the
- *	cached objects).
+ *	This internal function implements pool_cache_invalidate() and
+ *	pool_cache_reclaim().
  */
-void
-pool_cache_invalidate(struct pool_cache *pc)
+static void
+pool_cache_do_invalidate(struct pool_cache *pc, int free_groups,
+    void (*putit)(struct pool *, void *))
 {
-	struct pool_pagelist pq;
 	struct pool_cache_group *pcg, *npcg;
 	void *object;
-
-	LIST_INIT(&pq);
-
-	simple_lock(&pc->pc_slock);
-	simple_lock(&pc->pc_pool->pr_slock);
+	int s;
 
 	for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;
 	     pcg = npcg) {
@@ -2048,15 +2047,33 @@ pool_cache_invalidate(struct pool_cache 
 				pc->pc_allocfrom = NULL;
 			if (pc->pc_dtor != NULL)
 				(*pc->pc_dtor)(pc->pc_arg, object);
-			pool_do_put(pc->pc_pool, object, &pq);
+			(*putit)(pc->pc_pool, object);
+		}
+		if (free_groups) {
+			pc->pc_ngroups--;
+			TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list);
+			if (pc->pc_freeto == pcg)
+				pc->pc_freeto = NULL;
+			s = splvm();
+			pool_put(&pcgpool, pcg);
+			splx(s);
 		}
 	}
+}
 
-	simple_unlock(&pc->pc_pool->pr_slock);
-	simple_unlock(&pc->pc_slock);
+/*
+ * pool_cache_invalidate:
+ *
+ *	Invalidate a pool cache (destruct and release all of the
+ *	cached objects).
+ */
+void
+pool_cache_invalidate(struct pool_cache *pc)
+{
 
-	if (! LIST_EMPTY(&pq))
-		pr_pagelist_free(pc->pc_pool, &pq);
+	simple_lock(&pc->pc_slock);
+	pool_cache_do_invalidate(pc, 0, pool_put);
+	simple_unlock(&pc->pc_slock);
 }
 
 /*
@@ -2065,42 +2082,11 @@ pool_cache_invalidate(struct pool_cache 
  *	Reclaim a pool cache for pool_reclaim().
  */
 static void
-pool_cache_reclaim(struct pool_cache *pc, struct pool_pagelist *pq)
+pool_cache_reclaim(struct pool_cache *pc)
 {
-	struct pool_cache_group *pcg, *npcg;
-	void *object;
-	int s;
-
-	/*
-	 * We're locking in the wrong order (normally pool_cache -> pool,
-	 * but the pool is already locked when we get here), so we have
-	 * to use trylock.  If we can't lock the pool_cache, it's not really
-	 * a big deal here.
-	 */
-	if (simple_lock_try(&pc->pc_slock) == 0)
-		return;
-
-	for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;
-	     pcg = npcg) {
-		npcg = TAILQ_NEXT(pcg, pcg_list);
-		while (pcg->pcg_avail != 0) {
-			pc->pc_nitems--;
-			object = pcg_get(pcg, NULL);
-			if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg)
-				pc->pc_allocfrom = NULL;
-			if (pc->pc_dtor != NULL)
-				(*pc->pc_dtor)(pc->pc_arg, object);
-			pool_do_put(pc->pc_pool, object, pq);
-		}
-		pc->pc_ngroups--;
-		TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list);
-		if (pc->pc_freeto == pcg)
-			pc->pc_freeto = NULL;
-		s = splvm();
-		pool_put(&pcgpool, pcg);
-		splx(s);
-	}
 
+	simple_lock(&pc->pc_slock);
+	pool_cache_do_invalidate(pc, 1, pool_do_put);
 	simple_unlock(&pc->pc_slock);
 }
 
@@ -2120,42 +2106,23 @@ pool_cache_reclaim(struct pool_cache *pc
 void	*pool_page_alloc(struct pool *, int);
 void	pool_page_free(struct pool *, void *);
 
-#ifdef POOL_SUBPAGE
-struct pool_allocator pool_allocator_kmem_fullpage = {
-	pool_page_alloc, pool_page_free, 0,
-};
-#else
 struct pool_allocator pool_allocator_kmem = {
 	pool_page_alloc, pool_page_free, 0,
 };
-#endif
 
 void	*pool_page_alloc_nointr(struct pool *, int);
 void	pool_page_free_nointr(struct pool *, void *);
 
-#ifdef POOL_SUBPAGE
-struct pool_allocator pool_allocator_nointr_fullpage = {
-	pool_page_alloc_nointr, pool_page_free_nointr, 0,
-};
-#else
 struct pool_allocator pool_allocator_nointr = {
 	pool_page_alloc_nointr, pool_page_free_nointr, 0,
 };
-#endif
 
 #ifdef POOL_SUBPAGE
 void	*pool_subpage_alloc(struct pool *, int);
 void	pool_subpage_free(struct pool *, void *);
 
-struct pool_allocator pool_allocator_kmem = {
-	pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
-};
-
-void	*pool_subpage_alloc_nointr(struct pool *, int);
-void	pool_subpage_free_nointr(struct pool *, void *);
-
-struct pool_allocator pool_allocator_nointr = {
-	pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
+struct pool_allocator pool_allocator_kmem_subpage = {
+	pool_subpage_alloc, pool_subpage_free, 0,
 };
 #endif /* POOL_SUBPAGE */
 
@@ -2284,7 +2251,7 @@ pool_page_alloc(struct pool *pp, int fla
 {
 	boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
 
-	return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, NULL, waitok));
+	return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok));
 }
 
 void
@@ -2299,14 +2266,14 @@ pool_page_alloc_meta(struct pool *pp, in
 {
 	boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
 
-	return ((void *) uvm_km_alloc_poolpage1(kmem_map, NULL, waitok));
+	return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok));
 }
 
 static void
 pool_page_free_meta(struct pool *pp, void *v)
 {
 
-	uvm_km_free_poolpage1(kmem_map, (vaddr_t) v);
+	uvm_km_free_poolpage(kmem_map, (vaddr_t) v);
 }
 
 #ifdef POOL_SUBPAGE
@@ -2333,26 +2300,25 @@ pool_subpage_free(struct pool *pp, void 
 
 /* We don't provide a real nointr allocator.  Maybe later. */
 void *
-pool_subpage_alloc_nointr(struct pool *pp, int flags)
+pool_page_alloc_nointr(struct pool *pp, int flags)
 {
 
 	return (pool_subpage_alloc(pp, flags));
 }
 
 void
-pool_subpage_free_nointr(struct pool *pp, void *v)
+pool_page_free_nointr(struct pool *pp, void *v)
 {
 
 	pool_subpage_free(pp, v);
 }
-#endif /* POOL_SUBPAGE */
+#else
 void *
 pool_page_alloc_nointr(struct pool *pp, int flags)
 {
 	boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
 
-	return ((void *) uvm_km_alloc_poolpage_cache(kernel_map,
-	    uvm.kernel_object, waitok));
+	return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok));
 }
 
 void
@@ -2361,3 +2327,4 @@ pool_page_free_nointr(struct pool *pp, v
 
 	uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v);
 }
+#endif /* POOL_SUBPAGE */