Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

===================================================================
RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v
retrieving revision 1.161.2.1
retrieving revision 1.162
diff -u -p -r1.161.2.1 -r1.162
--- src/sys/kern/subr_pool.c	2008/07/18 16:37:49	1.161.2.1
+++ src/sys/kern/subr_pool.c	2008/07/04 13:28:08	1.162
@@ -1,4 +1,4 @@
-/*	$NetBSD: subr_pool.c,v 1.161.2.1 2008/07/18 16:37:49 simonb Exp $	*/
+/*	$NetBSD: subr_pool.c,v 1.162 2008/07/04 13:28:08 ad Exp $	*/
 
 /*-
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.161.2.1 2008/07/18 16:37:49 simonb Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.162 2008/07/04 13:28:08 ad Exp $");
 
 #include "opt_ddb.h"
 #include "opt_pool.h"
@@ -1253,7 +1253,10 @@ pool_do_put(struct pool *pp, void *v, st
 
 	if (pp->pr_flags & PR_WANTED) {
 		pp->pr_flags &= ~PR_WANTED;
+		if (ph->ph_nmissing == 0)
+			pp->pr_nidle++;
 		cv_broadcast(&pp->pr_cv);
+		return;
 	}
 
 	/*
@@ -2082,10 +2085,8 @@ pool_cache_bootstrap(pool_cache_t pc, si
 
 	if ((flags & PR_LARGECACHE) != 0) {
 		pc->pc_pcgsize = PCG_NOBJECTS_LARGE;
-		pc->pc_pcgpool = &pcg_large_pool;
 	} else {
 		pc->pc_pcgsize = PCG_NOBJECTS_NORMAL;
-		pc->pc_pcgpool = &pcg_normal_pool;
 	}
 
 	/* Allocate per-CPU caches. */
@@ -2468,7 +2469,7 @@ pool_cache_get_paddr(pool_cache_t pc, in
 
 	/* Lock out interrupts and disable preemption. */
 	s = splvm();
-	while (/* CONSTCOND */ true) {
+	do {
 		/* Try and allocate an object from the current group. */
 		cc = pc->pc_cpus[curcpu()->ci_index];
 		KASSERT(cc->cc_cache == pc);
@@ -2479,9 +2480,9 @@ pool_cache_get_paddr(pool_cache_t pc, in
 				*pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa;
 #if defined(DIAGNOSTIC)
 			pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL;
-			KASSERT(pcg->pcg_avail < pcg->pcg_size);
+#endif /* defined(DIAGNOSTIC) */
+			KASSERT(pcg->pcg_avail <= pcg->pcg_size);
 			KASSERT(object != NULL);
-#endif
 			cc->cc_hits++;
 			splx(s);
 			FREECHECK_OUT(&pc->pc_freecheck, object);
@@ -2505,9 +2506,7 @@ pool_cache_get_paddr(pool_cache_t pc, in
 		 * no more objects are available, it will return false.
 		 * Otherwise, we need to retry.
 		 */
-		if (!pool_cache_get_slow(cc, s, &object, pap, flags))
-			break;
-	}
+	} while (pool_cache_get_slow(cc, s, &object, pap, flags));
 
 	return object;
 }
@@ -2515,41 +2514,55 @@ pool_cache_get_paddr(pool_cache_t pc, in
 static bool __noinline
 pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object)
 {
-	pcg_t *pcg, *cur;
+	pcg_t *pcg, *cur, *empty;
 	uint64_t ncsw;
 	pool_cache_t pc;
 
 	pc = cc->cc_cache;
 	cc->cc_misses++;
 
+	/*
+	 * If there appear to be no empty groups in the cache then
+	 * allocate one in advance.
+	 */
+	empty = NULL;
+	if (__predict_false(pc->pc_emptygroups == NULL)) {
+		if (__predict_false(pool_cache_disable)) {
+			empty = NULL;
+		} else if (pc->pc_pcgsize == PCG_NOBJECTS_LARGE) {
+			empty = pool_get(&pcg_large_pool, PR_NOWAIT);
+		} else {
+			empty = pool_get(&pcg_normal_pool, PR_NOWAIT);
+		}
+	}
+
 	/* Lock the cache. */
+	ncsw = curlwp->l_ncsw;
 	if (__predict_false(!mutex_tryenter(&pc->pc_lock))) {
-		ncsw = curlwp->l_ncsw;
 		mutex_enter(&pc->pc_lock);
 		pc->pc_contended++;
+	}
 
-		/*
-		 * If we context switched while locking, then our view of
-		 * the per-CPU data is invalid: retry.
-		 */
-		if (__predict_false(curlwp->l_ncsw != ncsw)) {
-			mutex_exit(&pc->pc_lock);
-			return true;
-		}
+	/*
+	 * If we speculatively allocated an empty group, link it into
+	 * the cache's list.
+	 */
+	if (empty != NULL) {
+		empty->pcg_avail = 0;
+		empty->pcg_size = pc->pc_pcgsize;
+		empty->pcg_next = pc->pc_emptygroups;
+		pc->pc_emptygroups = empty;
+		pc->pc_nempty++;
+		pc->pc_misses++;
 	}
 
-	/* If there are no empty groups in the cache then allocate one. */
-	if (__predict_false((pcg = pc->pc_emptygroups) == NULL)) {
-		if (__predict_true(!pool_cache_disable)) {
-			pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT);
-		}
-		if (__predict_true(pcg != NULL)) {
-			pcg->pcg_avail = 0;
-			pcg->pcg_size = pc->pc_pcgsize;
-		}
-	} else {
-		pc->pc_emptygroups = pcg->pcg_next;
-		pc->pc_nempty--;
+	/*
+	 * If we context switched while locking, then our view of the
+	 * per-CPU data is invalid: retry.
+	 */
+	if (__predict_false(curlwp->l_ncsw != ncsw)) {
+		mutex_exit(&pc->pc_lock);
+		return true;
 	}
 
 	/*
@@ -2557,21 +2570,23 @@ pool_cache_put_slow(pool_cache_cpu_t *cc
 	 * to the cache.  Install the empty group to the local CPU
 	 * and return.
 	 */
-	if (pcg != NULL) {
+	if (__predict_true((pcg = pc->pc_emptygroups) != NULL)) {
 		KASSERT(pcg->pcg_avail == 0);
+		pc->pc_emptygroups = pcg->pcg_next;
 		if (__predict_false(cc->cc_previous == &pcg_dummy)) {
 			cc->cc_previous = pcg;
 		} else {
 			cur = cc->cc_current;
 			if (__predict_true(cur != &pcg_dummy)) {
-				KASSERT(cur->pcg_avail == cur->pcg_size);
+				KASSERT(cur->pcg_avail == pcg->pcg_size);
 				cur->pcg_next = pc->pc_fullgroups;
 				pc->pc_fullgroups = cur;
 				pc->pc_nfull++;
 			}
 			cc->cc_current = pcg;
 		}
-		pc->pc_hits++;
+		pc->pc_hits += (empty == NULL);
+		pc->pc_nempty--;
 		mutex_exit(&pc->pc_lock);
 		return true;
 	}
@@ -2606,7 +2621,7 @@ pool_cache_put_paddr(pool_cache_t pc, vo
 
 	/* Lock out interrupts and disable preemption. */
 	s = splvm();
-	while (/* CONSTCOND */ true) {
+	do {
 		/* If the current group isn't full, release it there. */
 		cc = pc->pc_cpus[curcpu()->ci_index];
 		KASSERT(cc->cc_cache == pc);
@@ -2636,9 +2651,7 @@ pool_cache_put_paddr(pool_cache_t pc, vo
 		 * If put_slow() releases the object for us, it
 		 * will return false.  Otherwise we need to retry.
 		 */
-		if (!pool_cache_put_slow(cc, s, object))
-			break;
-	}
+	} while (pool_cache_put_slow(cc, s, object));
 }
 
 /*