Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

===================================================================
RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v
retrieving revision 1.190.6.2
retrieving revision 1.191
diff -u -p -r1.190.6.2 -r1.191
--- src/sys/kern/subr_pool.c	2012/06/02 11:09:33	1.190.6.2
+++ src/sys/kern/subr_pool.c	2012/01/27 19:48:40	1.191
@@ -1,4 +1,4 @@
-/*	$NetBSD: subr_pool.c,v 1.190.6.2 2012/06/02 11:09:33 mrg Exp $	*/
+/*	$NetBSD: subr_pool.c,v 1.191 2012/01/27 19:48:40 para Exp $	*/
 
 /*-
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010
@@ -32,9 +32,11 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.190.6.2 2012/06/02 11:09:33 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.191 2012/01/27 19:48:40 para Exp $");
 
 #include "opt_ddb.h"
+#include "opt_pool.h"
+#include "opt_poollog.h"
 #include "opt_lockdebug.h"
 
 #include <sys/param.h>
@@ -43,6 +45,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/kernel.h>
+#include <sys/malloc.h>
 #include <sys/vmem.h>
 #include <sys/pool.h>
 #include <sys/syslog.h>
@@ -210,6 +213,136 @@ static void pool_print1(struct pool *, c
 static int pool_chk_page(struct pool *, const char *,
 			 struct pool_item_header *);
 
+/*
+ * Pool log entry. An array of these is allocated in pool_init().
+ */
+struct pool_log {
+	const char	*pl_file;
+	long		pl_line;
+	int		pl_action;
+#define	PRLOG_GET	1
+#define	PRLOG_PUT	2
+	void		*pl_addr;
+};
+
+#ifdef POOL_DIAGNOSTIC
+/* Number of entries in pool log buffers */
+#ifndef POOL_LOGSIZE
+#define	POOL_LOGSIZE	10
+#endif
+
+int pool_logsize = POOL_LOGSIZE;
+
+static inline void
+pr_log(struct pool *pp, void *v, int action, const char *file, long line)
+{
+	int n;
+	struct pool_log *pl;
+
+	if ((pp->pr_roflags & PR_LOGGING) == 0)
+		return;
+
+	if (pp->pr_log == NULL) {
+		if (kmem_map != NULL)
+			pp->pr_log = malloc(
+				pool_logsize * sizeof(struct pool_log),
+				M_TEMP, M_NOWAIT | M_ZERO);
+		if (pp->pr_log == NULL)
+			return;
+		pp->pr_curlogentry = 0;
+		pp->pr_logsize = pool_logsize;
+	}
+
+	/*
+	 * Fill in the current entry. Wrap around and overwrite
+	 * the oldest entry if necessary.
+	 */
+	n = pp->pr_curlogentry;
+	pl = &pp->pr_log[n];
+	pl->pl_file = file;
+	pl->pl_line = line;
+	pl->pl_action = action;
+	pl->pl_addr = v;
+	if (++n >= pp->pr_logsize)
+		n = 0;
+	pp->pr_curlogentry = n;
+}
+
+static void
+pr_printlog(struct pool *pp, struct pool_item *pi,
+    void (*pr)(const char *, ...))
+{
+	int i = pp->pr_logsize;
+	int n = pp->pr_curlogentry;
+
+	if (pp->pr_log == NULL)
+		return;
+
+	/*
+	 * Print all entries in this pool's log.
+	 */
+	while (i-- > 0) {
+		struct pool_log *pl = &pp->pr_log[n];
+		if (pl->pl_action != 0) {
+			if (pi == NULL || pi == pl->pl_addr) {
+				(*pr)("\tlog entry %d:\n", i);
+				(*pr)("\t\taction = %s, addr = %p\n",
+				    pl->pl_action == PRLOG_GET ? "get" : "put",
+				    pl->pl_addr);
+				(*pr)("\t\tfile: %s at line %lu\n",
+				    pl->pl_file, pl->pl_line);
+			}
+		}
+		if (++n >= pp->pr_logsize)
+			n = 0;
+	}
+}
+
+static inline void
+pr_enter(struct pool *pp, const char *file, long line)
+{
+
+	if (__predict_false(pp->pr_entered_file != NULL)) {
+		printf("pool %s: reentrancy at file %s line %ld\n",
+		    pp->pr_wchan, file, line);
+		printf("         previous entry at file %s line %ld\n",
+		    pp->pr_entered_file, pp->pr_entered_line);
+		panic("pr_enter");
+	}
+
+	pp->pr_entered_file = file;
+	pp->pr_entered_line = line;
+}
+
+static inline void
+pr_leave(struct pool *pp)
+{
+
+	if (__predict_false(pp->pr_entered_file == NULL)) {
+		printf("pool %s not entered?\n", pp->pr_wchan);
+		panic("pr_leave");
+	}
+
+	pp->pr_entered_file = NULL;
+	pp->pr_entered_line = 0;
+}
+
+static inline void
+pr_enter_check(struct pool *pp, void (*pr)(const char *, ...))
+{
+
+	if (pp->pr_entered_file != NULL)
+		(*pr)("\n\tcurrently entered from file %s line %ld\n",
+		    pp->pr_entered_file, pp->pr_entered_line);
+}
+#else
+#define	pr_log(pp, v, action, file, line)
+#define	pr_printlog(pp, pi, pr)
+#define	pr_enter(pp, file, line)
+#define	pr_leave(pp)
+#define	pr_enter_check(pp, pr)
+#endif /* POOL_DIAGNOSTIC */
+
 static inline unsigned int
 pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph,
     const void *v)
@@ -398,8 +531,8 @@ pr_rmpage(struct pool *pp, struct pool_i
 void
 pool_subsystem_init(void)
 {
-	size_t size;
 	int idx;
+	size_t size;
 
 	mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE);
 	mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE);
@@ -451,7 +584,7 @@ pool_subsystem_init(void)
  * Initialize the given pool resource structure.
  *
  * We export this routine to allow other kernel parts to declare
- * static pools that must be initialized before kmem(9) is available.
+ * static pools that must be initialized before malloc() is available.
  */
 void
 pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
@@ -473,6 +606,14 @@ pool_init(struct pool *pp, size_t size, 
 	}
 #endif
 
+#ifdef POOL_DIAGNOSTIC
+	/*
+	 * Always log if POOL_DIAGNOSTIC is defined.
+	 */
+	if (pool_logsize != 0)
+		flags |= PR_LOGGING;
+#endif
+
 	if (palloc == NULL)
 		palloc = &pool_allocator_kmem;
 #ifdef POOL_SUBPAGE
@@ -619,6 +760,11 @@ pool_init(struct pool *pp, size_t size, 
 	pp->pr_nidle = 0;
 	pp->pr_refcnt = 0;
 
+	pp->pr_log = NULL;
+
+	pp->pr_entered_file = NULL;
+	pp->pr_entered_line = 0;
+
 	mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl);
 	cv_init(&pp->pr_cv, wchan);
 	pp->pr_ipl = ipl;
@@ -679,6 +825,7 @@ pool_destroy(struct pool *pp)
 
 #ifdef DIAGNOSTIC
 	if (pp->pr_nout != 0) {
+		pr_printlog(pp, NULL, printf);
 		panic("pool_destroy: pool busy: still out: %u",
 		    pp->pr_nout);
 	}
@@ -695,6 +842,14 @@ pool_destroy(struct pool *pp)
 	mutex_exit(&pp->pr_lock);
 
 	pr_pagelist_free(pp, &pq);
+
+#ifdef POOL_DIAGNOSTIC
+	if (pp->pr_log != NULL) {
+		free(pp->pr_log, M_TEMP);
+		pp->pr_log = NULL;
+	}
+#endif
+
 	cv_destroy(&pp->pr_cv);
 	mutex_destroy(&pp->pr_lock);
 }
@@ -729,7 +884,11 @@ pool_alloc_item_header(struct pool *pp, 
  * Grab an item from the pool.
  */
 void *
+#ifdef POOL_DIAGNOSTIC
+_pool_get(struct pool *pp, int flags, const char *file, long line)
+#else
 pool_get(struct pool *pp, int flags)
+#endif
 {
 	struct pool_item *pi;
 	struct pool_item_header *ph;
@@ -749,6 +908,8 @@ pool_get(struct pool *pp, int flags)
 	}
 
 	mutex_enter(&pp->pr_lock);
+	pr_enter(pp, file, line);
+
  startover:
 	/*
 	 * Check to see if we've reached the hard limit.  If we have,
@@ -757,6 +918,7 @@ pool_get(struct pool *pp, int flags)
 	 */
 #ifdef DIAGNOSTIC
 	if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {
+		pr_leave(pp);
 		mutex_exit(&pp->pr_lock);
 		panic("pool_get: %s: crossed hard limit", pp->pr_wchan);
 	}
@@ -768,9 +930,11 @@ pool_get(struct pool *pp, int flags)
 			 * back to the pool, unlock, call the hook, re-lock,
 			 * and check the hardlimit condition again.
 			 */
+			pr_leave(pp);
 			mutex_exit(&pp->pr_lock);
 			(*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);
 			mutex_enter(&pp->pr_lock);
+			pr_enter(pp, file, line);
 			if (pp->pr_nout < pp->pr_hardlimit)
 				goto startover;
 		}
@@ -781,7 +945,9 @@ pool_get(struct pool *pp, int flags)
 			 * it be?
 			 */
 			pp->pr_flags |= PR_WANTED;
+			pr_leave(pp);
 			cv_wait(&pp->pr_cv, &pp->pr_lock);
+			pr_enter(pp, file, line);
 			goto startover;
 		}
 
@@ -795,6 +961,7 @@ pool_get(struct pool *pp, int flags)
 
 		pp->pr_nfail++;
 
+		pr_leave(pp);
 		mutex_exit(&pp->pr_lock);
 		return (NULL);
 	}
@@ -822,7 +989,9 @@ pool_get(struct pool *pp, int flags)
 		 * Release the pool lock, as the back-end page allocator
 		 * may block.
 		 */
+		pr_leave(pp);
 		error = pool_grow(pp, flags);
+		pr_enter(pp, file, line);
 		if (error != 0) {
 			/*
 			 * We were unable to allocate a page or item
@@ -834,6 +1003,7 @@ pool_get(struct pool *pp, int flags)
 				goto startover;
 
 			pp->pr_nfail++;
+			pr_leave(pp);
 			mutex_exit(&pp->pr_lock);
 			return (NULL);
 		}
@@ -844,19 +1014,25 @@ pool_get(struct pool *pp, int flags)
 	if (pp->pr_roflags & PR_NOTOUCH) {
 #ifdef DIAGNOSTIC
 		if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) {
+			pr_leave(pp);
 			mutex_exit(&pp->pr_lock);
 			panic("pool_get: %s: page empty", pp->pr_wchan);
 		}
 #endif
 		v = pr_item_notouch_get(pp, ph);
+#ifdef POOL_DIAGNOSTIC
+		pr_log(pp, v, PRLOG_GET, file, line);
+#endif
 	} else {
 		v = pi = LIST_FIRST(&ph->ph_itemlist);
 		if (__predict_false(v == NULL)) {
+			pr_leave(pp);
 			mutex_exit(&pp->pr_lock);
 			panic("pool_get: %s: page empty", pp->pr_wchan);
 		}
 #ifdef DIAGNOSTIC
 		if (__predict_false(pp->pr_nitems == 0)) {
+			pr_leave(pp);
 			mutex_exit(&pp->pr_lock);
 			printf("pool_get: %s: items on itemlist, nitems %u\n",
 			    pp->pr_wchan, pp->pr_nitems);
@@ -864,8 +1040,13 @@ pool_get(struct pool *pp, int flags)
 		}
 #endif
 
+#ifdef POOL_DIAGNOSTIC
+		pr_log(pp, v, PRLOG_GET, file, line);
+#endif
+
 #ifdef DIAGNOSTIC
 		if (__predict_false(pi->pi_magic != PI_MAGIC)) {
+			pr_printlog(pp, pi, printf);
 			panic("pool_get(%s): free list modified: "
 			    "magic=%x; page %p; item addr %p\n",
 			    pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
@@ -898,6 +1079,7 @@ pool_get(struct pool *pp, int flags)
 #ifdef DIAGNOSTIC
 		if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 &&
 		    !LIST_EMPTY(&ph->ph_itemlist))) {
+			pr_leave(pp);
 			mutex_exit(&pp->pr_lock);
 			panic("pool_get: %s: nmissing inconsistent",
 			    pp->pr_wchan);
@@ -913,6 +1095,7 @@ pool_get(struct pool *pp, int flags)
 	}
 
 	pp->pr_nget++;
+	pr_leave(pp);
 
 	/*
 	 * If we have a low water mark and we are now below that low
@@ -954,6 +1137,7 @@ pool_do_put(struct pool *pp, void *v, st
 #endif
 
 	if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) {
+		pr_printlog(pp, NULL, printf);
 		panic("pool_put: %s: page header missing", pp->pr_wchan);
 	}
 
@@ -1042,6 +1226,32 @@ pool_do_put(struct pool *pp, void *v, st
 	}
 }
 
+/*
+ * Return resource to the pool.
+ */
+#ifdef POOL_DIAGNOSTIC
+void
+_pool_put(struct pool *pp, void *v, const char *file, long line)
+{
+	struct pool_pagelist pq;
+
+	LIST_INIT(&pq);
+
+	mutex_enter(&pp->pr_lock);
+	pr_enter(pp, file, line);
+
+	pr_log(pp, v, PRLOG_PUT, file, line);
+
+	pool_do_put(pp, v, &pq);
+
+	pr_leave(pp);
+	mutex_exit(&pp->pr_lock);
+
+	pr_pagelist_free(pp, &pq);
+}
+#undef pool_put
+#endif /* POOL_DIAGNOSTIC */
+
 void
 pool_put(struct pool *pp, void *v)
 {
@@ -1056,6 +1266,10 @@ pool_put(struct pool *pp, void *v)
 	pr_pagelist_free(pp, &pq);
 }
 
+#ifdef POOL_DIAGNOSTIC
+#define		pool_put(h, v)	_pool_put((h), (v), __FILE__, __LINE__)
+#endif
+
 /*
  * pool_grow: grow a pool by a page.
  *
@@ -1303,7 +1517,11 @@ pool_sethardlimit(struct pool *pp, int n
  * Might be called from interrupt context.
  */
 int
+#ifdef POOL_DIAGNOSTIC
+_pool_reclaim(struct pool *pp, const char *file, long line)
+#else
 pool_reclaim(struct pool *pp)
+#endif
 {
 	struct pool_item_header *ph, *phnext;
 	struct pool_pagelist pq;
@@ -1343,6 +1561,7 @@ pool_reclaim(struct pool *pp)
 		}
 		return (0);
 	}
+	pr_enter(pp, file, line);
 
 	LIST_INIT(&pq);
 
@@ -1370,6 +1589,7 @@ pool_reclaim(struct pool *pp)
 		pr_rmpage(pp, ph, &pq);
 	}
 
+	pr_leave(pp);
 	mutex_exit(&pp->pr_lock);
 
 	if (LIST_EMPTY(&pq))
@@ -1459,6 +1679,12 @@ pool_drain_end(struct pool *pp, uint64_t
 /*
  * Diagnostic helpers.
  */
+void
+pool_print(struct pool *pp, const char *modif)
+{
+
+	pool_print1(pp, modif, printf);
+}
 
 void
 pool_printall(const char *modif, void (*pr)(const char *, ...))
@@ -1570,6 +1796,11 @@ pool_print1(struct pool *pp, const char 
 		goto skip_log;
 
 	(*pr)("\n");
+	if ((pp->pr_roflags & PR_LOGGING) == 0)
+		(*pr)("\tno log\n");
+	else {
+		pr_printlog(pp, NULL, pr);
+	}
 
  skip_log:
 
@@ -1619,6 +1850,8 @@ pool_print1(struct pool *pp, const char 
 		}
 	}
 #undef PR_GROUPLIST
+
+	pr_enter_check(pp, pr);
 }
 
 static int
@@ -2063,6 +2296,7 @@ pool_cache_invalidate(pool_cache_t pc)
 static void
 pool_cache_invalidate_cpu(pool_cache_t pc, u_int index)
 {
+
 	pool_cache_cpu_t *cc;
 	pcg_t *pcg;
 
@@ -2484,9 +2718,7 @@ void	pool_page_free(struct pool *, void 
 
 #ifdef POOL_SUBPAGE
 struct pool_allocator pool_allocator_kmem_fullpage = {
-	.pa_alloc = pool_page_alloc,
-	.pa_free = pool_page_free,
-	.pa_pagesz = 0
+	pool_page_alloc, pool_page_free, 0
 };
 #else
 struct pool_allocator pool_allocator_kmem = {
@@ -2496,11 +2728,12 @@ struct pool_allocator pool_allocator_kme
 };
 #endif
 
+void	*pool_page_alloc_nointr(struct pool *, int);
+void	pool_page_free_nointr(struct pool *, void *);
+
 #ifdef POOL_SUBPAGE
 struct pool_allocator pool_allocator_nointr_fullpage = {
-	.pa_alloc = pool_page_alloc,
-	.pa_free = pool_page_free,
-	.pa_pagesz = 0
+	pool_page_alloc_nointr, pool_page_free_nointr, 0,
 };
 #else
 struct pool_allocator pool_allocator_nointr = {
@@ -2515,15 +2748,14 @@ void	*pool_subpage_alloc(struct pool *, 
 void	pool_subpage_free(struct pool *, void *);
 
 struct pool_allocator pool_allocator_kmem = {
-	.pa_alloc = pool_subpage_alloc,
-	.pa_free = pool_subpage_free,
-	.pa_pagesz = POOL_SUBPAGE
+	pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
 };
 
+void	*pool_subpage_alloc_nointr(struct pool *, int);
+void	pool_subpage_free_nointr(struct pool *, void *);
+
 struct pool_allocator pool_allocator_nointr = {
-	.pa_alloc = pool_subpage_alloc,
-	.pa_free = pool_subpage_free,
-	.pa_pagesz = POOL_SUBPAGE
+	pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
 };
 #endif /* POOL_SUBPAGE */
 
@@ -2559,14 +2791,18 @@ pool_allocator_free(struct pool *pp, voi
 void *
 pool_page_alloc(struct pool *pp, int flags)
 {
-	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
+	bool waitok = (flags & PR_WAITOK) ? true : false;
+	int rc;
 	vmem_addr_t va;
-	int ret;
 
-	ret = uvm_km_kmem_alloc(kmem_va_arena, pp->pr_alloc->pa_pagesz,
-	    vflags | VM_INSTANTFIT, &va);
+	rc = uvm_km_kmem_alloc(kmem_va_arena,
+	    pp->pr_alloc->pa_pagesz,
+	    ((waitok ? VM_SLEEP : VM_NOSLEEP) | VM_INSTANTFIT), &va);
 
-	return ret ? NULL : (void *)va;
+	if (rc != 0)
+		return NULL;
+	else
+		return (void *)va;
 }
 
 void
@@ -2579,21 +2815,25 @@ pool_page_free(struct pool *pp, void *v)
 static void *
 pool_page_alloc_meta(struct pool *pp, int flags)
 {
-	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
-	vmem_addr_t va;
-	int ret;
+	bool waitok = (flags & PR_WAITOK) ? true : false;
+	int rc;
+	vmem_addr_t addr;
 
-	ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz,
-	    vflags | VM_INSTANTFIT, &va);
+	rc = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz,
+	    (waitok ? VM_SLEEP : VM_NOSLEEP) | VM_INSTANTFIT, &addr);
 
-	return ret ? NULL : (void *)va;
+	if (rc != 0)
+		return 0;
+	else
+		return (void *)addr;
 }
 
 static void
 pool_page_free_meta(struct pool *pp, void *v)
 {
 
-	vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz);
+	vmem_free(kmem_meta_arena, (vmem_addr_t)v,
+	    pp->pr_alloc->pa_pagesz);
 }
 
 #ifdef POOL_SUBPAGE
@@ -2610,6 +2850,20 @@ pool_subpage_free(struct pool *pp, void 
 	pool_put(&psppool, v);
 }
 
+/* We don't provide a real nointr allocator.  Maybe later. */
+void *
+pool_subpage_alloc_nointr(struct pool *pp, int flags)
+{
+
+	return (pool_subpage_alloc(pp, flags));
+}
+
+void
+pool_subpage_free_nointr(struct pool *pp, void *v)
+{
+
+	pool_subpage_free(pp, v);
+}
 #endif /* POOL_SUBPAGE */
 
 #if defined(DDB)