Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

===================================================================
RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v
rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present.
retrieving revision 1.207
retrieving revision 1.210
diff -u -p -r1.207 -r1.210
--- src/sys/kern/subr_pool.c	2017/03/14 03:13:50	1.207
+++ src/sys/kern/subr_pool.c	2017/11/05 07:49:45	1.210
@@ -1,4 +1,4 @@
-/*	$NetBSD: subr_pool.c,v 1.207 2017/03/14 03:13:50 riastradh Exp $	*/
+/*	$NetBSD: subr_pool.c,v 1.210 2017/11/05 07:49:45 mlelstv Exp $	*/
 
 /*-
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015
@@ -33,7 +33,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.207 2017/03/14 03:13:50 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.210 2017/11/05 07:49:45 mlelstv Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ddb.h"
@@ -106,6 +106,10 @@ struct pool_allocator pool_allocator_met
 	.pa_pagesz = 0
 };
 
+#define POOL_ALLOCATOR_BIG_BASE 13
+extern struct pool_allocator pool_allocator_big[];
+static int pool_bigidx(size_t);
+
 /* # of seconds to retain page after last use */
 int pool_inactive_time = 10;
 
@@ -826,6 +830,14 @@ pool_get(struct pool *pp, int flags)
 		error = pool_grow(pp, flags);
 		if (error != 0) {
 			/*
+			 * pool_grow aborts when another thread
+			 * is allocating a new page. Retry if it
+			 * waited for it.
+			 */
+			if (error == ERESTART)
+				goto startover;
+
+			/*
 			 * We were unable to allocate a page or item
 			 * header, but we released the lock during
 			 * allocation, so perhaps items were freed
@@ -1047,6 +1059,23 @@ pool_grow(struct pool *pp, int flags)
 {
 	struct pool_item_header *ph = NULL;
 	char *cp;
+	int error;
+
+	/*
+	 * If there's a pool_grow in progress, wait for it to complete
+	 * and try again from the top.
+	 */
+	if (pp->pr_flags & PR_GROWING) {
+		if (flags & PR_WAITOK) {
+			do {
+				cv_wait(&pp->pr_cv, &pp->pr_lock);
+			} while (pp->pr_flags & PR_GROWING);
+			return ERESTART;
+		} else {
+			return EWOULDBLOCK;
+		}
+	}
+	pp->pr_flags |= PR_GROWING;
 
 	mutex_exit(&pp->pr_lock);
 	cp = pool_allocator_alloc(pp, flags);
@@ -1058,13 +1087,25 @@ pool_grow(struct pool *pp, int flags)
 			pool_allocator_free(pp, cp);
 		}
 		mutex_enter(&pp->pr_lock);
-		return ENOMEM;
+		error = ENOMEM;
+		goto out;
 	}
 
 	mutex_enter(&pp->pr_lock);
 	pool_prime_page(pp, cp, ph);
 	pp->pr_npagealloc++;
-	return 0;
+	error = 0;
+
+out:
+	/*
+	 * If anyone was waiting for pool_grow, notify them that we
+	 * may have just done it.
+	 */
+	KASSERT(pp->pr_flags & PR_GROWING);
+	pp->pr_flags &= ~PR_GROWING;
+	cv_broadcast(&pp->pr_cv);
+
+	return error;
 }
 
 /*
@@ -1700,8 +1741,14 @@ pool_cache_bootstrap(pool_cache_t pc, si
 	struct pool *pp;
 
 	pp = &pc->pc_pool;
-	if (palloc == NULL && ipl == IPL_NONE)
-		palloc = &pool_allocator_nointr;
+	if (palloc == NULL && ipl == IPL_NONE) {
+		if (size > PAGE_SIZE) {
+			int bigidx = pool_bigidx(size);
+
+			palloc = &pool_allocator_big[bigidx];
+		} else
+			palloc = &pool_allocator_nointr;
+	}
 	pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl);
 	mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl);
 
@@ -2498,6 +2545,61 @@ struct pool_allocator pool_allocator_noi
 };
 #endif /* POOL_SUBPAGE */
 
+struct pool_allocator pool_allocator_big[] = {
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 0),
+	},
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 1),
+	},
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 2),
+	},
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 3),
+	},
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 4),
+	},
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 5),
+	},
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 6),
+	},
+	{
+		.pa_alloc = pool_page_alloc,
+		.pa_free = pool_page_free,
+		.pa_pagesz = 1 << (POOL_ALLOCATOR_BIG_BASE + 7),
+	}
+};
+
+static int
+pool_bigidx(size_t size)
+{
+	int i;
+
+	for (i = 0; i < __arraycount(pool_allocator_big); i++) {
+		if (1 << (i + POOL_ALLOCATOR_BIG_BASE) >= size)
+			return i;
+	}
+	panic("pool item size %zu too large, use a custom allocator", size);
+}
+
 static void *
 pool_allocator_alloc(struct pool *pp, int flags)
 {