Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.21.2.2.2.1 retrieving revision 1.42 diff -u -p -r1.21.2.2.2.1 -r1.42 --- src/sys/kern/subr_pool.c 1999/06/21 01:24:03 1.21.2.2.2.1 +++ src/sys/kern/subr_pool.c 2000/12/06 18:20:52 1.42 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.21.2.2.2.1 1999/06/21 01:24:03 thorpej Exp $ */ +/* $NetBSD: subr_pool.c,v 1.42 2000/12/06 18:20:52 thorpej Exp $ */ /*- * Copyright (c) 1997, 1999 The NetBSD Foundation, Inc. @@ -39,6 +39,7 @@ #include "opt_pool.h" #include "opt_poollog.h" +#include "opt_lockdebug.h" #include #include @@ -50,9 +51,6 @@ #include #include -#include -#include - #include /* @@ -96,8 +94,8 @@ struct pool_item_header { struct pool_item { #ifdef DIAGNOSTIC int pi_magic; -#define PI_MAGIC 0xdeadbeef #endif +#define PI_MAGIC 0xdeadbeef /* Other entries use only this list entry */ TAILQ_ENTRY(pool_item) pi_list; }; @@ -108,16 +106,13 @@ struct pool_item { -static struct pool_item_header - *pr_find_pagehead __P((struct pool *, caddr_t)); -static void pr_rmpage __P((struct pool *, struct pool_item_header *)); -static int pool_catchup __P((struct pool *)); -static void pool_prime_page __P((struct pool *, caddr_t)); -static void *pool_page_alloc __P((unsigned long, int, int)); -static void pool_page_free __P((void *, unsigned long, int)); +static int pool_catchup(struct pool *); +static void pool_prime_page(struct pool *, caddr_t); +static void *pool_page_alloc(unsigned long, int, int); +static void pool_page_free(void *, unsigned long, int); -static void pool_print1 __P((struct pool *, const char *, - void (*)(const char *, ...))); +static void pool_print1(struct pool *, const char *, + void (*)(const char *, ...)); /* * Pool log entry. An array of these is allocated in pool_create(). @@ -139,21 +134,8 @@ struct pool_log { int pool_logsize = POOL_LOGSIZE; #ifdef DIAGNOSTIC -static void pr_log __P((struct pool *, void *, int, const char *, long)); -static void pr_printlog __P((struct pool *, struct pool_item *, - void (*)(const char *, ...))); -static void pr_enter __P((struct pool *, const char *, long)); -static void pr_leave __P((struct pool *)); -static void pr_enter_check __P((struct pool *, - void (*)(const char *, ...))); - -static __inline__ void -pr_log(pp, v, action, file, line) - struct pool *pp; - void *v; - int action; - const char *file; - long line; +static __inline void +pr_log(struct pool *pp, void *v, int action, const char *file, long line) { int n = pp->pr_curlogentry; struct pool_log *pl; @@ -176,10 +158,8 @@ pr_log(pp, v, action, file, line) } static void -pr_printlog(pp, pi, pr) - struct pool *pp; - struct pool_item *pi; - void (*pr) __P((const char *, ...)); +pr_printlog(struct pool *pp, struct pool_item *pi, + void (*pr)(const char *, ...)) { int i = pp->pr_logsize; int n = pp->pr_curlogentry; @@ -207,14 +187,11 @@ pr_printlog(pp, pi, pr) } } -static __inline__ void -pr_enter(pp, file, line) - struct pool *pp; - const char *file; - long line; +static __inline void +pr_enter(struct pool *pp, const char *file, long line) { - if (pp->pr_entered_file != NULL) { + if (__predict_false(pp->pr_entered_file != NULL)) { printf("pool %s: reentrancy at file %s line %ld\n", pp->pr_wchan, file, line); printf(" previous entry at file %s line %ld\n", @@ -226,12 +203,11 @@ pr_enter(pp, file, line) pp->pr_entered_line = line; } -static __inline__ void -pr_leave(pp) - struct pool *pp; +static __inline void +pr_leave(struct pool *pp) { - if (pp->pr_entered_file == NULL) { + if (__predict_false(pp->pr_entered_file == NULL)) { printf("pool %s not entered?\n", pp->pr_wchan); panic("pr_leave"); } @@ -240,10 +216,8 @@ pr_leave(pp) pp->pr_entered_line = 0; } -static __inline__ void -pr_enter_check(pp, pr) - struct pool *pp; - void (*pr) __P((const char *, ...)); +static __inline void +pr_enter_check(struct pool *pp, void (*pr)(const char *, ...)) { if (pp->pr_entered_file != NULL) @@ -261,10 +235,8 @@ pr_enter_check(pp, pr) /* * Return the pool page header based on page address. */ -static __inline__ struct pool_item_header * -pr_find_pagehead(pp, page) - struct pool *pp; - caddr_t page; +static __inline struct pool_item_header * +pr_find_pagehead(struct pool *pp, caddr_t page) { struct pool_item_header *ph; @@ -283,10 +255,8 @@ pr_find_pagehead(pp, page) /* * Remove a page from the pool. */ -static __inline__ void -pr_rmpage(pp, ph) - struct pool *pp; - struct pool_item_header *ph; +static __inline void +pr_rmpage(struct pool *pp, struct pool_item_header *ph) { /* @@ -339,16 +309,11 @@ pr_rmpage(pp, ph) * Allocate and initialize a pool. */ struct pool * -pool_create(size, align, ioff, nitems, wchan, pagesz, alloc, release, mtype) - size_t size; - u_int align; - u_int ioff; - int nitems; - const char *wchan; - size_t pagesz; - void *(*alloc) __P((unsigned long, int, int)); - void (*release) __P((void *, unsigned long, int)); - int mtype; +pool_create(size_t size, u_int align, u_int ioff, int nitems, + const char *wchan, size_t pagesz, + void *(*alloc)(unsigned long, int, int), + void (*release)(void *, unsigned long, int), + int mtype) { struct pool *pp; int flags; @@ -378,17 +343,11 @@ pool_create(size, align, ioff, nitems, w * static pools that must be initialized before malloc() is available. */ void -pool_init(pp, size, align, ioff, flags, wchan, pagesz, alloc, release, mtype) - struct pool *pp; - size_t size; - u_int align; - u_int ioff; - int flags; - const char *wchan; - size_t pagesz; - void *(*alloc) __P((unsigned long, int, int)); - void (*release) __P((void *, unsigned long, int)); - int mtype; +pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, + const char *wchan, size_t pagesz, + void *(*alloc)(unsigned long, int, int), + void (*release)(void *, unsigned long, int), + int mtype) { int off, slack, i; @@ -403,7 +362,7 @@ pool_init(pp, size, align, ioff, flags, /* * Check arguments and construct default values. */ - if (!powerof2(pagesz) || pagesz > PAGE_SIZE) + if (!powerof2(pagesz)) panic("pool_init: page size invalid (%lx)\n", (u_long)pagesz); if (alloc == NULL && release == NULL) { @@ -424,6 +383,11 @@ pool_init(pp, size, align, ioff, flags, if (size < sizeof(struct pool_item)) size = sizeof(struct pool_item); + size = ALIGN(size); + if (size >= pagesz) + panic("pool_init: pool item size (%lu) too large", + (u_long)size); + /* * Initialize the pool structure. */ @@ -435,7 +399,7 @@ pool_init(pp, size, align, ioff, flags, pp->pr_maxpages = UINT_MAX; pp->pr_roflags = flags; pp->pr_flags = 0; - pp->pr_size = ALIGN(size); + pp->pr_size = size; pp->pr_align = align; pp->pr_wchan = wchan; pp->pr_mtype = mtype; @@ -448,9 +412,10 @@ pool_init(pp, size, align, ioff, flags, pp->pr_nout = 0; pp->pr_hardlimit = UINT_MAX; pp->pr_hardlimit_warning = NULL; - pp->pr_hardlimit_ratecap = 0; - memset(&pp->pr_hardlimit_warning_last, 0, - sizeof(pp->pr_hardlimit_warning_last)); + pp->pr_hardlimit_ratecap.tv_sec = 0; + pp->pr_hardlimit_ratecap.tv_usec = 0; + pp->pr_hardlimit_warning_last.tv_sec = 0; + pp->pr_hardlimit_warning_last.tv_usec = 0; /* * Decide whether to put the page header off page to avoid @@ -532,8 +497,7 @@ pool_init(pp, size, align, ioff, flags, * De-commision a pool resource. */ void -pool_destroy(pp) - struct pool *pp; +pool_destroy(struct pool *pp) { struct pool_item_header *ph; @@ -569,24 +533,22 @@ pool_destroy(pp) * Grab an item from the pool; must be called at appropriate spl level */ void * -_pool_get(pp, flags, file, line) - struct pool *pp; - int flags; - const char *file; - long line; +_pool_get(struct pool *pp, int flags, const char *file, long line) { void *v; struct pool_item *pi; struct pool_item_header *ph; #ifdef DIAGNOSTIC - if ((pp->pr_roflags & PR_STATIC) && (flags & PR_MALLOCOK)) { + if (__predict_false((pp->pr_roflags & PR_STATIC) && + (flags & PR_MALLOCOK))) { pr_printlog(pp, NULL, printf); panic("pool_get: static"); } #endif - if (curproc == NULL && (flags & PR_WAITOK) != 0) + if (__predict_false(curproc == NULL && doing_shutdown == 0 && + (flags & PR_WAITOK) != 0)) panic("pool_get: must have NOWAIT"); simple_lock(&pp->pr_slock); @@ -599,41 +561,32 @@ _pool_get(pp, flags, file, line) * the pool. */ #ifdef DIAGNOSTIC - if (pp->pr_nout > pp->pr_hardlimit) { + if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) { pr_leave(pp); simple_unlock(&pp->pr_slock); panic("pool_get: %s: crossed hard limit", pp->pr_wchan); } #endif - if (pp->pr_nout == pp->pr_hardlimit) { - if (flags & PR_WAITOK) { + if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { + if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) { /* * XXX: A warning isn't logged in this case. Should * it be? */ pp->pr_flags |= PR_WANTED; pr_leave(pp); - simple_unlock(&pp->pr_slock); - tsleep((caddr_t)pp, PSWP, pp->pr_wchan, 0); - simple_lock(&pp->pr_slock); + ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock); pr_enter(pp, file, line); goto startover; } - if (pp->pr_hardlimit_warning != NULL) { - /* - * Log a message that the hard limit has been hit. - */ - struct timeval curtime, logdiff; - int s = splclock(); - curtime = mono_time; - splx(s); - timersub(&curtime, &pp->pr_hardlimit_warning_last, - &logdiff); - if (logdiff.tv_sec >= pp->pr_hardlimit_ratecap) { - pp->pr_hardlimit_warning_last = curtime; - log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning); - } - } + + /* + * Log a message that the hard limit has been hit. + */ + if (pp->pr_hardlimit_warning != NULL && + ratecheck(&pp->pr_hardlimit_warning_last, + &pp->pr_hardlimit_ratecap)) + log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning); if (flags & PR_URGENT) panic("pool_get: urgent"); @@ -707,9 +660,7 @@ _pool_get(pp, flags, file, line) */ pp->pr_flags |= PR_WANTED; pr_leave(pp); - simple_unlock(&pp->pr_slock); - tsleep((caddr_t)pp, PSWP, pp->pr_wchan, 0); - simple_lock(&pp->pr_slock); + ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock); pr_enter(pp, file, line); goto startover; } @@ -722,13 +673,13 @@ _pool_get(pp, flags, file, line) goto startover; } - if ((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL) { + if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) { pr_leave(pp); simple_unlock(&pp->pr_slock); panic("pool_get: %s: page empty", pp->pr_wchan); } #ifdef DIAGNOSTIC - if (pp->pr_nitems == 0) { + if (__predict_false(pp->pr_nitems == 0)) { pr_leave(pp); simple_unlock(&pp->pr_slock); printf("pool_get: %s: items on itemlist, nitems %u\n", @@ -739,7 +690,7 @@ _pool_get(pp, flags, file, line) pr_log(pp, v, PRLOG_GET, file, line); #ifdef DIAGNOSTIC - if (pi->pi_magic != PI_MAGIC) { + if (__predict_false(pi->pi_magic != PI_MAGIC)) { pr_printlog(pp, pi, printf); panic("pool_get(%s): free list modified: magic=%x; page %p;" " item addr %p\n", @@ -755,7 +706,7 @@ _pool_get(pp, flags, file, line) pp->pr_nout++; if (ph->ph_nmissing == 0) { #ifdef DIAGNOSTIC - if (pp->pr_nidle == 0) + if (__predict_false(pp->pr_nidle == 0)) panic("pool_get: nidle inconsistent"); #endif pp->pr_nidle--; @@ -763,7 +714,7 @@ _pool_get(pp, flags, file, line) ph->ph_nmissing++; if (TAILQ_FIRST(&ph->ph_itemlist) == NULL) { #ifdef DIAGNOSTIC - if (ph->ph_nmissing != pp->pr_itemsperpage) { + if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) { pr_leave(pp); simple_unlock(&pp->pr_slock); panic("pool_get: %s: nmissing inconsistent", @@ -814,11 +765,7 @@ _pool_get(pp, flags, file, line) * Return resource to the pool; must be called at appropriate spl level */ void -_pool_put(pp, v, file, line) - struct pool *pp; - void *v; - const char *file; - long line; +_pool_put(struct pool *pp, void *v, const char *file, long line) { struct pool_item *pi = v; struct pool_item_header *ph; @@ -830,20 +777,44 @@ _pool_put(pp, v, file, line) simple_lock(&pp->pr_slock); pr_enter(pp, file, line); +#ifdef DIAGNOSTIC + if (__predict_false(pp->pr_nout == 0)) { + printf("pool %s: putting with none out\n", + pp->pr_wchan); + panic("pool_put"); + } +#endif + pr_log(pp, v, PRLOG_PUT, file, line); - if ((ph = pr_find_pagehead(pp, page)) == NULL) { + if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) { pr_printlog(pp, NULL, printf); panic("pool_put: %s: page header missing", pp->pr_wchan); } +#ifdef LOCKDEBUG + /* + * Check if we're freeing a locked simple lock. + */ + simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size); +#endif + /* * Return to item list. */ #ifdef DIAGNOSTIC - /* XXX Should fill the item. */ pi->pi_magic = PI_MAGIC; #endif +#ifdef DEBUG + { + int i, *ip = v; + + for (i = 0; i < pp->pr_size / sizeof(int); i++) { + *ip++ = PI_MAGIC; + } + } +#endif + TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); ph->ph_nmissing--; pp->pr_nput++; @@ -933,16 +904,13 @@ _pool_put(pp, v, file, line) * Add N items to the pool. */ int -pool_prime(pp, n, storage) - struct pool *pp; - int n; - caddr_t storage; +pool_prime(struct pool *pp, int n, caddr_t storage) { caddr_t cp; int newnitems, newpages; #ifdef DIAGNOSTIC - if (storage && !(pp->pr_roflags & PR_STATIC)) + if (__predict_false(storage && !(pp->pr_roflags & PR_STATIC))) panic("pool_prime: static"); /* !storage && static caught below */ #endif @@ -989,9 +957,7 @@ pool_prime(pp, n, storage) * Note, we must be called with the pool descriptor LOCKED. */ static void -pool_prime_page(pp, storage) - struct pool *pp; - caddr_t storage; +pool_prime_page(struct pool *pp, caddr_t storage) { struct pool_item *pi; struct pool_item_header *ph; @@ -1000,6 +966,9 @@ pool_prime_page(pp, storage) unsigned int ioff = pp->pr_itemoffset; int s, n; + if (((u_long)cp & (pp->pr_pagesz - 1)) != 0) + panic("pool_prime_page: %s: unaligned page", pp->pr_wchan); + if ((pp->pr_roflags & PR_PHINPAGE) != 0) { ph = (struct pool_item_header *)(cp + pp->pr_phoffset); } else { @@ -1074,8 +1043,7 @@ pool_prime_page(pp, storage) * with it locked. */ static int -pool_catchup(pp) - struct pool *pp; +pool_catchup(struct pool *pp) { caddr_t cp; int error = 0; @@ -1102,7 +1070,7 @@ pool_catchup(pp) simple_unlock(&pp->pr_slock); cp = (*pp->pr_alloc)(pp->pr_pagesz, 0, pp->pr_mtype); simple_lock(&pp->pr_slock); - if (cp == NULL) { + if (__predict_false(cp == NULL)) { error = ENOMEM; break; } @@ -1114,9 +1082,7 @@ pool_catchup(pp) } void -pool_setlowat(pp, n) - pool_handle_t pp; - int n; +pool_setlowat(struct pool *pp, int n) { int error; @@ -1128,7 +1094,8 @@ pool_setlowat(pp, n) : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; /* Make sure we're caught up with the newly-set low water mark. */ - if ((error = pool_catchup(pp)) != 0) { + if ((pp->pr_nitems < pp->pr_minitems) && + (error = pool_catchup(pp)) != 0) { /* * XXX: Should we log a warning? Should we set up a timeout * to try again in a second or so? The latter could break @@ -1140,9 +1107,7 @@ pool_setlowat(pp, n) } void -pool_sethiwat(pp, n) - pool_handle_t pp; - int n; +pool_sethiwat(struct pool *pp, int n) { simple_lock(&pp->pr_slock); @@ -1155,20 +1120,16 @@ pool_sethiwat(pp, n) } void -pool_sethardlimit(pp, n, warnmess, ratecap) - pool_handle_t pp; - int n; - const char *warnmess; - int ratecap; +pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap) { simple_lock(&pp->pr_slock); pp->pr_hardlimit = n; pp->pr_hardlimit_warning = warnmess; - pp->pr_hardlimit_ratecap = ratecap; - memset(&pp->pr_hardlimit_warning_last, 0, - sizeof(pp->pr_hardlimit_warning_last)); + pp->pr_hardlimit_ratecap.tv_sec = ratecap; + pp->pr_hardlimit_warning_last.tv_sec = 0; + pp->pr_hardlimit_warning_last.tv_usec = 0; /* * In-line version of pool_sethiwat(), because we don't want to @@ -1185,10 +1146,7 @@ pool_sethardlimit(pp, n, warnmess, ratec * Default page allocator. */ static void * -pool_page_alloc(sz, flags, mtype) - unsigned long sz; - int flags; - int mtype; +pool_page_alloc(unsigned long sz, int flags, int mtype) { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; @@ -1196,10 +1154,7 @@ pool_page_alloc(sz, flags, mtype) } static void -pool_page_free(v, sz, mtype) - void *v; - unsigned long sz; - int mtype; +pool_page_free(void *v, unsigned long sz, int mtype) { uvm_km_free_poolpage((vaddr_t)v); @@ -1210,10 +1165,7 @@ pool_page_free(v, sz, mtype) * never be accessed in interrupt context. */ void * -pool_page_alloc_nointr(sz, flags, mtype) - unsigned long sz; - int flags; - int mtype; +pool_page_alloc_nointr(unsigned long sz, int flags, int mtype) { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; @@ -1222,10 +1174,7 @@ pool_page_alloc_nointr(sz, flags, mtype) } void -pool_page_free_nointr(v, sz, mtype) - void *v; - unsigned long sz; - int mtype; +pool_page_free_nointr(void *v, unsigned long sz, int mtype) { uvm_km_free_poolpage1(kernel_map, (vaddr_t)v); @@ -1236,10 +1185,7 @@ pool_page_free_nointr(v, sz, mtype) * Release all complete pages that have not been used recently. */ void -_pool_reclaim(pp, file, line) - pool_handle_t pp; - const char *file; - long line; +_pool_reclaim(struct pool *pp, const char *file, long line) { struct pool_item_header *ph, *phnext; struct timeval curtime; @@ -1292,8 +1238,7 @@ _pool_reclaim(pp, file, line) * Note, we must never be called from an interrupt context. */ void -pool_drain(arg) - void *arg; +pool_drain(void *arg) { struct pool *pp; int s; @@ -1319,9 +1264,7 @@ pool_drain(arg) * Diagnostic helpers. */ void -pool_print(pp, modif) - struct pool *pp; - const char *modif; +pool_print(struct pool *pp, const char *modif) { int s; @@ -1338,10 +1281,7 @@ pool_print(pp, modif) } void -pool_printit(pp, modif, pr) - struct pool *pp; - const char *modif; - void (*pr) __P((const char *, ...)); +pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) { int didlock = 0; @@ -1371,10 +1311,7 @@ pool_printit(pp, modif, pr) } static void -pool_print1(pp, modif, pr) - struct pool *pp; - const char *modif; - void (*pr) __P((const char *, ...)); +pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) { struct pool_item_header *ph; #ifdef DIAGNOSTIC @@ -1448,9 +1385,7 @@ pool_print1(pp, modif, pr) } int -pool_chk(pp, label) - struct pool *pp; - char *label; +pool_chk(struct pool *pp, const char *label) { struct pool_item_header *ph; int r = 0;