[BACK]Return to subr_pool.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / kern

Annotation of src/sys/kern/subr_pool.c, Revision 1.4.2.1

1.4.2.1 ! eeh         1: /*     $NetBSD: subr_pool.c,v 1.4 1998/07/24 20:19:23 thorpej Exp $    */
1.1       pk          2:
                      3: /*-
                      4:  * Copyright (c) 1997 The NetBSD Foundation, Inc.
                      5:  * All rights reserved.
                      6:  *
                      7:  * This code is derived from software contributed to The NetBSD Foundation
                      8:  * by Paul Kranenburg.
                      9:  *
                     10:  * Redistribution and use in source and binary forms, with or without
                     11:  * modification, are permitted provided that the following conditions
                     12:  * are met:
                     13:  * 1. Redistributions of source code must retain the above copyright
                     14:  *    notice, this list of conditions and the following disclaimer.
                     15:  * 2. Redistributions in binary form must reproduce the above copyright
                     16:  *    notice, this list of conditions and the following disclaimer in the
                     17:  *    documentation and/or other materials provided with the distribution.
                     18:  * 3. All advertising materials mentioning features or use of this software
                     19:  *    must display the following acknowledgement:
                     20:  *        This product includes software developed by the NetBSD
                     21:  *        Foundation, Inc. and its contributors.
                     22:  * 4. Neither the name of The NetBSD Foundation nor the names of its
                     23:  *    contributors may be used to endorse or promote products derived
                     24:  *    from this software without specific prior written permission.
                     25:  *
                     26:  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     27:  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     28:  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     29:  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     30:  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     31:  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     32:  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     33:  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     34:  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     35:  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     36:  * POSSIBILITY OF SUCH DAMAGE.
                     37:  */
                     38:
                     39: #include <sys/param.h>
                     40: #include <sys/systm.h>
                     41: #include <sys/proc.h>
                     42: #include <sys/errno.h>
                     43: #include <sys/kernel.h>
                     44: #include <sys/malloc.h>
                     45: #include <sys/lock.h>
                     46: #include <sys/pool.h>
                     47:
1.3       pk         48: #include <vm/vm.h>
                     49: #include <vm/vm_kern.h>
                     50:
                     51: #if defined(UVM)
                     52: #include <uvm/uvm.h>
                     53: #endif
                     54:
1.1       pk         55: /*
                     56:  * Pool resource management utility.
1.3       pk         57:  *
                     58:  * Memory is allocated in pages which are split into pieces according
                     59:  * to the pool item size. Each page is kept on a list headed by `pr_pagelist'
                     60:  * in the pool structure and the individual pool items are on a linked list
                     61:  * headed by `ph_itemlist' in each page header. The memory for building
                     62:  * the page list is either taken from the allocated pages themselves (for
                     63:  * small pool items) or taken from an internal pool of page headers (`phpool').
                     64:  *
1.1       pk         65:  */
                     66:
1.3       pk         67: /* List of all pools */
                     68: static TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
                     69:
                     70: /* Private pool for page header structures */
                     71: static struct pool phpool;
                     72:
                     73: /* # of seconds to retain page after last use */
                     74: int pool_inactive_time = 10;
                     75:
                     76: /* Next candidate for drainage (see pool_drain()) */
                     77: static struct pool     *drainpp = NULL;
                     78:
                     79: struct pool_item_header {
                     80:        /* Page headers */
                     81:        TAILQ_ENTRY(pool_item_header)
                     82:                                ph_pagelist;    /* pool page list */
                     83:        TAILQ_HEAD(,pool_item)  ph_itemlist;    /* chunk list for this page */
                     84:        LIST_ENTRY(pool_item_header)
                     85:                                ph_hashlist;    /* Off-page page headers */
                     86:        int                     ph_nmissing;    /* # of chunks in use */
                     87:        caddr_t                 ph_page;        /* this page's address */
                     88:        struct timeval          ph_time;        /* last referenced */
                     89: };
                     90:
1.1       pk         91: struct pool_item {
1.3       pk         92: #ifdef DIAGNOSTIC
                     93:        int pi_magic;
                     94: #define PI_MAGIC 0xdeadbeef
                     95: #endif
                     96:        /* Other entries use only this list entry */
                     97:        TAILQ_ENTRY(pool_item)  pi_list;
                     98: };
                     99:
                    100:
                    101: #define PR_HASH_INDEX(pp,addr) \
                    102:        (((u_long)(addr) >> (pp)->pr_pageshift) & (PR_HASHTABSIZE - 1))
                    103:
                    104:
                    105:
                    106: static struct pool_item_header
                    107:                *pr_find_pagehead __P((struct pool *, caddr_t));
                    108: static void    pr_rmpage __P((struct pool *, struct pool_item_header *));
                    109: static int     pool_prime_page __P((struct pool *, caddr_t));
                    110: static void    *pool_page_alloc __P((unsigned long, int, int));
                    111: static void    pool_page_free __P((void *, unsigned long, int));
                    112: int pool_chk __P((struct pool *, char *));
                    113:
                    114:
                    115: #ifdef POOL_DIAGNOSTIC
                    116: /*
                    117:  * Pool log entry. An array of these is allocated in pool_create().
                    118:  */
                    119: struct pool_log {
                    120:        const char      *pl_file;
                    121:        long            pl_line;
                    122:        int             pl_action;
                    123: #define PRLOG_GET      1
                    124: #define PRLOG_PUT      2
                    125:        void            *pl_addr;
1.1       pk        126: };
                    127:
1.3       pk        128: /* Number of entries in pool log buffers */
                    129: int pool_logsize = 10;
1.1       pk        130:
1.3       pk        131: static void    pr_log __P((struct pool *, void *, int, const char *, long));
                    132: static void    pr_printlog __P((struct pool *));
                    133:
                    134: static __inline__ void
                    135: pr_log(pp, v, action, file, line)
                    136:        struct pool     *pp;
                    137:        void            *v;
                    138:        int             action;
                    139:        const char      *file;
                    140:        long            line;
                    141: {
                    142:        int n = pp->pr_curlogentry;
                    143:        struct pool_log *pl;
                    144:
                    145:        if ((pp->pr_flags & PR_LOGGING) == 0)
                    146:                return;
                    147:
                    148:        /*
                    149:         * Fill in the current entry. Wrap around and overwrite
                    150:         * the oldest entry if necessary.
                    151:         */
                    152:        pl = &pp->pr_log[n];
                    153:        pl->pl_file = file;
                    154:        pl->pl_line = line;
                    155:        pl->pl_action = action;
                    156:        pl->pl_addr = v;
                    157:        if (++n >= pp->pr_logsize)
                    158:                n = 0;
                    159:        pp->pr_curlogentry = n;
                    160: }
                    161:
                    162: static void
                    163: pr_printlog(pp)
                    164:        struct pool *pp;
                    165: {
                    166:        int i = pp->pr_logsize;
                    167:        int n = pp->pr_curlogentry;
                    168:
                    169:        if ((pp->pr_flags & PR_LOGGING) == 0)
                    170:                return;
                    171:
                    172:        pool_print(pp, "printlog");
                    173:
                    174:        /*
                    175:         * Print all entries in this pool's log.
                    176:         */
                    177:        while (i-- > 0) {
                    178:                struct pool_log *pl = &pp->pr_log[n];
                    179:                if (pl->pl_action != 0) {
                    180:                        printf("log entry %d:\n", i);
                    181:                        printf("\taction = %s, addr = %p\n",
                    182:                                pl->pl_action == PRLOG_GET ? "get" : "put",
                    183:                                pl->pl_addr);
                    184:                        printf("\tfile: %s at line %lu\n",
                    185:                                pl->pl_file, pl->pl_line);
                    186:                }
                    187:                if (++n >= pp->pr_logsize)
                    188:                        n = 0;
                    189:        }
                    190: }
                    191: #else
                    192: #define pr_log(pp, v, action, file, line)
                    193: #define pr_printlog(pp)
                    194: #endif
                    195:
                    196:
                    197: /*
                    198:  * Return the pool page header based on page address.
                    199:  */
                    200: static __inline__ struct pool_item_header *
                    201: pr_find_pagehead(pp, page)
                    202:        struct pool *pp;
                    203:        caddr_t page;
                    204: {
                    205:        struct pool_item_header *ph;
                    206:
                    207:        if ((pp->pr_flags & PR_PHINPAGE) != 0)
                    208:                return ((struct pool_item_header *)(page + pp->pr_phoffset));
                    209:
                    210:        for (ph = LIST_FIRST(&pp->pr_hashtab[PR_HASH_INDEX(pp, page)]);
                    211:             ph != NULL;
                    212:             ph = LIST_NEXT(ph, ph_hashlist)) {
                    213:                if (ph->ph_page == page)
                    214:                        return (ph);
                    215:        }
                    216:        return (NULL);
                    217: }
                    218:
                    219: /*
                    220:  * Remove a page from the pool.
                    221:  */
                    222: static __inline__ void
                    223: pr_rmpage(pp, ph)
                    224:        struct pool *pp;
                    225:        struct pool_item_header *ph;
                    226: {
                    227:
                    228:        /*
                    229:         * Unlink a page from the pool and release it.
                    230:         */
                    231:        TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);
                    232:        (*pp->pr_free)(ph->ph_page, pp->pr_pagesz, pp->pr_mtype);
                    233:        pp->pr_npages--;
                    234:        pp->pr_npagefree++;
                    235:
                    236:        if ((pp->pr_flags & PR_PHINPAGE) == 0) {
                    237:                LIST_REMOVE(ph, ph_hashlist);
                    238:                pool_put(&phpool, ph);
                    239:        }
                    240:
                    241:        if (pp->pr_curpage == ph) {
                    242:                /*
                    243:                 * Find a new non-empty page header, if any.
                    244:                 * Start search from the page head, to increase the
                    245:                 * chance for "high water" pages to be freed.
                    246:                 */
                    247:                for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;
                    248:                     ph = TAILQ_NEXT(ph, ph_pagelist))
                    249:                        if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)
                    250:                                break;
                    251:
                    252:                pp->pr_curpage = ph;
                    253:        }
                    254: }
                    255:
                    256: /*
                    257:  * Allocate and initialize a pool.
                    258:  */
1.1       pk        259: struct pool *
1.3       pk        260: pool_create(size, align, ioff, nitems, wchan, pagesz, alloc, release, mtype)
1.1       pk        261:        size_t  size;
1.3       pk        262:        u_int   align;
                    263:        u_int   ioff;
1.1       pk        264:        int     nitems;
                    265:        char    *wchan;
1.3       pk        266:        size_t  pagesz;
                    267:        void    *(*alloc) __P((unsigned long, int, int));
                    268:        void    (*release) __P((void *, unsigned long, int));
1.1       pk        269:        int     mtype;
                    270: {
                    271:        struct pool *pp;
1.3       pk        272:        int flags;
1.1       pk        273:
1.3       pk        274:        pp = (struct pool *)malloc(sizeof(*pp), M_POOL, M_NOWAIT);
                    275:        if (pp == NULL)
1.1       pk        276:                return (NULL);
1.3       pk        277:
                    278:        flags = PR_FREEHEADER;
                    279: #ifdef POOL_DIAGNOSTIC
                    280:        if (pool_logsize != 0)
                    281:                flags |= PR_LOGGING;
                    282: #endif
                    283:
                    284:        pool_init(pp, size, align, ioff, flags, wchan, pagesz,
                    285:                  alloc, release, mtype);
                    286:
                    287:        if (nitems != 0) {
                    288:                if (pool_prime(pp, nitems, NULL) != 0) {
                    289:                        pool_destroy(pp);
                    290:                        return (NULL);
                    291:                }
1.1       pk        292:        }
                    293:
1.3       pk        294:        return (pp);
                    295: }
                    296:
                    297: /*
                    298:  * Initialize the given pool resource structure.
                    299:  *
                    300:  * We export this routine to allow other kernel parts to declare
                    301:  * static pools that must be initialized before malloc() is available.
                    302:  */
                    303: void
                    304: pool_init(pp, size, align, ioff, flags, wchan, pagesz, alloc, release, mtype)
                    305:        struct pool     *pp;
                    306:        size_t          size;
                    307:        u_int           align;
                    308:        u_int           ioff;
                    309:        int             flags;
                    310:        char            *wchan;
                    311:        size_t          pagesz;
                    312:        void            *(*alloc) __P((unsigned long, int, int));
                    313:        void            (*release) __P((void *, unsigned long, int));
                    314:        int             mtype;
                    315: {
                    316:        int off, slack;
                    317:
                    318:        /*
                    319:         * Check arguments and construct default values.
                    320:         */
                    321:        if (!powerof2(pagesz) || pagesz > PAGE_SIZE)
                    322:                panic("pool_init: page size invalid (%lx)\n", (u_long)pagesz);
                    323:
1.4       thorpej   324:        if (alloc == NULL && release == NULL) {
1.3       pk        325:                alloc = pool_page_alloc;
                    326:                release = pool_page_free;
1.4       thorpej   327:                pagesz = PAGE_SIZE;     /* Rounds to PAGE_SIZE anyhow. */
                    328:        } else if ((alloc != NULL && release != NULL) == 0) {
                    329:                /* If you specifiy one, must specify both. */
                    330:                panic("pool_init: must specify alloc and release together");
                    331:        }
                    332:
1.3       pk        333:        if (pagesz == 0)
                    334:                pagesz = PAGE_SIZE;
                    335:
                    336:        if (align == 0)
                    337:                align = ALIGN(1);
                    338:
                    339:        /*
                    340:         * Initialize the pool structure.
                    341:         */
                    342:        TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
                    343:        TAILQ_INIT(&pp->pr_pagelist);
                    344:        pp->pr_curpage = NULL;
                    345:        pp->pr_npages = 0;
                    346:        pp->pr_minitems = 0;
                    347:        pp->pr_minpages = 0;
                    348:        pp->pr_maxpages = UINT_MAX;
                    349:        pp->pr_flags = flags;
                    350:        pp->pr_size = ALIGN(size);
                    351:        pp->pr_align = align;
                    352:        pp->pr_wchan = wchan;
                    353:        pp->pr_mtype = mtype;
                    354:        pp->pr_alloc = alloc;
                    355:        pp->pr_free = release;
                    356:        pp->pr_pagesz = pagesz;
                    357:        pp->pr_pagemask = ~(pagesz - 1);
                    358:        pp->pr_pageshift = ffs(pagesz) - 1;
                    359:
                    360:        /*
                    361:         * Decide whether to put the page header off page to avoid
                    362:         * wasting too large a part of the page. Off-page page headers
                    363:         * go on a hash table, so we can match a returned item
                    364:         * with its header based on the page address.
                    365:         * We use 1/16 of the page size as the threshold (XXX: tune)
                    366:         */
                    367:        if (pp->pr_size < pagesz/16) {
                    368:                /* Use the end of the page for the page header */
                    369:                pp->pr_flags |= PR_PHINPAGE;
                    370:                pp->pr_phoffset = off =
                    371:                        pagesz - ALIGN(sizeof(struct pool_item_header));
1.2       pk        372:        } else {
1.3       pk        373:                /* The page header will be taken from our page header pool */
                    374:                pp->pr_phoffset = 0;
                    375:                off = pagesz;
                    376:                bzero(pp->pr_hashtab, sizeof(pp->pr_hashtab));
1.2       pk        377:        }
1.1       pk        378:
1.3       pk        379:        /*
                    380:         * Alignment is to take place at `ioff' within the item. This means
                    381:         * we must reserve up to `align - 1' bytes on the page to allow
                    382:         * appropriate positioning of each item.
                    383:         *
                    384:         * Silently enforce `0 <= ioff < align'.
                    385:         */
                    386:        pp->pr_itemoffset = ioff = ioff % align;
                    387:        pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size;
                    388:
                    389:        /*
                    390:         * Use the slack between the chunks and the page header
                    391:         * for "cache coloring".
                    392:         */
                    393:        slack = off - pp->pr_itemsperpage * pp->pr_size;
                    394:        pp->pr_maxcolor = (slack / align) * align;
                    395:        pp->pr_curcolor = 0;
                    396:
                    397:        pp->pr_nget = 0;
                    398:        pp->pr_nfail = 0;
                    399:        pp->pr_nput = 0;
                    400:        pp->pr_npagealloc = 0;
                    401:        pp->pr_npagefree = 0;
1.1       pk        402:        pp->pr_hiwat = 0;
1.3       pk        403:
                    404: #ifdef POOL_DIAGNOSTIC
                    405:        if ((flags & PR_LOGGING) != 0) {
                    406:                pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log),
                    407:                                    M_TEMP, M_NOWAIT);
                    408:                if (pp->pr_log == NULL)
                    409:                        pp->pr_flags &= ~PR_LOGGING;
                    410:                pp->pr_curlogentry = 0;
                    411:                pp->pr_logsize = pool_logsize;
                    412:        }
                    413: #endif
                    414:
1.1       pk        415:        simple_lock_init(&pp->pr_lock);
                    416:
1.3       pk        417:        /*
                    418:         * Initialize private page header pool if we haven't done so yet.
                    419:         */
                    420:        if (phpool.pr_size == 0) {
                    421:                pool_init(&phpool, sizeof(struct pool_item_header), 0, 0,
                    422:                          0, "phpool", 0, 0, 0, 0);
1.1       pk        423:        }
                    424:
1.3       pk        425:        return;
1.1       pk        426: }
                    427:
                    428: /*
                    429:  * De-commision a pool resource.
                    430:  */
                    431: void
                    432: pool_destroy(pp)
                    433:        struct pool *pp;
                    434: {
1.3       pk        435:        struct pool_item_header *ph;
                    436:
                    437: #ifdef DIAGNOSTIC
                    438:        if (pp->pr_nget - pp->pr_nput != 0) {
                    439:                pr_printlog(pp);
                    440:                panic("pool_destroy: pool busy: still out: %lu\n",
                    441:                      pp->pr_nget - pp->pr_nput);
                    442:        }
                    443: #endif
1.1       pk        444:
1.3       pk        445:        /* Remove all pages */
                    446:        if ((pp->pr_flags & PR_STATIC) == 0)
                    447:                while ((ph = pp->pr_pagelist.tqh_first) != NULL)
                    448:                        pr_rmpage(pp, ph);
                    449:
                    450:        /* Remove from global pool list */
                    451:        TAILQ_REMOVE(&pool_head, pp, pr_poollist);
                    452:        drainpp = NULL;
                    453:
                    454: #ifdef POOL_DIAGNOSTIC
                    455:        if ((pp->pr_flags & PR_LOGGING) != 0)
                    456:                free(pp->pr_log, M_TEMP);
                    457: #endif
1.2       pk        458:
1.3       pk        459:        if (pp->pr_flags & PR_FREEHEADER)
                    460:                free(pp, M_POOL);
1.1       pk        461: }
                    462:
                    463:
                    464: /*
1.3       pk        465:  * Grab an item from the pool; must be called at appropriate spl level
1.1       pk        466:  */
1.3       pk        467: #ifdef POOL_DIAGNOSTIC
                    468: void *
                    469: _pool_get(pp, flags, file, line)
                    470:        struct pool *pp;
                    471:        int flags;
                    472:        const char *file;
                    473:        long line;
                    474: #else
1.1       pk        475: void *
                    476: pool_get(pp, flags)
                    477:        struct pool *pp;
                    478:        int flags;
1.3       pk        479: #endif
1.1       pk        480: {
                    481:        void *v;
                    482:        struct pool_item *pi;
1.3       pk        483:        struct pool_item_header *ph;
1.1       pk        484:
1.2       pk        485: #ifdef DIAGNOSTIC
1.3       pk        486:        if ((pp->pr_flags & PR_STATIC) && (flags & PR_MALLOCOK)) {
                    487:                pr_printlog(pp);
1.2       pk        488:                panic("pool_get: static");
1.3       pk        489:        }
1.2       pk        490: #endif
                    491:
1.1       pk        492:        simple_lock(&pp->pr_lock);
1.3       pk        493:        if (curproc == NULL && (flags & PR_WAITOK) != 0)
                    494:                panic("pool_get: must have NOWAIT");
1.1       pk        495:
1.3       pk        496:        /*
                    497:         * The convention we use is that if `curpage' is not NULL, then
                    498:         * it points at a non-empty bucket. In particular, `curpage'
                    499:         * never points at a page header which has PR_PHINPAGE set and
                    500:         * has no items in its bucket.
                    501:         */
                    502: again:
                    503:        if ((ph = pp->pr_curpage) == NULL) {
                    504:                void *v = (*pp->pr_alloc)(pp->pr_pagesz, flags, pp->pr_mtype);
1.1       pk        505:                if (v == NULL) {
1.3       pk        506:                        if (flags & PR_URGENT)
                    507:                                panic("pool_get: urgent");
                    508:                        if ((flags & PR_WAITOK) == 0) {
                    509:                                pp->pr_nfail++;
                    510:                                simple_unlock(&pp->pr_lock);
1.1       pk        511:                                return (NULL);
1.3       pk        512:                        }
                    513:
1.1       pk        514:                        pp->pr_flags |= PR_WANTED;
                    515:                        simple_unlock(&pp->pr_lock);
                    516:                        tsleep((caddr_t)pp, PSWP, pp->pr_wchan, 0);
1.3       pk        517:                        simple_lock(&pp->pr_lock);
                    518:                } else {
                    519:                        pp->pr_npagealloc++;
                    520:                        pool_prime_page(pp, v);
1.1       pk        521:                }
1.3       pk        522:
                    523:                goto again;
                    524:        }
                    525:
                    526:        if ((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)
                    527:                panic("pool_get: %s: page empty", pp->pr_wchan);
                    528:
                    529:        pr_log(pp, v, PRLOG_GET, file, line);
                    530:
                    531: #ifdef DIAGNOSTIC
                    532:        if (pi->pi_magic != PI_MAGIC) {
                    533:                pr_printlog(pp);
                    534:                panic("pool_get(%s): free list modified: magic=%x; page %p;"
                    535:                       " item addr %p\n",
                    536:                        pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
                    537:        }
                    538: #endif
                    539:
                    540:        /*
                    541:         * Remove from item list.
                    542:         */
                    543:        TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list);
                    544:        ph->ph_nmissing++;
                    545:        if (TAILQ_FIRST(&ph->ph_itemlist) == NULL) {
                    546:                /*
                    547:                 * Find a new non-empty page header, if any.
                    548:                 * Start search from the page head, to increase
                    549:                 * the chance for "high water" pages to be freed.
                    550:                 *
                    551:                 * First, move the now empty page to the head of
                    552:                 * the page list.
                    553:                 */
                    554:                TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);
                    555:                TAILQ_INSERT_HEAD(&pp->pr_pagelist, ph, ph_pagelist);
                    556:                while ((ph = TAILQ_NEXT(ph, ph_pagelist)) != NULL)
                    557:                        if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)
                    558:                                break;
                    559:
                    560:                pp->pr_curpage = ph;
1.1       pk        561:        }
1.3       pk        562:
                    563:        pp->pr_nget++;
1.1       pk        564:        simple_unlock(&pp->pr_lock);
                    565:        return (v);
                    566: }
                    567:
                    568: /*
1.3       pk        569:  * Return resource to the pool; must be called at appropriate spl level
1.1       pk        570:  */
1.3       pk        571: #ifdef POOL_DIAGNOSTIC
                    572: void
                    573: _pool_put(pp, v, file, line)
                    574:        struct pool *pp;
                    575:        void *v;
                    576:        const char *file;
                    577:        long line;
                    578: #else
1.1       pk        579: void
                    580: pool_put(pp, v)
                    581:        struct pool *pp;
                    582:        void *v;
1.3       pk        583: #endif
1.1       pk        584: {
                    585:        struct pool_item *pi = v;
1.3       pk        586:        struct pool_item_header *ph;
                    587:        caddr_t page;
                    588:
                    589:        page = (caddr_t)((u_long)v & pp->pr_pagemask);
1.1       pk        590:
                    591:        simple_lock(&pp->pr_lock);
1.3       pk        592:
                    593:        pr_log(pp, v, PRLOG_PUT, file, line);
                    594:
                    595:        if ((ph = pr_find_pagehead(pp, page)) == NULL) {
                    596:                pr_printlog(pp);
                    597:                panic("pool_put: %s: page header missing", pp->pr_wchan);
                    598:        }
                    599:
                    600:        /*
                    601:         * Return to item list.
                    602:         */
1.2       pk        603: #ifdef DIAGNOSTIC
1.3       pk        604:        pi->pi_magic = PI_MAGIC;
                    605: #endif
                    606:        TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
                    607:        ph->ph_nmissing--;
                    608:        pp->pr_nput++;
                    609:
                    610:        /* Cancel "pool empty" condition if it exists */
                    611:        if (pp->pr_curpage == NULL)
                    612:                pp->pr_curpage = ph;
                    613:
                    614:        if (pp->pr_flags & PR_WANTED) {
                    615:                pp->pr_flags &= ~PR_WANTED;
                    616:                wakeup((caddr_t)pp);
                    617:                simple_unlock(&pp->pr_lock);
                    618:                return;
                    619:        }
                    620:
                    621:        /*
                    622:         * If this page is now complete, move it to the end of the pagelist.
                    623:         * If this page has just become un-empty, move it the head.
                    624:         */
                    625:        if (ph->ph_nmissing == 0) {
                    626:                if (pp->pr_npages > pp->pr_maxpages) {
                    627: #if 0
                    628:                        timeout(pool_drain, 0, pool_inactive_time*hz);
                    629: #else
                    630:                        pr_rmpage(pp, ph);
1.2       pk        631: #endif
1.3       pk        632:                } else {
                    633:                        TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist);
                    634:                        TAILQ_INSERT_TAIL(&pp->pr_pagelist, ph, ph_pagelist);
                    635:                        ph->ph_time = time;
                    636:
                    637:                        /* XXX - update curpage */
                    638:                        for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;
                    639:                             ph = TAILQ_NEXT(ph, ph_pagelist))
                    640:                                if (TAILQ_FIRST(&ph->ph_itemlist) != NULL)
                    641:                                        break;
1.1       pk        642:
1.3       pk        643:                        pp->pr_curpage = ph;
1.1       pk        644:                }
                    645:        }
1.3       pk        646:
1.1       pk        647:        simple_unlock(&pp->pr_lock);
                    648: }
                    649:
                    650: /*
1.3       pk        651:  * Add N items to the pool.
1.1       pk        652:  */
                    653: int
1.2       pk        654: pool_prime(pp, n, storage)
1.1       pk        655:        struct pool *pp;
                    656:        int n;
1.2       pk        657:        caddr_t storage;
1.1       pk        658: {
1.3       pk        659:        caddr_t cp;
                    660:        int newnitems, newpages;
1.2       pk        661:
                    662: #ifdef DIAGNOSTIC
                    663:        if (storage && !(pp->pr_flags & PR_STATIC))
                    664:                panic("pool_prime: static");
                    665:        /* !storage && static caught below */
                    666: #endif
1.1       pk        667:
1.3       pk        668:        newnitems = pp->pr_minitems + n;
                    669:        newpages =
                    670:                roundup(pp->pr_itemsperpage,newnitems) / pp->pr_itemsperpage
                    671:                - pp->pr_minpages;
                    672:
1.1       pk        673:        simple_lock(&pp->pr_lock);
1.3       pk        674:        while (newpages-- > 0) {
                    675:
1.2       pk        676:                if (pp->pr_flags & PR_STATIC) {
1.3       pk        677:                        cp = storage;
                    678:                        storage += pp->pr_pagesz;
                    679:                } else {
                    680:                        cp = (*pp->pr_alloc)(pp->pr_pagesz, 0, pp->pr_mtype);
                    681:                }
1.2       pk        682:
1.3       pk        683:                if (cp == NULL) {
1.1       pk        684:                        simple_unlock(&pp->pr_lock);
                    685:                        return (ENOMEM);
                    686:                }
                    687:
1.3       pk        688:                pool_prime_page(pp, cp);
                    689:                pp->pr_minpages++;
1.1       pk        690:        }
1.3       pk        691:
                    692:        pp->pr_minitems = newnitems;
                    693:
                    694:        if (pp->pr_minpages >= pp->pr_maxpages)
                    695:                pp->pr_maxpages = pp->pr_minpages + 1;  /* XXX */
                    696:
1.1       pk        697:        simple_unlock(&pp->pr_lock);
                    698:        return (0);
                    699: }
1.3       pk        700:
                    701: /*
                    702:  * Add a page worth of items to the pool.
                    703:  */
                    704: int
                    705: pool_prime_page(pp, storage)
                    706:        struct pool *pp;
                    707:        caddr_t storage;
                    708: {
                    709:        struct pool_item *pi;
                    710:        struct pool_item_header *ph;
                    711:        caddr_t cp = storage;
                    712:        unsigned int align = pp->pr_align;
                    713:        unsigned int ioff = pp->pr_itemoffset;
                    714:        int n;
                    715:
                    716:        if ((pp->pr_flags & PR_PHINPAGE) != 0) {
                    717:                ph = (struct pool_item_header *)(cp + pp->pr_phoffset);
                    718:        } else {
                    719:                ph = pool_get(&phpool, PR_URGENT);
                    720:                LIST_INSERT_HEAD(&pp->pr_hashtab[PR_HASH_INDEX(pp, cp)],
                    721:                                 ph, ph_hashlist);
                    722:        }
                    723:
                    724:        /*
                    725:         * Insert page header.
                    726:         */
                    727:        TAILQ_INSERT_HEAD(&pp->pr_pagelist, ph, ph_pagelist);
                    728:        TAILQ_INIT(&ph->ph_itemlist);
                    729:        ph->ph_page = storage;
                    730:        ph->ph_nmissing = 0;
                    731:        ph->ph_time.tv_sec = ph->ph_time.tv_usec = 0;
                    732:
                    733:        /*
                    734:         * Color this page.
                    735:         */
                    736:        cp = (caddr_t)(cp + pp->pr_curcolor);
                    737:        if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
                    738:                pp->pr_curcolor = 0;
                    739:
                    740:        /*
                    741:         * Adjust storage to apply aligment to `pr_itemoffset' in each item.
                    742:         */
                    743:        if (ioff != 0)
                    744:                cp = (caddr_t)(cp + (align - ioff));
                    745:
                    746:        /*
                    747:         * Insert remaining chunks on the bucket list.
                    748:         */
                    749:        n = pp->pr_itemsperpage;
                    750:
                    751:        while (n--) {
                    752:                pi = (struct pool_item *)cp;
                    753:
                    754:                /* Insert on page list */
                    755:                TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list);
                    756: #ifdef DIAGNOSTIC
                    757:                pi->pi_magic = PI_MAGIC;
                    758: #endif
                    759:                cp = (caddr_t)(cp + pp->pr_size);
                    760:        }
                    761:
                    762:        /*
                    763:         * If the pool was depleted, point at the new page.
                    764:         */
                    765:        if (pp->pr_curpage == NULL)
                    766:                pp->pr_curpage = ph;
                    767:
                    768:        if (++pp->pr_npages > pp->pr_hiwat)
                    769:                pp->pr_hiwat = pp->pr_npages;
                    770:
                    771:        return (0);
                    772: }
                    773:
                    774: void
                    775: pool_setlowat(pp, n)
                    776:        pool_handle_t   pp;
                    777:        int n;
                    778: {
                    779:        pp->pr_minitems = n;
                    780:        if (n == 0) {
                    781:                pp->pr_minpages = 0;
                    782:                return;
                    783:        }
                    784:        pp->pr_minpages =
                    785:                roundup(pp->pr_itemsperpage,n) / pp->pr_itemsperpage;
                    786: }
                    787:
                    788: void
                    789: pool_sethiwat(pp, n)
                    790:        pool_handle_t   pp;
                    791:        int n;
                    792: {
                    793:        if (n == 0) {
                    794:                pp->pr_maxpages = 0;
                    795:                return;
                    796:        }
                    797:        pp->pr_maxpages =
                    798:                roundup(pp->pr_itemsperpage,n) / pp->pr_itemsperpage;
                    799: }
                    800:
                    801:
                    802: /*
                    803:  * Default page allocator.
                    804:  */
                    805: static void *
                    806: pool_page_alloc(sz, flags, mtype)
                    807:        unsigned long sz;
                    808:        int flags;
                    809:        int mtype;
                    810: {
                    811:
                    812: #if defined(UVM)
1.4       thorpej   813:        return ((void *)uvm_km_alloc_poolpage());
                    814: #else
                    815:        return ((void *)kmem_alloc_poolpage());
                    816: #endif
1.3       pk        817: }
                    818:
                    819: static void
                    820: pool_page_free(v, sz, mtype)
                    821:        void *v;
                    822:        unsigned long sz;
                    823:        int mtype;
                    824: {
                    825:
                    826: #if defined(UVM)
1.4.2.1 ! eeh       827:        uvm_km_free_poolpage((vaddr_t)v);
1.3       pk        828: #else
1.4.2.1 ! eeh       829:        kmem_free_poolpage((vaddr_t)v);
1.3       pk        830: #endif
                    831: }
                    832:
                    833: /*
                    834:  * Release all complete pages that have not been used recently.
                    835:  */
                    836: void
                    837: pool_reclaim (pp)
                    838:        pool_handle_t pp;
                    839: {
                    840:        struct pool_item_header *ph, *phnext;
                    841:        struct timeval curtime = time;
                    842:
                    843:        if (pp->pr_flags & PR_STATIC)
                    844:                return;
                    845:
                    846:        if (simple_lock_try(&pp->pr_lock) == 0)
                    847:                return;
                    848:
                    849:        for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL; ph = phnext) {
                    850:                phnext = TAILQ_NEXT(ph, ph_pagelist);
                    851:
                    852:                /* Check our minimum page claim */
                    853:                if (pp->pr_npages <= pp->pr_minpages)
                    854:                        break;
                    855:
                    856:                if (ph->ph_nmissing == 0) {
                    857:                        struct timeval diff;
                    858:                        timersub(&curtime, &ph->ph_time, &diff);
                    859:                        if (diff.tv_sec < pool_inactive_time)
                    860:                                continue;
                    861:                        pr_rmpage(pp, ph);
                    862:                }
                    863:        }
                    864:
                    865:        simple_unlock(&pp->pr_lock);
                    866: }
                    867:
                    868:
                    869: /*
                    870:  * Drain pools, one at a time.
                    871:  */
                    872: void
                    873: pool_drain(arg)
                    874:        void *arg;
                    875: {
                    876:        struct pool *pp;
                    877:        int s = splimp();
                    878:
                    879:        /* XXX:lock pool head */
                    880:        if (drainpp == NULL && (drainpp = TAILQ_FIRST(&pool_head)) == NULL) {
                    881:                splx(s);
                    882:                return;
                    883:        }
                    884:
                    885:        pp = drainpp;
                    886:        drainpp = TAILQ_NEXT(pp, pr_poollist);
                    887:        /* XXX:unlock pool head */
                    888:
                    889:        pool_reclaim(pp);
                    890:        splx(s);
                    891: }
                    892:
                    893:
                    894: #ifdef DEBUG
                    895: /*
                    896:  * Diagnostic helpers.
                    897:  */
                    898: void
                    899: pool_print(pp, label)
                    900:        struct pool *pp;
                    901:        char *label;
                    902: {
                    903:
                    904:        if (label != NULL)
                    905:                printf("%s: ", label);
                    906:
                    907:        printf("pool %s: nalloc %lu nfree %lu npagealloc %lu npagefree %lu\n"
                    908:               "         npages %u minitems %u itemsperpage %u itemoffset %u\n",
                    909:                pp->pr_wchan,
                    910:                pp->pr_nget,
                    911:                pp->pr_nput,
                    912:                pp->pr_npagealloc,
                    913:                pp->pr_npagefree,
                    914:                pp->pr_npages,
                    915:                pp->pr_minitems,
                    916:                pp->pr_itemsperpage,
                    917:                pp->pr_itemoffset);
                    918: }
                    919:
                    920: int
                    921: pool_chk(pp, label)
                    922:        struct pool *pp;
                    923:        char *label;
                    924: {
                    925:        struct pool_item_header *ph;
                    926:        int r = 0;
                    927:
                    928:        simple_lock(&pp->pr_lock);
                    929:
                    930:        for (ph = TAILQ_FIRST(&pp->pr_pagelist); ph != NULL;
                    931:             ph = TAILQ_NEXT(ph, ph_pagelist)) {
                    932:
                    933:                struct pool_item *pi;
                    934:                int n;
                    935:                caddr_t page;
                    936:
                    937:                page = (caddr_t)((u_long)ph & pp->pr_pagemask);
                    938:                if (page != ph->ph_page) {
                    939:                        if (label != NULL)
                    940:                                printf("%s: ", label);
                    941:                        printf("pool(%s): page inconsistency: page %p;"
                    942:                               " at page head addr %p (p %p)\n",
                    943:                                pp->pr_wchan, ph->ph_page,
                    944:                                ph, page);
                    945:                        r++;
                    946:                        goto out;
                    947:                }
                    948:
                    949:                for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0;
                    950:                     pi != NULL;
                    951:                     pi = TAILQ_NEXT(pi,pi_list), n++) {
                    952:
                    953: #ifdef DIAGNOSTIC
                    954:                        if (pi->pi_magic != PI_MAGIC) {
                    955:                                if (label != NULL)
                    956:                                        printf("%s: ", label);
                    957:                                printf("pool(%s): free list modified: magic=%x;"
                    958:                                       " page %p; item ordinal %d;"
                    959:                                       " addr %p (p %p)\n",
                    960:                                        pp->pr_wchan, pi->pi_magic, ph->ph_page,
                    961:                                        n, pi, page);
                    962:                                panic("pool");
                    963:                        }
                    964: #endif
                    965:                        page = (caddr_t)((u_long)pi & pp->pr_pagemask);
                    966:                        if (page == ph->ph_page)
                    967:                                continue;
                    968:
                    969:                        if (label != NULL)
                    970:                                printf("%s: ", label);
                    971:                        printf("pool(%s): page inconsistency: page %p;"
                    972:                               " item ordinal %d; addr %p (p %p)\n",
                    973:                                pp->pr_wchan, ph->ph_page,
                    974:                                n, pi, page);
                    975:                        r++;
                    976:                        goto out;
                    977:                }
                    978:        }
                    979: out:
                    980:        simple_unlock(&pp->pr_lock);
                    981:        return (r);
                    982: }
                    983: #endif

CVSweb <webmaster@jp.NetBSD.org>