Annotation of src/sys/kern/subr_pool.c, Revision 1.131.2.2
1.131.2.2! matt 1: /* $NetBSD$ */
1.1 pk 2:
3: /*-
1.131.2.2! matt 4: * Copyright (c) 1997, 1999, 2000, 2002, 2007 The NetBSD Foundation, Inc.
1.1 pk 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
1.20 thorpej 8: * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
1.131.2.2! matt 9: * Simulation Facility, NASA Ames Research Center, and by Andrew Doran.
1.1 pk 10: *
11: * Redistribution and use in source and binary forms, with or without
12: * modification, are permitted provided that the following conditions
13: * are met:
14: * 1. Redistributions of source code must retain the above copyright
15: * notice, this list of conditions and the following disclaimer.
16: * 2. Redistributions in binary form must reproduce the above copyright
17: * notice, this list of conditions and the following disclaimer in the
18: * documentation and/or other materials provided with the distribution.
19: * 3. All advertising materials mentioning features or use of this software
20: * must display the following acknowledgement:
1.13 christos 21: * This product includes software developed by the NetBSD
22: * Foundation, Inc. and its contributors.
1.1 pk 23: * 4. Neither the name of The NetBSD Foundation nor the names of its
24: * contributors may be used to endorse or promote products derived
25: * from this software without specific prior written permission.
26: *
27: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
28: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
31: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37: * POSSIBILITY OF SUCH DAMAGE.
38: */
1.64 lukem 39:
40: #include <sys/cdefs.h>
1.131.2.2! matt 41: __KERNEL_RCSID(0, "$NetBSD$");
1.24 scottr 42:
1.25 thorpej 43: #include "opt_pool.h"
1.24 scottr 44: #include "opt_poollog.h"
1.28 thorpej 45: #include "opt_lockdebug.h"
1.1 pk 46:
47: #include <sys/param.h>
48: #include <sys/systm.h>
49: #include <sys/proc.h>
50: #include <sys/errno.h>
51: #include <sys/kernel.h>
52: #include <sys/malloc.h>
53: #include <sys/lock.h>
54: #include <sys/pool.h>
1.20 thorpej 55: #include <sys/syslog.h>
1.125 ad 56: #include <sys/debug.h>
1.131.2.2! matt 57: #include <sys/lockdebug.h>
! 58: #include <sys/xcall.h>
! 59: #include <sys/cpu.h>
1.3 pk 60:
61: #include <uvm/uvm.h>
62:
1.1 pk 63: /*
64: * Pool resource management utility.
1.3 pk 65: *
1.88 chs 66: * Memory is allocated in pages which are split into pieces according to
67: * the pool item size. Each page is kept on one of three lists in the
68: * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages',
69: * for empty, full and partially-full pages respectively. The individual
70: * pool items are on a linked list headed by `ph_itemlist' in each page
71: * header. The memory for building the page list is either taken from
72: * the allocated pages themselves (for small pool items) or taken from
73: * an internal pool of page headers (`phpool').
1.1 pk 74: */
75:
1.3 pk 76: /* List of all pools */
1.102 chs 77: LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head);
1.3 pk 78:
1.131.2.2! matt 79: /* List of all caches. */
! 80: LIST_HEAD(,pool_cache) pool_cache_head =
! 81: LIST_HEAD_INITIALIZER(pool_cache_head);
! 82:
1.3 pk 83: /* Private pool for page header structures */
1.97 yamt 84: #define PHPOOL_MAX 8
85: static struct pool phpool[PHPOOL_MAX];
86: #define PHPOOL_FREELIST_NELEM(idx) (((idx) == 0) ? 0 : (1 << (idx)))
1.3 pk 87:
1.62 bjh21 88: #ifdef POOL_SUBPAGE
89: /* Pool of subpages for use by normal pools. */
90: static struct pool psppool;
91: #endif
92:
1.117 yamt 93: static SLIST_HEAD(, pool_allocator) pa_deferinitq =
94: SLIST_HEAD_INITIALIZER(pa_deferinitq);
95:
1.98 yamt 96: static void *pool_page_alloc_meta(struct pool *, int);
97: static void pool_page_free_meta(struct pool *, void *);
98:
99: /* allocator for pool metadata */
1.131.2.2! matt 100: struct pool_allocator pool_allocator_meta = {
1.117 yamt 101: pool_page_alloc_meta, pool_page_free_meta,
102: .pa_backingmapptr = &kmem_map,
1.98 yamt 103: };
104:
1.3 pk 105: /* # of seconds to retain page after last use */
106: int pool_inactive_time = 10;
107:
108: /* Next candidate for drainage (see pool_drain()) */
1.23 thorpej 109: static struct pool *drainpp;
110:
1.131.2.2! matt 111: /* This lock protects both pool_head and drainpp. */
! 112: static kmutex_t pool_head_lock;
! 113: static kcondvar_t pool_busy;
1.3 pk 114:
1.99 yamt 115: typedef uint8_t pool_item_freelist_t;
116:
1.3 pk 117: struct pool_item_header {
118: /* Page headers */
1.88 chs 119: LIST_ENTRY(pool_item_header)
1.3 pk 120: ph_pagelist; /* pool page list */
1.88 chs 121: SPLAY_ENTRY(pool_item_header)
122: ph_node; /* Off-page page headers */
1.128 christos 123: void * ph_page; /* this page's address */
1.3 pk 124: struct timeval ph_time; /* last referenced */
1.97 yamt 125: union {
126: /* !PR_NOTOUCH */
127: struct {
1.102 chs 128: LIST_HEAD(, pool_item)
1.97 yamt 129: phu_itemlist; /* chunk list for this page */
130: } phu_normal;
131: /* PR_NOTOUCH */
132: struct {
133: uint16_t
134: phu_off; /* start offset in page */
1.99 yamt 135: pool_item_freelist_t
1.97 yamt 136: phu_firstfree; /* first free item */
1.99 yamt 137: /*
138: * XXX it might be better to use
139: * a simple bitmap and ffs(3)
140: */
1.97 yamt 141: } phu_notouch;
142: } ph_u;
143: uint16_t ph_nmissing; /* # of chunks in use */
1.3 pk 144: };
1.97 yamt 145: #define ph_itemlist ph_u.phu_normal.phu_itemlist
146: #define ph_off ph_u.phu_notouch.phu_off
147: #define ph_firstfree ph_u.phu_notouch.phu_firstfree
1.3 pk 148:
1.1 pk 149: struct pool_item {
1.3 pk 150: #ifdef DIAGNOSTIC
1.82 thorpej 151: u_int pi_magic;
1.33 chs 152: #endif
1.131.2.2! matt 153: #define PI_MAGIC 0xdeaddeadU
1.3 pk 154: /* Other entries use only this list entry */
1.102 chs 155: LIST_ENTRY(pool_item) pi_list;
1.3 pk 156: };
157:
1.53 thorpej 158: #define POOL_NEEDS_CATCHUP(pp) \
159: ((pp)->pr_nitems < (pp)->pr_minitems)
160:
1.43 thorpej 161: /*
162: * Pool cache management.
163: *
164: * Pool caches provide a way for constructed objects to be cached by the
165: * pool subsystem. This can lead to performance improvements by avoiding
166: * needless object construction/destruction; it is deferred until absolutely
167: * necessary.
168: *
1.131.2.2! matt 169: * Caches are grouped into cache groups. Each cache group references up
! 170: * to PCG_NUMOBJECTS constructed objects. When a cache allocates an
! 171: * object from the pool, it calls the object's constructor and places it
! 172: * into a cache group. When a cache group frees an object back to the
! 173: * pool, it first calls the object's destructor. This allows the object
! 174: * to persist in constructed form while freed to the cache.
! 175: *
! 176: * The pool references each cache, so that when a pool is drained by the
! 177: * pagedaemon, it can drain each individual cache as well. Each time a
! 178: * cache is drained, the most idle cache group is freed to the pool in
! 179: * its entirety.
1.43 thorpej 180: *
181: * Pool caches are layed on top of pools. By layering them, we can avoid
182: * the complexity of cache management for pools which would not benefit
183: * from it.
184: */
185:
186: static struct pool pcgpool;
1.131.2.2! matt 187: static struct pool cache_pool;
! 188: static struct pool cache_cpu_pool;
1.3 pk 189:
1.131.2.2! matt 190: static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *,
! 191: void *, paddr_t);
! 192: static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *,
! 193: void **, paddr_t *, int);
! 194: static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t);
! 195: static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *);
! 196: static void pool_cache_xcall(pool_cache_t);
1.3 pk 197:
1.42 thorpej 198: static int pool_catchup(struct pool *);
1.128 christos 199: static void pool_prime_page(struct pool *, void *,
1.55 thorpej 200: struct pool_item_header *);
1.88 chs 201: static void pool_update_curpage(struct pool *);
1.66 thorpej 202:
1.113 yamt 203: static int pool_grow(struct pool *, int);
1.117 yamt 204: static void *pool_allocator_alloc(struct pool *, int);
205: static void pool_allocator_free(struct pool *, void *);
1.3 pk 206:
1.97 yamt 207: static void pool_print_pagelist(struct pool *, struct pool_pagelist *,
1.88 chs 208: void (*)(const char *, ...));
1.42 thorpej 209: static void pool_print1(struct pool *, const char *,
210: void (*)(const char *, ...));
1.3 pk 211:
1.88 chs 212: static int pool_chk_page(struct pool *, const char *,
213: struct pool_item_header *);
214:
1.3 pk 215: /*
1.52 thorpej 216: * Pool log entry. An array of these is allocated in pool_init().
1.3 pk 217: */
218: struct pool_log {
219: const char *pl_file;
220: long pl_line;
221: int pl_action;
1.25 thorpej 222: #define PRLOG_GET 1
223: #define PRLOG_PUT 2
1.3 pk 224: void *pl_addr;
1.1 pk 225: };
226:
1.86 matt 227: #ifdef POOL_DIAGNOSTIC
1.3 pk 228: /* Number of entries in pool log buffers */
1.17 thorpej 229: #ifndef POOL_LOGSIZE
230: #define POOL_LOGSIZE 10
231: #endif
232:
233: int pool_logsize = POOL_LOGSIZE;
1.1 pk 234:
1.110 perry 235: static inline void
1.42 thorpej 236: pr_log(struct pool *pp, void *v, int action, const char *file, long line)
1.3 pk 237: {
238: int n = pp->pr_curlogentry;
239: struct pool_log *pl;
240:
1.20 thorpej 241: if ((pp->pr_roflags & PR_LOGGING) == 0)
1.3 pk 242: return;
243:
244: /*
245: * Fill in the current entry. Wrap around and overwrite
246: * the oldest entry if necessary.
247: */
248: pl = &pp->pr_log[n];
249: pl->pl_file = file;
250: pl->pl_line = line;
251: pl->pl_action = action;
252: pl->pl_addr = v;
253: if (++n >= pp->pr_logsize)
254: n = 0;
255: pp->pr_curlogentry = n;
256: }
257:
258: static void
1.42 thorpej 259: pr_printlog(struct pool *pp, struct pool_item *pi,
260: void (*pr)(const char *, ...))
1.3 pk 261: {
262: int i = pp->pr_logsize;
263: int n = pp->pr_curlogentry;
264:
1.20 thorpej 265: if ((pp->pr_roflags & PR_LOGGING) == 0)
1.3 pk 266: return;
267:
268: /*
269: * Print all entries in this pool's log.
270: */
271: while (i-- > 0) {
272: struct pool_log *pl = &pp->pr_log[n];
273: if (pl->pl_action != 0) {
1.25 thorpej 274: if (pi == NULL || pi == pl->pl_addr) {
275: (*pr)("\tlog entry %d:\n", i);
276: (*pr)("\t\taction = %s, addr = %p\n",
277: pl->pl_action == PRLOG_GET ? "get" : "put",
278: pl->pl_addr);
279: (*pr)("\t\tfile: %s at line %lu\n",
280: pl->pl_file, pl->pl_line);
281: }
1.3 pk 282: }
283: if (++n >= pp->pr_logsize)
284: n = 0;
285: }
286: }
1.25 thorpej 287:
1.110 perry 288: static inline void
1.42 thorpej 289: pr_enter(struct pool *pp, const char *file, long line)
1.25 thorpej 290: {
291:
1.34 thorpej 292: if (__predict_false(pp->pr_entered_file != NULL)) {
1.25 thorpej 293: printf("pool %s: reentrancy at file %s line %ld\n",
294: pp->pr_wchan, file, line);
295: printf(" previous entry at file %s line %ld\n",
296: pp->pr_entered_file, pp->pr_entered_line);
297: panic("pr_enter");
298: }
299:
300: pp->pr_entered_file = file;
301: pp->pr_entered_line = line;
302: }
303:
1.110 perry 304: static inline void
1.42 thorpej 305: pr_leave(struct pool *pp)
1.25 thorpej 306: {
307:
1.34 thorpej 308: if (__predict_false(pp->pr_entered_file == NULL)) {
1.25 thorpej 309: printf("pool %s not entered?\n", pp->pr_wchan);
310: panic("pr_leave");
311: }
312:
313: pp->pr_entered_file = NULL;
314: pp->pr_entered_line = 0;
315: }
316:
1.110 perry 317: static inline void
1.42 thorpej 318: pr_enter_check(struct pool *pp, void (*pr)(const char *, ...))
1.25 thorpej 319: {
320:
321: if (pp->pr_entered_file != NULL)
322: (*pr)("\n\tcurrently entered from file %s line %ld\n",
323: pp->pr_entered_file, pp->pr_entered_line);
324: }
1.3 pk 325: #else
1.25 thorpej 326: #define pr_log(pp, v, action, file, line)
327: #define pr_printlog(pp, pi, pr)
328: #define pr_enter(pp, file, line)
329: #define pr_leave(pp)
330: #define pr_enter_check(pp, pr)
1.59 thorpej 331: #endif /* POOL_DIAGNOSTIC */
1.3 pk 332:
1.110 perry 333: static inline int
1.97 yamt 334: pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph,
335: const void *v)
336: {
337: const char *cp = v;
338: int idx;
339:
340: KASSERT(pp->pr_roflags & PR_NOTOUCH);
1.128 christos 341: idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size;
1.97 yamt 342: KASSERT(idx < pp->pr_itemsperpage);
343: return idx;
344: }
345:
1.99 yamt 346: #define PR_FREELIST_ALIGN(p) \
347: roundup((uintptr_t)(p), sizeof(pool_item_freelist_t))
348: #define PR_FREELIST(ph) ((pool_item_freelist_t *)PR_FREELIST_ALIGN((ph) + 1))
349: #define PR_INDEX_USED ((pool_item_freelist_t)-1)
350: #define PR_INDEX_EOL ((pool_item_freelist_t)-2)
1.97 yamt 351:
1.110 perry 352: static inline void
1.97 yamt 353: pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph,
354: void *obj)
355: {
356: int idx = pr_item_notouch_index(pp, ph, obj);
1.99 yamt 357: pool_item_freelist_t *freelist = PR_FREELIST(ph);
1.97 yamt 358:
359: KASSERT(freelist[idx] == PR_INDEX_USED);
360: freelist[idx] = ph->ph_firstfree;
361: ph->ph_firstfree = idx;
362: }
363:
1.110 perry 364: static inline void *
1.97 yamt 365: pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph)
366: {
367: int idx = ph->ph_firstfree;
1.99 yamt 368: pool_item_freelist_t *freelist = PR_FREELIST(ph);
1.97 yamt 369:
370: KASSERT(freelist[idx] != PR_INDEX_USED);
371: ph->ph_firstfree = freelist[idx];
372: freelist[idx] = PR_INDEX_USED;
373:
1.128 christos 374: return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size;
1.97 yamt 375: }
376:
1.110 perry 377: static inline int
1.88 chs 378: phtree_compare(struct pool_item_header *a, struct pool_item_header *b)
379: {
1.121 yamt 380:
381: /*
382: * we consider pool_item_header with smaller ph_page bigger.
383: * (this unnatural ordering is for the benefit of pr_find_pagehead.)
384: */
385:
1.88 chs 386: if (a->ph_page < b->ph_page)
1.121 yamt 387: return (1);
388: else if (a->ph_page > b->ph_page)
1.88 chs 389: return (-1);
390: else
391: return (0);
392: }
393:
394: SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare);
395: SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare);
396:
1.3 pk 397: /*
1.121 yamt 398: * Return the pool page header based on item address.
1.3 pk 399: */
1.110 perry 400: static inline struct pool_item_header *
1.121 yamt 401: pr_find_pagehead(struct pool *pp, void *v)
1.3 pk 402: {
1.88 chs 403: struct pool_item_header *ph, tmp;
1.3 pk 404:
1.121 yamt 405: if ((pp->pr_roflags & PR_NOALIGN) != 0) {
1.128 christos 406: tmp.ph_page = (void *)(uintptr_t)v;
1.121 yamt 407: ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
408: if (ph == NULL) {
409: ph = SPLAY_ROOT(&pp->pr_phtree);
410: if (ph != NULL && phtree_compare(&tmp, ph) >= 0) {
411: ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph);
412: }
413: KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0);
414: }
415: } else {
1.128 christos 416: void *page =
417: (void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask);
1.121 yamt 418:
419: if ((pp->pr_roflags & PR_PHINPAGE) != 0) {
1.128 christos 420: ph = (struct pool_item_header *)((char *)page + pp->pr_phoffset);
1.121 yamt 421: } else {
422: tmp.ph_page = page;
423: ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
424: }
425: }
1.3 pk 426:
1.121 yamt 427: KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) ||
1.128 christos 428: ((char *)ph->ph_page <= (char *)v &&
429: (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz));
1.88 chs 430: return ph;
1.3 pk 431: }
432:
1.101 thorpej 433: static void
434: pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq)
435: {
436: struct pool_item_header *ph;
437:
438: while ((ph = LIST_FIRST(pq)) != NULL) {
439: LIST_REMOVE(ph, ph_pagelist);
440: pool_allocator_free(pp, ph->ph_page);
1.131.2.2! matt 441: if ((pp->pr_roflags & PR_PHINPAGE) == 0)
1.101 thorpej 442: pool_put(pp->pr_phpool, ph);
443: }
444: }
445:
1.3 pk 446: /*
447: * Remove a page from the pool.
448: */
1.110 perry 449: static inline void
1.61 chs 450: pr_rmpage(struct pool *pp, struct pool_item_header *ph,
451: struct pool_pagelist *pq)
1.3 pk 452: {
453:
1.131.2.2! matt 454: KASSERT(mutex_owned(&pp->pr_lock));
1.91 yamt 455:
1.3 pk 456: /*
1.7 thorpej 457: * If the page was idle, decrement the idle page count.
1.3 pk 458: */
1.6 thorpej 459: if (ph->ph_nmissing == 0) {
460: #ifdef DIAGNOSTIC
461: if (pp->pr_nidle == 0)
462: panic("pr_rmpage: nidle inconsistent");
1.20 thorpej 463: if (pp->pr_nitems < pp->pr_itemsperpage)
464: panic("pr_rmpage: nitems inconsistent");
1.6 thorpej 465: #endif
466: pp->pr_nidle--;
467: }
1.7 thorpej 468:
1.20 thorpej 469: pp->pr_nitems -= pp->pr_itemsperpage;
470:
1.7 thorpej 471: /*
1.101 thorpej 472: * Unlink the page from the pool and queue it for release.
1.7 thorpej 473: */
1.88 chs 474: LIST_REMOVE(ph, ph_pagelist);
1.91 yamt 475: if ((pp->pr_roflags & PR_PHINPAGE) == 0)
476: SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
1.101 thorpej 477: LIST_INSERT_HEAD(pq, ph, ph_pagelist);
478:
1.7 thorpej 479: pp->pr_npages--;
480: pp->pr_npagefree++;
1.6 thorpej 481:
1.88 chs 482: pool_update_curpage(pp);
1.3 pk 483: }
484:
1.126 thorpej 485: static bool
1.117 yamt 486: pa_starved_p(struct pool_allocator *pa)
487: {
488:
489: if (pa->pa_backingmap != NULL) {
490: return vm_map_starved_p(pa->pa_backingmap);
491: }
1.127 thorpej 492: return false;
1.117 yamt 493: }
494:
495: static int
1.124 yamt 496: pool_reclaim_callback(struct callback_entry *ce, void *obj, void *arg)
1.117 yamt 497: {
498: struct pool *pp = obj;
499: struct pool_allocator *pa = pp->pr_alloc;
500:
501: KASSERT(&pp->pr_reclaimerentry == ce);
502: pool_reclaim(pp);
503: if (!pa_starved_p(pa)) {
504: return CALLBACK_CHAIN_ABORT;
505: }
506: return CALLBACK_CHAIN_CONTINUE;
507: }
508:
509: static void
510: pool_reclaim_register(struct pool *pp)
511: {
512: struct vm_map *map = pp->pr_alloc->pa_backingmap;
513: int s;
514:
515: if (map == NULL) {
516: return;
517: }
518:
519: s = splvm(); /* not necessary for INTRSAFE maps, but don't care. */
520: callback_register(&vm_map_to_kernel(map)->vmk_reclaim_callback,
521: &pp->pr_reclaimerentry, pp, pool_reclaim_callback);
522: splx(s);
523: }
524:
525: static void
526: pool_reclaim_unregister(struct pool *pp)
527: {
528: struct vm_map *map = pp->pr_alloc->pa_backingmap;
529: int s;
530:
531: if (map == NULL) {
532: return;
533: }
534:
535: s = splvm(); /* not necessary for INTRSAFE maps, but don't care. */
536: callback_unregister(&vm_map_to_kernel(map)->vmk_reclaim_callback,
537: &pp->pr_reclaimerentry);
538: splx(s);
539: }
540:
541: static void
542: pa_reclaim_register(struct pool_allocator *pa)
543: {
544: struct vm_map *map = *pa->pa_backingmapptr;
545: struct pool *pp;
546:
547: KASSERT(pa->pa_backingmap == NULL);
548: if (map == NULL) {
549: SLIST_INSERT_HEAD(&pa_deferinitq, pa, pa_q);
550: return;
551: }
552: pa->pa_backingmap = map;
553: TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) {
554: pool_reclaim_register(pp);
555: }
556: }
557:
1.3 pk 558: /*
1.94 simonb 559: * Initialize all the pools listed in the "pools" link set.
560: */
561: void
1.117 yamt 562: pool_subsystem_init(void)
1.94 simonb 563: {
1.117 yamt 564: struct pool_allocator *pa;
1.94 simonb 565: __link_set_decl(pools, struct link_pool_init);
566: struct link_pool_init * const *pi;
567:
1.131.2.2! matt 568: mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE);
! 569: cv_init(&pool_busy, "poolbusy");
! 570:
1.94 simonb 571: __link_set_foreach(pi, pools)
572: pool_init((*pi)->pp, (*pi)->size, (*pi)->align,
573: (*pi)->align_offset, (*pi)->flags, (*pi)->wchan,
1.129 ad 574: (*pi)->palloc, (*pi)->ipl);
1.117 yamt 575:
576: while ((pa = SLIST_FIRST(&pa_deferinitq)) != NULL) {
577: KASSERT(pa->pa_backingmapptr != NULL);
578: KASSERT(*pa->pa_backingmapptr != NULL);
579: SLIST_REMOVE_HEAD(&pa_deferinitq, pa_q);
580: pa_reclaim_register(pa);
581: }
1.131.2.2! matt 582:
! 583: pool_init(&cache_pool, sizeof(struct pool_cache), CACHE_LINE_SIZE,
! 584: 0, 0, "pcache", &pool_allocator_nointr, IPL_NONE);
! 585:
! 586: pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), CACHE_LINE_SIZE,
! 587: 0, 0, "pcachecpu", &pool_allocator_nointr, IPL_NONE);
1.94 simonb 588: }
589:
590: /*
1.3 pk 591: * Initialize the given pool resource structure.
592: *
593: * We export this routine to allow other kernel parts to declare
594: * static pools that must be initialized before malloc() is available.
595: */
596: void
1.42 thorpej 597: pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
1.129 ad 598: const char *wchan, struct pool_allocator *palloc, int ipl)
1.3 pk 599: {
1.116 simonb 600: #ifdef DEBUG
601: struct pool *pp1;
602: #endif
1.92 enami 603: size_t trysize, phsize;
1.131.2.2! matt 604: int off, slack;
1.3 pk 605:
1.99 yamt 606: KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >=
607: PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1));
608:
1.116 simonb 609: #ifdef DEBUG
610: /*
611: * Check that the pool hasn't already been initialised and
612: * added to the list of all pools.
613: */
614: LIST_FOREACH(pp1, &pool_head, pr_poollist) {
615: if (pp == pp1)
616: panic("pool_init: pool %s already initialised",
617: wchan);
618: }
619: #endif
620:
1.25 thorpej 621: #ifdef POOL_DIAGNOSTIC
622: /*
623: * Always log if POOL_DIAGNOSTIC is defined.
624: */
625: if (pool_logsize != 0)
626: flags |= PR_LOGGING;
627: #endif
628:
1.66 thorpej 629: if (palloc == NULL)
630: palloc = &pool_allocator_kmem;
1.112 bjh21 631: #ifdef POOL_SUBPAGE
632: if (size > palloc->pa_pagesz) {
633: if (palloc == &pool_allocator_kmem)
634: palloc = &pool_allocator_kmem_fullpage;
635: else if (palloc == &pool_allocator_nointr)
636: palloc = &pool_allocator_nointr_fullpage;
637: }
1.66 thorpej 638: #endif /* POOL_SUBPAGE */
639: if ((palloc->pa_flags & PA_INITIALIZED) == 0) {
1.112 bjh21 640: if (palloc->pa_pagesz == 0)
1.66 thorpej 641: palloc->pa_pagesz = PAGE_SIZE;
642:
643: TAILQ_INIT(&palloc->pa_list);
644:
1.131.2.2! matt 645: mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM);
1.66 thorpej 646: palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);
647: palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;
1.117 yamt 648:
649: if (palloc->pa_backingmapptr != NULL) {
650: pa_reclaim_register(palloc);
651: }
1.66 thorpej 652: palloc->pa_flags |= PA_INITIALIZED;
1.4 thorpej 653: }
1.3 pk 654:
655: if (align == 0)
656: align = ALIGN(1);
1.14 thorpej 657:
1.120 yamt 658: if ((flags & PR_NOTOUCH) == 0 && size < sizeof(struct pool_item))
1.14 thorpej 659: size = sizeof(struct pool_item);
1.3 pk 660:
1.78 thorpej 661: size = roundup(size, align);
1.66 thorpej 662: #ifdef DIAGNOSTIC
663: if (size > palloc->pa_pagesz)
1.121 yamt 664: panic("pool_init: pool item size (%zu) too large", size);
1.66 thorpej 665: #endif
1.35 pk 666:
1.3 pk 667: /*
668: * Initialize the pool structure.
669: */
1.88 chs 670: LIST_INIT(&pp->pr_emptypages);
671: LIST_INIT(&pp->pr_fullpages);
672: LIST_INIT(&pp->pr_partpages);
1.131.2.2! matt 673: pp->pr_cache = NULL;
1.3 pk 674: pp->pr_curpage = NULL;
675: pp->pr_npages = 0;
676: pp->pr_minitems = 0;
677: pp->pr_minpages = 0;
678: pp->pr_maxpages = UINT_MAX;
1.20 thorpej 679: pp->pr_roflags = flags;
680: pp->pr_flags = 0;
1.35 pk 681: pp->pr_size = size;
1.3 pk 682: pp->pr_align = align;
683: pp->pr_wchan = wchan;
1.66 thorpej 684: pp->pr_alloc = palloc;
1.20 thorpej 685: pp->pr_nitems = 0;
686: pp->pr_nout = 0;
687: pp->pr_hardlimit = UINT_MAX;
688: pp->pr_hardlimit_warning = NULL;
1.31 thorpej 689: pp->pr_hardlimit_ratecap.tv_sec = 0;
690: pp->pr_hardlimit_ratecap.tv_usec = 0;
691: pp->pr_hardlimit_warning_last.tv_sec = 0;
692: pp->pr_hardlimit_warning_last.tv_usec = 0;
1.68 thorpej 693: pp->pr_drain_hook = NULL;
694: pp->pr_drain_hook_arg = NULL;
1.125 ad 695: pp->pr_freecheck = NULL;
1.3 pk 696:
697: /*
698: * Decide whether to put the page header off page to avoid
1.92 enami 699: * wasting too large a part of the page or too big item.
700: * Off-page page headers go on a hash table, so we can match
701: * a returned item with its header based on the page address.
702: * We use 1/16 of the page size and about 8 times of the item
703: * size as the threshold (XXX: tune)
704: *
705: * However, we'll put the header into the page if we can put
706: * it without wasting any items.
707: *
708: * Silently enforce `0 <= ioff < align'.
1.3 pk 709: */
1.92 enami 710: pp->pr_itemoffset = ioff %= align;
711: /* See the comment below about reserved bytes. */
712: trysize = palloc->pa_pagesz - ((align - ioff) % align);
713: phsize = ALIGN(sizeof(struct pool_item_header));
1.121 yamt 714: if ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 &&
1.97 yamt 715: (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) ||
716: trysize / pp->pr_size == (trysize - phsize) / pp->pr_size)) {
1.3 pk 717: /* Use the end of the page for the page header */
1.20 thorpej 718: pp->pr_roflags |= PR_PHINPAGE;
1.92 enami 719: pp->pr_phoffset = off = palloc->pa_pagesz - phsize;
1.2 pk 720: } else {
1.3 pk 721: /* The page header will be taken from our page header pool */
722: pp->pr_phoffset = 0;
1.66 thorpej 723: off = palloc->pa_pagesz;
1.88 chs 724: SPLAY_INIT(&pp->pr_phtree);
1.2 pk 725: }
1.1 pk 726:
1.3 pk 727: /*
728: * Alignment is to take place at `ioff' within the item. This means
729: * we must reserve up to `align - 1' bytes on the page to allow
730: * appropriate positioning of each item.
731: */
732: pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size;
1.43 thorpej 733: KASSERT(pp->pr_itemsperpage != 0);
1.97 yamt 734: if ((pp->pr_roflags & PR_NOTOUCH)) {
735: int idx;
736:
737: for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx);
738: idx++) {
739: /* nothing */
740: }
741: if (idx >= PHPOOL_MAX) {
742: /*
743: * if you see this panic, consider to tweak
744: * PHPOOL_MAX and PHPOOL_FREELIST_NELEM.
745: */
746: panic("%s: too large itemsperpage(%d) for PR_NOTOUCH",
747: pp->pr_wchan, pp->pr_itemsperpage);
748: }
749: pp->pr_phpool = &phpool[idx];
750: } else if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
751: pp->pr_phpool = &phpool[0];
752: }
753: #if defined(DIAGNOSTIC)
754: else {
755: pp->pr_phpool = NULL;
756: }
757: #endif
1.3 pk 758:
759: /*
760: * Use the slack between the chunks and the page header
761: * for "cache coloring".
762: */
763: slack = off - pp->pr_itemsperpage * pp->pr_size;
764: pp->pr_maxcolor = (slack / align) * align;
765: pp->pr_curcolor = 0;
766:
767: pp->pr_nget = 0;
768: pp->pr_nfail = 0;
769: pp->pr_nput = 0;
770: pp->pr_npagealloc = 0;
771: pp->pr_npagefree = 0;
1.1 pk 772: pp->pr_hiwat = 0;
1.8 thorpej 773: pp->pr_nidle = 0;
1.131.2.2! matt 774: pp->pr_refcnt = 0;
1.3 pk 775:
1.59 thorpej 776: #ifdef POOL_DIAGNOSTIC
1.25 thorpej 777: if (flags & PR_LOGGING) {
778: if (kmem_map == NULL ||
779: (pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log),
780: M_TEMP, M_NOWAIT)) == NULL)
1.20 thorpej 781: pp->pr_roflags &= ~PR_LOGGING;
1.3 pk 782: pp->pr_curlogentry = 0;
783: pp->pr_logsize = pool_logsize;
784: }
1.59 thorpej 785: #endif
1.25 thorpej 786:
787: pp->pr_entered_file = NULL;
788: pp->pr_entered_line = 0;
1.3 pk 789:
1.131.2.2! matt 790: mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl);
! 791: cv_init(&pp->pr_cv, wchan);
! 792: pp->pr_ipl = ipl;
1.1 pk 793:
1.3 pk 794: /*
1.43 thorpej 795: * Initialize private page header pool and cache magazine pool if we
796: * haven't done so yet.
1.23 thorpej 797: * XXX LOCKING.
1.3 pk 798: */
1.97 yamt 799: if (phpool[0].pr_size == 0) {
800: int idx;
801: for (idx = 0; idx < PHPOOL_MAX; idx++) {
802: static char phpool_names[PHPOOL_MAX][6+1+6+1];
803: int nelem;
804: size_t sz;
805:
806: nelem = PHPOOL_FREELIST_NELEM(idx);
807: snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
808: "phpool-%d", nelem);
809: sz = sizeof(struct pool_item_header);
810: if (nelem) {
811: sz = PR_FREELIST_ALIGN(sz)
1.99 yamt 812: + nelem * sizeof(pool_item_freelist_t);
1.97 yamt 813: }
814: pool_init(&phpool[idx], sz, 0, 0, 0,
1.129 ad 815: phpool_names[idx], &pool_allocator_meta, IPL_VM);
1.97 yamt 816: }
1.62 bjh21 817: #ifdef POOL_SUBPAGE
818: pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0,
1.129 ad 819: PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM);
1.62 bjh21 820: #endif
1.131.2.2! matt 821: pool_init(&pcgpool, sizeof(pcg_t), CACHE_LINE_SIZE, 0, 0,
! 822: "cachegrp", &pool_allocator_meta, IPL_VM);
1.1 pk 823: }
824:
1.131.2.2! matt 825: if (__predict_true(!cold)) {
! 826: /* Insert into the list of all pools. */
! 827: mutex_enter(&pool_head_lock);
! 828: LIST_INSERT_HEAD(&pool_head, pp, pr_poollist);
! 829: mutex_exit(&pool_head_lock);
! 830:
! 831: /* Insert this into the list of pools using this allocator. */
! 832: mutex_enter(&palloc->pa_lock);
! 833: TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
! 834: mutex_exit(&palloc->pa_lock);
! 835: } else {
! 836: LIST_INSERT_HEAD(&pool_head, pp, pr_poollist);
! 837: TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
! 838: }
1.66 thorpej 839:
1.117 yamt 840: pool_reclaim_register(pp);
1.1 pk 841: }
842:
843: /*
844: * De-commision a pool resource.
845: */
846: void
1.42 thorpej 847: pool_destroy(struct pool *pp)
1.1 pk 848: {
1.101 thorpej 849: struct pool_pagelist pq;
1.3 pk 850: struct pool_item_header *ph;
1.43 thorpej 851:
1.101 thorpej 852: /* Remove from global pool list */
1.131.2.2! matt 853: mutex_enter(&pool_head_lock);
! 854: while (pp->pr_refcnt != 0)
! 855: cv_wait(&pool_busy, &pool_head_lock);
1.102 chs 856: LIST_REMOVE(pp, pr_poollist);
1.101 thorpej 857: if (drainpp == pp)
858: drainpp = NULL;
1.131.2.2! matt 859: mutex_exit(&pool_head_lock);
1.101 thorpej 860:
861: /* Remove this pool from its allocator's list of pools. */
1.117 yamt 862: pool_reclaim_unregister(pp);
1.131.2.2! matt 863: mutex_enter(&pp->pr_alloc->pa_lock);
1.66 thorpej 864: TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);
1.131.2.2! matt 865: mutex_exit(&pp->pr_alloc->pa_lock);
1.66 thorpej 866:
1.131.2.2! matt 867: mutex_enter(&pp->pr_lock);
1.101 thorpej 868:
1.131.2.2! matt 869: KASSERT(pp->pr_cache == NULL);
1.3 pk 870:
871: #ifdef DIAGNOSTIC
1.20 thorpej 872: if (pp->pr_nout != 0) {
1.25 thorpej 873: pr_printlog(pp, NULL, printf);
1.80 provos 874: panic("pool_destroy: pool busy: still out: %u",
1.20 thorpej 875: pp->pr_nout);
1.3 pk 876: }
877: #endif
1.1 pk 878:
1.101 thorpej 879: KASSERT(LIST_EMPTY(&pp->pr_fullpages));
880: KASSERT(LIST_EMPTY(&pp->pr_partpages));
881:
1.3 pk 882: /* Remove all pages */
1.101 thorpej 883: LIST_INIT(&pq);
1.88 chs 884: while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
1.101 thorpej 885: pr_rmpage(pp, ph, &pq);
886:
1.131.2.2! matt 887: mutex_exit(&pp->pr_lock);
1.3 pk 888:
1.101 thorpej 889: pr_pagelist_free(pp, &pq);
1.3 pk 890:
1.59 thorpej 891: #ifdef POOL_DIAGNOSTIC
1.20 thorpej 892: if ((pp->pr_roflags & PR_LOGGING) != 0)
1.3 pk 893: free(pp->pr_log, M_TEMP);
1.59 thorpej 894: #endif
1.131.2.2! matt 895:
! 896: cv_destroy(&pp->pr_cv);
! 897: mutex_destroy(&pp->pr_lock);
1.1 pk 898: }
899:
1.68 thorpej 900: void
901: pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg)
902: {
903:
904: /* XXX no locking -- must be used just after pool_init() */
905: #ifdef DIAGNOSTIC
906: if (pp->pr_drain_hook != NULL)
907: panic("pool_set_drain_hook(%s): already set", pp->pr_wchan);
908: #endif
909: pp->pr_drain_hook = fn;
910: pp->pr_drain_hook_arg = arg;
911: }
912:
1.88 chs 913: static struct pool_item_header *
1.128 christos 914: pool_alloc_item_header(struct pool *pp, void *storage, int flags)
1.55 thorpej 915: {
916: struct pool_item_header *ph;
917:
918: if ((pp->pr_roflags & PR_PHINPAGE) != 0)
1.128 christos 919: ph = (struct pool_item_header *) ((char *)storage + pp->pr_phoffset);
1.131.2.2! matt 920: else
1.97 yamt 921: ph = pool_get(pp->pr_phpool, flags);
1.55 thorpej 922:
923: return (ph);
924: }
1.1 pk 925:
926: /*
1.131.2.2! matt 927: * Grab an item from the pool.
1.1 pk 928: */
1.3 pk 929: void *
1.59 thorpej 930: #ifdef POOL_DIAGNOSTIC
1.42 thorpej 931: _pool_get(struct pool *pp, int flags, const char *file, long line)
1.56 sommerfe 932: #else
933: pool_get(struct pool *pp, int flags)
934: #endif
1.1 pk 935: {
936: struct pool_item *pi;
1.3 pk 937: struct pool_item_header *ph;
1.55 thorpej 938: void *v;
1.1 pk 939:
1.2 pk 940: #ifdef DIAGNOSTIC
1.95 atatat 941: if (__predict_false(pp->pr_itemsperpage == 0))
942: panic("pool_get: pool %p: pr_itemsperpage is zero, "
943: "pool not initialized?", pp);
1.84 thorpej 944: if (__predict_false(curlwp == NULL && doing_shutdown == 0 &&
1.37 sommerfe 945: (flags & PR_WAITOK) != 0))
1.77 matt 946: panic("pool_get: %s: must have NOWAIT", pp->pr_wchan);
1.58 thorpej 947:
1.102 chs 948: #endif /* DIAGNOSTIC */
1.58 thorpej 949: #ifdef LOCKDEBUG
950: if (flags & PR_WAITOK)
1.119 yamt 951: ASSERT_SLEEPABLE(NULL, "pool_get(PR_WAITOK)");
1.56 sommerfe 952: #endif
1.1 pk 953:
1.131.2.2! matt 954: mutex_enter(&pp->pr_lock);
1.25 thorpej 955: pr_enter(pp, file, line);
1.20 thorpej 956:
957: startover:
958: /*
959: * Check to see if we've reached the hard limit. If we have,
960: * and we can wait, then wait until an item has been returned to
961: * the pool.
962: */
963: #ifdef DIAGNOSTIC
1.34 thorpej 964: if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {
1.25 thorpej 965: pr_leave(pp);
1.131.2.2! matt 966: mutex_exit(&pp->pr_lock);
1.20 thorpej 967: panic("pool_get: %s: crossed hard limit", pp->pr_wchan);
968: }
969: #endif
1.34 thorpej 970: if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {
1.68 thorpej 971: if (pp->pr_drain_hook != NULL) {
972: /*
973: * Since the drain hook is going to free things
974: * back to the pool, unlock, call the hook, re-lock,
975: * and check the hardlimit condition again.
976: */
977: pr_leave(pp);
1.131.2.2! matt 978: mutex_exit(&pp->pr_lock);
1.68 thorpej 979: (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);
1.131.2.2! matt 980: mutex_enter(&pp->pr_lock);
1.68 thorpej 981: pr_enter(pp, file, line);
982: if (pp->pr_nout < pp->pr_hardlimit)
983: goto startover;
984: }
985:
1.29 sommerfe 986: if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {
1.20 thorpej 987: /*
988: * XXX: A warning isn't logged in this case. Should
989: * it be?
990: */
991: pp->pr_flags |= PR_WANTED;
1.25 thorpej 992: pr_leave(pp);
1.131.2.2! matt 993: cv_wait(&pp->pr_cv, &pp->pr_lock);
1.25 thorpej 994: pr_enter(pp, file, line);
1.20 thorpej 995: goto startover;
996: }
1.31 thorpej 997:
998: /*
999: * Log a message that the hard limit has been hit.
1000: */
1001: if (pp->pr_hardlimit_warning != NULL &&
1002: ratecheck(&pp->pr_hardlimit_warning_last,
1003: &pp->pr_hardlimit_ratecap))
1004: log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);
1.21 thorpej 1005:
1006: pp->pr_nfail++;
1007:
1.25 thorpej 1008: pr_leave(pp);
1.131.2.2! matt 1009: mutex_exit(&pp->pr_lock);
1.20 thorpej 1010: return (NULL);
1011: }
1012:
1.3 pk 1013: /*
1014: * The convention we use is that if `curpage' is not NULL, then
1015: * it points at a non-empty bucket. In particular, `curpage'
1016: * never points at a page header which has PR_PHINPAGE set and
1017: * has no items in its bucket.
1018: */
1.20 thorpej 1019: if ((ph = pp->pr_curpage) == NULL) {
1.113 yamt 1020: int error;
1021:
1.20 thorpej 1022: #ifdef DIAGNOSTIC
1023: if (pp->pr_nitems != 0) {
1.131.2.2! matt 1024: mutex_exit(&pp->pr_lock);
1.20 thorpej 1025: printf("pool_get: %s: curpage NULL, nitems %u\n",
1026: pp->pr_wchan, pp->pr_nitems);
1.80 provos 1027: panic("pool_get: nitems inconsistent");
1.20 thorpej 1028: }
1029: #endif
1030:
1.21 thorpej 1031: /*
1032: * Call the back-end page allocator for more memory.
1033: * Release the pool lock, as the back-end page allocator
1034: * may block.
1035: */
1.25 thorpej 1036: pr_leave(pp);
1.113 yamt 1037: error = pool_grow(pp, flags);
1038: pr_enter(pp, file, line);
1039: if (error != 0) {
1.21 thorpej 1040: /*
1.55 thorpej 1041: * We were unable to allocate a page or item
1042: * header, but we released the lock during
1043: * allocation, so perhaps items were freed
1044: * back to the pool. Check for this case.
1.21 thorpej 1045: */
1046: if (pp->pr_curpage != NULL)
1047: goto startover;
1.15 pk 1048:
1.117 yamt 1049: pp->pr_nfail++;
1.25 thorpej 1050: pr_leave(pp);
1.131.2.2! matt 1051: mutex_exit(&pp->pr_lock);
1.117 yamt 1052: return (NULL);
1.1 pk 1053: }
1.3 pk 1054:
1.20 thorpej 1055: /* Start the allocation process over. */
1056: goto startover;
1.3 pk 1057: }
1.97 yamt 1058: if (pp->pr_roflags & PR_NOTOUCH) {
1059: #ifdef DIAGNOSTIC
1060: if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) {
1061: pr_leave(pp);
1.131.2.2! matt 1062: mutex_exit(&pp->pr_lock);
1.97 yamt 1063: panic("pool_get: %s: page empty", pp->pr_wchan);
1064: }
1065: #endif
1066: v = pr_item_notouch_get(pp, ph);
1067: #ifdef POOL_DIAGNOSTIC
1068: pr_log(pp, v, PRLOG_GET, file, line);
1069: #endif
1070: } else {
1.102 chs 1071: v = pi = LIST_FIRST(&ph->ph_itemlist);
1.97 yamt 1072: if (__predict_false(v == NULL)) {
1073: pr_leave(pp);
1.131.2.2! matt 1074: mutex_exit(&pp->pr_lock);
1.97 yamt 1075: panic("pool_get: %s: page empty", pp->pr_wchan);
1076: }
1.20 thorpej 1077: #ifdef DIAGNOSTIC
1.97 yamt 1078: if (__predict_false(pp->pr_nitems == 0)) {
1079: pr_leave(pp);
1.131.2.2! matt 1080: mutex_exit(&pp->pr_lock);
1.97 yamt 1081: printf("pool_get: %s: items on itemlist, nitems %u\n",
1082: pp->pr_wchan, pp->pr_nitems);
1083: panic("pool_get: nitems inconsistent");
1084: }
1.65 enami 1085: #endif
1.56 sommerfe 1086:
1.65 enami 1087: #ifdef POOL_DIAGNOSTIC
1.97 yamt 1088: pr_log(pp, v, PRLOG_GET, file, line);
1.65 enami 1089: #endif
1.3 pk 1090:
1.65 enami 1091: #ifdef DIAGNOSTIC
1.97 yamt 1092: if (__predict_false(pi->pi_magic != PI_MAGIC)) {
1093: pr_printlog(pp, pi, printf);
1094: panic("pool_get(%s): free list modified: "
1095: "magic=%x; page %p; item addr %p\n",
1096: pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
1097: }
1.3 pk 1098: #endif
1099:
1.97 yamt 1100: /*
1101: * Remove from item list.
1102: */
1.102 chs 1103: LIST_REMOVE(pi, pi_list);
1.97 yamt 1104: }
1.20 thorpej 1105: pp->pr_nitems--;
1106: pp->pr_nout++;
1.6 thorpej 1107: if (ph->ph_nmissing == 0) {
1108: #ifdef DIAGNOSTIC
1.34 thorpej 1109: if (__predict_false(pp->pr_nidle == 0))
1.6 thorpej 1110: panic("pool_get: nidle inconsistent");
1111: #endif
1112: pp->pr_nidle--;
1.88 chs 1113:
1114: /*
1115: * This page was previously empty. Move it to the list of
1116: * partially-full pages. This page is already curpage.
1117: */
1118: LIST_REMOVE(ph, ph_pagelist);
1119: LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
1.6 thorpej 1120: }
1.3 pk 1121: ph->ph_nmissing++;
1.97 yamt 1122: if (ph->ph_nmissing == pp->pr_itemsperpage) {
1.21 thorpej 1123: #ifdef DIAGNOSTIC
1.97 yamt 1124: if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 &&
1.102 chs 1125: !LIST_EMPTY(&ph->ph_itemlist))) {
1.25 thorpej 1126: pr_leave(pp);
1.131.2.2! matt 1127: mutex_exit(&pp->pr_lock);
1.21 thorpej 1128: panic("pool_get: %s: nmissing inconsistent",
1129: pp->pr_wchan);
1130: }
1131: #endif
1.3 pk 1132: /*
1.88 chs 1133: * This page is now full. Move it to the full list
1134: * and select a new current page.
1.3 pk 1135: */
1.88 chs 1136: LIST_REMOVE(ph, ph_pagelist);
1137: LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
1138: pool_update_curpage(pp);
1.1 pk 1139: }
1.3 pk 1140:
1141: pp->pr_nget++;
1.111 christos 1142: pr_leave(pp);
1.20 thorpej 1143:
1144: /*
1145: * If we have a low water mark and we are now below that low
1146: * water mark, add more items to the pool.
1147: */
1.53 thorpej 1148: if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
1.20 thorpej 1149: /*
1150: * XXX: Should we log a warning? Should we set up a timeout
1151: * to try again in a second or so? The latter could break
1152: * a caller's assumptions about interrupt protection, etc.
1153: */
1154: }
1155:
1.131.2.2! matt 1156: mutex_exit(&pp->pr_lock);
1.125 ad 1157: KASSERT((((vaddr_t)v + pp->pr_itemoffset) & (pp->pr_align - 1)) == 0);
1158: FREECHECK_OUT(&pp->pr_freecheck, v);
1.1 pk 1159: return (v);
1160: }
1161:
1162: /*
1.43 thorpej 1163: * Internal version of pool_put(). Pool is already locked/entered.
1.1 pk 1164: */
1.43 thorpej 1165: static void
1.101 thorpej 1166: pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq)
1.1 pk 1167: {
1168: struct pool_item *pi = v;
1.3 pk 1169: struct pool_item_header *ph;
1170:
1.131.2.2! matt 1171: KASSERT(mutex_owned(&pp->pr_lock));
1.125 ad 1172: FREECHECK_IN(&pp->pr_freecheck, v);
1.131.2.2! matt 1173: LOCKDEBUG_MEM_CHECK(v, pp->pr_size);
1.61 chs 1174:
1.30 thorpej 1175: #ifdef DIAGNOSTIC
1.34 thorpej 1176: if (__predict_false(pp->pr_nout == 0)) {
1.30 thorpej 1177: printf("pool %s: putting with none out\n",
1178: pp->pr_wchan);
1179: panic("pool_put");
1180: }
1181: #endif
1.3 pk 1182:
1.121 yamt 1183: if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) {
1.25 thorpej 1184: pr_printlog(pp, NULL, printf);
1.3 pk 1185: panic("pool_put: %s: page header missing", pp->pr_wchan);
1186: }
1.28 thorpej 1187:
1.3 pk 1188: /*
1189: * Return to item list.
1190: */
1.97 yamt 1191: if (pp->pr_roflags & PR_NOTOUCH) {
1192: pr_item_notouch_put(pp, ph, v);
1193: } else {
1.2 pk 1194: #ifdef DIAGNOSTIC
1.97 yamt 1195: pi->pi_magic = PI_MAGIC;
1.3 pk 1196: #endif
1.32 chs 1197: #ifdef DEBUG
1.97 yamt 1198: {
1199: int i, *ip = v;
1.32 chs 1200:
1.97 yamt 1201: for (i = 0; i < pp->pr_size / sizeof(int); i++) {
1202: *ip++ = PI_MAGIC;
1203: }
1.32 chs 1204: }
1205: #endif
1206:
1.102 chs 1207: LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
1.97 yamt 1208: }
1.79 thorpej 1209: KDASSERT(ph->ph_nmissing != 0);
1.3 pk 1210: ph->ph_nmissing--;
1211: pp->pr_nput++;
1.20 thorpej 1212: pp->pr_nitems++;
1213: pp->pr_nout--;
1.3 pk 1214:
1215: /* Cancel "pool empty" condition if it exists */
1216: if (pp->pr_curpage == NULL)
1217: pp->pr_curpage = ph;
1218:
1219: if (pp->pr_flags & PR_WANTED) {
1220: pp->pr_flags &= ~PR_WANTED;
1.15 pk 1221: if (ph->ph_nmissing == 0)
1222: pp->pr_nidle++;
1.131.2.2! matt 1223: cv_broadcast(&pp->pr_cv);
1.3 pk 1224: return;
1225: }
1226:
1227: /*
1.88 chs 1228: * If this page is now empty, do one of two things:
1.21 thorpej 1229: *
1.88 chs 1230: * (1) If we have more pages than the page high water mark,
1.96 thorpej 1231: * free the page back to the system. ONLY CONSIDER
1.90 thorpej 1232: * FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE
1233: * CLAIM.
1.21 thorpej 1234: *
1.88 chs 1235: * (2) Otherwise, move the page to the empty page list.
1236: *
1237: * Either way, select a new current page (so we use a partially-full
1238: * page if one is available).
1.3 pk 1239: */
1240: if (ph->ph_nmissing == 0) {
1.6 thorpej 1241: pp->pr_nidle++;
1.90 thorpej 1242: if (pp->pr_npages > pp->pr_minpages &&
1243: (pp->pr_npages > pp->pr_maxpages ||
1.117 yamt 1244: pa_starved_p(pp->pr_alloc))) {
1.101 thorpej 1245: pr_rmpage(pp, ph, pq);
1.3 pk 1246: } else {
1.88 chs 1247: LIST_REMOVE(ph, ph_pagelist);
1248: LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
1.3 pk 1249:
1.21 thorpej 1250: /*
1251: * Update the timestamp on the page. A page must
1252: * be idle for some period of time before it can
1253: * be reclaimed by the pagedaemon. This minimizes
1254: * ping-pong'ing for memory.
1255: */
1.118 kardel 1256: getmicrotime(&ph->ph_time);
1.1 pk 1257: }
1.88 chs 1258: pool_update_curpage(pp);
1.1 pk 1259: }
1.88 chs 1260:
1.21 thorpej 1261: /*
1.88 chs 1262: * If the page was previously completely full, move it to the
1263: * partially-full list and make it the current page. The next
1264: * allocation will get the item from this page, instead of
1265: * further fragmenting the pool.
1.21 thorpej 1266: */
1267: else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
1.88 chs 1268: LIST_REMOVE(ph, ph_pagelist);
1269: LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
1.21 thorpej 1270: pp->pr_curpage = ph;
1271: }
1.43 thorpej 1272: }
1273:
1274: /*
1.131.2.2! matt 1275: * Return resource to the pool.
1.43 thorpej 1276: */
1.59 thorpej 1277: #ifdef POOL_DIAGNOSTIC
1.43 thorpej 1278: void
1279: _pool_put(struct pool *pp, void *v, const char *file, long line)
1280: {
1.101 thorpej 1281: struct pool_pagelist pq;
1282:
1283: LIST_INIT(&pq);
1.43 thorpej 1284:
1.131.2.2! matt 1285: mutex_enter(&pp->pr_lock);
1.43 thorpej 1286: pr_enter(pp, file, line);
1287:
1.56 sommerfe 1288: pr_log(pp, v, PRLOG_PUT, file, line);
1289:
1.101 thorpej 1290: pool_do_put(pp, v, &pq);
1.21 thorpej 1291:
1.25 thorpej 1292: pr_leave(pp);
1.131.2.2! matt 1293: mutex_exit(&pp->pr_lock);
1.101 thorpej 1294:
1.102 chs 1295: pr_pagelist_free(pp, &pq);
1.1 pk 1296: }
1.57 sommerfe 1297: #undef pool_put
1.59 thorpej 1298: #endif /* POOL_DIAGNOSTIC */
1.1 pk 1299:
1.56 sommerfe 1300: void
1301: pool_put(struct pool *pp, void *v)
1302: {
1.101 thorpej 1303: struct pool_pagelist pq;
1304:
1305: LIST_INIT(&pq);
1.56 sommerfe 1306:
1.131.2.2! matt 1307: mutex_enter(&pp->pr_lock);
1.101 thorpej 1308: pool_do_put(pp, v, &pq);
1.131.2.2! matt 1309: mutex_exit(&pp->pr_lock);
1.56 sommerfe 1310:
1.102 chs 1311: pr_pagelist_free(pp, &pq);
1.56 sommerfe 1312: }
1.57 sommerfe 1313:
1.59 thorpej 1314: #ifdef POOL_DIAGNOSTIC
1.57 sommerfe 1315: #define pool_put(h, v) _pool_put((h), (v), __FILE__, __LINE__)
1.56 sommerfe 1316: #endif
1.74 thorpej 1317:
1318: /*
1.113 yamt 1319: * pool_grow: grow a pool by a page.
1320: *
1321: * => called with pool locked.
1322: * => unlock and relock the pool.
1323: * => return with pool locked.
1324: */
1325:
1326: static int
1327: pool_grow(struct pool *pp, int flags)
1328: {
1329: struct pool_item_header *ph = NULL;
1330: char *cp;
1331:
1.131.2.2! matt 1332: mutex_exit(&pp->pr_lock);
1.113 yamt 1333: cp = pool_allocator_alloc(pp, flags);
1334: if (__predict_true(cp != NULL)) {
1335: ph = pool_alloc_item_header(pp, cp, flags);
1336: }
1337: if (__predict_false(cp == NULL || ph == NULL)) {
1338: if (cp != NULL) {
1339: pool_allocator_free(pp, cp);
1340: }
1.131.2.2! matt 1341: mutex_enter(&pp->pr_lock);
1.113 yamt 1342: return ENOMEM;
1343: }
1344:
1.131.2.2! matt 1345: mutex_enter(&pp->pr_lock);
1.113 yamt 1346: pool_prime_page(pp, cp, ph);
1347: pp->pr_npagealloc++;
1348: return 0;
1349: }
1350:
1351: /*
1.74 thorpej 1352: * Add N items to the pool.
1353: */
1354: int
1355: pool_prime(struct pool *pp, int n)
1356: {
1.75 simonb 1357: int newpages;
1.113 yamt 1358: int error = 0;
1.74 thorpej 1359:
1.131.2.2! matt 1360: mutex_enter(&pp->pr_lock);
1.74 thorpej 1361:
1362: newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1363:
1364: while (newpages-- > 0) {
1.113 yamt 1365: error = pool_grow(pp, PR_NOWAIT);
1366: if (error) {
1.74 thorpej 1367: break;
1368: }
1369: pp->pr_minpages++;
1370: }
1371:
1372: if (pp->pr_minpages >= pp->pr_maxpages)
1373: pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */
1374:
1.131.2.2! matt 1375: mutex_exit(&pp->pr_lock);
1.113 yamt 1376: return error;
1.74 thorpej 1377: }
1.55 thorpej 1378:
1379: /*
1.3 pk 1380: * Add a page worth of items to the pool.
1.21 thorpej 1381: *
1382: * Note, we must be called with the pool descriptor LOCKED.
1.3 pk 1383: */
1.55 thorpej 1384: static void
1.128 christos 1385: pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph)
1.3 pk 1386: {
1387: struct pool_item *pi;
1.128 christos 1388: void *cp = storage;
1.125 ad 1389: const unsigned int align = pp->pr_align;
1390: const unsigned int ioff = pp->pr_itemoffset;
1.55 thorpej 1391: int n;
1.36 pk 1392:
1.131.2.2! matt 1393: KASSERT(mutex_owned(&pp->pr_lock));
1.91 yamt 1394:
1.66 thorpej 1395: #ifdef DIAGNOSTIC
1.121 yamt 1396: if ((pp->pr_roflags & PR_NOALIGN) == 0 &&
1397: ((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0)
1.36 pk 1398: panic("pool_prime_page: %s: unaligned page", pp->pr_wchan);
1.66 thorpej 1399: #endif
1.3 pk 1400:
1401: /*
1402: * Insert page header.
1403: */
1.88 chs 1404: LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
1.102 chs 1405: LIST_INIT(&ph->ph_itemlist);
1.3 pk 1406: ph->ph_page = storage;
1407: ph->ph_nmissing = 0;
1.118 kardel 1408: getmicrotime(&ph->ph_time);
1.88 chs 1409: if ((pp->pr_roflags & PR_PHINPAGE) == 0)
1410: SPLAY_INSERT(phtree, &pp->pr_phtree, ph);
1.3 pk 1411:
1.6 thorpej 1412: pp->pr_nidle++;
1413:
1.3 pk 1414: /*
1415: * Color this page.
1416: */
1.128 christos 1417: cp = (char *)cp + pp->pr_curcolor;
1.3 pk 1418: if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
1419: pp->pr_curcolor = 0;
1420:
1421: /*
1422: * Adjust storage to apply aligment to `pr_itemoffset' in each item.
1423: */
1424: if (ioff != 0)
1.128 christos 1425: cp = (char *)cp + align - ioff;
1.3 pk 1426:
1.125 ad 1427: KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0);
1428:
1.3 pk 1429: /*
1430: * Insert remaining chunks on the bucket list.
1431: */
1432: n = pp->pr_itemsperpage;
1.20 thorpej 1433: pp->pr_nitems += n;
1.3 pk 1434:
1.97 yamt 1435: if (pp->pr_roflags & PR_NOTOUCH) {
1.99 yamt 1436: pool_item_freelist_t *freelist = PR_FREELIST(ph);
1.97 yamt 1437: int i;
1438:
1.128 christos 1439: ph->ph_off = (char *)cp - (char *)storage;
1.97 yamt 1440: ph->ph_firstfree = 0;
1441: for (i = 0; i < n - 1; i++)
1442: freelist[i] = i + 1;
1443: freelist[n - 1] = PR_INDEX_EOL;
1444: } else {
1445: while (n--) {
1446: pi = (struct pool_item *)cp;
1.78 thorpej 1447:
1.97 yamt 1448: KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0);
1.3 pk 1449:
1.97 yamt 1450: /* Insert on page list */
1.102 chs 1451: LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
1.3 pk 1452: #ifdef DIAGNOSTIC
1.97 yamt 1453: pi->pi_magic = PI_MAGIC;
1.3 pk 1454: #endif
1.128 christos 1455: cp = (char *)cp + pp->pr_size;
1.125 ad 1456:
1457: KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0);
1.97 yamt 1458: }
1.3 pk 1459: }
1460:
1461: /*
1462: * If the pool was depleted, point at the new page.
1463: */
1464: if (pp->pr_curpage == NULL)
1465: pp->pr_curpage = ph;
1466:
1467: if (++pp->pr_npages > pp->pr_hiwat)
1468: pp->pr_hiwat = pp->pr_npages;
1469: }
1470:
1.20 thorpej 1471: /*
1.52 thorpej 1472: * Used by pool_get() when nitems drops below the low water mark. This
1.88 chs 1473: * is used to catch up pr_nitems with the low water mark.
1.20 thorpej 1474: *
1.21 thorpej 1475: * Note 1, we never wait for memory here, we let the caller decide what to do.
1.20 thorpej 1476: *
1.73 thorpej 1477: * Note 2, we must be called with the pool already locked, and we return
1.20 thorpej 1478: * with it locked.
1479: */
1480: static int
1.42 thorpej 1481: pool_catchup(struct pool *pp)
1.20 thorpej 1482: {
1483: int error = 0;
1484:
1.54 thorpej 1485: while (POOL_NEEDS_CATCHUP(pp)) {
1.113 yamt 1486: error = pool_grow(pp, PR_NOWAIT);
1487: if (error) {
1.20 thorpej 1488: break;
1489: }
1490: }
1.113 yamt 1491: return error;
1.20 thorpej 1492: }
1493:
1.88 chs 1494: static void
1495: pool_update_curpage(struct pool *pp)
1496: {
1497:
1498: pp->pr_curpage = LIST_FIRST(&pp->pr_partpages);
1499: if (pp->pr_curpage == NULL) {
1500: pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages);
1501: }
1502: }
1503:
1.3 pk 1504: void
1.42 thorpej 1505: pool_setlowat(struct pool *pp, int n)
1.3 pk 1506: {
1.15 pk 1507:
1.131.2.2! matt 1508: mutex_enter(&pp->pr_lock);
1.21 thorpej 1509:
1.3 pk 1510: pp->pr_minitems = n;
1.15 pk 1511: pp->pr_minpages = (n == 0)
1512: ? 0
1.18 thorpej 1513: : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1.20 thorpej 1514:
1515: /* Make sure we're caught up with the newly-set low water mark. */
1.75 simonb 1516: if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
1.20 thorpej 1517: /*
1518: * XXX: Should we log a warning? Should we set up a timeout
1519: * to try again in a second or so? The latter could break
1520: * a caller's assumptions about interrupt protection, etc.
1521: */
1522: }
1.21 thorpej 1523:
1.131.2.2! matt 1524: mutex_exit(&pp->pr_lock);
1.3 pk 1525: }
1526:
1527: void
1.42 thorpej 1528: pool_sethiwat(struct pool *pp, int n)
1.3 pk 1529: {
1.15 pk 1530:
1.131.2.2! matt 1531: mutex_enter(&pp->pr_lock);
1.21 thorpej 1532:
1.15 pk 1533: pp->pr_maxpages = (n == 0)
1534: ? 0
1.18 thorpej 1535: : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1.21 thorpej 1536:
1.131.2.2! matt 1537: mutex_exit(&pp->pr_lock);
1.3 pk 1538: }
1539:
1.20 thorpej 1540: void
1.42 thorpej 1541: pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap)
1.20 thorpej 1542: {
1543:
1.131.2.2! matt 1544: mutex_enter(&pp->pr_lock);
1.20 thorpej 1545:
1546: pp->pr_hardlimit = n;
1547: pp->pr_hardlimit_warning = warnmess;
1.31 thorpej 1548: pp->pr_hardlimit_ratecap.tv_sec = ratecap;
1549: pp->pr_hardlimit_warning_last.tv_sec = 0;
1550: pp->pr_hardlimit_warning_last.tv_usec = 0;
1.20 thorpej 1551:
1552: /*
1.21 thorpej 1553: * In-line version of pool_sethiwat(), because we don't want to
1554: * release the lock.
1.20 thorpej 1555: */
1556: pp->pr_maxpages = (n == 0)
1557: ? 0
1558: : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1.21 thorpej 1559:
1.131.2.2! matt 1560: mutex_exit(&pp->pr_lock);
1.20 thorpej 1561: }
1.3 pk 1562:
1563: /*
1564: * Release all complete pages that have not been used recently.
1565: */
1.66 thorpej 1566: int
1.59 thorpej 1567: #ifdef POOL_DIAGNOSTIC
1.42 thorpej 1568: _pool_reclaim(struct pool *pp, const char *file, long line)
1.56 sommerfe 1569: #else
1570: pool_reclaim(struct pool *pp)
1571: #endif
1.3 pk 1572: {
1573: struct pool_item_header *ph, *phnext;
1.61 chs 1574: struct pool_pagelist pq;
1.102 chs 1575: struct timeval curtime, diff;
1.131.2.2! matt 1576: bool klock;
! 1577: int rv;
1.3 pk 1578:
1.68 thorpej 1579: if (pp->pr_drain_hook != NULL) {
1580: /*
1581: * The drain hook must be called with the pool unlocked.
1582: */
1583: (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT);
1584: }
1585:
1.131.2.2! matt 1586: /*
! 1587: * XXXSMP Because mutexes at IPL_SOFTXXX are still spinlocks,
! 1588: * and we are called from the pagedaemon without kernel_lock.
! 1589: * Does not apply to IPL_SOFTBIO.
! 1590: */
! 1591: if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK ||
! 1592: pp->pr_ipl == IPL_SOFTSERIAL) {
! 1593: KERNEL_LOCK(1, NULL);
! 1594: klock = true;
! 1595: } else
! 1596: klock = false;
! 1597:
! 1598: /* Reclaim items from the pool's cache (if any). */
! 1599: if (pp->pr_cache != NULL)
! 1600: pool_cache_invalidate(pp->pr_cache);
! 1601:
! 1602: if (mutex_tryenter(&pp->pr_lock) == 0) {
! 1603: if (klock) {
! 1604: KERNEL_UNLOCK_ONE(NULL);
! 1605: }
1.66 thorpej 1606: return (0);
1.131.2.2! matt 1607: }
1.25 thorpej 1608: pr_enter(pp, file, line);
1.68 thorpej 1609:
1.88 chs 1610: LIST_INIT(&pq);
1.43 thorpej 1611:
1.118 kardel 1612: getmicrotime(&curtime);
1.21 thorpej 1613:
1.88 chs 1614: for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
1615: phnext = LIST_NEXT(ph, ph_pagelist);
1.3 pk 1616:
1617: /* Check our minimum page claim */
1618: if (pp->pr_npages <= pp->pr_minpages)
1619: break;
1620:
1.88 chs 1621: KASSERT(ph->ph_nmissing == 0);
1622: timersub(&curtime, &ph->ph_time, &diff);
1.117 yamt 1623: if (diff.tv_sec < pool_inactive_time
1624: && !pa_starved_p(pp->pr_alloc))
1.88 chs 1625: continue;
1.21 thorpej 1626:
1.88 chs 1627: /*
1628: * If freeing this page would put us below
1629: * the low water mark, stop now.
1630: */
1631: if ((pp->pr_nitems - pp->pr_itemsperpage) <
1632: pp->pr_minitems)
1633: break;
1.21 thorpej 1634:
1.88 chs 1635: pr_rmpage(pp, ph, &pq);
1.3 pk 1636: }
1637:
1.25 thorpej 1638: pr_leave(pp);
1.131.2.2! matt 1639: mutex_exit(&pp->pr_lock);
1.66 thorpej 1640:
1.131.2.2! matt 1641: if (LIST_EMPTY(&pq))
! 1642: rv = 0;
! 1643: else {
! 1644: pr_pagelist_free(pp, &pq);
! 1645: rv = 1;
! 1646: }
! 1647:
! 1648: if (klock) {
! 1649: KERNEL_UNLOCK_ONE(NULL);
! 1650: }
! 1651:
! 1652: return (rv);
1.3 pk 1653: }
1654:
1655: /*
1.131.2.2! matt 1656: * Drain pools, one at a time. This is a two stage process;
! 1657: * drain_start kicks off a cross call to drain CPU-level caches
! 1658: * if the pool has an associated pool_cache. drain_end waits
! 1659: * for those cross calls to finish, and then drains the cache
! 1660: * (if any) and pool.
1.131 ad 1661: *
1.131.2.2! matt 1662: * Note, must never be called from interrupt context.
1.3 pk 1663: */
1664: void
1.131.2.2! matt 1665: pool_drain_start(struct pool **ppp, uint64_t *wp)
1.3 pk 1666: {
1667: struct pool *pp;
1.131.2.2! matt 1668:
! 1669: KASSERT(!LIST_EMPTY(&pool_head));
1.3 pk 1670:
1.61 chs 1671: pp = NULL;
1.131.2.2! matt 1672:
! 1673: /* Find next pool to drain, and add a reference. */
! 1674: mutex_enter(&pool_head_lock);
! 1675: do {
! 1676: if (drainpp == NULL) {
! 1677: drainpp = LIST_FIRST(&pool_head);
! 1678: }
! 1679: if (drainpp != NULL) {
! 1680: pp = drainpp;
! 1681: drainpp = LIST_NEXT(pp, pr_poollist);
! 1682: }
! 1683: /*
! 1684: * Skip completely idle pools. We depend on at least
! 1685: * one pool in the system being active.
! 1686: */
! 1687: } while (pp == NULL || pp->pr_npages == 0);
! 1688: pp->pr_refcnt++;
! 1689: mutex_exit(&pool_head_lock);
! 1690:
! 1691: /* If there is a pool_cache, drain CPU level caches. */
! 1692: *ppp = pp;
! 1693: if (pp->pr_cache != NULL) {
! 1694: *wp = xc_broadcast(0, (xcfunc_t)pool_cache_xcall,
! 1695: pp->pr_cache, NULL);
! 1696: }
! 1697: }
! 1698:
! 1699: void
! 1700: pool_drain_end(struct pool *pp, uint64_t where)
! 1701: {
! 1702:
! 1703: if (pp == NULL)
! 1704: return;
! 1705:
! 1706: KASSERT(pp->pr_refcnt > 0);
! 1707:
! 1708: /* Wait for remote draining to complete. */
! 1709: if (pp->pr_cache != NULL)
! 1710: xc_wait(where);
! 1711:
! 1712: /* Drain the cache (if any) and pool.. */
! 1713: pool_reclaim(pp);
! 1714:
! 1715: /* Finally, unlock the pool. */
! 1716: mutex_enter(&pool_head_lock);
! 1717: pp->pr_refcnt--;
! 1718: cv_broadcast(&pool_busy);
! 1719: mutex_exit(&pool_head_lock);
1.3 pk 1720: }
1721:
1722: /*
1723: * Diagnostic helpers.
1724: */
1725: void
1.42 thorpej 1726: pool_print(struct pool *pp, const char *modif)
1.21 thorpej 1727: {
1728:
1.25 thorpej 1729: pool_print1(pp, modif, printf);
1.21 thorpej 1730: }
1731:
1.25 thorpej 1732: void
1.108 yamt 1733: pool_printall(const char *modif, void (*pr)(const char *, ...))
1734: {
1735: struct pool *pp;
1736:
1737: LIST_FOREACH(pp, &pool_head, pr_poollist) {
1738: pool_printit(pp, modif, pr);
1739: }
1740: }
1741:
1742: void
1.42 thorpej 1743: pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
1.25 thorpej 1744: {
1745:
1746: if (pp == NULL) {
1747: (*pr)("Must specify a pool to print.\n");
1748: return;
1749: }
1750:
1751: pool_print1(pp, modif, pr);
1752: }
1753:
1.21 thorpej 1754: static void
1.124 yamt 1755: pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl,
1.97 yamt 1756: void (*pr)(const char *, ...))
1.88 chs 1757: {
1758: struct pool_item_header *ph;
1759: #ifdef DIAGNOSTIC
1760: struct pool_item *pi;
1761: #endif
1762:
1763: LIST_FOREACH(ph, pl, ph_pagelist) {
1764: (*pr)("\t\tpage %p, nmissing %d, time %lu,%lu\n",
1765: ph->ph_page, ph->ph_nmissing,
1766: (u_long)ph->ph_time.tv_sec,
1767: (u_long)ph->ph_time.tv_usec);
1768: #ifdef DIAGNOSTIC
1.97 yamt 1769: if (!(pp->pr_roflags & PR_NOTOUCH)) {
1.102 chs 1770: LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) {
1.97 yamt 1771: if (pi->pi_magic != PI_MAGIC) {
1772: (*pr)("\t\t\titem %p, magic 0x%x\n",
1773: pi, pi->pi_magic);
1774: }
1.88 chs 1775: }
1776: }
1777: #endif
1778: }
1779: }
1780:
1781: static void
1.42 thorpej 1782: pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...))
1.3 pk 1783: {
1.25 thorpej 1784: struct pool_item_header *ph;
1.131.2.2! matt 1785: pool_cache_t pc;
! 1786: pcg_t *pcg;
! 1787: pool_cache_cpu_t *cc;
! 1788: uint64_t cpuhit, cpumiss;
1.44 thorpej 1789: int i, print_log = 0, print_pagelist = 0, print_cache = 0;
1.25 thorpej 1790: char c;
1791:
1792: while ((c = *modif++) != '\0') {
1793: if (c == 'l')
1794: print_log = 1;
1795: if (c == 'p')
1796: print_pagelist = 1;
1.44 thorpej 1797: if (c == 'c')
1798: print_cache = 1;
1.25 thorpej 1799: }
1800:
1.131.2.2! matt 1801: if ((pc = pp->pr_cache) != NULL) {
! 1802: (*pr)("POOL CACHE");
! 1803: } else {
! 1804: (*pr)("POOL");
! 1805: }
! 1806:
! 1807: (*pr)(" %s: size %u, align %u, ioff %u, roflags 0x%08x\n",
1.25 thorpej 1808: pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,
1809: pp->pr_roflags);
1.66 thorpej 1810: (*pr)("\talloc %p\n", pp->pr_alloc);
1.25 thorpej 1811: (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",
1812: pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);
1813: (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",
1814: pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);
1815:
1.131.2.2! matt 1816: (*pr)("\tnget %lu, nfail %lu, nput %lu\n",
1.25 thorpej 1817: pp->pr_nget, pp->pr_nfail, pp->pr_nput);
1818: (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",
1819: pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);
1820:
1821: if (print_pagelist == 0)
1822: goto skip_pagelist;
1823:
1.88 chs 1824: if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
1825: (*pr)("\n\tempty page list:\n");
1.97 yamt 1826: pool_print_pagelist(pp, &pp->pr_emptypages, pr);
1.88 chs 1827: if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
1828: (*pr)("\n\tfull page list:\n");
1.97 yamt 1829: pool_print_pagelist(pp, &pp->pr_fullpages, pr);
1.88 chs 1830: if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
1831: (*pr)("\n\tpartial-page list:\n");
1.97 yamt 1832: pool_print_pagelist(pp, &pp->pr_partpages, pr);
1.88 chs 1833:
1.25 thorpej 1834: if (pp->pr_curpage == NULL)
1835: (*pr)("\tno current page\n");
1836: else
1837: (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page);
1838:
1839: skip_pagelist:
1840: if (print_log == 0)
1841: goto skip_log;
1842:
1843: (*pr)("\n");
1844: if ((pp->pr_roflags & PR_LOGGING) == 0)
1845: (*pr)("\tno log\n");
1.122 christos 1846: else {
1.25 thorpej 1847: pr_printlog(pp, NULL, pr);
1.122 christos 1848: }
1.3 pk 1849:
1.25 thorpej 1850: skip_log:
1.44 thorpej 1851:
1.102 chs 1852: #define PR_GROUPLIST(pcg) \
1853: (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \
1854: for (i = 0; i < PCG_NOBJECTS; i++) { \
1855: if (pcg->pcg_objects[i].pcgo_pa != \
1856: POOL_PADDR_INVALID) { \
1857: (*pr)("\t\t\t%p, 0x%llx\n", \
1858: pcg->pcg_objects[i].pcgo_va, \
1859: (unsigned long long) \
1860: pcg->pcg_objects[i].pcgo_pa); \
1861: } else { \
1862: (*pr)("\t\t\t%p\n", \
1863: pcg->pcg_objects[i].pcgo_va); \
1864: } \
1865: }
1866:
1.131.2.2! matt 1867: if (pc != NULL) {
! 1868: cpuhit = 0;
! 1869: cpumiss = 0;
! 1870: for (i = 0; i < MAXCPUS; i++) {
! 1871: if ((cc = pc->pc_cpus[i]) == NULL)
! 1872: continue;
! 1873: cpuhit += cc->cc_hits;
! 1874: cpumiss += cc->cc_misses;
! 1875: }
! 1876: (*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss);
! 1877: (*pr)("\tcache layer hits %llu misses %llu\n",
! 1878: pc->pc_hits, pc->pc_misses);
! 1879: (*pr)("\tcache layer entry uncontended %llu contended %llu\n",
! 1880: pc->pc_hits + pc->pc_misses - pc->pc_contended,
! 1881: pc->pc_contended);
! 1882: (*pr)("\tcache layer empty groups %u full groups %u\n",
! 1883: pc->pc_nempty, pc->pc_nfull);
! 1884: if (print_cache) {
! 1885: (*pr)("\tfull cache groups:\n");
! 1886: for (pcg = pc->pc_fullgroups; pcg != NULL;
! 1887: pcg = pcg->pcg_next) {
! 1888: PR_GROUPLIST(pcg);
! 1889: }
! 1890: (*pr)("\tempty cache groups:\n");
! 1891: for (pcg = pc->pc_emptygroups; pcg != NULL;
! 1892: pcg = pcg->pcg_next) {
! 1893: PR_GROUPLIST(pcg);
! 1894: }
1.103 chs 1895: }
1.44 thorpej 1896: }
1.102 chs 1897: #undef PR_GROUPLIST
1.44 thorpej 1898:
1.88 chs 1899: pr_enter_check(pp, pr);
1900: }
1901:
1902: static int
1903: pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph)
1904: {
1905: struct pool_item *pi;
1.128 christos 1906: void *page;
1.88 chs 1907: int n;
1908:
1.121 yamt 1909: if ((pp->pr_roflags & PR_NOALIGN) == 0) {
1.128 christos 1910: page = (void *)((uintptr_t)ph & pp->pr_alloc->pa_pagemask);
1.121 yamt 1911: if (page != ph->ph_page &&
1912: (pp->pr_roflags & PR_PHINPAGE) != 0) {
1913: if (label != NULL)
1914: printf("%s: ", label);
1915: printf("pool(%p:%s): page inconsistency: page %p;"
1916: " at page head addr %p (p %p)\n", pp,
1917: pp->pr_wchan, ph->ph_page,
1918: ph, page);
1919: return 1;
1920: }
1.88 chs 1921: }
1.3 pk 1922:
1.97 yamt 1923: if ((pp->pr_roflags & PR_NOTOUCH) != 0)
1924: return 0;
1925:
1.102 chs 1926: for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0;
1.88 chs 1927: pi != NULL;
1.102 chs 1928: pi = LIST_NEXT(pi,pi_list), n++) {
1.88 chs 1929:
1930: #ifdef DIAGNOSTIC
1931: if (pi->pi_magic != PI_MAGIC) {
1932: if (label != NULL)
1933: printf("%s: ", label);
1934: printf("pool(%s): free list modified: magic=%x;"
1.121 yamt 1935: " page %p; item ordinal %d; addr %p\n",
1.88 chs 1936: pp->pr_wchan, pi->pi_magic, ph->ph_page,
1.121 yamt 1937: n, pi);
1.88 chs 1938: panic("pool");
1939: }
1940: #endif
1.121 yamt 1941: if ((pp->pr_roflags & PR_NOALIGN) != 0) {
1942: continue;
1943: }
1.128 christos 1944: page = (void *)((uintptr_t)pi & pp->pr_alloc->pa_pagemask);
1.88 chs 1945: if (page == ph->ph_page)
1946: continue;
1947:
1948: if (label != NULL)
1949: printf("%s: ", label);
1950: printf("pool(%p:%s): page inconsistency: page %p;"
1951: " item ordinal %d; addr %p (p %p)\n", pp,
1952: pp->pr_wchan, ph->ph_page,
1953: n, pi, page);
1954: return 1;
1955: }
1956: return 0;
1.3 pk 1957: }
1958:
1.88 chs 1959:
1.3 pk 1960: int
1.42 thorpej 1961: pool_chk(struct pool *pp, const char *label)
1.3 pk 1962: {
1963: struct pool_item_header *ph;
1964: int r = 0;
1965:
1.131.2.2! matt 1966: mutex_enter(&pp->pr_lock);
1.88 chs 1967: LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
1968: r = pool_chk_page(pp, label, ph);
1969: if (r) {
1970: goto out;
1971: }
1972: }
1973: LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
1974: r = pool_chk_page(pp, label, ph);
1975: if (r) {
1.3 pk 1976: goto out;
1977: }
1.88 chs 1978: }
1979: LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
1980: r = pool_chk_page(pp, label, ph);
1981: if (r) {
1.3 pk 1982: goto out;
1983: }
1984: }
1.88 chs 1985:
1.3 pk 1986: out:
1.131.2.2! matt 1987: mutex_exit(&pp->pr_lock);
1.3 pk 1988: return (r);
1.43 thorpej 1989: }
1990:
1991: /*
1992: * pool_cache_init:
1993: *
1994: * Initialize a pool cache.
1.131.2.2! matt 1995: */
! 1996: pool_cache_t
! 1997: pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags,
! 1998: const char *wchan, struct pool_allocator *palloc, int ipl,
! 1999: int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg)
! 2000: {
! 2001: pool_cache_t pc;
! 2002:
! 2003: pc = pool_get(&cache_pool, PR_WAITOK);
! 2004: if (pc == NULL)
! 2005: return NULL;
! 2006:
! 2007: pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan,
! 2008: palloc, ipl, ctor, dtor, arg);
! 2009:
! 2010: return pc;
! 2011: }
! 2012:
! 2013: /*
! 2014: * pool_cache_bootstrap:
1.43 thorpej 2015: *
1.131.2.2! matt 2016: * Kernel-private version of pool_cache_init(). The caller
! 2017: * provides initial storage.
1.43 thorpej 2018: */
2019: void
1.131.2.2! matt 2020: pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align,
! 2021: u_int align_offset, u_int flags, const char *wchan,
! 2022: struct pool_allocator *palloc, int ipl,
! 2023: int (*ctor)(void *, void *, int), void (*dtor)(void *, void *),
1.43 thorpej 2024: void *arg)
2025: {
1.131.2.2! matt 2026: CPU_INFO_ITERATOR cii;
! 2027: struct cpu_info *ci;
! 2028: struct pool *pp;
! 2029:
! 2030: pp = &pc->pc_pool;
! 2031: if (palloc == NULL && ipl == IPL_NONE)
! 2032: palloc = &pool_allocator_nointr;
! 2033: pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl);
1.43 thorpej 2034:
1.131.2.2! matt 2035: mutex_init(&pc->pc_lock, MUTEX_DEFAULT, pp->pr_ipl);
1.43 thorpej 2036:
1.131.2.2! matt 2037: if (ctor == NULL) {
! 2038: ctor = (int (*)(void *, void *, int))nullop;
! 2039: }
! 2040: if (dtor == NULL) {
! 2041: dtor = (void (*)(void *, void *))nullop;
! 2042: }
1.43 thorpej 2043:
1.131.2.2! matt 2044: pc->pc_emptygroups = NULL;
! 2045: pc->pc_fullgroups = NULL;
! 2046: pc->pc_partgroups = NULL;
1.43 thorpej 2047: pc->pc_ctor = ctor;
2048: pc->pc_dtor = dtor;
2049: pc->pc_arg = arg;
1.131.2.2! matt 2050: pc->pc_hits = 0;
1.48 thorpej 2051: pc->pc_misses = 0;
1.131.2.2! matt 2052: pc->pc_nempty = 0;
! 2053: pc->pc_npart = 0;
! 2054: pc->pc_nfull = 0;
! 2055: pc->pc_contended = 0;
! 2056: pc->pc_refcnt = 0;
! 2057:
! 2058: /* Allocate per-CPU caches. */
! 2059: memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus));
! 2060: pc->pc_ncpu = 0;
! 2061: for (CPU_INFO_FOREACH(cii, ci)) {
! 2062: pool_cache_cpu_init1(ci, pc);
! 2063: }
! 2064:
! 2065: if (__predict_true(!cold)) {
! 2066: mutex_enter(&pp->pr_lock);
! 2067: pp->pr_cache = pc;
! 2068: mutex_exit(&pp->pr_lock);
! 2069: mutex_enter(&pool_head_lock);
! 2070: LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist);
! 2071: mutex_exit(&pool_head_lock);
! 2072: } else {
! 2073: pp->pr_cache = pc;
! 2074: LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist);
! 2075: }
1.43 thorpej 2076: }
2077:
2078: /*
2079: * pool_cache_destroy:
2080: *
2081: * Destroy a pool cache.
2082: */
2083: void
1.131.2.2! matt 2084: pool_cache_destroy(pool_cache_t pc)
1.43 thorpej 2085: {
1.131.2.2! matt 2086: struct pool *pp = &pc->pc_pool;
! 2087: pool_cache_cpu_t *cc;
! 2088: pcg_t *pcg;
! 2089: int i;
! 2090:
! 2091: /* Remove it from the global list. */
! 2092: mutex_enter(&pool_head_lock);
! 2093: while (pc->pc_refcnt != 0)
! 2094: cv_wait(&pool_busy, &pool_head_lock);
! 2095: LIST_REMOVE(pc, pc_cachelist);
! 2096: mutex_exit(&pool_head_lock);
1.43 thorpej 2097:
2098: /* First, invalidate the entire cache. */
2099: pool_cache_invalidate(pc);
2100:
1.131.2.2! matt 2101: /* Disassociate it from the pool. */
! 2102: mutex_enter(&pp->pr_lock);
! 2103: pp->pr_cache = NULL;
! 2104: mutex_exit(&pp->pr_lock);
! 2105:
! 2106: /* Destroy per-CPU data */
! 2107: for (i = 0; i < MAXCPUS; i++) {
! 2108: if ((cc = pc->pc_cpus[i]) == NULL)
! 2109: continue;
! 2110: if ((pcg = cc->cc_current) != NULL) {
! 2111: pcg->pcg_next = NULL;
! 2112: pool_cache_invalidate_groups(pc, pcg);
! 2113: }
! 2114: if ((pcg = cc->cc_previous) != NULL) {
! 2115: pcg->pcg_next = NULL;
! 2116: pool_cache_invalidate_groups(pc, pcg);
! 2117: }
! 2118: if (cc != &pc->pc_cpu0)
! 2119: pool_put(&cache_cpu_pool, cc);
! 2120: }
! 2121:
! 2122: /* Finally, destroy it. */
! 2123: mutex_destroy(&pc->pc_lock);
! 2124: pool_destroy(pp);
! 2125: pool_put(&cache_pool, pc);
1.43 thorpej 2126: }
2127:
1.131.2.2! matt 2128: /*
! 2129: * pool_cache_cpu_init1:
! 2130: *
! 2131: * Called for each pool_cache whenever a new CPU is attached.
! 2132: */
! 2133: static void
! 2134: pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc)
1.43 thorpej 2135: {
1.131.2.2! matt 2136: pool_cache_cpu_t *cc;
1.43 thorpej 2137:
1.131.2.2! matt 2138: KASSERT(((uintptr_t)pc->pc_cpus & (CACHE_LINE_SIZE - 1)) == 0);
1.43 thorpej 2139:
1.131.2.2! matt 2140: if ((cc = pc->pc_cpus[ci->ci_index]) != NULL) {
! 2141: KASSERT(cc->cc_cpu = ci);
! 2142: return;
! 2143: }
! 2144:
! 2145: /*
! 2146: * The first CPU is 'free'. This needs to be the case for
! 2147: * bootstrap - we may not be able to allocate yet.
! 2148: */
! 2149: if (pc->pc_ncpu == 0) {
! 2150: cc = &pc->pc_cpu0;
! 2151: pc->pc_ncpu = 1;
! 2152: } else {
! 2153: mutex_enter(&pc->pc_lock);
! 2154: pc->pc_ncpu++;
! 2155: mutex_exit(&pc->pc_lock);
! 2156: cc = pool_get(&cache_cpu_pool, PR_WAITOK);
! 2157: }
! 2158:
! 2159: cc->cc_ipl = pc->pc_pool.pr_ipl;
! 2160: cc->cc_iplcookie = makeiplcookie(cc->cc_ipl);
! 2161: cc->cc_cache = pc;
! 2162: cc->cc_cpu = ci;
! 2163: cc->cc_hits = 0;
! 2164: cc->cc_misses = 0;
! 2165: cc->cc_current = NULL;
! 2166: cc->cc_previous = NULL;
! 2167:
! 2168: pc->pc_cpus[ci->ci_index] = cc;
1.43 thorpej 2169: }
2170:
1.131.2.2! matt 2171: /*
! 2172: * pool_cache_cpu_init:
! 2173: *
! 2174: * Called whenever a new CPU is attached.
! 2175: */
! 2176: void
! 2177: pool_cache_cpu_init(struct cpu_info *ci)
! 2178: {
! 2179: pool_cache_t pc;
! 2180:
! 2181: mutex_enter(&pool_head_lock);
! 2182: LIST_FOREACH(pc, &pool_cache_head, pc_cachelist) {
! 2183: pc->pc_refcnt++;
! 2184: mutex_exit(&pool_head_lock);
! 2185:
! 2186: pool_cache_cpu_init1(ci, pc);
! 2187:
! 2188: mutex_enter(&pool_head_lock);
! 2189: pc->pc_refcnt--;
! 2190: cv_broadcast(&pool_busy);
! 2191: }
! 2192: mutex_exit(&pool_head_lock);
! 2193: }
! 2194:
! 2195: /*
! 2196: * pool_cache_reclaim:
! 2197: *
! 2198: * Reclaim memory from a pool cache.
! 2199: */
! 2200: bool
! 2201: pool_cache_reclaim(pool_cache_t pc)
1.43 thorpej 2202: {
2203:
1.131.2.2! matt 2204: return pool_reclaim(&pc->pc_pool);
! 2205: }
1.43 thorpej 2206:
1.131.2.2! matt 2207: /*
! 2208: * pool_cache_destruct_object:
! 2209: *
! 2210: * Force destruction of an object and its release back into
! 2211: * the pool.
! 2212: */
! 2213: void
! 2214: pool_cache_destruct_object(pool_cache_t pc, void *object)
! 2215: {
! 2216:
! 2217: (*pc->pc_dtor)(pc->pc_arg, object);
! 2218: pool_put(&pc->pc_pool, object);
1.43 thorpej 2219: }
2220:
1.131.2.2! matt 2221: /*
! 2222: * pool_cache_invalidate_groups:
! 2223: *
! 2224: * Invalidate a chain of groups and destruct all objects.
! 2225: */
1.102 chs 2226: static void
1.131.2.2! matt 2227: pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg)
1.102 chs 2228: {
1.131.2.2! matt 2229: void *object;
! 2230: pcg_t *next;
! 2231: int i;
! 2232:
! 2233: for (; pcg != NULL; pcg = next) {
! 2234: next = pcg->pcg_next;
! 2235:
! 2236: for (i = 0; i < pcg->pcg_avail; i++) {
! 2237: object = pcg->pcg_objects[i].pcgo_va;
! 2238: pool_cache_destruct_object(pc, object);
! 2239: }
1.102 chs 2240:
2241: pool_put(&pcgpool, pcg);
2242: }
2243: }
2244:
1.43 thorpej 2245: /*
1.131.2.2! matt 2246: * pool_cache_invalidate:
1.43 thorpej 2247: *
1.131.2.2! matt 2248: * Invalidate a pool cache (destruct and release all of the
! 2249: * cached objects). Does not reclaim objects from the pool.
1.43 thorpej 2250: */
1.131.2.2! matt 2251: void
! 2252: pool_cache_invalidate(pool_cache_t pc)
1.43 thorpej 2253: {
1.131.2.2! matt 2254: pcg_t *full, *empty, *part;
1.58 thorpej 2255:
1.131.2.2! matt 2256: mutex_enter(&pc->pc_lock);
! 2257: full = pc->pc_fullgroups;
! 2258: empty = pc->pc_emptygroups;
! 2259: part = pc->pc_partgroups;
! 2260: pc->pc_fullgroups = NULL;
! 2261: pc->pc_emptygroups = NULL;
! 2262: pc->pc_partgroups = NULL;
! 2263: pc->pc_nfull = 0;
! 2264: pc->pc_nempty = 0;
! 2265: pc->pc_npart = 0;
! 2266: mutex_exit(&pc->pc_lock);
1.43 thorpej 2267:
1.131.2.2! matt 2268: pool_cache_invalidate_groups(pc, full);
! 2269: pool_cache_invalidate_groups(pc, empty);
! 2270: pool_cache_invalidate_groups(pc, part);
! 2271: }
1.43 thorpej 2272:
1.131.2.2! matt 2273: void
! 2274: pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg)
! 2275: {
1.43 thorpej 2276:
1.131.2.2! matt 2277: pool_set_drain_hook(&pc->pc_pool, fn, arg);
! 2278: }
1.125 ad 2279:
1.131.2.2! matt 2280: void
! 2281: pool_cache_setlowat(pool_cache_t pc, int n)
! 2282: {
1.43 thorpej 2283:
1.131.2.2! matt 2284: pool_setlowat(&pc->pc_pool, n);
! 2285: }
1.43 thorpej 2286:
1.131.2.2! matt 2287: void
! 2288: pool_cache_sethiwat(pool_cache_t pc, int n)
! 2289: {
1.43 thorpej 2290:
1.131.2.2! matt 2291: pool_sethiwat(&pc->pc_pool, n);
1.43 thorpej 2292: }
2293:
2294: void
1.131.2.2! matt 2295: pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap)
1.43 thorpej 2296: {
2297:
1.131.2.2! matt 2298: pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap);
! 2299: }
! 2300:
! 2301: static inline pool_cache_cpu_t *
! 2302: pool_cache_cpu_enter(pool_cache_t pc, int *s)
! 2303: {
! 2304: pool_cache_cpu_t *cc;
! 2305: struct cpu_info *ci;
1.125 ad 2306:
1.131.2.2! matt 2307: /*
! 2308: * Prevent other users of the cache from accessing our
! 2309: * CPU-local data. To avoid touching shared state, we
! 2310: * pull the neccessary information from CPU local data.
! 2311: */
! 2312: ci = curcpu();
! 2313: KASSERT(ci->ci_data.cpu_index < MAXCPUS);
! 2314: cc = pc->pc_cpus[ci->ci_data.cpu_index];
! 2315: KASSERT(cc->cc_cache == pc);
! 2316: if (cc->cc_ipl == IPL_NONE) {
! 2317: crit_enter();
! 2318: } else {
! 2319: *s = splraiseipl(cc->cc_iplcookie);
1.109 christos 2320: }
2321:
1.131.2.2! matt 2322: /* Moved to another CPU before disabling preemption? */
! 2323: if (__predict_false(ci != curcpu())) {
! 2324: ci = curcpu();
! 2325: cc = pc->pc_cpus[ci->ci_data.cpu_index];
! 2326: }
1.43 thorpej 2327:
1.131.2.2! matt 2328: #ifdef DIAGNOSTIC
! 2329: KASSERT(cc->cc_cpu == ci);
! 2330: KASSERT(((uintptr_t)cc & (CACHE_LINE_SIZE - 1)) == 0);
! 2331: #endif
! 2332:
! 2333: return cc;
! 2334: }
! 2335:
! 2336: static inline void
! 2337: pool_cache_cpu_exit(pool_cache_cpu_t *cc, int *s)
! 2338: {
! 2339:
! 2340: /* No longer need exclusive access to the per-CPU data. */
! 2341: if (cc->cc_ipl == IPL_NONE) {
! 2342: crit_exit();
! 2343: } else {
! 2344: splx(*s);
1.102 chs 2345: }
1.131.2.2! matt 2346: }
! 2347:
! 2348: #if __GNUC_PREREQ__(3, 0)
! 2349: __attribute ((noinline))
! 2350: #endif
! 2351: pool_cache_cpu_t *
! 2352: pool_cache_get_slow(pool_cache_cpu_t *cc, int *s, void **objectp,
! 2353: paddr_t *pap, int flags)
! 2354: {
! 2355: pcg_t *pcg, *cur;
! 2356: uint64_t ncsw;
! 2357: pool_cache_t pc;
! 2358: void *object;
! 2359:
! 2360: pc = cc->cc_cache;
! 2361: cc->cc_misses++;
! 2362:
! 2363: /*
! 2364: * Nothing was available locally. Try and grab a group
! 2365: * from the cache.
! 2366: */
! 2367: if (!mutex_tryenter(&pc->pc_lock)) {
! 2368: ncsw = curlwp->l_ncsw;
! 2369: mutex_enter(&pc->pc_lock);
! 2370: pc->pc_contended++;
1.43 thorpej 2371:
2372: /*
1.131.2.2! matt 2373: * If we context switched while locking, then
! 2374: * our view of the per-CPU data is invalid:
! 2375: * retry.
1.43 thorpej 2376: */
1.131.2.2! matt 2377: if (curlwp->l_ncsw != ncsw) {
! 2378: mutex_exit(&pc->pc_lock);
! 2379: pool_cache_cpu_exit(cc, s);
! 2380: return pool_cache_cpu_enter(pc, s);
! 2381: }
! 2382: }
1.102 chs 2383:
1.131.2.2! matt 2384: if ((pcg = pc->pc_fullgroups) != NULL) {
! 2385: /*
! 2386: * If there's a full group, release our empty
! 2387: * group back to the cache. Install the full
! 2388: * group as cc_current and return.
! 2389: */
! 2390: if ((cur = cc->cc_current) != NULL) {
! 2391: KASSERT(cur->pcg_avail == 0);
! 2392: cur->pcg_next = pc->pc_emptygroups;
! 2393: pc->pc_emptygroups = cur;
! 2394: pc->pc_nempty++;
1.43 thorpej 2395: }
1.131.2.2! matt 2396: KASSERT(pcg->pcg_avail == PCG_NOBJECTS);
! 2397: cc->cc_current = pcg;
! 2398: pc->pc_fullgroups = pcg->pcg_next;
! 2399: pc->pc_hits++;
! 2400: pc->pc_nfull--;
! 2401: mutex_exit(&pc->pc_lock);
! 2402: return cc;
1.43 thorpej 2403: }
2404:
1.131.2.2! matt 2405: /*
! 2406: * Nothing available locally or in cache. Take the slow
! 2407: * path: fetch a new object from the pool and construct
! 2408: * it.
! 2409: */
! 2410: pc->pc_misses++;
! 2411: mutex_exit(&pc->pc_lock);
! 2412: pool_cache_cpu_exit(cc, s);
! 2413:
! 2414: object = pool_get(&pc->pc_pool, flags);
! 2415: *objectp = object;
! 2416: if (object == NULL)
! 2417: return NULL;
1.43 thorpej 2418:
1.131.2.2! matt 2419: if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) {
! 2420: pool_put(&pc->pc_pool, object);
! 2421: *objectp = NULL;
! 2422: return NULL;
1.102 chs 2423: }
1.51 thorpej 2424:
1.131.2.2! matt 2425: KASSERT((((vaddr_t)object + pc->pc_pool.pr_itemoffset) &
! 2426: (pc->pc_pool.pr_align - 1)) == 0);
! 2427:
! 2428: if (pap != NULL) {
! 2429: #ifdef POOL_VTOPHYS
! 2430: *pap = POOL_VTOPHYS(object);
! 2431: #else
! 2432: *pap = POOL_PADDR_INVALID;
! 2433: #endif
! 2434: }
1.51 thorpej 2435:
1.131.2.2! matt 2436: FREECHECK_OUT(&pc->pc_freecheck, object);
! 2437: return NULL;
1.43 thorpej 2438: }
2439:
1.130 ad 2440: /*
1.131.2.2! matt 2441: * pool_cache_get{,_paddr}:
1.130 ad 2442: *
1.131.2.2! matt 2443: * Get an object from a pool cache (optionally returning
! 2444: * the physical address of the object).
1.130 ad 2445: */
1.131.2.2! matt 2446: void *
! 2447: pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap)
1.102 chs 2448: {
1.131.2.2! matt 2449: pool_cache_cpu_t *cc;
! 2450: pcg_t *pcg;
1.102 chs 2451: void *object;
1.131.2.2! matt 2452: int s;
1.102 chs 2453:
1.131.2.2! matt 2454: #ifdef LOCKDEBUG
! 2455: if (flags & PR_WAITOK)
! 2456: ASSERT_SLEEPABLE(NULL, "pool_cache_get(PR_WAITOK)");
! 2457: #endif
1.130 ad 2458:
1.131.2.2! matt 2459: cc = pool_cache_cpu_enter(pc, &s);
! 2460: do {
! 2461: /* Try and allocate an object from the current group. */
! 2462: pcg = cc->cc_current;
! 2463: if (pcg != NULL && pcg->pcg_avail > 0) {
! 2464: object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va;
! 2465: if (pap != NULL)
! 2466: *pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa;
! 2467: pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL;
! 2468: KASSERT(pcg->pcg_avail <= PCG_NOBJECTS);
! 2469: KASSERT(object != NULL);
! 2470: cc->cc_hits++;
! 2471: pool_cache_cpu_exit(cc, &s);
! 2472: FREECHECK_OUT(&pc->pc_freecheck, object);
! 2473: return object;
1.102 chs 2474: }
1.130 ad 2475:
1.131.2.2! matt 2476: /*
! 2477: * That failed. If the previous group isn't empty, swap
! 2478: * it with the current group and allocate from there.
! 2479: */
! 2480: pcg = cc->cc_previous;
! 2481: if (pcg != NULL && pcg->pcg_avail > 0) {
! 2482: cc->cc_previous = cc->cc_current;
! 2483: cc->cc_current = pcg;
! 2484: continue;
! 2485: }
! 2486:
! 2487: /*
! 2488: * Can't allocate from either group: try the slow path.
! 2489: * If get_slow() allocated an object for us, or if
! 2490: * no more objects are available, it will return NULL.
! 2491: * Otherwise, we need to retry.
! 2492: */
! 2493: cc = pool_cache_get_slow(cc, &s, &object, pap, flags);
! 2494: } while (cc != NULL);
! 2495:
! 2496: return object;
1.105 christos 2497: }
2498:
1.131.2.2! matt 2499: #if __GNUC_PREREQ__(3, 0)
! 2500: __attribute ((noinline))
! 2501: #endif
! 2502: pool_cache_cpu_t *
! 2503: pool_cache_put_slow(pool_cache_cpu_t *cc, int *s, void *object, paddr_t pa)
1.105 christos 2504: {
1.131.2.2! matt 2505: pcg_t *pcg, *cur;
! 2506: uint64_t ncsw;
! 2507: pool_cache_t pc;
1.105 christos 2508:
1.131.2.2! matt 2509: pc = cc->cc_cache;
! 2510: cc->cc_misses++;
1.105 christos 2511:
1.131.2.2! matt 2512: /*
! 2513: * No free slots locally. Try to grab an empty, unused
! 2514: * group from the cache.
! 2515: */
! 2516: if (!mutex_tryenter(&pc->pc_lock)) {
! 2517: ncsw = curlwp->l_ncsw;
! 2518: mutex_enter(&pc->pc_lock);
! 2519: pc->pc_contended++;
1.103 chs 2520:
1.131.2.2! matt 2521: /*
! 2522: * If we context switched while locking, then
! 2523: * our view of the per-CPU data is invalid:
! 2524: * retry.
! 2525: */
! 2526: if (curlwp->l_ncsw != ncsw) {
! 2527: mutex_exit(&pc->pc_lock);
! 2528: pool_cache_cpu_exit(cc, s);
! 2529: return pool_cache_cpu_enter(pc, s);
! 2530: }
! 2531: }
! 2532:
! 2533: if ((pcg = pc->pc_emptygroups) != NULL) {
! 2534: /*
! 2535: * If there's a empty group, release our full
! 2536: * group back to the cache. Install the empty
! 2537: * group as cc_current and return.
! 2538: */
! 2539: if ((cur = cc->cc_current) != NULL) {
! 2540: KASSERT(cur->pcg_avail == PCG_NOBJECTS);
! 2541: cur->pcg_next = pc->pc_fullgroups;
! 2542: pc->pc_fullgroups = cur;
! 2543: pc->pc_nfull++;
! 2544: }
! 2545: KASSERT(pcg->pcg_avail == 0);
! 2546: cc->cc_current = pcg;
! 2547: pc->pc_emptygroups = pcg->pcg_next;
! 2548: pc->pc_hits++;
! 2549: pc->pc_nempty--;
! 2550: mutex_exit(&pc->pc_lock);
! 2551: return cc;
! 2552: }
! 2553:
! 2554: /*
! 2555: * Nothing available locally or in cache. Take the
! 2556: * slow path and try to allocate a new group that we
! 2557: * can release to.
! 2558: */
! 2559: pc->pc_misses++;
! 2560: mutex_exit(&pc->pc_lock);
! 2561: pool_cache_cpu_exit(cc, s);
! 2562:
! 2563: /*
! 2564: * If we can't allocate a new group, just throw the
! 2565: * object away.
! 2566: */
! 2567: pcg = pool_get(&pcgpool, PR_NOWAIT);
! 2568: if (pcg == NULL) {
! 2569: pool_cache_destruct_object(pc, object);
! 2570: return NULL;
! 2571: }
! 2572: #ifdef DIAGNOSTIC
! 2573: memset(pcg, 0, sizeof(*pcg));
! 2574: #else
! 2575: pcg->pcg_avail = 0;
! 2576: #endif
! 2577:
! 2578: /*
! 2579: * Add the empty group to the cache and try again.
! 2580: */
! 2581: mutex_enter(&pc->pc_lock);
! 2582: pcg->pcg_next = pc->pc_emptygroups;
! 2583: pc->pc_emptygroups = pcg;
! 2584: pc->pc_nempty++;
! 2585: mutex_exit(&pc->pc_lock);
! 2586:
! 2587: return pool_cache_cpu_enter(pc, s);
! 2588: }
1.102 chs 2589:
1.43 thorpej 2590: /*
1.131.2.2! matt 2591: * pool_cache_put{,_paddr}:
1.43 thorpej 2592: *
1.131.2.2! matt 2593: * Put an object back to the pool cache (optionally caching the
! 2594: * physical address of the object).
1.43 thorpej 2595: */
1.101 thorpej 2596: void
1.131.2.2! matt 2597: pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa)
1.43 thorpej 2598: {
1.131.2.2! matt 2599: pool_cache_cpu_t *cc;
! 2600: pcg_t *pcg;
! 2601: int s;
1.101 thorpej 2602:
1.131.2.2! matt 2603: FREECHECK_IN(&pc->pc_freecheck, object);
1.43 thorpej 2604:
1.131.2.2! matt 2605: cc = pool_cache_cpu_enter(pc, &s);
! 2606: do {
! 2607: /* If the current group isn't full, release it there. */
! 2608: pcg = cc->cc_current;
! 2609: if (pcg != NULL && pcg->pcg_avail < PCG_NOBJECTS) {
! 2610: KASSERT(pcg->pcg_objects[pcg->pcg_avail].pcgo_va
! 2611: == NULL);
! 2612: pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object;
! 2613: pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa;
! 2614: pcg->pcg_avail++;
! 2615: cc->cc_hits++;
! 2616: pool_cache_cpu_exit(cc, &s);
! 2617: return;
! 2618: }
1.43 thorpej 2619:
1.131.2.2! matt 2620: /*
! 2621: * That failed. If the previous group is empty, swap
! 2622: * it with the current group and try again.
! 2623: */
! 2624: pcg = cc->cc_previous;
! 2625: if (pcg != NULL && pcg->pcg_avail == 0) {
! 2626: cc->cc_previous = cc->cc_current;
! 2627: cc->cc_current = pcg;
! 2628: continue;
! 2629: }
1.43 thorpej 2630:
1.131.2.2! matt 2631: /*
! 2632: * Can't free to either group: try the slow path.
! 2633: * If put_slow() releases the object for us, it
! 2634: * will return NULL. Otherwise we need to retry.
! 2635: */
! 2636: cc = pool_cache_put_slow(cc, &s, object, pa);
! 2637: } while (cc != NULL);
1.43 thorpej 2638: }
2639:
2640: /*
1.131.2.2! matt 2641: * pool_cache_xcall:
1.43 thorpej 2642: *
1.131.2.2! matt 2643: * Transfer objects from the per-CPU cache to the global cache.
! 2644: * Run within a cross-call thread.
1.43 thorpej 2645: */
2646: static void
1.131.2.2! matt 2647: pool_cache_xcall(pool_cache_t pc)
1.43 thorpej 2648: {
1.131.2.2! matt 2649: pool_cache_cpu_t *cc;
! 2650: pcg_t *prev, *cur, **list;
! 2651: int s = 0; /* XXXgcc */
! 2652:
! 2653: cc = pool_cache_cpu_enter(pc, &s);
! 2654: cur = cc->cc_current;
! 2655: cc->cc_current = NULL;
! 2656: prev = cc->cc_previous;
! 2657: cc->cc_previous = NULL;
! 2658: pool_cache_cpu_exit(cc, &s);
! 2659:
! 2660: /*
! 2661: * XXXSMP Go to splvm to prevent kernel_lock from being taken,
! 2662: * because locks at IPL_SOFTXXX are still spinlocks. Does not
! 2663: * apply to IPL_SOFTBIO. Cross-call threads do not take the
! 2664: * kernel_lock.
1.101 thorpej 2665: */
1.131.2.2! matt 2666: s = splvm();
! 2667: mutex_enter(&pc->pc_lock);
! 2668: if (cur != NULL) {
! 2669: if (cur->pcg_avail == PCG_NOBJECTS) {
! 2670: list = &pc->pc_fullgroups;
! 2671: pc->pc_nfull++;
! 2672: } else if (cur->pcg_avail == 0) {
! 2673: list = &pc->pc_emptygroups;
! 2674: pc->pc_nempty++;
! 2675: } else {
! 2676: list = &pc->pc_partgroups;
! 2677: pc->pc_npart++;
! 2678: }
! 2679: cur->pcg_next = *list;
! 2680: *list = cur;
! 2681: }
! 2682: if (prev != NULL) {
! 2683: if (prev->pcg_avail == PCG_NOBJECTS) {
! 2684: list = &pc->pc_fullgroups;
! 2685: pc->pc_nfull++;
! 2686: } else if (prev->pcg_avail == 0) {
! 2687: list = &pc->pc_emptygroups;
! 2688: pc->pc_nempty++;
! 2689: } else {
! 2690: list = &pc->pc_partgroups;
! 2691: pc->pc_npart++;
! 2692: }
! 2693: prev->pcg_next = *list;
! 2694: *list = prev;
! 2695: }
! 2696: mutex_exit(&pc->pc_lock);
! 2697: splx(s);
1.3 pk 2698: }
1.66 thorpej 2699:
2700: /*
2701: * Pool backend allocators.
2702: *
2703: * Each pool has a backend allocator that handles allocation, deallocation,
2704: * and any additional draining that might be needed.
2705: *
2706: * We provide two standard allocators:
2707: *
2708: * pool_allocator_kmem - the default when no allocator is specified
2709: *
2710: * pool_allocator_nointr - used for pools that will not be accessed
2711: * in interrupt context.
2712: */
2713: void *pool_page_alloc(struct pool *, int);
2714: void pool_page_free(struct pool *, void *);
2715:
1.112 bjh21 2716: #ifdef POOL_SUBPAGE
2717: struct pool_allocator pool_allocator_kmem_fullpage = {
2718: pool_page_alloc, pool_page_free, 0,
1.117 yamt 2719: .pa_backingmapptr = &kmem_map,
1.112 bjh21 2720: };
2721: #else
1.66 thorpej 2722: struct pool_allocator pool_allocator_kmem = {
2723: pool_page_alloc, pool_page_free, 0,
1.117 yamt 2724: .pa_backingmapptr = &kmem_map,
1.66 thorpej 2725: };
1.112 bjh21 2726: #endif
1.66 thorpej 2727:
2728: void *pool_page_alloc_nointr(struct pool *, int);
2729: void pool_page_free_nointr(struct pool *, void *);
2730:
1.112 bjh21 2731: #ifdef POOL_SUBPAGE
2732: struct pool_allocator pool_allocator_nointr_fullpage = {
2733: pool_page_alloc_nointr, pool_page_free_nointr, 0,
1.117 yamt 2734: .pa_backingmapptr = &kernel_map,
1.112 bjh21 2735: };
2736: #else
1.66 thorpej 2737: struct pool_allocator pool_allocator_nointr = {
2738: pool_page_alloc_nointr, pool_page_free_nointr, 0,
1.117 yamt 2739: .pa_backingmapptr = &kernel_map,
1.66 thorpej 2740: };
1.112 bjh21 2741: #endif
1.66 thorpej 2742:
2743: #ifdef POOL_SUBPAGE
2744: void *pool_subpage_alloc(struct pool *, int);
2745: void pool_subpage_free(struct pool *, void *);
2746:
1.112 bjh21 2747: struct pool_allocator pool_allocator_kmem = {
2748: pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
1.117 yamt 2749: .pa_backingmapptr = &kmem_map,
1.112 bjh21 2750: };
2751:
2752: void *pool_subpage_alloc_nointr(struct pool *, int);
2753: void pool_subpage_free_nointr(struct pool *, void *);
2754:
2755: struct pool_allocator pool_allocator_nointr = {
2756: pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE,
1.117 yamt 2757: .pa_backingmapptr = &kmem_map,
1.66 thorpej 2758: };
2759: #endif /* POOL_SUBPAGE */
2760:
1.117 yamt 2761: static void *
2762: pool_allocator_alloc(struct pool *pp, int flags)
1.66 thorpej 2763: {
1.117 yamt 2764: struct pool_allocator *pa = pp->pr_alloc;
1.66 thorpej 2765: void *res;
2766:
1.117 yamt 2767: res = (*pa->pa_alloc)(pp, flags);
2768: if (res == NULL && (flags & PR_WAITOK) == 0) {
1.66 thorpej 2769: /*
1.117 yamt 2770: * We only run the drain hook here if PR_NOWAIT.
2771: * In other cases, the hook will be run in
2772: * pool_reclaim().
1.66 thorpej 2773: */
1.117 yamt 2774: if (pp->pr_drain_hook != NULL) {
2775: (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags);
2776: res = (*pa->pa_alloc)(pp, flags);
1.66 thorpej 2777: }
1.117 yamt 2778: }
2779: return res;
1.66 thorpej 2780: }
2781:
1.117 yamt 2782: static void
1.66 thorpej 2783: pool_allocator_free(struct pool *pp, void *v)
2784: {
2785: struct pool_allocator *pa = pp->pr_alloc;
2786:
2787: (*pa->pa_free)(pp, v);
2788: }
2789:
2790: void *
1.124 yamt 2791: pool_page_alloc(struct pool *pp, int flags)
1.66 thorpej 2792: {
1.127 thorpej 2793: bool waitok = (flags & PR_WAITOK) ? true : false;
1.66 thorpej 2794:
1.100 yamt 2795: return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok));
1.66 thorpej 2796: }
2797:
2798: void
1.124 yamt 2799: pool_page_free(struct pool *pp, void *v)
1.66 thorpej 2800: {
2801:
1.98 yamt 2802: uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v);
2803: }
2804:
2805: static void *
1.124 yamt 2806: pool_page_alloc_meta(struct pool *pp, int flags)
1.98 yamt 2807: {
1.127 thorpej 2808: bool waitok = (flags & PR_WAITOK) ? true : false;
1.98 yamt 2809:
1.100 yamt 2810: return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok));
1.98 yamt 2811: }
2812:
2813: static void
1.124 yamt 2814: pool_page_free_meta(struct pool *pp, void *v)
1.98 yamt 2815: {
2816:
1.100 yamt 2817: uvm_km_free_poolpage(kmem_map, (vaddr_t) v);
1.66 thorpej 2818: }
2819:
2820: #ifdef POOL_SUBPAGE
2821: /* Sub-page allocator, for machines with large hardware pages. */
2822: void *
2823: pool_subpage_alloc(struct pool *pp, int flags)
2824: {
1.131.2.2! matt 2825: return pool_get(&psppool, flags);
1.66 thorpej 2826: }
2827:
2828: void
2829: pool_subpage_free(struct pool *pp, void *v)
2830: {
2831: pool_put(&psppool, v);
2832: }
2833:
2834: /* We don't provide a real nointr allocator. Maybe later. */
2835: void *
1.112 bjh21 2836: pool_subpage_alloc_nointr(struct pool *pp, int flags)
1.66 thorpej 2837: {
2838:
2839: return (pool_subpage_alloc(pp, flags));
2840: }
2841:
2842: void
1.112 bjh21 2843: pool_subpage_free_nointr(struct pool *pp, void *v)
1.66 thorpej 2844: {
2845:
2846: pool_subpage_free(pp, v);
2847: }
1.112 bjh21 2848: #endif /* POOL_SUBPAGE */
1.66 thorpej 2849: void *
1.124 yamt 2850: pool_page_alloc_nointr(struct pool *pp, int flags)
1.66 thorpej 2851: {
1.127 thorpej 2852: bool waitok = (flags & PR_WAITOK) ? true : false;
1.66 thorpej 2853:
1.100 yamt 2854: return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok));
1.66 thorpej 2855: }
2856:
2857: void
1.124 yamt 2858: pool_page_free_nointr(struct pool *pp, void *v)
1.66 thorpej 2859: {
2860:
1.98 yamt 2861: uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v);
1.66 thorpej 2862: }
CVSweb <webmaster@jp.NetBSD.org>