Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.87.2.5 retrieving revision 1.94 diff -u -p -r1.87.2.5 -r1.94 --- src/sys/kern/subr_pool.c 2005/04/01 14:30:56 1.87.2.5 +++ src/sys/kern/subr_pool.c 2004/04/25 16:42:41 1.94 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.87.2.5 2005/04/01 14:30:56 skrll Exp $ */ +/* $NetBSD: subr_pool.c,v 1.94 2004/04/25 16:42:41 simonb Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.87.2.5 2005/04/01 14:30:56 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.94 2004/04/25 16:42:41 simonb Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -73,23 +73,13 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); /* Private pool for page header structures */ -#define PHPOOL_MAX 8 -static struct pool phpool[PHPOOL_MAX]; -#define PHPOOL_FREELIST_NELEM(idx) (((idx) == 0) ? 0 : (1 << (idx))) +static struct pool phpool; #ifdef POOL_SUBPAGE /* Pool of subpages for use by normal pools. */ static struct pool psppool; #endif -static void *pool_page_alloc_meta(struct pool *, int); -static void pool_page_free_meta(struct pool *, void *); - -/* allocator for pool metadata */ -static struct pool_allocator pool_allocator_meta = { - pool_page_alloc_meta, pool_page_free_meta -}; - /* # of seconds to retain page after last use */ int pool_inactive_time = 10; @@ -99,39 +89,17 @@ static struct pool *drainpp; /* This spin lock protects both pool_head and drainpp. */ struct simplelock pool_head_slock = SIMPLELOCK_INITIALIZER; -typedef uint8_t pool_item_freelist_t; - struct pool_item_header { /* Page headers */ LIST_ENTRY(pool_item_header) ph_pagelist; /* pool page list */ + TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */ SPLAY_ENTRY(pool_item_header) ph_node; /* Off-page page headers */ + unsigned int ph_nmissing; /* # of chunks in use */ caddr_t ph_page; /* this page's address */ struct timeval ph_time; /* last referenced */ - union { - /* !PR_NOTOUCH */ - struct { - TAILQ_HEAD(, pool_item) - phu_itemlist; /* chunk list for this page */ - } phu_normal; - /* PR_NOTOUCH */ - struct { - uint16_t - phu_off; /* start offset in page */ - pool_item_freelist_t - phu_firstfree; /* first free item */ - /* - * XXX it might be better to use - * a simple bitmap and ffs(3) - */ - } phu_notouch; - } ph_u; - uint16_t ph_nmissing; /* # of chunks in use */ }; -#define ph_itemlist ph_u.phu_normal.phu_itemlist -#define ph_off ph_u.phu_notouch.phu_off -#define ph_firstfree ph_u.phu_notouch.phu_firstfree struct pool_item { #ifdef DIAGNOSTIC @@ -184,7 +152,7 @@ static void pool_update_curpage(struct p void *pool_allocator_alloc(struct pool *, int); void pool_allocator_free(struct pool *, void *); -static void pool_print_pagelist(struct pool *, struct pool_pagelist *, +static void pool_print_pagelist(struct pool_pagelist *, void (*)(const char *, ...)); static void pool_print1(struct pool *, const char *, void (*)(const char *, ...)); @@ -311,50 +279,6 @@ pr_enter_check(struct pool *pp, void (*p #endif /* POOL_DIAGNOSTIC */ static __inline int -pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, - const void *v) -{ - const char *cp = v; - int idx; - - KASSERT(pp->pr_roflags & PR_NOTOUCH); - idx = (cp - ph->ph_page - ph->ph_off) / pp->pr_size; - KASSERT(idx < pp->pr_itemsperpage); - return idx; -} - -#define PR_FREELIST_ALIGN(p) \ - roundup((uintptr_t)(p), sizeof(pool_item_freelist_t)) -#define PR_FREELIST(ph) ((pool_item_freelist_t *)PR_FREELIST_ALIGN((ph) + 1)) -#define PR_INDEX_USED ((pool_item_freelist_t)-1) -#define PR_INDEX_EOL ((pool_item_freelist_t)-2) - -static __inline void -pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, - void *obj) -{ - int idx = pr_item_notouch_index(pp, ph, obj); - pool_item_freelist_t *freelist = PR_FREELIST(ph); - - KASSERT(freelist[idx] == PR_INDEX_USED); - freelist[idx] = ph->ph_firstfree; - ph->ph_firstfree = idx; -} - -static __inline void * -pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) -{ - int idx = ph->ph_firstfree; - pool_item_freelist_t *freelist = PR_FREELIST(ph); - - KASSERT(freelist[idx] != PR_INDEX_USED); - ph->ph_firstfree = freelist[idx]; - freelist[idx] = PR_INDEX_USED; - - return ph->ph_page + ph->ph_off + idx * pp->pr_size; -} - -static __inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { if (a->ph_page < b->ph_page) @@ -422,7 +346,7 @@ pr_rmpage(struct pool *pp, struct pool_i pool_allocator_free(pp, ph->ph_page); if ((pp->pr_roflags & PR_PHINPAGE) == 0) { s = splvm(); - pool_put(pp->pr_phpool, ph); + pool_put(&phpool, ph); splx(s); } } @@ -461,9 +385,6 @@ pool_init(struct pool *pp, size_t size, size_t trysize, phsize; int s; - KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >= - PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1)); - #ifdef POOL_DIAGNOSTIC /* * Always log if POOL_DIAGNOSTIC is defined. @@ -569,9 +490,8 @@ pool_init(struct pool *pp, size_t size, /* See the comment below about reserved bytes. */ trysize = palloc->pa_pagesz - ((align - ioff) % align); phsize = ALIGN(sizeof(struct pool_item_header)); - if ((pp->pr_roflags & PR_NOTOUCH) == 0 && - (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) || - trysize / pp->pr_size == (trysize - phsize) / pp->pr_size)) { + if (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) || + trysize / pp->pr_size == (trysize - phsize) / pp->pr_size) { /* Use the end of the page for the page header */ pp->pr_roflags |= PR_PHINPAGE; pp->pr_phoffset = off = palloc->pa_pagesz - phsize; @@ -589,30 +509,6 @@ pool_init(struct pool *pp, size_t size, */ pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size; KASSERT(pp->pr_itemsperpage != 0); - if ((pp->pr_roflags & PR_NOTOUCH)) { - int idx; - - for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx); - idx++) { - /* nothing */ - } - if (idx >= PHPOOL_MAX) { - /* - * if you see this panic, consider to tweak - * PHPOOL_MAX and PHPOOL_FREELIST_NELEM. - */ - panic("%s: too large itemsperpage(%d) for PR_NOTOUCH", - pp->pr_wchan, pp->pr_itemsperpage); - } - pp->pr_phpool = &phpool[idx]; - } else if ((pp->pr_roflags & PR_PHINPAGE) == 0) { - pp->pr_phpool = &phpool[0]; - } -#if defined(DIAGNOSTIC) - else { - pp->pr_phpool = NULL; - } -#endif /* * Use the slack between the chunks and the page header @@ -651,30 +547,18 @@ pool_init(struct pool *pp, size_t size, * haven't done so yet. * XXX LOCKING. */ - if (phpool[0].pr_size == 0) { - int idx; - for (idx = 0; idx < PHPOOL_MAX; idx++) { - static char phpool_names[PHPOOL_MAX][6+1+6+1]; - int nelem; - size_t sz; - - nelem = PHPOOL_FREELIST_NELEM(idx); - snprintf(phpool_names[idx], sizeof(phpool_names[idx]), - "phpool-%d", nelem); - sz = sizeof(struct pool_item_header); - if (nelem) { - sz = PR_FREELIST_ALIGN(sz) - + nelem * sizeof(pool_item_freelist_t); - } - pool_init(&phpool[idx], sz, 0, 0, 0, - phpool_names[idx], &pool_allocator_meta); - } + if (phpool.pr_size == 0) { #ifdef POOL_SUBPAGE + pool_init(&phpool, sizeof(struct pool_item_header), 0, 0, 0, + "phpool", &pool_allocator_kmem); pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, - PR_RECURSIVE, "psppool", &pool_allocator_meta); + PR_RECURSIVE, "psppool", &pool_allocator_kmem); +#else + pool_init(&phpool, sizeof(struct pool_item_header), 0, 0, + 0, "phpool", NULL); #endif pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0, - 0, "pcgpool", &pool_allocator_meta); + 0, "pcgpool", NULL); } /* Insert into the list of all pools. */ @@ -764,7 +648,7 @@ pool_alloc_item_header(struct pool *pp, ph = (struct pool_item_header *) (storage + pp->pr_phoffset); else { s = splvm(); - ph = pool_get(pp->pr_phpool, flags); + ph = pool_get(&phpool, flags); splx(s); } @@ -786,9 +670,6 @@ pool_get(struct pool *pp, int flags) void *v; #ifdef DIAGNOSTIC - if (__predict_false(pp->pr_itemsperpage == 0)) - panic("pool_get: pool %p: pr_itemsperpage is zero, " - "pool not initialized?", pp); if (__predict_false(curlwp == NULL && doing_shutdown == 0 && (flags & PR_WAITOK) != 0)) panic("pool_get: %s: must have NOWAIT", pp->pr_wchan); @@ -931,53 +812,38 @@ pool_get(struct pool *pp, int flags) /* Start the allocation process over. */ goto startover; } - if (pp->pr_roflags & PR_NOTOUCH) { -#ifdef DIAGNOSTIC - if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) { - pr_leave(pp); - simple_unlock(&pp->pr_slock); - panic("pool_get: %s: page empty", pp->pr_wchan); - } -#endif - v = pr_item_notouch_get(pp, ph); -#ifdef POOL_DIAGNOSTIC - pr_log(pp, v, PRLOG_GET, file, line); -#endif - } else { - v = pi = TAILQ_FIRST(&ph->ph_itemlist); - if (__predict_false(v == NULL)) { - pr_leave(pp); - simple_unlock(&pp->pr_slock); - panic("pool_get: %s: page empty", pp->pr_wchan); - } + if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) { + pr_leave(pp); + simple_unlock(&pp->pr_slock); + panic("pool_get: %s: page empty", pp->pr_wchan); + } #ifdef DIAGNOSTIC - if (__predict_false(pp->pr_nitems == 0)) { - pr_leave(pp); - simple_unlock(&pp->pr_slock); - printf("pool_get: %s: items on itemlist, nitems %u\n", - pp->pr_wchan, pp->pr_nitems); - panic("pool_get: nitems inconsistent"); - } + if (__predict_false(pp->pr_nitems == 0)) { + pr_leave(pp); + simple_unlock(&pp->pr_slock); + printf("pool_get: %s: items on itemlist, nitems %u\n", + pp->pr_wchan, pp->pr_nitems); + panic("pool_get: nitems inconsistent"); + } #endif #ifdef POOL_DIAGNOSTIC - pr_log(pp, v, PRLOG_GET, file, line); + pr_log(pp, v, PRLOG_GET, file, line); #endif #ifdef DIAGNOSTIC - if (__predict_false(pi->pi_magic != PI_MAGIC)) { - pr_printlog(pp, pi, printf); - panic("pool_get(%s): free list modified: " - "magic=%x; page %p; item addr %p\n", - pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); - } + if (__predict_false(pi->pi_magic != PI_MAGIC)) { + pr_printlog(pp, pi, printf); + panic("pool_get(%s): free list modified: magic=%x; page %p;" + " item addr %p\n", + pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); + } #endif - /* - * Remove from item list. - */ - TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); - } + /* + * Remove from item list. + */ + TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list); pp->pr_nitems--; pp->pr_nout++; if (ph->ph_nmissing == 0) { @@ -995,10 +861,9 @@ pool_get(struct pool *pp, int flags) LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); } ph->ph_nmissing++; - if (ph->ph_nmissing == pp->pr_itemsperpage) { + if (TAILQ_EMPTY(&ph->ph_itemlist)) { #ifdef DIAGNOSTIC - if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 && - !TAILQ_EMPTY(&ph->ph_itemlist))) { + if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) { pr_leave(pp); simple_unlock(&pp->pr_slock); panic("pool_get: %s: nmissing inconsistent", @@ -1071,24 +936,20 @@ pool_do_put(struct pool *pp, void *v) /* * Return to item list. */ - if (pp->pr_roflags & PR_NOTOUCH) { - pr_item_notouch_put(pp, ph, v); - } else { #ifdef DIAGNOSTIC - pi->pi_magic = PI_MAGIC; + pi->pi_magic = PI_MAGIC; #endif #ifdef DEBUG - { - int i, *ip = v; + { + int i, *ip = v; - for (i = 0; i < pp->pr_size / sizeof(int); i++) { - *ip++ = PI_MAGIC; - } + for (i = 0; i < pp->pr_size / sizeof(int); i++) { + *ip++ = PI_MAGIC; } + } #endif - TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); - } + TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); KDASSERT(ph->ph_nmissing != 0); ph->ph_nmissing--; pp->pr_nput++; @@ -1111,7 +972,8 @@ pool_do_put(struct pool *pp, void *v) * If this page is now empty, do one of two things: * * (1) If we have more pages than the page high water mark, - * free the page back to the system. ONLY CONSIDER + * or if we are flagged as immediately freeing back idle + * pages, free the page back to the system. ONLY CONSIDER * FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE * CLAIM. * @@ -1124,6 +986,7 @@ pool_do_put(struct pool *pp, void *v) pp->pr_nidle++; if (pp->pr_npages > pp->pr_minpages && (pp->pr_npages > pp->pr_maxpages || + (pp->pr_roflags & PR_IMMEDRELEASE) != 0 || (pp->pr_alloc->pa_flags & PA_WANT) != 0)) { simple_unlock(&pp->pr_slock); pr_rmpage(pp, ph, NULL); @@ -1290,28 +1153,17 @@ pool_prime_page(struct pool *pp, caddr_t n = pp->pr_itemsperpage; pp->pr_nitems += n; - if (pp->pr_roflags & PR_NOTOUCH) { - pool_item_freelist_t *freelist = PR_FREELIST(ph); - int i; - - ph->ph_off = cp - storage; - ph->ph_firstfree = 0; - for (i = 0; i < n - 1; i++) - freelist[i] = i + 1; - freelist[n - 1] = PR_INDEX_EOL; - } else { - while (n--) { - pi = (struct pool_item *)cp; + while (n--) { + pi = (struct pool_item *)cp; - KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); + KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); - /* Insert on page list */ - TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); + /* Insert on page list */ + TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); #ifdef DIAGNOSTIC - pi->pi_magic = PI_MAGIC; + pi->pi_magic = PI_MAGIC; #endif - cp = (caddr_t)(cp + pp->pr_size); - } + cp = (caddr_t)(cp + pp->pr_size); } /* @@ -1510,7 +1362,7 @@ pool_reclaim(struct pool *pp) continue; } s = splvm(); - pool_put(pp->pr_phpool, ph); + pool_put(&phpool, ph); splx(s); } @@ -1594,8 +1446,7 @@ pool_printit(struct pool *pp, const char } static void -pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl, - void (*pr)(const char *, ...)) +pool_print_pagelist(struct pool_pagelist *pl, void (*pr)(const char *, ...)) { struct pool_item_header *ph; #ifdef DIAGNOSTIC @@ -1608,12 +1459,10 @@ pool_print_pagelist(struct pool *pp, str (u_long)ph->ph_time.tv_sec, (u_long)ph->ph_time.tv_usec); #ifdef DIAGNOSTIC - if (!(pp->pr_roflags & PR_NOTOUCH)) { - TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) { - if (pi->pi_magic != PI_MAGIC) { - (*pr)("\t\t\titem %p, magic 0x%x\n", - pi, pi->pi_magic); - } + TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) { + if (pi->pi_magic != PI_MAGIC) { + (*pr)("\t\t\titem %p, magic 0x%x\n", + pi, pi->pi_magic); } } #endif @@ -1657,13 +1506,13 @@ pool_print1(struct pool *pp, const char if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) (*pr)("\n\tempty page list:\n"); - pool_print_pagelist(pp, &pp->pr_emptypages, pr); + pool_print_pagelist(&pp->pr_emptypages, pr); if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL) (*pr)("\n\tfull page list:\n"); - pool_print_pagelist(pp, &pp->pr_fullpages, pr); + pool_print_pagelist(&pp->pr_fullpages, pr); if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL) (*pr)("\n\tpartial-page list:\n"); - pool_print_pagelist(pp, &pp->pr_partpages, pr); + pool_print_pagelist(&pp->pr_partpages, pr); if (pp->pr_curpage == NULL) (*pr)("\tno current page\n"); @@ -1729,9 +1578,6 @@ pool_chk_page(struct pool *pp, const cha return 1; } - if ((pp->pr_roflags & PR_NOTOUCH) != 0) - return 0; - for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0; pi != NULL; pi = TAILQ_NEXT(pi,pi_list), n++) { @@ -2251,29 +2097,14 @@ pool_page_alloc(struct pool *pp, int fla { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); + return ((void *) uvm_km_alloc_poolpage(waitok)); } void pool_page_free(struct pool *pp, void *v) { - uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v); -} - -static void * -pool_page_alloc_meta(struct pool *pp, int flags) -{ - boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - - return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); -} - -static void -pool_page_free_meta(struct pool *pp, void *v) -{ - - uvm_km_free_poolpage(kmem_map, (vaddr_t) v); + uvm_km_free_poolpage((vaddr_t) v); } #ifdef POOL_SUBPAGE @@ -2318,13 +2149,14 @@ pool_page_alloc_nointr(struct pool *pp, { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); + return ((void *) uvm_km_alloc_poolpage1(kernel_map, + uvm.kernel_object, waitok)); } void pool_page_free_nointr(struct pool *pp, void *v) { - uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); + uvm_km_free_poolpage1(kernel_map, (vaddr_t) v); } #endif /* POOL_SUBPAGE */