Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.68 retrieving revision 1.87 diff -u -p -r1.68 -r1.87 --- src/sys/kern/subr_pool.c 2002/03/08 21:41:59 1.68 +++ src/sys/kern/subr_pool.c 2003/04/09 18:22:13 1.87 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.68 2002/03/08 21:41:59 thorpej Exp $ */ +/* $NetBSD: subr_pool.c,v 1.87 2003/04/09 18:22:13 thorpej Exp $ */ /*- * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.68 2002/03/08 21:41:59 thorpej Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.87 2003/04/09 18:22:13 thorpej Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -94,7 +94,7 @@ struct pool_item_header { TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */ LIST_ENTRY(pool_item_header) ph_hashlist; /* Off-page page headers */ - int ph_nmissing; /* # of chunks in use */ + unsigned int ph_nmissing; /* # of chunks in use */ caddr_t ph_page; /* this page's address */ struct timeval ph_time; /* last referenced */ }; @@ -102,9 +102,9 @@ TAILQ_HEAD(pool_pagelist,pool_item_heade struct pool_item { #ifdef DIAGNOSTIC - int pi_magic; + u_int pi_magic; #endif -#define PI_MAGIC 0xdeadbeef +#define PI_MAGIC 0xdeadbeefU /* Other entries use only this list entry */ TAILQ_ENTRY(pool_item) pi_list; }; @@ -145,16 +145,6 @@ struct pool_item { /* The cache group pool. */ static struct pool pcgpool; -/* The pool cache group. */ -#define PCG_NOBJECTS 16 -struct pool_cache_group { - TAILQ_ENTRY(pool_cache_group) - pcg_list; /* link in the pool cache's group list */ - u_int pcg_avail; /* # available objects */ - /* pointers to the objects */ - void *pcg_objects[PCG_NOBJECTS]; -}; - static void pool_cache_reclaim(struct pool_cache *); static int pool_catchup(struct pool *); @@ -179,6 +169,7 @@ struct pool_log { void *pl_addr; }; +#ifdef POOL_DIAGNOSTIC /* Number of entries in pool log buffers */ #ifndef POOL_LOGSIZE #define POOL_LOGSIZE 10 @@ -186,7 +177,6 @@ struct pool_log { int pool_logsize = POOL_LOGSIZE; -#ifdef POOL_DIAGNOSTIC static __inline void pr_log(struct pool *pp, void *v, int action, const char *file, long line) { @@ -339,7 +329,7 @@ pr_rmpage(struct pool *pp, struct pool_i pool_allocator_free(pp, ph->ph_page); if ((pp->pr_roflags & PR_PHINPAGE) == 0) { LIST_REMOVE(ph, ph_hashlist); - s = splhigh(); + s = splvm(); pool_put(&phpool, ph); splx(s); } @@ -425,7 +415,7 @@ pool_init(struct pool *pp, size_t size, if (size < sizeof(struct pool_item)) size = sizeof(struct pool_item); - size = ALIGN(size); + size = roundup(size, align); #ifdef DIAGNOSTIC if (size > palloc->pa_pagesz) panic("pool_init: pool item size (%lu) too large", @@ -574,15 +564,14 @@ pool_destroy(struct pool *pp) #ifdef DIAGNOSTIC if (pp->pr_nout != 0) { pr_printlog(pp, NULL, printf); - panic("pool_destroy: pool busy: still out: %u\n", + panic("pool_destroy: pool busy: still out: %u", pp->pr_nout); } #endif /* Remove all pages */ - if ((pp->pr_roflags & PR_STATIC) == 0) - while ((ph = TAILQ_FIRST(&pp->pr_pagelist)) != NULL) - pr_rmpage(pp, ph, NULL); + while ((ph = TAILQ_FIRST(&pp->pr_pagelist)) != NULL) + pr_rmpage(pp, ph, NULL); /* Remove from global pool list */ simple_lock(&pool_head_slock); @@ -622,7 +611,7 @@ pool_alloc_item_header(struct pool *pp, if ((pp->pr_roflags & PR_PHINPAGE) != 0) ph = (struct pool_item_header *) (storage + pp->pr_phoffset); else { - s = splhigh(); + s = splvm(); ph = pool_get(&phpool, flags); splx(s); } @@ -645,15 +634,9 @@ pool_get(struct pool *pp, int flags) void *v; #ifdef DIAGNOSTIC - if (__predict_false((pp->pr_roflags & PR_STATIC) && - (flags & PR_MALLOCOK))) { - pr_printlog(pp, NULL, printf); - panic("pool_get: static"); - } - - if (__predict_false(curproc == NULL && doing_shutdown == 0 && + if (__predict_false(curlwp == NULL && doing_shutdown == 0 && (flags & PR_WAITOK) != 0)) - panic("pool_get: must have NOWAIT"); + panic("pool_get: %s: must have NOWAIT", pp->pr_wchan); #ifdef LOCKDEBUG if (flags & PR_WAITOK) @@ -732,7 +715,7 @@ pool_get(struct pool *pp, int flags) simple_unlock(&pp->pr_slock); printf("pool_get: %s: curpage NULL, nitems %u\n", pp->pr_wchan, pp->pr_nitems); - panic("pool_get: nitems inconsistent\n"); + panic("pool_get: nitems inconsistent"); } #endif @@ -802,7 +785,7 @@ pool_get(struct pool *pp, int flags) simple_unlock(&pp->pr_slock); printf("pool_get: %s: items on itemlist, nitems %u\n", pp->pr_wchan, pp->pr_nitems); - panic("pool_get: nitems inconsistent\n"); + panic("pool_get: nitems inconsistent"); } #endif @@ -933,6 +916,7 @@ pool_do_put(struct pool *pp, void *v) #endif TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); + KDASSERT(ph->ph_nmissing != 0); ph->ph_nmissing--; pp->pr_nput++; pp->pr_nitems++; @@ -966,7 +950,8 @@ pool_do_put(struct pool *pp, void *v) */ if (ph->ph_nmissing == 0) { pp->pr_nidle++; - if (pp->pr_npages > pp->pr_maxpages) { + if (pp->pr_npages > pp->pr_maxpages || + (pp->pr_alloc->pa_flags & PA_WANT) != 0) { pr_rmpage(pp, ph, NULL); } else { TAILQ_REMOVE(&pp->pr_pagelist, ph, ph_pagelist); @@ -1052,9 +1037,9 @@ pool_put(struct pool *pp, void *v) int pool_prime(struct pool *pp, int n) { - struct pool_item_header *ph; + struct pool_item_header *ph = NULL; caddr_t cp; - int newpages, error = 0; + int newpages; simple_lock(&pp->pr_slock); @@ -1068,7 +1053,6 @@ pool_prime(struct pool *pp, int n) simple_lock(&pp->pr_slock); if (__predict_false(cp == NULL || ph == NULL)) { - error = ENOMEM; if (cp != NULL) pool_allocator_free(pp, cp); break; @@ -1142,6 +1126,8 @@ pool_prime_page(struct pool *pp, caddr_t while (n--) { pi = (struct pool_item *)cp; + KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); + /* Insert on page list */ TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list); #ifdef DIAGNOSTIC @@ -1166,30 +1152,16 @@ pool_prime_page(struct pool *pp, caddr_t * * Note 1, we never wait for memory here, we let the caller decide what to do. * - * Note 2, this doesn't work with static pools. - * - * Note 3, we must be called with the pool already locked, and we return + * Note 2, we must be called with the pool already locked, and we return * with it locked. */ static int pool_catchup(struct pool *pp) { - struct pool_item_header *ph; + struct pool_item_header *ph = NULL; caddr_t cp; int error = 0; - if (pp->pr_roflags & PR_STATIC) { - /* - * We dropped below the low water mark, and this is not a - * good thing. Log a warning. - * - * XXX: rate-limit this? - */ - printf("WARNING: static pool `%s' dropped below low water " - "mark\n", pp->pr_wchan); - return (0); - } - while (POOL_NEEDS_CATCHUP(pp)) { /* * Call the page back-end allocator for more memory. @@ -1218,7 +1190,6 @@ pool_catchup(struct pool *pp) void pool_setlowat(struct pool *pp, int n) { - int error; simple_lock(&pp->pr_slock); @@ -1228,7 +1199,7 @@ pool_setlowat(struct pool *pp, int n) : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; /* Make sure we're caught up with the newly-set low water mark. */ - if (POOL_NEEDS_CATCHUP(pp) && (error = pool_catchup(pp) != 0)) { + if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { /* * XXX: Should we log a warning? Should we set up a timeout * to try again in a second or so? The latter could break @@ -1291,9 +1262,6 @@ pool_reclaim(struct pool *pp) struct pool_pagelist pq; int s; - if (pp->pr_roflags & PR_STATIC) - return (0); - if (pp->pr_drain_hook != NULL) { /* * The drain hook must be called with the pool unlocked. @@ -1354,7 +1322,7 @@ pool_reclaim(struct pool *pp) continue; } LIST_REMOVE(ph, ph_hashlist); - s = splhigh(); + s = splvm(); pool_put(&phpool, ph); splx(s); } @@ -1457,7 +1425,6 @@ pool_print1(struct pool *pp, const char print_pagelist = 1; if (c == 'c') print_cache = 1; - modif++; } (*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n", @@ -1521,8 +1488,18 @@ pool_print1(struct pool *pp, const char pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems); TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) { (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); - for (i = 0; i < PCG_NOBJECTS; i++) - (*pr)("\t\t\t%p\n", pcg->pcg_objects[i]); + for (i = 0; i < PCG_NOBJECTS; i++) { + if (pcg->pcg_objects[i].pcgo_pa != + POOL_PADDR_INVALID) { + (*pr)("\t\t\t%p, 0x%llx\n", + pcg->pcg_objects[i].pcgo_va, + (unsigned long long) + pcg->pcg_objects[i].pcgo_pa); + } else { + (*pr)("\t\t\t%p\n", + pcg->pcg_objects[i].pcgo_va); + } + } } } @@ -1651,7 +1628,7 @@ pool_cache_destroy(struct pool_cache *pc } static __inline void * -pcg_get(struct pool_cache_group *pcg) +pcg_get(struct pool_cache_group *pcg, paddr_t *pap) { void *object; u_int idx; @@ -1660,32 +1637,36 @@ pcg_get(struct pool_cache_group *pcg) KASSERT(pcg->pcg_avail != 0); idx = --pcg->pcg_avail; - KASSERT(pcg->pcg_objects[idx] != NULL); - object = pcg->pcg_objects[idx]; - pcg->pcg_objects[idx] = NULL; + KASSERT(pcg->pcg_objects[idx].pcgo_va != NULL); + object = pcg->pcg_objects[idx].pcgo_va; + if (pap != NULL) + *pap = pcg->pcg_objects[idx].pcgo_pa; + pcg->pcg_objects[idx].pcgo_va = NULL; return (object); } static __inline void -pcg_put(struct pool_cache_group *pcg, void *object) +pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa) { u_int idx; KASSERT(pcg->pcg_avail < PCG_NOBJECTS); idx = pcg->pcg_avail++; - KASSERT(pcg->pcg_objects[idx] == NULL); - pcg->pcg_objects[idx] = object; + KASSERT(pcg->pcg_objects[idx].pcgo_va == NULL); + pcg->pcg_objects[idx].pcgo_va = object; + pcg->pcg_objects[idx].pcgo_pa = pa; } /* - * pool_cache_get: + * pool_cache_get{,_paddr}: * - * Get an object from a pool cache. + * Get an object from a pool cache (optionally returning + * the physical address of the object). */ void * -pool_cache_get(struct pool_cache *pc, int flags) +pool_cache_get_paddr(struct pool_cache *pc, int flags, paddr_t *pap) { struct pool_cache_group *pcg; void *object; @@ -1720,13 +1701,20 @@ pool_cache_get(struct pool_cache *pc, in return (NULL); } } + if (object != NULL && pap != NULL) { +#ifdef POOL_VTOPHYS + *pap = POOL_VTOPHYS(object); +#else + *pap = POOL_PADDR_INVALID; +#endif + } return (object); } have_group: pc->pc_hits++; pc->pc_nitems--; - object = pcg_get(pcg); + object = pcg_get(pcg, pap); if (pcg->pcg_avail == 0) pc->pc_allocfrom = NULL; @@ -1737,12 +1725,13 @@ pool_cache_get(struct pool_cache *pc, in } /* - * pool_cache_put: + * pool_cache_put{,_paddr}: * - * Put an object back to the pool cache. + * Put an object back to the pool cache (optionally caching the + * physical address of the object). */ void -pool_cache_put(struct pool_cache *pc, void *object) +pool_cache_put_paddr(struct pool_cache *pc, void *object, paddr_t pa) { struct pool_cache_group *pcg; int s; @@ -1785,7 +1774,7 @@ pool_cache_put(struct pool_cache *pc, vo have_group: pc->pc_nitems++; - pcg_put(pcg, object); + pcg_put(pcg, object, pa); if (pcg->pcg_avail == PCG_NOBJECTS) pc->pc_freeto = NULL; @@ -1827,7 +1816,7 @@ pool_cache_do_invalidate(struct pool_cac npcg = TAILQ_NEXT(pcg, pcg_list); while (pcg->pcg_avail != 0) { pc->pc_nitems--; - object = pcg_get(pcg); + object = pcg_get(pcg, NULL); if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg) pc->pc_allocfrom = NULL; if (pc->pc_dtor != NULL) @@ -1977,9 +1966,9 @@ pool_allocator_alloc(struct pool *org, i TAILQ_INSERT_TAIL(&pa->pa_list, pp, pr_alloc_list); if (pp == org) continue; - simple_unlock(&pa->pa_list); + simple_unlock(&pa->pa_slock); freed = pool_reclaim(pp); - simple_lock(&pa->pa_list); + simple_lock(&pa->pa_slock); } while ((pp = TAILQ_FIRST(&pa->pa_list)) != start && freed == 0); @@ -2020,6 +2009,7 @@ pool_allocator_free(struct pool *pp, voi pp->pr_flags &= ~PR_WANTED; wakeup(pp); } + simple_unlock(&pp->pr_slock); } pa->pa_flags &= ~PA_WANT; simple_unlock(&pa->pa_slock);