Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.117 retrieving revision 1.126 diff -u -p -r1.117 -r1.126 --- src/sys/kern/subr_pool.c 2006/05/25 14:27:28 1.117 +++ src/sys/kern/subr_pool.c 2007/02/21 23:00:05 1.126 @@ -1,7 +1,7 @@ -/* $NetBSD: subr_pool.c,v 1.117 2006/05/25 14:27:28 yamt Exp $ */ +/* $NetBSD: subr_pool.c,v 1.126 2007/02/21 23:00:05 thorpej Exp $ */ /*- - * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc. + * Copyright (c) 1997, 1999, 2000, 2002 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.117 2006/05/25 14:27:28 yamt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.126 2007/02/21 23:00:05 thorpej Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -53,6 +53,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, #include #include #include +#include #include @@ -364,10 +365,16 @@ pr_item_notouch_get(const struct pool *p static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { + + /* + * we consider pool_item_header with smaller ph_page bigger. + * (this unnatural ordering is for the benefit of pr_find_pagehead.) + */ + if (a->ph_page < b->ph_page) - return (-1); - else if (a->ph_page > b->ph_page) return (1); + else if (a->ph_page > b->ph_page) + return (-1); else return (0); } @@ -376,18 +383,38 @@ SPLAY_PROTOTYPE(phtree, pool_item_header SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); /* - * Return the pool page header based on page address. + * Return the pool page header based on item address. */ static inline struct pool_item_header * -pr_find_pagehead(struct pool *pp, caddr_t page) +pr_find_pagehead(struct pool *pp, void *v) { struct pool_item_header *ph, tmp; - if ((pp->pr_roflags & PR_PHINPAGE) != 0) - return ((struct pool_item_header *)(page + pp->pr_phoffset)); + if ((pp->pr_roflags & PR_NOALIGN) != 0) { + tmp.ph_page = (caddr_t)(uintptr_t)v; + ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); + if (ph == NULL) { + ph = SPLAY_ROOT(&pp->pr_phtree); + if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { + ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); + } + KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); + } + } else { + caddr_t page = + (caddr_t)((uintptr_t)v & pp->pr_alloc->pa_pagemask); + + if ((pp->pr_roflags & PR_PHINPAGE) != 0) { + ph = (void *)(page + pp->pr_phoffset); + } else { + tmp.ph_page = page; + ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); + } + } - tmp.ph_page = page; - ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); + KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || + (ph->ph_page <= (char *)v && + (char *)v < ph->ph_page + pp->pr_alloc->pa_pagesz)); return ph; } @@ -447,7 +474,7 @@ pr_rmpage(struct pool *pp, struct pool_i pool_update_curpage(pp); } -static boolean_t +static bool pa_starved_p(struct pool_allocator *pa) { @@ -611,14 +638,13 @@ pool_init(struct pool *pp, size_t size, if (align == 0) align = ALIGN(1); - if (size < sizeof(struct pool_item)) + if ((flags & PR_NOTOUCH) == 0 && size < sizeof(struct pool_item)) size = sizeof(struct pool_item); size = roundup(size, align); #ifdef DIAGNOSTIC if (size > palloc->pa_pagesz) - panic("pool_init: pool item size (%lu) too large", - (u_long)size); + panic("pool_init: pool item size (%zu) too large", size); #endif /* @@ -649,6 +675,7 @@ pool_init(struct pool *pp, size_t size, pp->pr_hardlimit_warning_last.tv_usec = 0; pp->pr_drain_hook = NULL; pp->pr_drain_hook_arg = NULL; + pp->pr_freecheck = NULL; /* * Decide whether to put the page header off page to avoid @@ -667,7 +694,7 @@ pool_init(struct pool *pp, size_t size, /* See the comment below about reserved bytes. */ trysize = palloc->pa_pagesz - ((align - ioff) % align); phsize = ALIGN(sizeof(struct pool_item_header)); - if ((pp->pr_roflags & PR_NOTOUCH) == 0 && + if ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 && (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) || trysize / pp->pr_size == (trysize - phsize) / pp->pr_size)) { /* Use the end of the page for the page header */ @@ -903,8 +930,7 @@ pool_get(struct pool *pp, int flags) #endif /* DIAGNOSTIC */ #ifdef LOCKDEBUG if (flags & PR_WAITOK) - simple_lock_only_held(NULL, "pool_get(PR_WAITOK)"); - SCHED_ASSERT_UNLOCKED(); + ASSERT_SLEEPABLE(NULL, "pool_get(PR_WAITOK)"); #endif simple_lock(&pp->pr_slock); @@ -1110,6 +1136,8 @@ pool_get(struct pool *pp, int flags) } simple_unlock(&pp->pr_slock); + KASSERT((((vaddr_t)v + pp->pr_itemoffset) & (pp->pr_align - 1)) == 0); + FREECHECK_OUT(&pp->pr_freecheck, v); return (v); } @@ -1121,13 +1149,9 @@ pool_do_put(struct pool *pp, void *v, st { struct pool_item *pi = v; struct pool_item_header *ph; - caddr_t page; - int s; LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); - SCHED_ASSERT_UNLOCKED(); - - page = (caddr_t)((u_long)v & pp->pr_alloc->pa_pagemask); + FREECHECK_IN(&pp->pr_freecheck, v); #ifdef DIAGNOSTIC if (__predict_false(pp->pr_nout == 0)) { @@ -1137,7 +1161,7 @@ pool_do_put(struct pool *pp, void *v, st } #endif - if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) { + if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { pr_printlog(pp, NULL, printf); panic("pool_put: %s: page header missing", pp->pr_wchan); } @@ -1217,9 +1241,7 @@ pool_do_put(struct pool *pp, void *v, st * be reclaimed by the pagedaemon. This minimizes * ping-pong'ing for memory. */ - s = splclock(); - ph->ph_time = mono_time; - splx(s); + getmicrotime(&ph->ph_time); } pool_update_curpage(pp); } @@ -1352,15 +1374,15 @@ pool_prime_page(struct pool *pp, caddr_t { struct pool_item *pi; caddr_t cp = storage; - unsigned int align = pp->pr_align; - unsigned int ioff = pp->pr_itemoffset; + const unsigned int align = pp->pr_align; + const unsigned int ioff = pp->pr_itemoffset; int n; - int s; LOCK_ASSERT(simple_lock_held(&pp->pr_slock)); #ifdef DIAGNOSTIC - if (((u_long)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0) + if ((pp->pr_roflags & PR_NOALIGN) == 0 && + ((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0) panic("pool_prime_page: %s: unaligned page", pp->pr_wchan); #endif @@ -1371,9 +1393,7 @@ pool_prime_page(struct pool *pp, caddr_t LIST_INIT(&ph->ph_itemlist); ph->ph_page = storage; ph->ph_nmissing = 0; - s = splclock(); - ph->ph_time = mono_time; - splx(s); + getmicrotime(&ph->ph_time); if ((pp->pr_roflags & PR_PHINPAGE) == 0) SPLAY_INSERT(phtree, &pp->pr_phtree, ph); @@ -1392,6 +1412,8 @@ pool_prime_page(struct pool *pp, caddr_t if (ioff != 0) cp = (caddr_t)(cp + (align - ioff)); + KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); + /* * Insert remaining chunks on the bucket list. */ @@ -1419,6 +1441,8 @@ pool_prime_page(struct pool *pp, caddr_t pi->pi_magic = PI_MAGIC; #endif cp = (caddr_t)(cp + pp->pr_size); + + KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); } } @@ -1539,7 +1563,6 @@ pool_reclaim(struct pool *pp) struct pool_pagelist pq; struct pool_cache_grouplist pcgl; struct timeval curtime, diff; - int s; if (pp->pr_drain_hook != NULL) { /* @@ -1561,9 +1584,7 @@ pool_reclaim(struct pool *pp) LIST_FOREACH(pc, &pp->pr_cachelist, pc_poollist) pool_cache_reclaim(pc, &pq, &pcgl); - s = splclock(); - curtime = mono_time; - splx(s); + getmicrotime(&curtime); for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { phnext = LIST_NEXT(ph, ph_pagelist); @@ -1772,8 +1793,9 @@ pool_print1(struct pool *pp, const char (*pr)("\n"); if ((pp->pr_roflags & PR_LOGGING) == 0) (*pr)("\tno log\n"); - else + else { pr_printlog(pp, NULL, pr); + } skip_log: if (print_cache == 0) @@ -1824,16 +1846,18 @@ pool_chk_page(struct pool *pp, const cha caddr_t page; int n; - page = (caddr_t)((u_long)ph & pp->pr_alloc->pa_pagemask); - if (page != ph->ph_page && - (pp->pr_roflags & PR_PHINPAGE) != 0) { - if (label != NULL) - printf("%s: ", label); - printf("pool(%p:%s): page inconsistency: page %p;" - " at page head addr %p (p %p)\n", pp, - pp->pr_wchan, ph->ph_page, - ph, page); - return 1; + if ((pp->pr_roflags & PR_NOALIGN) == 0) { + page = (caddr_t)((uintptr_t)ph & pp->pr_alloc->pa_pagemask); + if (page != ph->ph_page && + (pp->pr_roflags & PR_PHINPAGE) != 0) { + if (label != NULL) + printf("%s: ", label); + printf("pool(%p:%s): page inconsistency: page %p;" + " at page head addr %p (p %p)\n", pp, + pp->pr_wchan, ph->ph_page, + ph, page); + return 1; + } } if ((pp->pr_roflags & PR_NOTOUCH) != 0) @@ -1848,15 +1872,16 @@ pool_chk_page(struct pool *pp, const cha if (label != NULL) printf("%s: ", label); printf("pool(%s): free list modified: magic=%x;" - " page %p; item ordinal %d;" - " addr %p (p %p)\n", + " page %p; item ordinal %d; addr %p\n", pp->pr_wchan, pi->pi_magic, ph->ph_page, - n, pi, page); + n, pi); panic("pool"); } #endif - page = - (caddr_t)((u_long)pi & pp->pr_alloc->pa_pagemask); + if ((pp->pr_roflags & PR_NOALIGN) != 0) { + continue; + } + page = (caddr_t)((uintptr_t)pi & pp->pr_alloc->pa_pagemask); if (page == ph->ph_page) continue; @@ -2020,7 +2045,7 @@ pool_cache_get_paddr(struct pool_cache * #ifdef LOCKDEBUG if (flags & PR_WAITOK) - simple_lock_only_held(NULL, "pool_cache_get(PR_WAITOK)"); + ASSERT_SLEEPABLE(NULL, "pool_cache_get(PR_WAITOK)"); #endif simple_lock(&pc->pc_slock); @@ -2050,6 +2075,8 @@ pool_cache_get_paddr(struct pool_cache * return (NULL); } } + KASSERT((((vaddr_t)object + pc->pc_pool->pr_itemoffset) & + (pc->pc_pool->pr_align - 1)) == 0); if (object != NULL && pap != NULL) { #ifdef POOL_VTOPHYS *pap = POOL_VTOPHYS(object); @@ -2057,6 +2084,8 @@ pool_cache_get_paddr(struct pool_cache * *pap = POOL_PADDR_INVALID; #endif } + + FREECHECK_OUT(&pc->pc_freecheck, object); return (object); } @@ -2070,6 +2099,9 @@ pool_cache_get_paddr(struct pool_cache * } simple_unlock(&pc->pc_slock); + KASSERT((((vaddr_t)object + pc->pc_pool->pr_itemoffset) & + (pc->pc_pool->pr_align - 1)) == 0); + FREECHECK_OUT(&pc->pc_freecheck, object); return (object); } @@ -2085,6 +2117,8 @@ pool_cache_put_paddr(struct pool_cache * struct pool_cache_group *pcg; int s; + FREECHECK_IN(&pc->pc_freecheck, object); + if (__predict_false((pc->pc_pool->pr_flags & PR_WANTED) != 0)) { goto destruct; } @@ -2337,7 +2371,7 @@ pool_allocator_free(struct pool *pp, voi void * pool_page_alloc(struct pool *pp, int flags) { - boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; + bool waitok = (flags & PR_WAITOK) ? TRUE : FALSE; return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); } @@ -2352,7 +2386,7 @@ pool_page_free(struct pool *pp, void *v) static void * pool_page_alloc_meta(struct pool *pp, int flags) { - boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; + bool waitok = (flags & PR_WAITOK) ? TRUE : FALSE; return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); } @@ -2404,7 +2438,7 @@ pool_subpage_free_nointr(struct pool *pp void * pool_page_alloc_nointr(struct pool *pp, int flags) { - boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; + bool waitok = (flags & PR_WAITOK) ? TRUE : FALSE; return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); }