Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v retrieving revision 1.133.2.2 retrieving revision 1.134 diff -u -p -r1.133.2.2 -r1.134 --- src/sys/kern/subr_pool.c 2007/11/18 19:35:50 1.133.2.2 +++ src/sys/kern/subr_pool.c 2007/11/07 00:23:23 1.134 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.133.2.2 2007/11/18 19:35:50 bouyer Exp $ */ +/* $NetBSD: subr_pool.c,v 1.134 2007/11/07 00:23:23 ad Exp $ */ /*- * Copyright (c) 1997, 1999, 2000, 2002, 2007 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.133.2.2 2007/11/18 19:35:50 bouyer Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.134 2007/11/07 00:23:23 ad Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -46,7 +46,6 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, #include #include -#include #include #include #include @@ -84,8 +83,7 @@ LIST_HEAD(,pool_cache) pool_cache_head = /* Private pool for page header structures */ #define PHPOOL_MAX 8 static struct pool phpool[PHPOOL_MAX]; -#define PHPOOL_FREELIST_NELEM(idx) \ - (((idx) == 0) ? 0 : BITMAP_SIZE * (1 << (idx))) +#define PHPOOL_FREELIST_NELEM(idx) (((idx) == 0) ? 0 : (1 << (idx))) #ifdef POOL_SUBPAGE /* Pool of subpages for use by normal pools. */ @@ -114,9 +112,7 @@ static struct pool *drainpp; static kmutex_t pool_head_lock; static kcondvar_t pool_busy; -typedef uint32_t pool_item_bitmap_t; -#define BITMAP_SIZE (CHAR_BIT * sizeof(pool_item_bitmap_t)) -#define BITMAP_MASK (BITMAP_SIZE - 1) +typedef uint8_t pool_item_freelist_t; struct pool_item_header { /* Page headers */ @@ -126,7 +122,6 @@ struct pool_item_header { ph_node; /* Off-page page headers */ void * ph_page; /* this page's address */ struct timeval ph_time; /* last referenced */ - uint16_t ph_nmissing; /* # of chunks in use */ union { /* !PR_NOTOUCH */ struct { @@ -135,14 +130,21 @@ struct pool_item_header { } phu_normal; /* PR_NOTOUCH */ struct { - uint16_t phu_off; /* start offset in page */ - pool_item_bitmap_t phu_bitmap[]; + uint16_t + phu_off; /* start offset in page */ + pool_item_freelist_t + phu_firstfree; /* first free item */ + /* + * XXX it might be better to use + * a simple bitmap and ffs(3) + */ } phu_notouch; } ph_u; + uint16_t ph_nmissing; /* # of chunks in use */ }; #define ph_itemlist ph_u.phu_normal.phu_itemlist #define ph_off ph_u.phu_notouch.phu_off -#define ph_bitmap ph_u.phu_notouch.phu_bitmap +#define ph_firstfree ph_u.phu_notouch.phu_firstfree struct pool_item { #ifdef DIAGNOSTIC @@ -328,12 +330,12 @@ pr_enter_check(struct pool *pp, void (*p #define pr_enter_check(pp, pr) #endif /* POOL_DIAGNOSTIC */ -static inline unsigned int +static inline int pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, const void *v) { const char *cp = v; - unsigned int idx; + int idx; KASSERT(pp->pr_roflags & PR_NOTOUCH); idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size; @@ -341,57 +343,37 @@ pr_item_notouch_index(const struct pool return idx; } +#define PR_FREELIST_ALIGN(p) \ + roundup((uintptr_t)(p), sizeof(pool_item_freelist_t)) +#define PR_FREELIST(ph) ((pool_item_freelist_t *)PR_FREELIST_ALIGN((ph) + 1)) +#define PR_INDEX_USED ((pool_item_freelist_t)-1) +#define PR_INDEX_EOL ((pool_item_freelist_t)-2) + static inline void pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, void *obj) { - unsigned int idx = pr_item_notouch_index(pp, ph, obj); - pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE); - pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); + int idx = pr_item_notouch_index(pp, ph, obj); + pool_item_freelist_t *freelist = PR_FREELIST(ph); - KASSERT((*bitmap & mask) == 0); - *bitmap |= mask; + KASSERT(freelist[idx] == PR_INDEX_USED); + freelist[idx] = ph->ph_firstfree; + ph->ph_firstfree = idx; } static inline void * pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) { - pool_item_bitmap_t *bitmap = ph->ph_bitmap; - unsigned int idx; - int i; + int idx = ph->ph_firstfree; + pool_item_freelist_t *freelist = PR_FREELIST(ph); - for (i = 0; ; i++) { - int bit; + KASSERT(freelist[idx] != PR_INDEX_USED); + ph->ph_firstfree = freelist[idx]; + freelist[idx] = PR_INDEX_USED; - KASSERT((i * BITMAP_SIZE) < pp->pr_itemsperpage); - bit = ffs32(bitmap[i]); - if (bit) { - pool_item_bitmap_t mask; - - bit--; - idx = (i * BITMAP_SIZE) + bit; - mask = 1 << bit; - KASSERT((bitmap[i] & mask) != 0); - bitmap[i] &= ~mask; - break; - } - } - KASSERT(idx < pp->pr_itemsperpage); return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size; } -static inline void -pr_item_notouch_init(const struct pool *pp, struct pool_item_header *ph) -{ - pool_item_bitmap_t *bitmap = ph->ph_bitmap; - const int n = howmany(pp->pr_itemsperpage, BITMAP_SIZE); - int i; - - for (i = 0; i < n; i++) { - bitmap[i] = (pool_item_bitmap_t)-1; - } -} - static inline int phtree_compare(struct pool_item_header *a, struct pool_item_header *b) { @@ -621,6 +603,9 @@ pool_init(struct pool *pp, size_t size, size_t trysize, phsize; int off, slack; + KASSERT((1UL << (CHAR_BIT * sizeof(pool_item_freelist_t))) - 2 >= + PHPOOL_FREELIST_NELEM(PHPOOL_MAX - 1)); + #ifdef DEBUG /* * Check that the pool hasn't already been initialised and @@ -823,8 +808,8 @@ pool_init(struct pool *pp, size_t size, "phpool-%d", nelem); sz = sizeof(struct pool_item_header); if (nelem) { - sz = offsetof(struct pool_item_header, - ph_bitmap[howmany(nelem, BITMAP_SIZE)]); + sz = PR_FREELIST_ALIGN(sz) + + nelem * sizeof(pool_item_freelist_t); } pool_init(&phpool[idx], sz, 0, 0, 0, phpool_names[idx], &pool_allocator_meta, IPL_VM); @@ -1448,7 +1433,14 @@ pool_prime_page(struct pool *pp, void *s pp->pr_nitems += n; if (pp->pr_roflags & PR_NOTOUCH) { - pr_item_notouch_init(pp, ph); + pool_item_freelist_t *freelist = PR_FREELIST(ph); + int i; + + ph->ph_off = (char *)cp - (char *)storage; + ph->ph_firstfree = 0; + for (i = 0; i < n - 1; i++) + freelist[i] = i + 1; + freelist[n - 1] = PR_INDEX_EOL; } else { while (n--) { pi = (struct pool_item *)cp; @@ -2062,18 +2054,12 @@ pool_cache_bootstrap(pool_cache_t pc, si pc->pc_nfull = 0; pc->pc_contended = 0; pc->pc_refcnt = 0; - pc->pc_freecheck = NULL; /* Allocate per-CPU caches. */ memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); pc->pc_ncpu = 0; - if (ncpu == 0) { - /* XXX For sparc: boot CPU is not attached yet. */ - pool_cache_cpu_init1(curcpu(), pc); - } else { - for (CPU_INFO_FOREACH(cii, ci)) { - pool_cache_cpu_init1(ci, pc); - } + for (CPU_INFO_FOREACH(cii, ci)) { + pool_cache_cpu_init1(ci, pc); } if (__predict_true(!cold)) { @@ -2148,15 +2134,11 @@ static void pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) { pool_cache_cpu_t *cc; - int index; - - index = ci->ci_index; - KASSERT(index < MAXCPUS); KASSERT(((uintptr_t)pc->pc_cpus & (CACHE_LINE_SIZE - 1)) == 0); - if ((cc = pc->pc_cpus[index]) != NULL) { - KASSERT(cc->cc_cpuindex == index); + if ((cc = pc->pc_cpus[ci->ci_index]) != NULL) { + KASSERT(cc->cc_cpu = ci); return; } @@ -2177,13 +2159,13 @@ pool_cache_cpu_init1(struct cpu_info *ci cc->cc_ipl = pc->pc_pool.pr_ipl; cc->cc_iplcookie = makeiplcookie(cc->cc_ipl); cc->cc_cache = pc; - cc->cc_cpuindex = index; + cc->cc_cpu = ci; cc->cc_hits = 0; cc->cc_misses = 0; cc->cc_current = NULL; cc->cc_previous = NULL; - pc->pc_cpus[index] = cc; + pc->pc_cpus[ci->ci_index] = cc; } /* @@ -2222,14 +2204,6 @@ pool_cache_reclaim(pool_cache_t pc) return pool_reclaim(&pc->pc_pool); } -static void -pool_cache_destruct_object1(pool_cache_t pc, void *object) -{ - - (*pc->pc_dtor)(pc->pc_arg, object); - pool_put(&pc->pc_pool, object); -} - /* * pool_cache_destruct_object: * @@ -2240,9 +2214,8 @@ void pool_cache_destruct_object(pool_cache_t pc, void *object) { - FREECHECK_IN(&pc->pc_freecheck, object); - - pool_cache_destruct_object1(pc, object); + (*pc->pc_dtor)(pc->pc_arg, object); + pool_put(&pc->pc_pool, object); } /* @@ -2262,7 +2235,7 @@ pool_cache_invalidate_groups(pool_cache_ for (i = 0; i < pcg->pcg_avail; i++) { object = pcg->pcg_objects[i].pcgo_va; - pool_cache_destruct_object1(pc, object); + pool_cache_destruct_object(pc, object); } pool_put(&pcgpool, pcg); @@ -2329,19 +2302,33 @@ static inline pool_cache_cpu_t * pool_cache_cpu_enter(pool_cache_t pc, int *s) { pool_cache_cpu_t *cc; + struct cpu_info *ci; /* * Prevent other users of the cache from accessing our * CPU-local data. To avoid touching shared state, we * pull the neccessary information from CPU local data. */ - crit_enter(); - cc = pc->pc_cpus[curcpu()->ci_index]; + ci = curcpu(); + KASSERT(ci->ci_data.cpu_index < MAXCPUS); + cc = pc->pc_cpus[ci->ci_data.cpu_index]; KASSERT(cc->cc_cache == pc); - if (cc->cc_ipl != IPL_NONE) { + if (cc->cc_ipl == IPL_NONE) { + crit_enter(); + } else { *s = splraiseipl(cc->cc_iplcookie); } + + /* Moved to another CPU before disabling preemption? */ + if (__predict_false(ci != curcpu())) { + ci = curcpu(); + cc = pc->pc_cpus[ci->ci_data.cpu_index]; + } + +#ifdef DIAGNOSTIC + KASSERT(cc->cc_cpu == ci); KASSERT(((uintptr_t)cc & (CACHE_LINE_SIZE - 1)) == 0); +#endif return cc; } @@ -2351,10 +2338,11 @@ pool_cache_cpu_exit(pool_cache_cpu_t *cc { /* No longer need exclusive access to the per-CPU data. */ - if (cc->cc_ipl != IPL_NONE) { + if (cc->cc_ipl == IPL_NONE) { + crit_exit(); + } else { splx(*s); } - crit_exit(); } #if __GNUC_PREREQ__(3, 0)