Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.136 retrieving revision 1.137.2.4 diff -u -p -r1.136 -r1.137.2.4 --- src/sys/kern/subr_pool.c 2007/11/14 11:14:13 1.136 +++ src/sys/kern/subr_pool.c 2007/12/26 17:55:57 1.137.2.4 @@ -1,4 +1,4 @@ -/* $NetBSD: subr_pool.c,v 1.136 2007/11/14 11:14:13 yamt Exp $ */ +/* $NetBSD: subr_pool.c,v 1.137.2.4 2007/12/26 17:55:57 ad Exp $ */ /*- * Copyright (c) 1997, 1999, 2000, 2002, 2007 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.136 2007/11/14 11:14:13 yamt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.137.2.4 2007/12/26 17:55:57 ad Exp $"); #include "opt_pool.h" #include "opt_poollog.h" @@ -58,6 +58,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, #include #include #include +#include #include @@ -75,11 +76,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c, */ /* List of all pools */ -LIST_HEAD(,pool) pool_head = LIST_HEAD_INITIALIZER(pool_head); - -/* List of all caches. */ -LIST_HEAD(,pool_cache) pool_cache_head = - LIST_HEAD_INITIALIZER(pool_cache_head); +TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); /* Private pool for page header structures */ #define PHPOOL_MAX 8 @@ -185,6 +182,13 @@ static struct pool pcgpool; static struct pool cache_pool; static struct pool cache_cpu_pool; +/* List of all caches. */ +TAILQ_HEAD(,pool_cache) pool_cache_head = + TAILQ_HEAD_INITIALIZER(pool_cache_head); + +int pool_cache_disable; + + static pool_cache_cpu_t *pool_cache_put_slow(pool_cache_cpu_t *, int *, void *, paddr_t); static pool_cache_cpu_t *pool_cache_get_slow(pool_cache_cpu_t *, int *, @@ -615,9 +619,7 @@ void pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, const char *wchan, struct pool_allocator *palloc, int ipl) { -#ifdef DEBUG struct pool *pp1; -#endif size_t trysize, phsize; int off, slack; @@ -626,7 +628,7 @@ pool_init(struct pool *pp, size_t size, * Check that the pool hasn't already been initialised and * added to the list of all pools. */ - LIST_FOREACH(pp1, &pool_head, pr_poollist) { + TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { if (pp == pp1) panic("pool_init: pool %s already initialised", wchan); @@ -802,7 +804,14 @@ pool_init(struct pool *pp, size_t size, pp->pr_entered_file = NULL; pp->pr_entered_line = 0; - mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); + /* + * XXXAD hack to prevent IP input processing from blocking. + */ + if (ipl == IPL_SOFTNET) { + mutex_init(&pp->pr_lock, MUTEX_DEFAULT, IPL_VM); + } else { + mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); + } cv_init(&pp->pr_cv, wchan); pp->pr_ipl = ipl; @@ -837,20 +846,26 @@ pool_init(struct pool *pp, size_t size, "cachegrp", &pool_allocator_meta, IPL_VM); } - if (__predict_true(!cold)) { - /* Insert into the list of all pools. */ + /* Insert into the list of all pools. */ + if (__predict_true(!cold)) mutex_enter(&pool_head_lock); - LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); + TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { + if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) + break; + } + if (pp1 == NULL) + TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); + else + TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); + if (__predict_true(!cold)) mutex_exit(&pool_head_lock); /* Insert this into the list of pools using this allocator. */ + if (__predict_true(!cold)) mutex_enter(&palloc->pa_lock); - TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); + TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); + if (__predict_true(!cold)) mutex_exit(&palloc->pa_lock); - } else { - LIST_INSERT_HEAD(&pool_head, pp, pr_poollist); - TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); - } pool_reclaim_register(pp); } @@ -868,7 +883,7 @@ pool_destroy(struct pool *pp) mutex_enter(&pool_head_lock); while (pp->pr_refcnt != 0) cv_wait(&pool_busy, &pool_head_lock); - LIST_REMOVE(pp, pr_poollist); + TAILQ_REMOVE(&pool_head, pp, pr_poollist); if (drainpp == pp) drainpp = NULL; mutex_exit(&pool_head_lock); @@ -1674,7 +1689,7 @@ pool_drain_start(struct pool **ppp, uint { struct pool *pp; - KASSERT(!LIST_EMPTY(&pool_head)); + KASSERT(!TAILQ_EMPTY(&pool_head)); pp = NULL; @@ -1682,11 +1697,11 @@ pool_drain_start(struct pool **ppp, uint mutex_enter(&pool_head_lock); do { if (drainpp == NULL) { - drainpp = LIST_FIRST(&pool_head); + drainpp = TAILQ_FIRST(&pool_head); } if (drainpp != NULL) { pp = drainpp; - drainpp = LIST_NEXT(pp, pr_poollist); + drainpp = TAILQ_NEXT(pp, pr_poollist); } /* * Skip completely idle pools. We depend on at least @@ -1742,7 +1757,7 @@ pool_printall(const char *modif, void (* { struct pool *pp; - LIST_FOREACH(pp, &pool_head, pr_poollist) { + TAILQ_FOREACH(pp, &pool_head, pr_poollist) { pool_printit(pp, modif, pr); } } @@ -2032,6 +2047,7 @@ pool_cache_bootstrap(pool_cache_t pc, si void *arg) { CPU_INFO_ITERATOR cii; + pool_cache_t pc1; struct cpu_info *ci; struct pool *pp; @@ -2040,7 +2056,14 @@ pool_cache_bootstrap(pool_cache_t pc, si palloc = &pool_allocator_nointr; pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl); - mutex_init(&pc->pc_lock, MUTEX_DEFAULT, pp->pr_ipl); + /* + * XXXAD hack to prevent IP input processing from blocking. + */ + if (ipl == IPL_SOFTNET) { + mutex_init(&pc->pc_lock, MUTEX_DEFAULT, IPL_VM); + } else { + mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl); + } if (ctor == NULL) { ctor = (int (*)(void *, void *, int))nullop; @@ -2067,21 +2090,31 @@ pool_cache_bootstrap(pool_cache_t pc, si /* Allocate per-CPU caches. */ memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); pc->pc_ncpu = 0; - for (CPU_INFO_FOREACH(cii, ci)) { - pool_cache_cpu_init1(ci, pc); + if (ncpu == 0) { + /* XXX For sparc: boot CPU is not attached yet. */ + pool_cache_cpu_init1(curcpu(), pc); + } else { + for (CPU_INFO_FOREACH(cii, ci)) { + pool_cache_cpu_init1(ci, pc); + } } - - if (__predict_true(!cold)) { - mutex_enter(&pp->pr_lock); - pp->pr_cache = pc; - mutex_exit(&pp->pr_lock); + + /* Add to list of all pools. */ + if (__predict_true(!cold)) mutex_enter(&pool_head_lock); - LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); - mutex_exit(&pool_head_lock); - } else { - pp->pr_cache = pc; - LIST_INSERT_HEAD(&pool_cache_head, pc, pc_cachelist); + TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { + if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) + break; } + if (pc1 == NULL) + TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); + else + TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); + if (__predict_true(!cold)) + mutex_exit(&pool_head_lock); + + membar_sync(); + pp->pr_cache = pc; } /* @@ -2101,7 +2134,7 @@ pool_cache_destroy(pool_cache_t pc) mutex_enter(&pool_head_lock); while (pc->pc_refcnt != 0) cv_wait(&pool_busy, &pool_head_lock); - LIST_REMOVE(pc, pc_cachelist); + TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); mutex_exit(&pool_head_lock); /* First, invalidate the entire cache. */ @@ -2143,11 +2176,15 @@ static void pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) { pool_cache_cpu_t *cc; + int index; + index = ci->ci_index; + + KASSERT(index < MAXCPUS); KASSERT(((uintptr_t)pc->pc_cpus & (CACHE_LINE_SIZE - 1)) == 0); - if ((cc = pc->pc_cpus[ci->ci_index]) != NULL) { - KASSERT(cc->cc_cpu = ci); + if ((cc = pc->pc_cpus[index]) != NULL) { + KASSERT(cc->cc_cpuindex == index); return; } @@ -2168,13 +2205,13 @@ pool_cache_cpu_init1(struct cpu_info *ci cc->cc_ipl = pc->pc_pool.pr_ipl; cc->cc_iplcookie = makeiplcookie(cc->cc_ipl); cc->cc_cache = pc; - cc->cc_cpu = ci; + cc->cc_cpuindex = index; cc->cc_hits = 0; cc->cc_misses = 0; cc->cc_current = NULL; cc->cc_previous = NULL; - pc->pc_cpus[ci->ci_index] = cc; + pc->pc_cpus[index] = cc; } /* @@ -2188,7 +2225,7 @@ pool_cache_cpu_init(struct cpu_info *ci) pool_cache_t pc; mutex_enter(&pool_head_lock); - LIST_FOREACH(pc, &pool_cache_head, pc_cachelist) { + TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { pc->pc_refcnt++; mutex_exit(&pool_head_lock); @@ -2320,33 +2357,19 @@ static inline pool_cache_cpu_t * pool_cache_cpu_enter(pool_cache_t pc, int *s) { pool_cache_cpu_t *cc; - struct cpu_info *ci; /* * Prevent other users of the cache from accessing our * CPU-local data. To avoid touching shared state, we * pull the neccessary information from CPU local data. */ - ci = curcpu(); - KASSERT(ci->ci_data.cpu_index < MAXCPUS); - cc = pc->pc_cpus[ci->ci_data.cpu_index]; + crit_enter(); + cc = pc->pc_cpus[curcpu()->ci_index]; KASSERT(cc->cc_cache == pc); - if (cc->cc_ipl == IPL_NONE) { - crit_enter(); - } else { + if (cc->cc_ipl != IPL_NONE) { *s = splraiseipl(cc->cc_iplcookie); } - - /* Moved to another CPU before disabling preemption? */ - if (__predict_false(ci != curcpu())) { - ci = curcpu(); - cc = pc->pc_cpus[ci->ci_data.cpu_index]; - } - -#ifdef DIAGNOSTIC - KASSERT(cc->cc_cpu == ci); KASSERT(((uintptr_t)cc & (CACHE_LINE_SIZE - 1)) == 0); -#endif return cc; } @@ -2356,11 +2379,10 @@ pool_cache_cpu_exit(pool_cache_cpu_t *cc { /* No longer need exclusive access to the per-CPU data. */ - if (cc->cc_ipl == IPL_NONE) { - crit_exit(); - } else { + if (cc->cc_ipl != IPL_NONE) { splx(*s); } + crit_exit(); } #if __GNUC_PREREQ__(3, 0) @@ -2582,7 +2604,11 @@ pool_cache_put_slow(pool_cache_cpu_t *cc * If we can't allocate a new group, just throw the * object away. */ - pcg = pool_get(&pcgpool, PR_NOWAIT); + if (pool_cache_disable) { + pcg = NULL; + } else { + pcg = pool_get(&pcgpool, PR_NOWAIT); + } if (pcg == NULL) { pool_cache_destruct_object(pc, object); return NULL;