Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/subr_pool.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.151.6.2 retrieving revision 1.153 diff -u -p -r1.151.6.2 -r1.153 --- src/sys/kern/subr_pool.c 2008/06/02 13:24:11 1.151.6.2 +++ src/sys/kern/subr_pool.c 2008/03/10 22:20:14 1.153 @@ -1,7 +1,7 @@ -/* $NetBSD: subr_pool.c,v 1.151.6.2 2008/06/02 13:24:11 mjf Exp $ */ +/* $NetBSD: subr_pool.c,v 1.153 2008/03/10 22:20:14 martin Exp $ */ /*- - * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008 The NetBSD Foundation, Inc. + * Copyright (c) 1997, 1999, 2000, 2002, 2007 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -16,6 +16,13 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED @@ -31,7 +38,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.151.6.2 2008/06/02 13:24:11 mjf Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.153 2008/03/10 22:20:14 martin Exp $"); #include "opt_ddb.h" #include "opt_pool.h" @@ -605,10 +612,10 @@ pool_subsystem_init(void) pa_reclaim_register(pa); } - pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit, + pool_init(&cache_pool, sizeof(struct pool_cache), CACHE_LINE_SIZE, 0, 0, "pcache", &pool_allocator_nointr, IPL_NONE); - pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), coherency_unit, + pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), CACHE_LINE_SIZE, 0, 0, "pcachecpu", &pool_allocator_nointr, IPL_NONE); } @@ -807,7 +814,14 @@ pool_init(struct pool *pp, size_t size, pp->pr_entered_file = NULL; pp->pr_entered_line = 0; - mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); + /* + * XXXAD hack to prevent IP input processing from blocking. + */ + if (ipl == IPL_SOFTNET) { + mutex_init(&pp->pr_lock, MUTEX_DEFAULT, IPL_VM); + } else { + mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); + } cv_init(&pp->pr_cv, wchan); pp->pr_ipl = ipl; @@ -841,12 +855,12 @@ pool_init(struct pool *pp, size_t size, size = sizeof(pcg_t) + (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); - pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0, + pool_init(&pcg_normal_pool, size, CACHE_LINE_SIZE, 0, 0, "pcgnormal", &pool_allocator_meta, IPL_VM); size = sizeof(pcg_t) + (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); - pool_init(&pcg_large_pool, size, coherency_unit, 0, 0, + pool_init(&pcg_large_pool, size, CACHE_LINE_SIZE, 0, 0, "pcglarge", &pool_allocator_meta, IPL_VM); } @@ -981,9 +995,8 @@ pool_get(struct pool *pp, int flags) #endif /* DIAGNOSTIC */ #ifdef LOCKDEBUG - if (flags & PR_WAITOK) { - ASSERT_SLEEPABLE(); - } + if (flags & PR_WAITOK) + ASSERT_SLEEPABLE(NULL, "pool_get(PR_WAITOK)"); #endif mutex_enter(&pp->pr_lock); @@ -1615,8 +1628,9 @@ pool_reclaim(struct pool *pp) } /* - * XXXSMP Because we do not want to cause non-MPSAFE code - * to block. + * XXXSMP Because mutexes at IPL_SOFTXXX are still spinlocks, + * and we are called from the pagedaemon without kernel_lock. + * Does not apply to IPL_SOFTBIO. */ if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK || pp->pr_ipl == IPL_SOFTSERIAL) { @@ -2059,7 +2073,15 @@ pool_cache_bootstrap(pool_cache_t pc, si if (palloc == NULL && ipl == IPL_NONE) palloc = &pool_allocator_nointr; pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl); - mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl); + + /* + * XXXAD hack to prevent IP input processing from blocking. + */ + if (ipl == IPL_SOFTNET) { + mutex_init(&pc->pc_lock, MUTEX_DEFAULT, IPL_VM); + } else { + mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl); + } if (ctor == NULL) { ctor = (int (*)(void *, void *, int))nullop; @@ -2183,6 +2205,7 @@ pool_cache_cpu_init1(struct cpu_info *ci index = ci->ci_index; KASSERT(index < MAXCPUS); + KASSERT(((uintptr_t)pc->pc_cpus & (CACHE_LINE_SIZE - 1)) == 0); if ((cc = pc->pc_cpus[index]) != NULL) { KASSERT(cc->cc_cpuindex == index); @@ -2369,12 +2392,13 @@ pool_cache_cpu_enter(pool_cache_t pc, in * CPU-local data. To avoid touching shared state, we * pull the neccessary information from CPU local data. */ - KPREEMPT_DISABLE(curlwp); + crit_enter(); cc = pc->pc_cpus[curcpu()->ci_index]; KASSERT(cc->cc_cache == pc); if (cc->cc_ipl != IPL_NONE) { *s = splraiseipl(cc->cc_iplcookie); } + KASSERT(((uintptr_t)cc & (CACHE_LINE_SIZE - 1)) == 0); return cc; } @@ -2387,10 +2411,13 @@ pool_cache_cpu_exit(pool_cache_cpu_t *cc if (cc->cc_ipl != IPL_NONE) { splx(*s); } - KPREEMPT_ENABLE(curlwp); + crit_exit(); } -pool_cache_cpu_t * __noinline +#if __GNUC_PREREQ__(3, 0) +__attribute ((noinline)) +#endif +pool_cache_cpu_t * pool_cache_get_slow(pool_cache_cpu_t *cc, int *s, void **objectp, paddr_t *pap, int flags) { @@ -2494,9 +2521,8 @@ pool_cache_get_paddr(pool_cache_t pc, in int s; #ifdef LOCKDEBUG - if (flags & PR_WAITOK) { - ASSERT_SLEEPABLE(); - } + if (flags & PR_WAITOK) + ASSERT_SLEEPABLE(NULL, "pool_cache_get(PR_WAITOK)"); #endif cc = pool_cache_cpu_enter(pc, &s); @@ -2541,7 +2567,10 @@ pool_cache_get_paddr(pool_cache_t pc, in return object; } -pool_cache_cpu_t * __noinline +#if __GNUC_PREREQ__(3, 0) +__attribute ((noinline)) +#endif +pool_cache_cpu_t * pool_cache_put_slow(pool_cache_cpu_t *cc, int *s, void *object, paddr_t pa) { pcg_t *pcg, *cur;