Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/kern_mutex.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/kern_mutex.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.44 retrieving revision 1.62 diff -u -p -r1.44 -r1.62 --- src/sys/kern/kern_mutex.c 2008/10/15 06:51:20 1.44 +++ src/sys/kern/kern_mutex.c 2015/05/25 21:02:37 1.62 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_mutex.c,v 1.44 2008/10/15 06:51:20 wrstuden Exp $ */ +/* $NetBSD: kern_mutex.c,v 1.62 2015/05/25 21:02:37 prlw1 Exp $ */ /*- * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. @@ -40,9 +40,10 @@ #define __MUTEX_PRIVATE #include -__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.44 2008/10/15 06:51:20 wrstuden Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.62 2015/05/25 21:02:37 prlw1 Exp $"); #include +#include #include #include #include @@ -50,17 +51,14 @@ __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c #include #include #include -#include #include #include -#include +#include #include #include -#include "opt_sa.h" - /* * When not running a debug kernel, spin mutexes are not much * more than an splraiseipl() and splx() pair. @@ -76,7 +74,7 @@ __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c #define MUTEX_WANTLOCK(mtx) \ LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ - (uintptr_t)__builtin_return_address(0), false, false) + (uintptr_t)__builtin_return_address(0), 0) #define MUTEX_LOCKED(mtx) \ LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \ (uintptr_t)__builtin_return_address(0), 0) @@ -115,21 +113,40 @@ do { \ #endif /* DIAGNOSTIC */ /* - * Spin mutex SPL save / restore. + * Some architectures can't use __cpu_simple_lock as is so allow a way + * for them to use an alternate definition. */ -#ifndef MUTEX_COUNT_BIAS -#define MUTEX_COUNT_BIAS 0 +#ifndef MUTEX_SPINBIT_LOCK_INIT +#define MUTEX_SPINBIT_LOCK_INIT(mtx) __cpu_simple_lock_init(&(mtx)->mtx_lock) +#endif +#ifndef MUTEX_SPINBIT_LOCKED_P +#define MUTEX_SPINBIT_LOCKED_P(mtx) __SIMPLELOCK_LOCKED_P(&(mtx)->mtx_lock) +#endif +#ifndef MUTEX_SPINBIT_LOCK_TRY +#define MUTEX_SPINBIT_LOCK_TRY(mtx) __cpu_simple_lock_try(&(mtx)->mtx_lock) +#endif +#ifndef MUTEX_SPINBIT_LOCK_UNLOCK +#define MUTEX_SPINBIT_LOCK_UNLOCK(mtx) __cpu_simple_unlock(&(mtx)->mtx_lock) +#endif + +#ifndef MUTEX_INITIALIZE_SPIN_IPL +#define MUTEX_INITIALIZE_SPIN_IPL(mtx, ipl) \ + ((mtx)->mtx_ipl = makeiplcookie((ipl))) #endif +/* + * Spin mutex SPL save / restore. + */ + #define MUTEX_SPIN_SPLRAISE(mtx) \ do { \ struct cpu_info *x__ci; \ int x__cnt, s; \ - s = splraiseipl(mtx->mtx_ipl); \ + s = splraiseipl(MUTEX_SPIN_IPL(mtx)); \ x__ci = curcpu(); \ x__cnt = x__ci->ci_mtx_count--; \ __insn_barrier(); \ - if (x__cnt == MUTEX_COUNT_BIAS) \ + if (x__cnt == 0) \ x__ci->ci_mtx_oldspl = (s); \ } while (/* CONSTCOND */ 0) @@ -138,7 +155,7 @@ do { \ struct cpu_info *x__ci = curcpu(); \ int s = x__ci->ci_mtx_oldspl; \ __insn_barrier(); \ - if (++(x__ci->ci_mtx_count) == MUTEX_COUNT_BIAS) \ + if (++(x__ci->ci_mtx_count) == 0) \ splx(s); \ } while (/* CONSTCOND */ 0) @@ -157,18 +174,18 @@ do { \ (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) #define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \ + if (!dodebug) \ + (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ do { \ - if (dodebug) \ - (mtx)->mtx_owner |= MUTEX_BIT_DEBUG; \ } while (/* CONSTCOND */ 0); #define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \ do { \ (mtx)->mtx_owner = MUTEX_BIT_SPIN; \ - if (dodebug) \ - (mtx)->mtx_owner |= MUTEX_BIT_DEBUG; \ - (mtx)->mtx_ipl = makeiplcookie((ipl)); \ - __cpu_simple_lock_init(&(mtx)->mtx_lock); \ + if (!dodebug) \ + (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ + MUTEX_INITIALIZE_SPIN_IPL((mtx), (ipl)); \ + MUTEX_SPINBIT_LOCK_INIT((mtx)); \ } while (/* CONSTCOND */ 0) #define MUTEX_DESTROY(mtx) \ @@ -181,25 +198,25 @@ do { \ #define MUTEX_ADAPTIVE_P(mtx) \ (((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0) -#define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_DEBUG) != 0) +#define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_NODEBUG) == 0) #if defined(LOCKDEBUG) -#define MUTEX_OWNED(owner) (((owner) & ~MUTEX_BIT_DEBUG) != 0) -#define MUTEX_INHERITDEBUG(new, old) (new) |= (old) & MUTEX_BIT_DEBUG +#define MUTEX_OWNED(owner) (((owner) & ~MUTEX_BIT_NODEBUG) != 0) +#define MUTEX_INHERITDEBUG(n, o) (n) |= (o) & MUTEX_BIT_NODEBUG #else /* defined(LOCKDEBUG) */ #define MUTEX_OWNED(owner) ((owner) != 0) -#define MUTEX_INHERITDEBUG(new, old) /* nothing */ +#define MUTEX_INHERITDEBUG(n, o) /* nothing */ #endif /* defined(LOCKDEBUG) */ static inline int MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread) { int rv; - uintptr_t old = 0; - uintptr_t new = curthread; + uintptr_t oldown = 0; + uintptr_t newown = curthread; - MUTEX_INHERITDEBUG(old, mtx->mtx_owner); - MUTEX_INHERITDEBUG(new, old); - rv = MUTEX_CAS(&mtx->mtx_owner, old, new); + MUTEX_INHERITDEBUG(oldown, mtx->mtx_owner); + MUTEX_INHERITDEBUG(newown, oldown); + rv = MUTEX_CAS(&mtx->mtx_owner, oldown, newown); MUTEX_RECEIVE(mtx); return rv; } @@ -216,18 +233,12 @@ MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr static inline void MUTEX_RELEASE(kmutex_t *mtx) { - uintptr_t new; + uintptr_t newown; MUTEX_GIVE(mtx); - new = 0; - MUTEX_INHERITDEBUG(new, mtx->mtx_owner); - mtx->mtx_owner = new; -} - -static inline void -MUTEX_CLEAR_WAITERS(kmutex_t *mtx) -{ - /* nothing */ + newown = 0; + MUTEX_INHERITDEBUG(newown, mtx->mtx_owner); + mtx->mtx_owner = newown; } #endif /* __HAVE_SIMPLE_MUTEXES */ @@ -250,9 +261,8 @@ __strong_alias(mutex_spin_enter,mutex_ve __strong_alias(mutex_spin_exit,mutex_vector_exit); #endif -void mutex_abort(kmutex_t *, const char *, const char *); -void mutex_dump(volatile void *); -int mutex_onproc(uintptr_t, struct cpu_info **); +static void mutex_abort(kmutex_t *, const char *, const char *); +static void mutex_dump(volatile void *); lockops_t mutex_spin_lockops = { "Mutex", @@ -274,18 +284,6 @@ syncobj_t mutex_syncobj = { (void *)mutex_owner, }; -/* Mutex cache */ -#define MUTEX_OBJ_MAGIC 0x5aa3c85d -struct kmutexobj { - kmutex_t mo_lock; - u_int mo_magic; - u_int mo_refcnt; -}; - -static int mutex_obj_ctor(void *, void *, int); - -static pool_cache_t mutex_obj_cache; - /* * mutex_dump: * @@ -385,60 +383,56 @@ mutex_destroy(kmutex_t *mtx) MUTEX_ASSERT(mtx, !MUTEX_OWNED(mtx->mtx_owner) && !MUTEX_HAS_WAITERS(mtx)); } else { - MUTEX_ASSERT(mtx, !__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)); + MUTEX_ASSERT(mtx, !MUTEX_SPINBIT_LOCKED_P(mtx)); } LOCKDEBUG_FREE(MUTEX_DEBUG_P(mtx), mtx); MUTEX_DESTROY(mtx); } +#ifdef MULTIPROCESSOR /* - * mutex_onproc: + * mutex_oncpu: * * Return true if an adaptive mutex owner is running on a CPU in the * system. If the target is waiting on the kernel big lock, then we * must release it. This is necessary to avoid deadlock. - * - * Note that we can't use the mutex owner field as an LWP pointer. We - * don't have full control over the timing of our execution, and so the - * pointer could be completely invalid by the time we dereference it. */ -#ifdef MULTIPROCESSOR -int -mutex_onproc(uintptr_t owner, struct cpu_info **cip) +static bool +mutex_oncpu(uintptr_t owner) { - CPU_INFO_ITERATOR cii; struct cpu_info *ci; - struct lwp *l; + lwp_t *l; - if (!MUTEX_OWNED(owner)) - return 0; - l = (struct lwp *)MUTEX_OWNER(owner); + KASSERT(kpreempt_disabled()); - /* See if the target is running on a CPU somewhere. */ - if ((ci = *cip) != NULL && ci->ci_curlwp == l) - goto run; - for (CPU_INFO_FOREACH(cii, ci)) - if (ci->ci_curlwp == l) - goto run; + if (!MUTEX_OWNED(owner)) { + return false; + } - /* No: it may be safe to block now. */ - *cip = NULL; - return 0; + /* + * See lwp_dtor() why dereference of the LWP pointer is safe. + * We must have kernel preemption disabled for that. + */ + l = (lwp_t *)MUTEX_OWNER(owner); + ci = l->l_cpu; + + if (ci && ci->ci_curlwp == l) { + /* Target is running; do we need to block? */ + return (ci->ci_biglock_wanted != l); + } - run: - /* Target is running; do we need to block? */ - *cip = ci; - return ci->ci_biglock_wanted != l; + /* Not running. It may be safe to block now. */ + return false; } #endif /* MULTIPROCESSOR */ /* * mutex_vector_enter: * - * Support routine for mutex_enter() that must handles all cases. In + * Support routine for mutex_enter() that must handle all cases. In * the LOCKDEBUG case, mutex_enter() is always aliased here, even if - * fast-path stubs are available. If an mutex_spin_enter() stub is + * fast-path stubs are available. If a mutex_spin_enter() stub is * not available, then it is also aliased directly here. */ void @@ -447,12 +441,8 @@ mutex_vector_enter(kmutex_t *mtx) uintptr_t owner, curthread; turnstile_t *ts; #ifdef MULTIPROCESSOR - struct cpu_info *ci = NULL; u_int count; #endif -#ifdef KERN_SA - int f; -#endif LOCKSTAT_COUNTER(spincnt); LOCKSTAT_COUNTER(slpcnt); LOCKSTAT_TIMER(spintime); @@ -469,7 +459,7 @@ mutex_vector_enter(kmutex_t *mtx) MUTEX_SPIN_SPLRAISE(mtx); MUTEX_WANTLOCK(mtx); #ifdef FULL - if (__cpu_simple_lock_try(&mtx->mtx_lock)) { + if (MUTEX_SPINBIT_LOCK_TRY(mtx)) { MUTEX_LOCKED(mtx); return; } @@ -488,14 +478,14 @@ mutex_vector_enter(kmutex_t *mtx) do { if (panicstr != NULL) break; - while (__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) { + while (MUTEX_SPINBIT_LOCKED_P(mtx)) { SPINLOCK_BACKOFF(count); #ifdef LOCKDEBUG if (SPINLOCK_SPINOUT(spins)) MUTEX_ABORT(mtx, "spinout"); #endif /* LOCKDEBUG */ } - } while (!__cpu_simple_lock_try(&mtx->mtx_lock)); + } while (!MUTEX_SPINBIT_LOCK_TRY(mtx)); if (count != SPINLOCK_BACKOFF_MIN) { LOCKSTAT_STOP_TIMER(lsflag, spintime); @@ -526,6 +516,7 @@ mutex_vector_enter(kmutex_t *mtx) * determine that the owner is not running on a processor, * then we stop spinning, and sleep instead. */ + KPREEMPT_DISABLE(curlwp); for (owner = mtx->mtx_owner;;) { if (!MUTEX_OWNED(owner)) { /* @@ -542,27 +533,28 @@ mutex_vector_enter(kmutex_t *mtx) owner = mtx->mtx_owner; continue; } - - if (panicstr != NULL) + if (__predict_false(panicstr != NULL)) { + KPREEMPT_ENABLE(curlwp); return; - if (MUTEX_OWNER(owner) == curthread) + } + if (__predict_false(MUTEX_OWNER(owner) == curthread)) { MUTEX_ABORT(mtx, "locking against myself"); - + } #ifdef MULTIPROCESSOR /* * Check to see if the owner is running on a processor. * If so, then we should just spin, as the owner will * likely release the lock very soon. */ - if (mutex_onproc(owner, &ci)) { + if (mutex_oncpu(owner)) { LOCKSTAT_START_TIMER(lsflag, spintime); count = SPINLOCK_BACKOFF_MIN; - for (;;) { + do { + KPREEMPT_ENABLE(curlwp); SPINLOCK_BACKOFF(count); + KPREEMPT_DISABLE(curlwp); owner = mtx->mtx_owner; - if (!mutex_onproc(owner, &ci)) - break; - } + } while (mutex_oncpu(owner)); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_COUNT(spincnt, 1); if (!MUTEX_OWNED(owner)) @@ -602,7 +594,7 @@ mutex_vector_enter(kmutex_t *mtx) * .. clear lock word, waiters * return success * - * There is a another race that can occur: a third CPU could + * There is another race that can occur: a third CPU could * acquire the mutex as soon as it is released. Since * adaptive mutexes are primarily spin mutexes, this is not * something that we need to worry about too much. What we @@ -647,23 +639,23 @@ mutex_vector_enter(kmutex_t *mtx) * waiters field) and check the lock holder's status again. * Some of the possible outcomes (not an exhaustive list): * - * 1. The onproc check returns true: the holding LWP is + * 1. The on-CPU check returns true: the holding LWP is * running again. The lock may be released soon and * we should spin. Importantly, we can't trust the * value of the waiters flag. * - * 2. The onproc check returns false: the holding LWP is + * 2. The on-CPU check returns false: the holding LWP is * not running. We now have the opportunity to check * if mutex_exit() has blatted the modifications made * by MUTEX_SET_WAITERS(). * - * 3. The onproc check returns false: the holding LWP may + * 3. The on-CPU check returns false: the holding LWP may * or may not be running. It has context switched at * some point during our check. Again, we have the * chance to see if the waiters bit is still set or * has been overwritten. * - * 4. The onproc check returns false: the holding LWP is + * 4. The on-CPU check returns false: the holding LWP is * running on a CPU, but wants the big lock. It's OK * to check the waiters field in this case. * @@ -677,7 +669,7 @@ mutex_vector_enter(kmutex_t *mtx) * If the waiters bit is not set it's unsafe to go asleep, * as we might never be awoken. */ - if ((membar_consumer(), mutex_onproc(owner, &ci)) || + if ((membar_consumer(), mutex_oncpu(owner)) || (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) { turnstile_exit(mtx); owner = mtx->mtx_owner; @@ -685,16 +677,6 @@ mutex_vector_enter(kmutex_t *mtx) } #endif /* MULTIPROCESSOR */ -#ifdef KERN_SA - /* - * Sleeping for a mutex should not generate an upcall. - * So set LP_SA_NOBLOCK to indicate this. - * f indicates if we should clear LP_SA_NOBLOCK when done. - */ - f = ~curlwp->l_pflag & LP_SA_NOBLOCK; - curlwp->l_pflag |= LP_SA_NOBLOCK; -#endif /* KERN_SA */ - LOCKSTAT_START_TIMER(lsflag, slptime); turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj); @@ -702,12 +684,9 @@ mutex_vector_enter(kmutex_t *mtx) LOCKSTAT_STOP_TIMER(lsflag, slptime); LOCKSTAT_COUNT(slpcnt, 1); -#ifdef KERN_SA - curlwp->l_pflag ^= f; -#endif /* KERN_SA */ - owner = mtx->mtx_owner; } + KPREEMPT_ENABLE(curlwp); LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1, slpcnt, slptime); @@ -732,13 +711,13 @@ mutex_vector_exit(kmutex_t *mtx) if (MUTEX_SPIN_P(mtx)) { #ifdef FULL - if (__predict_false(!__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock))) { + if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) { if (panicstr != NULL) return; MUTEX_ABORT(mtx, "exiting unheld spin mutex"); } MUTEX_UNLOCKED(mtx); - __cpu_simple_unlock(&mtx->mtx_lock); + MUTEX_SPINBIT_LOCK_UNLOCK(mtx); #endif MUTEX_SPIN_SPLRESTORE(mtx); return; @@ -754,6 +733,9 @@ mutex_vector_exit(kmutex_t *mtx) MUTEX_DASSERT(mtx, curthread != 0); MUTEX_ASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread); MUTEX_UNLOCKED(mtx); +#if !defined(LOCKDEBUG) + __USE(curthread); +#endif #ifdef LOCKDEBUG /* @@ -828,7 +810,7 @@ mutex_owned(kmutex_t *mtx) if (MUTEX_ADAPTIVE_P(mtx)) return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp; #ifdef FULL - return __SIMPLELOCK_LOCKED_P(&mtx->mtx_lock); + return MUTEX_SPINBIT_LOCKED_P(mtx); #else return 1; #endif @@ -864,7 +846,7 @@ mutex_tryenter(kmutex_t *mtx) if (MUTEX_SPIN_P(mtx)) { MUTEX_SPIN_SPLRAISE(mtx); #ifdef FULL - if (__cpu_simple_lock_try(&mtx->mtx_lock)) { + if (MUTEX_SPINBIT_LOCK_TRY(mtx)) { MUTEX_WANTLOCK(mtx); MUTEX_LOCKED(mtx); return 1; @@ -921,14 +903,14 @@ mutex_spin_retry(kmutex_t *mtx) do { if (panicstr != NULL) break; - while (__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) { + while (MUTEX_SPINBIT_LOCKED_P(mtx)) { SPINLOCK_BACKOFF(count); #ifdef LOCKDEBUG if (SPINLOCK_SPINOUT(spins)) MUTEX_ABORT(mtx, "spinout"); #endif /* LOCKDEBUG */ } - } while (!__cpu_simple_lock_try(&mtx->mtx_lock)); + } while (!MUTEX_SPINBIT_LOCK_TRY(mtx)); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime); @@ -940,88 +922,3 @@ mutex_spin_retry(kmutex_t *mtx) #endif /* MULTIPROCESSOR */ } #endif /* defined(__HAVE_SPIN_MUTEX_STUBS) || defined(FULL) */ - -/* - * mutex_obj_init: - * - * Initialize the mutex object store. - */ -void -mutex_obj_init(void) -{ - - mutex_obj_cache = pool_cache_init(sizeof(struct kmutexobj), - coherency_unit, 0, 0, "mutex", NULL, IPL_NONE, mutex_obj_ctor, - NULL, NULL); -} - -/* - * mutex_obj_ctor: - * - * Initialize a new lock for the cache. - */ -static int -mutex_obj_ctor(void *arg, void *obj, int flags) -{ - struct kmutexobj * mo = obj; - - mo->mo_magic = MUTEX_OBJ_MAGIC; - - return 0; -} - -/* - * mutex_obj_alloc: - * - * Allocate a single lock object. - */ -kmutex_t * -mutex_obj_alloc(kmutex_type_t type, int ipl) -{ - struct kmutexobj *mo; - - mo = pool_cache_get(mutex_obj_cache, PR_WAITOK); - mutex_init(&mo->mo_lock, type, ipl); - mo->mo_refcnt = 1; - - return (kmutex_t *)mo; -} - -/* - * mutex_obj_hold: - * - * Add a single reference to a lock object. A reference to the object - * must already be held, and must be held across this call. - */ -void -mutex_obj_hold(kmutex_t *lock) -{ - struct kmutexobj *mo = (struct kmutexobj *)lock; - - KASSERT(mo->mo_magic == MUTEX_OBJ_MAGIC); - KASSERT(mo->mo_refcnt > 0); - - atomic_inc_uint(&mo->mo_refcnt); -} - -/* - * mutex_obj_free: - * - * Drop a reference from a lock object. If the last reference is being - * dropped, free the object and return true. Otherwise, return false. - */ -bool -mutex_obj_free(kmutex_t *lock) -{ - struct kmutexobj *mo = (struct kmutexobj *)lock; - - KASSERT(mo->mo_magic == MUTEX_OBJ_MAGIC); - KASSERT(mo->mo_refcnt > 0); - - if (atomic_dec_uint_nv(&mo->mo_refcnt) > 0) { - return false; - } - mutex_destroy(&mo->mo_lock); - pool_cache_put(mutex_obj_cache, mo); - return true; -}