Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/kern_mutex.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/kern_mutex.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.16 retrieving revision 1.81 diff -u -p -r1.16 -r1.81 --- src/sys/kern/kern_mutex.c 2007/09/10 11:34:10 1.16 +++ src/sys/kern/kern_mutex.c 2019/12/09 21:05:23 1.81 @@ -1,7 +1,7 @@ -/* $NetBSD: kern_mutex.c,v 1.16 2007/09/10 11:34:10 skrll Exp $ */ +/* $NetBSD: kern_mutex.c,v 1.81 2019/12/09 21:05:23 ad Exp $ */ /*- - * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. + * Copyright (c) 2002, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -15,13 +15,6 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED @@ -44,14 +37,13 @@ * Richard McDougall. */ -#include "opt_multiprocessor.h" - #define __MUTEX_PRIVATE #include -__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.16 2007/09/10 11:34:10 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.81 2019/12/09 21:05:23 ad Exp $"); #include +#include #include #include #include @@ -59,10 +51,15 @@ __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c #include #include #include +#include +#include +#include +#include +#include #include -#include +#include /* * When not running a debug kernel, spin mutexes are not much @@ -78,24 +75,27 @@ __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c */ #define MUTEX_WANTLOCK(mtx) \ - LOCKDEBUG_WANTLOCK(MUTEX_GETID(mtx), \ + LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ (uintptr_t)__builtin_return_address(0), 0) +#define MUTEX_TESTLOCK(mtx) \ + LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ + (uintptr_t)__builtin_return_address(0), -1) #define MUTEX_LOCKED(mtx) \ - LOCKDEBUG_LOCKED(MUTEX_GETID(mtx), \ + LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \ (uintptr_t)__builtin_return_address(0), 0) #define MUTEX_UNLOCKED(mtx) \ - LOCKDEBUG_UNLOCKED(MUTEX_GETID(mtx), \ + LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ (uintptr_t)__builtin_return_address(0), 0) #define MUTEX_ABORT(mtx, msg) \ - mutex_abort(mtx, __FUNCTION__, msg) + mutex_abort(__func__, __LINE__, mtx, msg) #if defined(LOCKDEBUG) #define MUTEX_DASSERT(mtx, cond) \ do { \ - if (!(cond)) \ + if (__predict_false(!(cond))) \ MUTEX_ABORT(mtx, "assertion failed: " #cond); \ -} while (/* CONSTCOND */ 0); +} while (/* CONSTCOND */ 0) #else /* LOCKDEBUG */ @@ -107,7 +107,7 @@ do { \ #define MUTEX_ASSERT(mtx, cond) \ do { \ - if (!(cond)) \ + if (__predict_false(!(cond))) \ MUTEX_ABORT(mtx, "assertion failed: " #cond); \ } while (/* CONSTCOND */ 0) @@ -118,19 +118,40 @@ do { \ #endif /* DIAGNOSTIC */ /* - * Spin mutex SPL save / restore. + * Some architectures can't use __cpu_simple_lock as is so allow a way + * for them to use an alternate definition. */ -#ifndef MUTEX_COUNT_BIAS -#define MUTEX_COUNT_BIAS 0 +#ifndef MUTEX_SPINBIT_LOCK_INIT +#define MUTEX_SPINBIT_LOCK_INIT(mtx) __cpu_simple_lock_init(&(mtx)->mtx_lock) +#endif +#ifndef MUTEX_SPINBIT_LOCKED_P +#define MUTEX_SPINBIT_LOCKED_P(mtx) __SIMPLELOCK_LOCKED_P(&(mtx)->mtx_lock) +#endif +#ifndef MUTEX_SPINBIT_LOCK_TRY +#define MUTEX_SPINBIT_LOCK_TRY(mtx) __cpu_simple_lock_try(&(mtx)->mtx_lock) +#endif +#ifndef MUTEX_SPINBIT_LOCK_UNLOCK +#define MUTEX_SPINBIT_LOCK_UNLOCK(mtx) __cpu_simple_unlock(&(mtx)->mtx_lock) #endif +#ifndef MUTEX_INITIALIZE_SPIN_IPL +#define MUTEX_INITIALIZE_SPIN_IPL(mtx, ipl) \ + ((mtx)->mtx_ipl = makeiplcookie((ipl))) +#endif + +/* + * Spin mutex SPL save / restore. + */ + #define MUTEX_SPIN_SPLRAISE(mtx) \ do { \ - struct cpu_info *x__ci = curcpu(); \ + struct cpu_info *x__ci; \ int x__cnt, s; \ + s = splraiseipl(MUTEX_SPIN_IPL(mtx)); \ + x__ci = curcpu(); \ x__cnt = x__ci->ci_mtx_count--; \ - s = splraiseipl(mtx->mtx_ipl); \ - if (x__cnt == MUTEX_COUNT_BIAS) \ + __insn_barrier(); \ + if (x__cnt == 0) \ x__ci->ci_mtx_oldspl = (s); \ } while (/* CONSTCOND */ 0) @@ -139,11 +160,22 @@ do { \ struct cpu_info *x__ci = curcpu(); \ int s = x__ci->ci_mtx_oldspl; \ __insn_barrier(); \ - if (++(x__ci->ci_mtx_count) == MUTEX_COUNT_BIAS) \ + if (++(x__ci->ci_mtx_count) == 0) \ splx(s); \ } while (/* CONSTCOND */ 0) /* + * Memory barriers. + */ +#ifdef __HAVE_ATOMIC_AS_MEMBAR +#define MUTEX_MEMBAR_ENTER() +#define MUTEX_MEMBAR_EXIT() +#else +#define MUTEX_MEMBAR_ENTER() membar_enter() +#define MUTEX_MEMBAR_EXIT() membar_exit() +#endif + +/* * For architectures that provide 'simple' mutexes: they provide a * CAS function that is either MP-safe, or does not need to be MP * safe. Adaptive mutexes on these architectures do not require an @@ -154,43 +186,54 @@ do { \ #define MUTEX_OWNER(owner) \ (owner & MUTEX_THREAD) -#define MUTEX_OWNED(owner) \ - (owner != 0) #define MUTEX_HAS_WAITERS(mtx) \ (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) -#define MUTEX_INITIALIZE_ADAPTIVE(mtx, id) \ +#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \ do { \ - (mtx)->mtx_id = (id); \ -} while (/* CONSTCOND */ 0); + if (!dodebug) \ + (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ +} while (/* CONSTCOND */ 0) -#define MUTEX_INITIALIZE_SPIN(mtx, id, ipl) \ +#define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \ do { \ (mtx)->mtx_owner = MUTEX_BIT_SPIN; \ - (mtx)->mtx_ipl = makeiplcookie((ipl)); \ - (mtx)->mtx_id = (id); \ - __cpu_simple_lock_init(&(mtx)->mtx_lock); \ + if (!dodebug) \ + (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ + MUTEX_INITIALIZE_SPIN_IPL((mtx), (ipl)); \ + MUTEX_SPINBIT_LOCK_INIT((mtx)); \ } while (/* CONSTCOND */ 0) #define MUTEX_DESTROY(mtx) \ do { \ (mtx)->mtx_owner = MUTEX_THREAD; \ - (mtx)->mtx_id = -1; \ -} while (/* CONSTCOND */ 0); +} while (/* CONSTCOND */ 0) #define MUTEX_SPIN_P(mtx) \ (((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0) #define MUTEX_ADAPTIVE_P(mtx) \ (((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0) -#define MUTEX_GETID(mtx) ((mtx)->mtx_id) +#define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_NODEBUG) == 0) +#if defined(LOCKDEBUG) +#define MUTEX_OWNED(owner) (((owner) & ~MUTEX_BIT_NODEBUG) != 0) +#define MUTEX_INHERITDEBUG(n, o) (n) |= (o) & MUTEX_BIT_NODEBUG +#else /* defined(LOCKDEBUG) */ +#define MUTEX_OWNED(owner) ((owner) != 0) +#define MUTEX_INHERITDEBUG(n, o) /* nothing */ +#endif /* defined(LOCKDEBUG) */ static inline int MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread) { int rv; - rv = MUTEX_CAS(&mtx->mtx_owner, 0UL, curthread); - MUTEX_RECEIVE(mtx); + uintptr_t oldown = 0; + uintptr_t newown = curthread; + + MUTEX_INHERITDEBUG(oldown, mtx->mtx_owner); + MUTEX_INHERITDEBUG(newown, oldown); + rv = MUTEX_CAS(&mtx->mtx_owner, oldown, newown); + MUTEX_MEMBAR_ENTER(); return rv; } @@ -199,21 +242,19 @@ MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr { int rv; rv = MUTEX_CAS(&mtx->mtx_owner, owner, owner | MUTEX_BIT_WAITERS); - MUTEX_RECEIVE(mtx); + MUTEX_MEMBAR_ENTER(); return rv; } static inline void MUTEX_RELEASE(kmutex_t *mtx) { - MUTEX_GIVE(mtx); - mtx->mtx_owner = 0; -} + uintptr_t newown; -static inline void -MUTEX_CLEAR_WAITERS(kmutex_t *mtx) -{ - /* nothing */ + MUTEX_MEMBAR_EXIT(); + newown = 0; + MUTEX_INHERITDEBUG(newown, mtx->mtx_owner); + mtx->mtx_owner = newown; } #endif /* __HAVE_SIMPLE_MUTEXES */ @@ -236,29 +277,28 @@ __strong_alias(mutex_spin_enter,mutex_ve __strong_alias(mutex_spin_exit,mutex_vector_exit); #endif -void mutex_abort(kmutex_t *, const char *, const char *); -void mutex_dump(volatile void *); -int mutex_onproc(uintptr_t, struct cpu_info **); -static struct lwp *mutex_owner(wchan_t); +static void mutex_abort(const char *, size_t, const kmutex_t *, + const char *); +static void mutex_dump(const volatile void *, lockop_printer_t); lockops_t mutex_spin_lockops = { - "Mutex", - 0, - mutex_dump + .lo_name = "Mutex", + .lo_type = LOCKOPS_SPIN, + .lo_dump = mutex_dump, }; lockops_t mutex_adaptive_lockops = { - "Mutex", - 1, - mutex_dump + .lo_name = "Mutex", + .lo_type = LOCKOPS_SLEEP, + .lo_dump = mutex_dump, }; syncobj_t mutex_syncobj = { - SOBJ_SLEEPQ_SORTED, - turnstile_unsleep, - turnstile_changepri, - sleepq_lendpri, - mutex_owner, + .sobj_flag = SOBJ_SLEEPQ_SORTED, + .sobj_unsleep = turnstile_unsleep, + .sobj_changepri = turnstile_changepri, + .sobj_lendpri = sleepq_lendpri, + .sobj_owner = (void *)mutex_owner, }; /* @@ -266,12 +306,12 @@ syncobj_t mutex_syncobj = { * * Dump the contents of a mutex structure. */ -void -mutex_dump(volatile void *cookie) +static void +mutex_dump(const volatile void *cookie, lockop_printer_t pr) { - volatile kmutex_t *mtx = cookie; + const volatile kmutex_t *mtx = cookie; - printf_nolog("owner field : %#018lx wait/spin: %16d/%d\n", + pr("owner field : %#018lx wait/spin: %16d/%d\n", (long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx), MUTEX_SPIN_P(mtx)); } @@ -283,17 +323,12 @@ mutex_dump(volatile void *cookie) * generates a lot of machine code in the DIAGNOSTIC case, so * we ask the compiler to not inline it. */ - -#if __GNUC_PREREQ__(3, 0) -__attribute ((noinline)) __attribute ((noreturn)) -#endif -void -mutex_abort(kmutex_t *mtx, const char *func, const char *msg) +static void __noinline +mutex_abort(const char *func, size_t line, const kmutex_t *mtx, const char *msg) { - LOCKDEBUG_ABORT(MUTEX_GETID(mtx), mtx, (MUTEX_SPIN_P(mtx) ? - &mutex_spin_lockops : &mutex_adaptive_lockops), func, msg); - /* NOTREACHED */ + LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx) ? + &mutex_spin_lockops : &mutex_adaptive_lockops), msg); } /* @@ -305,45 +340,38 @@ mutex_abort(kmutex_t *mtx, const char *f * sleeps - see comments in mutex_vector_enter() about releasing * mutexes unlocked. */ +void _mutex_init(kmutex_t *, kmutex_type_t, int, uintptr_t); void -mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) +_mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl, + uintptr_t return_address) { - u_int id; + lockops_t *lockops __unused; + bool dodebug; memset(mtx, 0, sizeof(*mtx)); - switch (type) { - case MUTEX_ADAPTIVE: - case MUTEX_DEFAULT: - KASSERT(ipl == IPL_NONE); - break; - case MUTEX_DRIVER: - type = (ipl == IPL_NONE ? MUTEX_ADAPTIVE : MUTEX_SPIN); - break; - default: - break; - } - - switch (type) { - case MUTEX_NODEBUG: - id = LOCKDEBUG_ALLOC(mtx, NULL); - MUTEX_INITIALIZE_SPIN(mtx, id, ipl); - break; - case MUTEX_ADAPTIVE: - case MUTEX_DEFAULT: - id = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops); - MUTEX_INITIALIZE_ADAPTIVE(mtx, id); - break; - case MUTEX_SPIN: - id = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops); - MUTEX_INITIALIZE_SPIN(mtx, id, ipl); - break; - default: - panic("mutex_init: impossible type"); - break; + if (ipl == IPL_NONE || ipl == IPL_SOFTCLOCK || + ipl == IPL_SOFTBIO || ipl == IPL_SOFTNET || + ipl == IPL_SOFTSERIAL) { + lockops = (type == MUTEX_NODEBUG ? + NULL : &mutex_adaptive_lockops); + dodebug = LOCKDEBUG_ALLOC(mtx, lockops, return_address); + MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug); + } else { + lockops = (type == MUTEX_NODEBUG ? + NULL : &mutex_spin_lockops); + dodebug = LOCKDEBUG_ALLOC(mtx, lockops, return_address); + MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); } } +void +mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) +{ + + _mutex_init(mtx, type, ipl, (uintptr_t)__builtin_return_address(0)); +} + /* * mutex_destroy: * @@ -357,60 +385,56 @@ mutex_destroy(kmutex_t *mtx) MUTEX_ASSERT(mtx, !MUTEX_OWNED(mtx->mtx_owner) && !MUTEX_HAS_WAITERS(mtx)); } else { - MUTEX_ASSERT(mtx, !__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)); + MUTEX_ASSERT(mtx, !MUTEX_SPINBIT_LOCKED_P(mtx)); } - LOCKDEBUG_FREE(mtx, MUTEX_GETID(mtx)); + LOCKDEBUG_FREE(MUTEX_DEBUG_P(mtx), mtx); MUTEX_DESTROY(mtx); } +#ifdef MULTIPROCESSOR /* - * mutex_onproc: + * mutex_oncpu: * * Return true if an adaptive mutex owner is running on a CPU in the * system. If the target is waiting on the kernel big lock, then we * must release it. This is necessary to avoid deadlock. - * - * Note that we can't use the mutex owner field as an LWP pointer. We - * don't have full control over the timing of our execution, and so the - * pointer could be completely invalid by the time we dereference it. */ -#ifdef MULTIPROCESSOR -int -mutex_onproc(uintptr_t owner, struct cpu_info **cip) +static bool +mutex_oncpu(uintptr_t owner) { - CPU_INFO_ITERATOR cii; struct cpu_info *ci; - struct lwp *l; + lwp_t *l; - if (!MUTEX_OWNED(owner)) - return 0; - l = (struct lwp *)MUTEX_OWNER(owner); + KASSERT(kpreempt_disabled()); - /* See if the target is running on a CPU somewhere. */ - if ((ci = *cip) != NULL && ci->ci_curlwp == l) - goto run; - for (CPU_INFO_FOREACH(cii, ci)) - if (ci->ci_curlwp == l) - goto run; + if (!MUTEX_OWNED(owner)) { + return false; + } - /* No: it may be safe to block now. */ - *cip = NULL; - return 0; + /* + * See lwp_dtor() why dereference of the LWP pointer is safe. + * We must have kernel preemption disabled for that. + */ + l = (lwp_t *)MUTEX_OWNER(owner); + ci = l->l_cpu; - run: - /* Target is running; do we need to block? */ - *cip = ci; - return ci->ci_biglock_wanted != l; + if (ci && ci->ci_curlwp == l) { + /* Target is running; do we need to block? */ + return (ci->ci_biglock_wanted != l); + } + + /* Not running. It may be safe to block now. */ + return false; } #endif /* MULTIPROCESSOR */ /* * mutex_vector_enter: * - * Support routine for mutex_enter() that must handles all cases. In + * Support routine for mutex_enter() that must handle all cases. In * the LOCKDEBUG case, mutex_enter() is always aliased here, even if - * fast-path stubs are available. If an mutex_spin_enter() stub is + * fast-path stubs are available. If a mutex_spin_enter() stub is * not available, then it is also aliased directly here. */ void @@ -419,7 +443,6 @@ mutex_vector_enter(kmutex_t *mtx) uintptr_t owner, curthread; turnstile_t *ts; #ifdef MULTIPROCESSOR - struct cpu_info *ci = NULL; u_int count; #endif LOCKSTAT_COUNTER(spincnt); @@ -438,7 +461,7 @@ mutex_vector_enter(kmutex_t *mtx) MUTEX_SPIN_SPLRAISE(mtx); MUTEX_WANTLOCK(mtx); #ifdef FULL - if (__cpu_simple_lock_try(&mtx->mtx_lock)) { + if (MUTEX_SPINBIT_LOCK_TRY(mtx)) { MUTEX_LOCKED(mtx); return; } @@ -455,16 +478,14 @@ mutex_vector_enter(kmutex_t *mtx) * to reduce cache line ping-ponging between CPUs. */ do { - if (panicstr != NULL) - break; - while (__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) { - SPINLOCK_BACKOFF(count); + while (MUTEX_SPINBIT_LOCKED_P(mtx)) { + SPINLOCK_BACKOFF(count); #ifdef LOCKDEBUG if (SPINLOCK_SPINOUT(spins)) MUTEX_ABORT(mtx, "spinout"); #endif /* LOCKDEBUG */ } - } while (!__cpu_simple_lock_try(&mtx->mtx_lock)); + } while (!MUTEX_SPINBIT_LOCK_TRY(mtx)); if (count != SPINLOCK_BACKOFF_MIN) { LOCKSTAT_STOP_TIMER(lsflag, spintime); @@ -482,18 +503,13 @@ mutex_vector_enter(kmutex_t *mtx) MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); MUTEX_ASSERT(mtx, curthread != 0); + MUTEX_ASSERT(mtx, !cpu_intr_p()); MUTEX_WANTLOCK(mtx); -#ifdef LOCKDEBUG if (panicstr == NULL) { - simple_lock_only_held(NULL, "mutex_enter"); -#ifdef MULTIPROCESSOR + KDASSERT(pserialize_not_in_read_section()); LOCKDEBUG_BARRIER(&kernel_lock, 1); -#else - LOCKDEBUG_BARRIER(NULL, 1); -#endif } -#endif LOCKSTAT_ENTER(lsflag); @@ -502,8 +518,8 @@ mutex_vector_enter(kmutex_t *mtx) * determine that the owner is not running on a processor, * then we stop spinning, and sleep instead. */ - for (;;) { - owner = mtx->mtx_owner; + KPREEMPT_DISABLE(curlwp); + for (owner = mtx->mtx_owner;;) { if (!MUTEX_OWNED(owner)) { /* * Mutex owner clear could mean two things: @@ -516,29 +532,27 @@ mutex_vector_enter(kmutex_t *mtx) */ if (MUTEX_ACQUIRE(mtx, curthread)) break; + owner = mtx->mtx_owner; continue; } - - if (panicstr != NULL) - return; - if (MUTEX_OWNER(owner) == curthread) + if (__predict_false(MUTEX_OWNER(owner) == curthread)) { MUTEX_ABORT(mtx, "locking against myself"); - + } #ifdef MULTIPROCESSOR /* * Check to see if the owner is running on a processor. * If so, then we should just spin, as the owner will * likely release the lock very soon. */ - if (mutex_onproc(owner, &ci)) { + if (mutex_oncpu(owner)) { LOCKSTAT_START_TIMER(lsflag, spintime); count = SPINLOCK_BACKOFF_MIN; - for (;;) { - owner = mtx->mtx_owner; - if (!mutex_onproc(owner, &ci)) - break; + do { + KPREEMPT_ENABLE(curlwp); SPINLOCK_BACKOFF(count); - } + KPREEMPT_DISABLE(curlwp); + owner = mtx->mtx_owner; + } while (mutex_oncpu(owner)); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_COUNT(spincnt, 1); if (!MUTEX_OWNED(owner)) @@ -550,11 +564,12 @@ mutex_vector_enter(kmutex_t *mtx) /* * Once we have the turnstile chain interlock, mark the - * mutex has having waiters. If that fails, spin again: + * mutex as having waiters. If that fails, spin again: * chances are that the mutex has been released. */ if (!MUTEX_SET_WAITERS(mtx, owner)) { turnstile_exit(mtx); + owner = mtx->mtx_owner; continue; } @@ -577,7 +592,7 @@ mutex_vector_enter(kmutex_t *mtx) * .. clear lock word, waiters * return success * - * There is a another race that can occur: a third CPU could + * There is another race that can occur: a third CPU could * acquire the mutex as soon as it is released. Since * adaptive mutexes are primarily spin mutexes, this is not * something that we need to worry about too much. What we @@ -592,7 +607,7 @@ mutex_vector_enter(kmutex_t *mtx) * or preempted). * * o At any given time, MUTEX_SET_WAITERS() can only ever - * be in progress on one CPU in the system - guarenteed + * be in progress on one CPU in the system - guaranteed * by the turnstile chain lock. * * o No other operations other than MUTEX_SET_WAITERS() @@ -622,23 +637,23 @@ mutex_vector_enter(kmutex_t *mtx) * waiters field) and check the lock holder's status again. * Some of the possible outcomes (not an exhaustive list): * - * 1. The onproc check returns true: the holding LWP is + * 1. The on-CPU check returns true: the holding LWP is * running again. The lock may be released soon and * we should spin. Importantly, we can't trust the * value of the waiters flag. * - * 2. The onproc check returns false: the holding LWP is - * not running. We now have the oppertunity to check + * 2. The on-CPU check returns false: the holding LWP is + * not running. We now have the opportunity to check * if mutex_exit() has blatted the modifications made * by MUTEX_SET_WAITERS(). * - * 3. The onproc check returns false: the holding LWP may + * 3. The on-CPU check returns false: the holding LWP may * or may not be running. It has context switched at * some point during our check. Again, we have the * chance to see if the waiters bit is still set or * has been overwritten. * - * 4. The onproc check returns false: the holding LWP is + * 4. The on-CPU check returns false: the holding LWP is * running on a CPU, but wants the big lock. It's OK * to check the waiters field in this case. * @@ -652,9 +667,10 @@ mutex_vector_enter(kmutex_t *mtx) * If the waiters bit is not set it's unsafe to go asleep, * as we might never be awoken. */ - if ((mb_read(), mutex_onproc(owner, &ci)) || - (mb_read(), !MUTEX_HAS_WAITERS(mtx))) { + if ((membar_consumer(), mutex_oncpu(owner)) || + (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) { turnstile_exit(mtx); + owner = mtx->mtx_owner; continue; } #endif /* MULTIPROCESSOR */ @@ -665,7 +681,10 @@ mutex_vector_enter(kmutex_t *mtx) LOCKSTAT_STOP_TIMER(lsflag, slptime); LOCKSTAT_COUNT(slpcnt, 1); + + owner = mtx->mtx_owner; } + KPREEMPT_ENABLE(curlwp); LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1, slpcnt, slptime); @@ -690,25 +709,23 @@ mutex_vector_exit(kmutex_t *mtx) if (MUTEX_SPIN_P(mtx)) { #ifdef FULL - if (!__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) + if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) { MUTEX_ABORT(mtx, "exiting unheld spin mutex"); + } MUTEX_UNLOCKED(mtx); - __cpu_simple_unlock(&mtx->mtx_lock); + MUTEX_SPINBIT_LOCK_UNLOCK(mtx); #endif MUTEX_SPIN_SPLRESTORE(mtx); return; } - if (__predict_false((uintptr_t)panicstr | cold)) { - MUTEX_UNLOCKED(mtx); - MUTEX_RELEASE(mtx); - return; - } - curthread = (uintptr_t)curlwp; MUTEX_DASSERT(mtx, curthread != 0); MUTEX_ASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread); MUTEX_UNLOCKED(mtx); +#if !defined(LOCKDEBUG) + __USE(curthread); +#endif #ifdef LOCKDEBUG /* @@ -775,13 +792,15 @@ mutex_wakeup(kmutex_t *mtx) * holds the mutex. */ int -mutex_owned(kmutex_t *mtx) +mutex_owned(const kmutex_t *mtx) { + if (mtx == NULL) + return 0; if (MUTEX_ADAPTIVE_P(mtx)) return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp; #ifdef FULL - return __SIMPLELOCK_LOCKED_P(&mtx->mtx_lock); + return MUTEX_SPINBIT_LOCKED_P(mtx); #else return 1; #endif @@ -793,16 +812,50 @@ mutex_owned(kmutex_t *mtx) * Return the current owner of an adaptive mutex. Used for * priority inheritance. */ -static struct lwp * -mutex_owner(wchan_t obj) +lwp_t * +mutex_owner(const kmutex_t *mtx) { - kmutex_t *mtx = (void *)(uintptr_t)obj; /* discard qualifiers */ MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner); } /* + * mutex_owner_running: + * + * Return true if an adaptive mutex is held and the owner is running + * on a CPU. For the pagedaemon. + */ +bool +mutex_owner_running(const kmutex_t *mtx) +{ + bool rv; + + MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); + kpreempt_disable(); + rv = mutex_oncpu(MUTEX_OWNER(mtx->mtx_owner)); + kpreempt_enable(); + return rv; +} + +/* + * mutex_ownable: + * + * When compiled with DEBUG and LOCKDEBUG defined, ensure that + * the mutex is available. We cannot use !mutex_owned() since + * that won't work correctly for spin mutexes. + */ +int +mutex_ownable(const kmutex_t *mtx) +{ + +#ifdef LOCKDEBUG + MUTEX_TESTLOCK(mtx); +#endif + return 1; +} + +/* * mutex_tryenter: * * Try to acquire the mutex; return non-zero if we did. @@ -818,7 +871,7 @@ mutex_tryenter(kmutex_t *mtx) if (MUTEX_SPIN_P(mtx)) { MUTEX_SPIN_SPLRAISE(mtx); #ifdef FULL - if (__cpu_simple_lock_try(&mtx->mtx_lock)) { + if (MUTEX_SPINBIT_LOCK_TRY(mtx)) { MUTEX_WANTLOCK(mtx); MUTEX_LOCKED(mtx); return 1; @@ -873,16 +926,14 @@ mutex_spin_retry(kmutex_t *mtx) * to reduce cache line ping-ponging between CPUs. */ do { - if (panicstr != NULL) - break; - while (__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) { - SPINLOCK_BACKOFF(count); + while (MUTEX_SPINBIT_LOCKED_P(mtx)) { + SPINLOCK_BACKOFF(count); #ifdef LOCKDEBUG if (SPINLOCK_SPINOUT(spins)) MUTEX_ABORT(mtx, "spinout"); #endif /* LOCKDEBUG */ } - } while (!__cpu_simple_lock_try(&mtx->mtx_lock)); + } while (!MUTEX_SPINBIT_LOCK_TRY(mtx)); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime);