version 1.62, 2015/05/25 21:02:37 |
version 1.79, 2019/05/09 05:00:31 |
Line 54 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 54 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include <sys/intr.h> |
#include <sys/intr.h> |
#include <sys/lock.h> |
#include <sys/lock.h> |
#include <sys/types.h> |
#include <sys/types.h> |
|
#include <sys/cpu.h> |
|
#include <sys/pserialize.h> |
|
|
#include <dev/lockstat.h> |
#include <dev/lockstat.h> |
|
|
#include <machine/lock.h> |
#include <machine/lock.h> |
|
|
|
#define MUTEX_PANIC_SKIP_SPIN 1 |
|
#define MUTEX_PANIC_SKIP_ADAPTIVE 1 |
|
|
/* |
/* |
* When not running a debug kernel, spin mutexes are not much |
* When not running a debug kernel, spin mutexes are not much |
* more than an splraiseipl() and splx() pair. |
* more than an splraiseipl() and splx() pair. |
Line 75 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 80 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#define MUTEX_WANTLOCK(mtx) \ |
#define MUTEX_WANTLOCK(mtx) \ |
LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ |
LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ |
(uintptr_t)__builtin_return_address(0), 0) |
(uintptr_t)__builtin_return_address(0), 0) |
|
#define MUTEX_TESTLOCK(mtx) \ |
|
LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ |
|
(uintptr_t)__builtin_return_address(0), -1) |
#define MUTEX_LOCKED(mtx) \ |
#define MUTEX_LOCKED(mtx) \ |
LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \ |
LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \ |
(uintptr_t)__builtin_return_address(0), 0) |
(uintptr_t)__builtin_return_address(0), 0) |
Line 82 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 90 __KERNEL_RCSID(0, "$NetBSD$"); |
|
LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ |
LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ |
(uintptr_t)__builtin_return_address(0), 0) |
(uintptr_t)__builtin_return_address(0), 0) |
#define MUTEX_ABORT(mtx, msg) \ |
#define MUTEX_ABORT(mtx, msg) \ |
mutex_abort(mtx, __func__, msg) |
mutex_abort(__func__, __LINE__, mtx, msg) |
|
|
#if defined(LOCKDEBUG) |
#if defined(LOCKDEBUG) |
|
|
#define MUTEX_DASSERT(mtx, cond) \ |
#define MUTEX_DASSERT(mtx, cond) \ |
do { \ |
do { \ |
if (!(cond)) \ |
if (__predict_false(!(cond))) \ |
MUTEX_ABORT(mtx, "assertion failed: " #cond); \ |
MUTEX_ABORT(mtx, "assertion failed: " #cond); \ |
} while (/* CONSTCOND */ 0); |
} while (/* CONSTCOND */ 0) |
|
|
#else /* LOCKDEBUG */ |
#else /* LOCKDEBUG */ |
|
|
|
|
|
|
#define MUTEX_ASSERT(mtx, cond) \ |
#define MUTEX_ASSERT(mtx, cond) \ |
do { \ |
do { \ |
if (!(cond)) \ |
if (__predict_false(!(cond))) \ |
MUTEX_ABORT(mtx, "assertion failed: " #cond); \ |
MUTEX_ABORT(mtx, "assertion failed: " #cond); \ |
} while (/* CONSTCOND */ 0) |
} while (/* CONSTCOND */ 0) |
|
|
|
|
(((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) |
(((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) |
|
|
#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \ |
#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \ |
|
do { \ |
if (!dodebug) \ |
if (!dodebug) \ |
(mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ |
(mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ |
do { \ |
} while (/* CONSTCOND */ 0) |
} while (/* CONSTCOND */ 0); |
|
|
|
#define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \ |
#define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \ |
do { \ |
do { \ |
|
|
#define MUTEX_DESTROY(mtx) \ |
#define MUTEX_DESTROY(mtx) \ |
do { \ |
do { \ |
(mtx)->mtx_owner = MUTEX_THREAD; \ |
(mtx)->mtx_owner = MUTEX_THREAD; \ |
} while (/* CONSTCOND */ 0); |
} while (/* CONSTCOND */ 0) |
|
|
#define MUTEX_SPIN_P(mtx) \ |
#define MUTEX_SPIN_P(mtx) \ |
(((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0) |
(((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0) |
Line 261 __strong_alias(mutex_spin_enter,mutex_ve |
|
Line 269 __strong_alias(mutex_spin_enter,mutex_ve |
|
__strong_alias(mutex_spin_exit,mutex_vector_exit); |
__strong_alias(mutex_spin_exit,mutex_vector_exit); |
#endif |
#endif |
|
|
static void mutex_abort(kmutex_t *, const char *, const char *); |
static void mutex_abort(const char *, size_t, const kmutex_t *, |
static void mutex_dump(volatile void *); |
const char *); |
|
static void mutex_dump(const volatile void *, lockop_printer_t); |
|
|
lockops_t mutex_spin_lockops = { |
lockops_t mutex_spin_lockops = { |
"Mutex", |
.lo_name = "Mutex", |
LOCKOPS_SPIN, |
.lo_type = LOCKOPS_SPIN, |
mutex_dump |
.lo_dump = mutex_dump, |
}; |
}; |
|
|
lockops_t mutex_adaptive_lockops = { |
lockops_t mutex_adaptive_lockops = { |
"Mutex", |
.lo_name = "Mutex", |
LOCKOPS_SLEEP, |
.lo_type = LOCKOPS_SLEEP, |
mutex_dump |
.lo_dump = mutex_dump, |
}; |
}; |
|
|
syncobj_t mutex_syncobj = { |
syncobj_t mutex_syncobj = { |
SOBJ_SLEEPQ_SORTED, |
.sobj_flag = SOBJ_SLEEPQ_SORTED, |
turnstile_unsleep, |
.sobj_unsleep = turnstile_unsleep, |
turnstile_changepri, |
.sobj_changepri = turnstile_changepri, |
sleepq_lendpri, |
.sobj_lendpri = sleepq_lendpri, |
(void *)mutex_owner, |
.sobj_owner = (void *)mutex_owner, |
}; |
}; |
|
|
/* |
/* |
Line 289 syncobj_t mutex_syncobj = { |
|
Line 298 syncobj_t mutex_syncobj = { |
|
* |
* |
* Dump the contents of a mutex structure. |
* Dump the contents of a mutex structure. |
*/ |
*/ |
void |
static void |
mutex_dump(volatile void *cookie) |
mutex_dump(const volatile void *cookie, lockop_printer_t pr) |
{ |
{ |
volatile kmutex_t *mtx = cookie; |
const volatile kmutex_t *mtx = cookie; |
|
|
printf_nolog("owner field : %#018lx wait/spin: %16d/%d\n", |
pr("owner field : %#018lx wait/spin: %16d/%d\n", |
(long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx), |
(long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx), |
MUTEX_SPIN_P(mtx)); |
MUTEX_SPIN_P(mtx)); |
} |
} |
Line 306 mutex_dump(volatile void *cookie) |
|
Line 315 mutex_dump(volatile void *cookie) |
|
* generates a lot of machine code in the DIAGNOSTIC case, so |
* generates a lot of machine code in the DIAGNOSTIC case, so |
* we ask the compiler to not inline it. |
* we ask the compiler to not inline it. |
*/ |
*/ |
void __noinline |
static void __noinline |
mutex_abort(kmutex_t *mtx, const char *func, const char *msg) |
mutex_abort(const char *func, size_t line, const kmutex_t *mtx, const char *msg) |
{ |
{ |
|
|
LOCKDEBUG_ABORT(mtx, (MUTEX_SPIN_P(mtx) ? |
LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx) ? |
&mutex_spin_lockops : &mutex_adaptive_lockops), func, msg); |
&mutex_spin_lockops : &mutex_adaptive_lockops), msg); |
} |
} |
|
|
/* |
/* |
Line 323 mutex_abort(kmutex_t *mtx, const char *f |
|
Line 332 mutex_abort(kmutex_t *mtx, const char *f |
|
* sleeps - see comments in mutex_vector_enter() about releasing |
* sleeps - see comments in mutex_vector_enter() about releasing |
* mutexes unlocked. |
* mutexes unlocked. |
*/ |
*/ |
|
void _mutex_init(kmutex_t *, kmutex_type_t, int, uintptr_t); |
void |
void |
mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) |
_mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl, |
|
uintptr_t return_address) |
{ |
{ |
bool dodebug; |
bool dodebug; |
|
|
Line 350 mutex_init(kmutex_t *mtx, kmutex_type_t |
|
Line 361 mutex_init(kmutex_t *mtx, kmutex_type_t |
|
|
|
switch (type) { |
switch (type) { |
case MUTEX_NODEBUG: |
case MUTEX_NODEBUG: |
dodebug = LOCKDEBUG_ALLOC(mtx, NULL, |
dodebug = LOCKDEBUG_ALLOC(mtx, NULL, return_address); |
(uintptr_t)__builtin_return_address(0)); |
|
MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); |
MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); |
break; |
break; |
case MUTEX_ADAPTIVE: |
case MUTEX_ADAPTIVE: |
dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops, |
dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops, |
(uintptr_t)__builtin_return_address(0)); |
return_address); |
MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug); |
MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug); |
break; |
break; |
case MUTEX_SPIN: |
case MUTEX_SPIN: |
dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops, |
dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops, |
(uintptr_t)__builtin_return_address(0)); |
return_address); |
MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); |
MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); |
break; |
break; |
default: |
default: |
Line 370 mutex_init(kmutex_t *mtx, kmutex_type_t |
|
Line 380 mutex_init(kmutex_t *mtx, kmutex_type_t |
|
} |
} |
} |
} |
|
|
|
void |
|
mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) |
|
{ |
|
|
|
_mutex_init(mtx, type, ipl, (uintptr_t)__builtin_return_address(0)); |
|
} |
|
|
/* |
/* |
* mutex_destroy: |
* mutex_destroy: |
* |
* |
Line 476 mutex_vector_enter(kmutex_t *mtx) |
|
Line 493 mutex_vector_enter(kmutex_t *mtx) |
|
* to reduce cache line ping-ponging between CPUs. |
* to reduce cache line ping-ponging between CPUs. |
*/ |
*/ |
do { |
do { |
|
#if MUTEX_PANIC_SKIP_SPIN |
if (panicstr != NULL) |
if (panicstr != NULL) |
break; |
break; |
|
#endif |
while (MUTEX_SPINBIT_LOCKED_P(mtx)) { |
while (MUTEX_SPINBIT_LOCKED_P(mtx)) { |
SPINLOCK_BACKOFF(count); |
SPINLOCK_BACKOFF(count); |
#ifdef LOCKDEBUG |
#ifdef LOCKDEBUG |
if (SPINLOCK_SPINOUT(spins)) |
if (SPINLOCK_SPINOUT(spins)) |
MUTEX_ABORT(mtx, "spinout"); |
MUTEX_ABORT(mtx, "spinout"); |
Line 503 mutex_vector_enter(kmutex_t *mtx) |
|
Line 522 mutex_vector_enter(kmutex_t *mtx) |
|
|
|
MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); |
MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); |
MUTEX_ASSERT(mtx, curthread != 0); |
MUTEX_ASSERT(mtx, curthread != 0); |
|
MUTEX_ASSERT(mtx, !cpu_intr_p()); |
MUTEX_WANTLOCK(mtx); |
MUTEX_WANTLOCK(mtx); |
|
|
if (panicstr == NULL) { |
if (panicstr == NULL) { |
|
KDASSERT(pserialize_not_in_read_section()); |
LOCKDEBUG_BARRIER(&kernel_lock, 1); |
LOCKDEBUG_BARRIER(&kernel_lock, 1); |
} |
} |
|
|
Line 533 mutex_vector_enter(kmutex_t *mtx) |
|
Line 554 mutex_vector_enter(kmutex_t *mtx) |
|
owner = mtx->mtx_owner; |
owner = mtx->mtx_owner; |
continue; |
continue; |
} |
} |
|
#if MUTEX_PANIC_SKIP_ADAPTIVE |
if (__predict_false(panicstr != NULL)) { |
if (__predict_false(panicstr != NULL)) { |
KPREEMPT_ENABLE(curlwp); |
KPREEMPT_ENABLE(curlwp); |
return; |
return; |
} |
} |
|
#endif |
if (__predict_false(MUTEX_OWNER(owner) == curthread)) { |
if (__predict_false(MUTEX_OWNER(owner) == curthread)) { |
MUTEX_ABORT(mtx, "locking against myself"); |
MUTEX_ABORT(mtx, "locking against myself"); |
} |
} |
Line 566 mutex_vector_enter(kmutex_t *mtx) |
|
Line 589 mutex_vector_enter(kmutex_t *mtx) |
|
|
|
/* |
/* |
* Once we have the turnstile chain interlock, mark the |
* Once we have the turnstile chain interlock, mark the |
* mutex has having waiters. If that fails, spin again: |
* mutex as having waiters. If that fails, spin again: |
* chances are that the mutex has been released. |
* chances are that the mutex has been released. |
*/ |
*/ |
if (!MUTEX_SET_WAITERS(mtx, owner)) { |
if (!MUTEX_SET_WAITERS(mtx, owner)) { |
Line 712 mutex_vector_exit(kmutex_t *mtx) |
|
Line 735 mutex_vector_exit(kmutex_t *mtx) |
|
if (MUTEX_SPIN_P(mtx)) { |
if (MUTEX_SPIN_P(mtx)) { |
#ifdef FULL |
#ifdef FULL |
if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) { |
if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) { |
|
#if MUTEX_PANIC_SKIP_SPIN |
if (panicstr != NULL) |
if (panicstr != NULL) |
return; |
return; |
|
#endif |
MUTEX_ABORT(mtx, "exiting unheld spin mutex"); |
MUTEX_ABORT(mtx, "exiting unheld spin mutex"); |
} |
} |
MUTEX_UNLOCKED(mtx); |
MUTEX_UNLOCKED(mtx); |
Line 723 mutex_vector_exit(kmutex_t *mtx) |
|
Line 748 mutex_vector_exit(kmutex_t *mtx) |
|
return; |
return; |
} |
} |
|
|
|
#ifdef MUTEX_PANIC_SKIP_ADAPTIVE |
if (__predict_false((uintptr_t)panicstr | cold)) { |
if (__predict_false((uintptr_t)panicstr | cold)) { |
MUTEX_UNLOCKED(mtx); |
MUTEX_UNLOCKED(mtx); |
MUTEX_RELEASE(mtx); |
MUTEX_RELEASE(mtx); |
return; |
return; |
} |
} |
|
#endif |
|
|
curthread = (uintptr_t)curlwp; |
curthread = (uintptr_t)curlwp; |
MUTEX_DASSERT(mtx, curthread != 0); |
MUTEX_DASSERT(mtx, curthread != 0); |
Line 802 mutex_wakeup(kmutex_t *mtx) |
|
Line 829 mutex_wakeup(kmutex_t *mtx) |
|
* holds the mutex. |
* holds the mutex. |
*/ |
*/ |
int |
int |
mutex_owned(kmutex_t *mtx) |
mutex_owned(const kmutex_t *mtx) |
{ |
{ |
|
|
if (mtx == NULL) |
if (mtx == NULL) |
Line 823 mutex_owned(kmutex_t *mtx) |
|
Line 850 mutex_owned(kmutex_t *mtx) |
|
* priority inheritance. |
* priority inheritance. |
*/ |
*/ |
lwp_t * |
lwp_t * |
mutex_owner(kmutex_t *mtx) |
mutex_owner(const kmutex_t *mtx) |
{ |
{ |
|
|
MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); |
MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); |
Line 831 mutex_owner(kmutex_t *mtx) |
|
Line 858 mutex_owner(kmutex_t *mtx) |
|
} |
} |
|
|
/* |
/* |
|
* mutex_ownable: |
|
* |
|
* When compiled with DEBUG and LOCKDEBUG defined, ensure that |
|
* the mutex is available. We cannot use !mutex_owned() since |
|
* that won't work correctly for spin mutexes. |
|
*/ |
|
int |
|
mutex_ownable(const kmutex_t *mtx) |
|
{ |
|
|
|
#ifdef LOCKDEBUG |
|
MUTEX_TESTLOCK(mtx); |
|
#endif |
|
return 1; |
|
} |
|
|
|
/* |
* mutex_tryenter: |
* mutex_tryenter: |
* |
* |
* Try to acquire the mutex; return non-zero if we did. |
* Try to acquire the mutex; return non-zero if we did. |
Line 901 mutex_spin_retry(kmutex_t *mtx) |
|
Line 945 mutex_spin_retry(kmutex_t *mtx) |
|
* to reduce cache line ping-ponging between CPUs. |
* to reduce cache line ping-ponging between CPUs. |
*/ |
*/ |
do { |
do { |
|
#if MUTEX_PANIC_SKIP_SPIN |
if (panicstr != NULL) |
if (panicstr != NULL) |
break; |
break; |
|
#endif |
while (MUTEX_SPINBIT_LOCKED_P(mtx)) { |
while (MUTEX_SPINBIT_LOCKED_P(mtx)) { |
SPINLOCK_BACKOFF(count); |
SPINLOCK_BACKOFF(count); |
#ifdef LOCKDEBUG |
#ifdef LOCKDEBUG |
if (SPINLOCK_SPINOUT(spins)) |
if (SPINLOCK_SPINOUT(spins)) |
MUTEX_ABORT(mtx, "spinout"); |
MUTEX_ABORT(mtx, "spinout"); |