version 1.16, 2007/09/10 11:34:10 |
version 1.39, 2008/04/30 00:40:13 |
|
|
/* $NetBSD$ */ |
/* $NetBSD$ */ |
|
|
/*- |
/*- |
* Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. |
* Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. |
* All rights reserved. |
* All rights reserved. |
* |
* |
* This code is derived from software contributed to The NetBSD Foundation |
* This code is derived from software contributed to The NetBSD Foundation |
|
|
* 2. Redistributions in binary form must reproduce the above copyright |
* 2. Redistributions in binary form must reproduce the above copyright |
* notice, this list of conditions and the following disclaimer in the |
* notice, this list of conditions and the following disclaimer in the |
* documentation and/or other materials provided with the distribution. |
* documentation and/or other materials provided with the distribution. |
* 3. All advertising materials mentioning features or use of this software |
|
* must display the following acknowledgement: |
|
* This product includes software developed by the NetBSD |
|
* Foundation, Inc. and its contributors. |
|
* 4. Neither the name of The NetBSD Foundation nor the names of its |
|
* contributors may be used to endorse or promote products derived |
|
* from this software without specific prior written permission. |
|
* |
* |
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
|
|
* Richard McDougall. |
* Richard McDougall. |
*/ |
*/ |
|
|
#include "opt_multiprocessor.h" |
|
|
|
#define __MUTEX_PRIVATE |
#define __MUTEX_PRIVATE |
|
|
#include <sys/cdefs.h> |
#include <sys/cdefs.h> |
__KERNEL_RCSID(0, "$NetBSD$"); |
__KERNEL_RCSID(0, "$NetBSD$"); |
|
|
|
#include "opt_multiprocessor.h" |
|
|
#include <sys/param.h> |
#include <sys/param.h> |
#include <sys/proc.h> |
#include <sys/proc.h> |
#include <sys/mutex.h> |
#include <sys/mutex.h> |
Line 59 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 52 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include <sys/systm.h> |
#include <sys/systm.h> |
#include <sys/lockdebug.h> |
#include <sys/lockdebug.h> |
#include <sys/kernel.h> |
#include <sys/kernel.h> |
|
#include <sys/atomic.h> |
|
#include <sys/intr.h> |
|
#include <sys/lock.h> |
|
#include <sys/pool.h> |
|
|
#include <dev/lockstat.h> |
#include <dev/lockstat.h> |
|
|
#include <machine/intr.h> |
#include <machine/lock.h> |
|
|
/* |
/* |
* When not running a debug kernel, spin mutexes are not much |
* When not running a debug kernel, spin mutexes are not much |
Line 78 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 75 __KERNEL_RCSID(0, "$NetBSD$"); |
|
*/ |
*/ |
|
|
#define MUTEX_WANTLOCK(mtx) \ |
#define MUTEX_WANTLOCK(mtx) \ |
LOCKDEBUG_WANTLOCK(MUTEX_GETID(mtx), \ |
LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ |
(uintptr_t)__builtin_return_address(0), 0) |
(uintptr_t)__builtin_return_address(0), 0) |
#define MUTEX_LOCKED(mtx) \ |
#define MUTEX_LOCKED(mtx) \ |
LOCKDEBUG_LOCKED(MUTEX_GETID(mtx), \ |
LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ |
(uintptr_t)__builtin_return_address(0), 0) |
(uintptr_t)__builtin_return_address(0), 0) |
#define MUTEX_UNLOCKED(mtx) \ |
#define MUTEX_UNLOCKED(mtx) \ |
LOCKDEBUG_UNLOCKED(MUTEX_GETID(mtx), \ |
LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ |
(uintptr_t)__builtin_return_address(0), 0) |
(uintptr_t)__builtin_return_address(0), 0) |
#define MUTEX_ABORT(mtx, msg) \ |
#define MUTEX_ABORT(mtx, msg) \ |
mutex_abort(mtx, __FUNCTION__, msg) |
mutex_abort(mtx, __func__, msg) |
|
|
#if defined(LOCKDEBUG) |
#if defined(LOCKDEBUG) |
|
|
|
|
|
|
#define MUTEX_SPIN_SPLRAISE(mtx) \ |
#define MUTEX_SPIN_SPLRAISE(mtx) \ |
do { \ |
do { \ |
struct cpu_info *x__ci = curcpu(); \ |
struct cpu_info *x__ci; \ |
int x__cnt, s; \ |
int x__cnt, s; \ |
x__cnt = x__ci->ci_mtx_count--; \ |
|
s = splraiseipl(mtx->mtx_ipl); \ |
s = splraiseipl(mtx->mtx_ipl); \ |
|
x__ci = curcpu(); \ |
|
x__cnt = x__ci->ci_mtx_count--; \ |
|
__insn_barrier(); \ |
if (x__cnt == MUTEX_COUNT_BIAS) \ |
if (x__cnt == MUTEX_COUNT_BIAS) \ |
x__ci->ci_mtx_oldspl = (s); \ |
x__ci->ci_mtx_oldspl = (s); \ |
} while (/* CONSTCOND */ 0) |
} while (/* CONSTCOND */ 0) |
|
|
|
|
#define MUTEX_OWNER(owner) \ |
#define MUTEX_OWNER(owner) \ |
(owner & MUTEX_THREAD) |
(owner & MUTEX_THREAD) |
#define MUTEX_OWNED(owner) \ |
|
(owner != 0) |
|
#define MUTEX_HAS_WAITERS(mtx) \ |
#define MUTEX_HAS_WAITERS(mtx) \ |
(((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) |
(((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) |
|
|
#define MUTEX_INITIALIZE_ADAPTIVE(mtx, id) \ |
#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \ |
do { \ |
do { \ |
(mtx)->mtx_id = (id); \ |
if (dodebug) \ |
|
(mtx)->mtx_owner |= MUTEX_BIT_DEBUG; \ |
} while (/* CONSTCOND */ 0); |
} while (/* CONSTCOND */ 0); |
|
|
#define MUTEX_INITIALIZE_SPIN(mtx, id, ipl) \ |
#define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \ |
do { \ |
do { \ |
(mtx)->mtx_owner = MUTEX_BIT_SPIN; \ |
(mtx)->mtx_owner = MUTEX_BIT_SPIN; \ |
|
if (dodebug) \ |
|
(mtx)->mtx_owner |= MUTEX_BIT_DEBUG; \ |
(mtx)->mtx_ipl = makeiplcookie((ipl)); \ |
(mtx)->mtx_ipl = makeiplcookie((ipl)); \ |
(mtx)->mtx_id = (id); \ |
|
__cpu_simple_lock_init(&(mtx)->mtx_lock); \ |
__cpu_simple_lock_init(&(mtx)->mtx_lock); \ |
} while (/* CONSTCOND */ 0) |
} while (/* CONSTCOND */ 0) |
|
|
#define MUTEX_DESTROY(mtx) \ |
#define MUTEX_DESTROY(mtx) \ |
do { \ |
do { \ |
(mtx)->mtx_owner = MUTEX_THREAD; \ |
(mtx)->mtx_owner = MUTEX_THREAD; \ |
(mtx)->mtx_id = -1; \ |
|
} while (/* CONSTCOND */ 0); |
} while (/* CONSTCOND */ 0); |
|
|
#define MUTEX_SPIN_P(mtx) \ |
#define MUTEX_SPIN_P(mtx) \ |
|
|
#define MUTEX_ADAPTIVE_P(mtx) \ |
#define MUTEX_ADAPTIVE_P(mtx) \ |
(((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0) |
(((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0) |
|
|
#define MUTEX_GETID(mtx) ((mtx)->mtx_id) |
#define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_DEBUG) != 0) |
|
#if defined(LOCKDEBUG) |
|
#define MUTEX_OWNED(owner) (((owner) & ~MUTEX_BIT_DEBUG) != 0) |
|
#define MUTEX_INHERITDEBUG(new, old) (new) |= (old) & MUTEX_BIT_DEBUG |
|
#else /* defined(LOCKDEBUG) */ |
|
#define MUTEX_OWNED(owner) ((owner) != 0) |
|
#define MUTEX_INHERITDEBUG(new, old) /* nothing */ |
|
#endif /* defined(LOCKDEBUG) */ |
|
|
static inline int |
static inline int |
MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread) |
MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread) |
{ |
{ |
int rv; |
int rv; |
rv = MUTEX_CAS(&mtx->mtx_owner, 0UL, curthread); |
uintptr_t old = 0; |
|
uintptr_t new = curthread; |
|
|
|
MUTEX_INHERITDEBUG(old, mtx->mtx_owner); |
|
MUTEX_INHERITDEBUG(new, old); |
|
rv = MUTEX_CAS(&mtx->mtx_owner, old, new); |
MUTEX_RECEIVE(mtx); |
MUTEX_RECEIVE(mtx); |
return rv; |
return rv; |
} |
} |
Line 206 MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr |
|
Line 216 MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr |
|
static inline void |
static inline void |
MUTEX_RELEASE(kmutex_t *mtx) |
MUTEX_RELEASE(kmutex_t *mtx) |
{ |
{ |
|
uintptr_t new; |
|
|
MUTEX_GIVE(mtx); |
MUTEX_GIVE(mtx); |
mtx->mtx_owner = 0; |
new = 0; |
|
MUTEX_INHERITDEBUG(new, mtx->mtx_owner); |
|
mtx->mtx_owner = new; |
} |
} |
|
|
static inline void |
static inline void |
Line 239 __strong_alias(mutex_spin_exit,mutex_vec |
|
Line 253 __strong_alias(mutex_spin_exit,mutex_vec |
|
void mutex_abort(kmutex_t *, const char *, const char *); |
void mutex_abort(kmutex_t *, const char *, const char *); |
void mutex_dump(volatile void *); |
void mutex_dump(volatile void *); |
int mutex_onproc(uintptr_t, struct cpu_info **); |
int mutex_onproc(uintptr_t, struct cpu_info **); |
static struct lwp *mutex_owner(wchan_t); |
|
|
|
lockops_t mutex_spin_lockops = { |
lockops_t mutex_spin_lockops = { |
"Mutex", |
"Mutex", |
Line 258 syncobj_t mutex_syncobj = { |
|
Line 271 syncobj_t mutex_syncobj = { |
|
turnstile_unsleep, |
turnstile_unsleep, |
turnstile_changepri, |
turnstile_changepri, |
sleepq_lendpri, |
sleepq_lendpri, |
mutex_owner, |
(void *)mutex_owner, |
|
}; |
|
|
|
/* Mutex cache */ |
|
#define MUTEX_OBJ_MAGIC 0x5aa3c85d |
|
struct kmutexobj { |
|
kmutex_t mo_lock; |
|
u_int mo_magic; |
|
u_int mo_refcnt; |
}; |
}; |
|
|
|
static int mutex_obj_ctor(void *, void *, int); |
|
|
|
static pool_cache_t mutex_obj_cache; |
|
|
/* |
/* |
* mutex_dump: |
* mutex_dump: |
* |
* |
|
|
mutex_abort(kmutex_t *mtx, const char *func, const char *msg) |
mutex_abort(kmutex_t *mtx, const char *func, const char *msg) |
{ |
{ |
|
|
LOCKDEBUG_ABORT(MUTEX_GETID(mtx), mtx, (MUTEX_SPIN_P(mtx) ? |
LOCKDEBUG_ABORT(mtx, (MUTEX_SPIN_P(mtx) ? |
&mutex_spin_lockops : &mutex_adaptive_lockops), func, msg); |
&mutex_spin_lockops : &mutex_adaptive_lockops), func, msg); |
/* NOTREACHED */ |
/* NOTREACHED */ |
} |
} |
Line 308 mutex_abort(kmutex_t *mtx, const char *f |
|
Line 333 mutex_abort(kmutex_t *mtx, const char *f |
|
void |
void |
mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) |
mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) |
{ |
{ |
u_int id; |
bool dodebug; |
|
|
memset(mtx, 0, sizeof(*mtx)); |
memset(mtx, 0, sizeof(*mtx)); |
|
|
switch (type) { |
switch (type) { |
case MUTEX_ADAPTIVE: |
case MUTEX_ADAPTIVE: |
case MUTEX_DEFAULT: |
|
KASSERT(ipl == IPL_NONE); |
KASSERT(ipl == IPL_NONE); |
break; |
break; |
|
case MUTEX_DEFAULT: |
case MUTEX_DRIVER: |
case MUTEX_DRIVER: |
type = (ipl == IPL_NONE ? MUTEX_ADAPTIVE : MUTEX_SPIN); |
if (ipl == IPL_NONE || ipl == IPL_SOFTCLOCK || |
|
ipl == IPL_SOFTBIO || ipl == IPL_SOFTNET || |
|
ipl == IPL_SOFTSERIAL) { |
|
type = MUTEX_ADAPTIVE; |
|
} else { |
|
type = MUTEX_SPIN; |
|
} |
break; |
break; |
default: |
default: |
break; |
break; |
Line 326 mutex_init(kmutex_t *mtx, kmutex_type_t |
|
Line 357 mutex_init(kmutex_t *mtx, kmutex_type_t |
|
|
|
switch (type) { |
switch (type) { |
case MUTEX_NODEBUG: |
case MUTEX_NODEBUG: |
id = LOCKDEBUG_ALLOC(mtx, NULL); |
dodebug = LOCKDEBUG_ALLOC(mtx, NULL, |
MUTEX_INITIALIZE_SPIN(mtx, id, ipl); |
(uintptr_t)__builtin_return_address(0)); |
|
MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); |
break; |
break; |
case MUTEX_ADAPTIVE: |
case MUTEX_ADAPTIVE: |
case MUTEX_DEFAULT: |
dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops, |
id = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops); |
(uintptr_t)__builtin_return_address(0)); |
MUTEX_INITIALIZE_ADAPTIVE(mtx, id); |
MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug); |
break; |
break; |
case MUTEX_SPIN: |
case MUTEX_SPIN: |
id = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops); |
dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops, |
MUTEX_INITIALIZE_SPIN(mtx, id, ipl); |
(uintptr_t)__builtin_return_address(0)); |
|
MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); |
break; |
break; |
default: |
default: |
panic("mutex_init: impossible type"); |
panic("mutex_init: impossible type"); |
Line 360 mutex_destroy(kmutex_t *mtx) |
|
Line 393 mutex_destroy(kmutex_t *mtx) |
|
MUTEX_ASSERT(mtx, !__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)); |
MUTEX_ASSERT(mtx, !__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)); |
} |
} |
|
|
LOCKDEBUG_FREE(mtx, MUTEX_GETID(mtx)); |
LOCKDEBUG_FREE(MUTEX_DEBUG_P(mtx), mtx); |
MUTEX_DESTROY(mtx); |
MUTEX_DESTROY(mtx); |
} |
} |
|
|
Line 484 mutex_vector_enter(kmutex_t *mtx) |
|
Line 517 mutex_vector_enter(kmutex_t *mtx) |
|
MUTEX_ASSERT(mtx, curthread != 0); |
MUTEX_ASSERT(mtx, curthread != 0); |
MUTEX_WANTLOCK(mtx); |
MUTEX_WANTLOCK(mtx); |
|
|
#ifdef LOCKDEBUG |
|
if (panicstr == NULL) { |
if (panicstr == NULL) { |
simple_lock_only_held(NULL, "mutex_enter"); |
|
#ifdef MULTIPROCESSOR |
|
LOCKDEBUG_BARRIER(&kernel_lock, 1); |
LOCKDEBUG_BARRIER(&kernel_lock, 1); |
#else |
|
LOCKDEBUG_BARRIER(NULL, 1); |
|
#endif |
|
} |
} |
#endif |
|
|
|
LOCKSTAT_ENTER(lsflag); |
LOCKSTAT_ENTER(lsflag); |
|
|
Line 502 mutex_vector_enter(kmutex_t *mtx) |
|
Line 528 mutex_vector_enter(kmutex_t *mtx) |
|
* determine that the owner is not running on a processor, |
* determine that the owner is not running on a processor, |
* then we stop spinning, and sleep instead. |
* then we stop spinning, and sleep instead. |
*/ |
*/ |
for (;;) { |
for (owner = mtx->mtx_owner;;) { |
owner = mtx->mtx_owner; |
|
if (!MUTEX_OWNED(owner)) { |
if (!MUTEX_OWNED(owner)) { |
/* |
/* |
* Mutex owner clear could mean two things: |
* Mutex owner clear could mean two things: |
Line 516 mutex_vector_enter(kmutex_t *mtx) |
|
Line 541 mutex_vector_enter(kmutex_t *mtx) |
|
*/ |
*/ |
if (MUTEX_ACQUIRE(mtx, curthread)) |
if (MUTEX_ACQUIRE(mtx, curthread)) |
break; |
break; |
|
owner = mtx->mtx_owner; |
continue; |
continue; |
} |
} |
|
|
Line 534 mutex_vector_enter(kmutex_t *mtx) |
|
Line 560 mutex_vector_enter(kmutex_t *mtx) |
|
LOCKSTAT_START_TIMER(lsflag, spintime); |
LOCKSTAT_START_TIMER(lsflag, spintime); |
count = SPINLOCK_BACKOFF_MIN; |
count = SPINLOCK_BACKOFF_MIN; |
for (;;) { |
for (;;) { |
|
SPINLOCK_BACKOFF(count); |
owner = mtx->mtx_owner; |
owner = mtx->mtx_owner; |
if (!mutex_onproc(owner, &ci)) |
if (!mutex_onproc(owner, &ci)) |
break; |
break; |
SPINLOCK_BACKOFF(count); |
|
} |
} |
LOCKSTAT_STOP_TIMER(lsflag, spintime); |
LOCKSTAT_STOP_TIMER(lsflag, spintime); |
LOCKSTAT_COUNT(spincnt, 1); |
LOCKSTAT_COUNT(spincnt, 1); |
Line 555 mutex_vector_enter(kmutex_t *mtx) |
|
Line 581 mutex_vector_enter(kmutex_t *mtx) |
|
*/ |
*/ |
if (!MUTEX_SET_WAITERS(mtx, owner)) { |
if (!MUTEX_SET_WAITERS(mtx, owner)) { |
turnstile_exit(mtx); |
turnstile_exit(mtx); |
|
owner = mtx->mtx_owner; |
continue; |
continue; |
} |
} |
|
|
Line 592 mutex_vector_enter(kmutex_t *mtx) |
|
Line 619 mutex_vector_enter(kmutex_t *mtx) |
|
* or preempted). |
* or preempted). |
* |
* |
* o At any given time, MUTEX_SET_WAITERS() can only ever |
* o At any given time, MUTEX_SET_WAITERS() can only ever |
* be in progress on one CPU in the system - guarenteed |
* be in progress on one CPU in the system - guaranteed |
* by the turnstile chain lock. |
* by the turnstile chain lock. |
* |
* |
* o No other operations other than MUTEX_SET_WAITERS() |
* o No other operations other than MUTEX_SET_WAITERS() |
Line 628 mutex_vector_enter(kmutex_t *mtx) |
|
Line 655 mutex_vector_enter(kmutex_t *mtx) |
|
* value of the waiters flag. |
* value of the waiters flag. |
* |
* |
* 2. The onproc check returns false: the holding LWP is |
* 2. The onproc check returns false: the holding LWP is |
* not running. We now have the oppertunity to check |
* not running. We now have the opportunity to check |
* if mutex_exit() has blatted the modifications made |
* if mutex_exit() has blatted the modifications made |
* by MUTEX_SET_WAITERS(). |
* by MUTEX_SET_WAITERS(). |
* |
* |
Line 652 mutex_vector_enter(kmutex_t *mtx) |
|
Line 679 mutex_vector_enter(kmutex_t *mtx) |
|
* If the waiters bit is not set it's unsafe to go asleep, |
* If the waiters bit is not set it's unsafe to go asleep, |
* as we might never be awoken. |
* as we might never be awoken. |
*/ |
*/ |
if ((mb_read(), mutex_onproc(owner, &ci)) || |
if ((membar_consumer(), mutex_onproc(owner, &ci)) || |
(mb_read(), !MUTEX_HAS_WAITERS(mtx))) { |
(membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) { |
turnstile_exit(mtx); |
turnstile_exit(mtx); |
|
owner = mtx->mtx_owner; |
continue; |
continue; |
} |
} |
#endif /* MULTIPROCESSOR */ |
#endif /* MULTIPROCESSOR */ |
Line 665 mutex_vector_enter(kmutex_t *mtx) |
|
Line 693 mutex_vector_enter(kmutex_t *mtx) |
|
|
|
LOCKSTAT_STOP_TIMER(lsflag, slptime); |
LOCKSTAT_STOP_TIMER(lsflag, slptime); |
LOCKSTAT_COUNT(slpcnt, 1); |
LOCKSTAT_COUNT(slpcnt, 1); |
|
|
|
owner = mtx->mtx_owner; |
} |
} |
|
|
LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1, |
LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1, |
Line 690 mutex_vector_exit(kmutex_t *mtx) |
|
Line 720 mutex_vector_exit(kmutex_t *mtx) |
|
|
|
if (MUTEX_SPIN_P(mtx)) { |
if (MUTEX_SPIN_P(mtx)) { |
#ifdef FULL |
#ifdef FULL |
if (!__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) |
if (__predict_false(!__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock))) { |
|
if (panicstr != NULL) |
|
return; |
MUTEX_ABORT(mtx, "exiting unheld spin mutex"); |
MUTEX_ABORT(mtx, "exiting unheld spin mutex"); |
|
} |
MUTEX_UNLOCKED(mtx); |
MUTEX_UNLOCKED(mtx); |
__cpu_simple_unlock(&mtx->mtx_lock); |
__cpu_simple_unlock(&mtx->mtx_lock); |
#endif |
#endif |
|
|
mutex_owned(kmutex_t *mtx) |
mutex_owned(kmutex_t *mtx) |
{ |
{ |
|
|
|
if (mtx == NULL) |
|
return 0; |
if (MUTEX_ADAPTIVE_P(mtx)) |
if (MUTEX_ADAPTIVE_P(mtx)) |
return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp; |
return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp; |
#ifdef FULL |
#ifdef FULL |
Line 793 mutex_owned(kmutex_t *mtx) |
|
Line 828 mutex_owned(kmutex_t *mtx) |
|
* Return the current owner of an adaptive mutex. Used for |
* Return the current owner of an adaptive mutex. Used for |
* priority inheritance. |
* priority inheritance. |
*/ |
*/ |
static struct lwp * |
lwp_t * |
mutex_owner(wchan_t obj) |
mutex_owner(kmutex_t *mtx) |
{ |
{ |
kmutex_t *mtx = (void *)(uintptr_t)obj; /* discard qualifiers */ |
|
|
|
MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); |
MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); |
return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner); |
return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner); |
Line 894 mutex_spin_retry(kmutex_t *mtx) |
|
Line 928 mutex_spin_retry(kmutex_t *mtx) |
|
#endif /* MULTIPROCESSOR */ |
#endif /* MULTIPROCESSOR */ |
} |
} |
#endif /* defined(__HAVE_SPIN_MUTEX_STUBS) || defined(FULL) */ |
#endif /* defined(__HAVE_SPIN_MUTEX_STUBS) || defined(FULL) */ |
|
|
|
/* |
|
* mutex_obj_init: |
|
* |
|
* Initialize the mutex object store. |
|
*/ |
|
void |
|
mutex_obj_init(void) |
|
{ |
|
|
|
mutex_obj_cache = pool_cache_init(sizeof(struct kmutexobj), |
|
coherency_unit, 0, 0, "mutex", NULL, IPL_NONE, mutex_obj_ctor, |
|
NULL, NULL); |
|
} |
|
|
|
/* |
|
* mutex_obj_ctor: |
|
* |
|
* Initialize a new lock for the cache. |
|
*/ |
|
static int |
|
mutex_obj_ctor(void *arg, void *obj, int flags) |
|
{ |
|
struct kmutexobj * mo = obj; |
|
|
|
mo->mo_magic = MUTEX_OBJ_MAGIC; |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* mutex_obj_alloc: |
|
* |
|
* Allocate a single lock object. |
|
*/ |
|
kmutex_t * |
|
mutex_obj_alloc(kmutex_type_t type, int ipl) |
|
{ |
|
struct kmutexobj *mo; |
|
|
|
mo = pool_cache_get(mutex_obj_cache, PR_WAITOK); |
|
mutex_init(&mo->mo_lock, type, ipl); |
|
mo->mo_refcnt = 1; |
|
|
|
return (kmutex_t *)mo; |
|
} |
|
|
|
/* |
|
* mutex_obj_hold: |
|
* |
|
* Add a single reference to a lock object. A reference to the object |
|
* must already be held, and must be held across this call. |
|
*/ |
|
void |
|
mutex_obj_hold(kmutex_t *lock) |
|
{ |
|
struct kmutexobj *mo = (struct kmutexobj *)lock; |
|
|
|
KASSERT(mo->mo_magic == MUTEX_OBJ_MAGIC); |
|
KASSERT(mo->mo_refcnt > 0); |
|
|
|
atomic_inc_uint(&mo->mo_refcnt); |
|
} |
|
|
|
/* |
|
* mutex_obj_free: |
|
* |
|
* Drop a reference from a lock object. If the last reference is being |
|
* dropped, free the object and return true. Otherwise, return false. |
|
*/ |
|
bool |
|
mutex_obj_free(kmutex_t *lock) |
|
{ |
|
struct kmutexobj *mo = (struct kmutexobj *)lock; |
|
|
|
KASSERT(mo->mo_magic == MUTEX_OBJ_MAGIC); |
|
KASSERT(mo->mo_refcnt > 0); |
|
|
|
if (atomic_dec_uint_nv(&mo->mo_refcnt) > 0) { |
|
return false; |
|
} |
|
mutex_destroy(&mo->mo_lock); |
|
pool_cache_put(mutex_obj_cache, mo); |
|
return true; |
|
} |