Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/kern_mutex.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/kern_mutex.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.17 retrieving revision 1.25 diff -u -p -r1.17 -r1.25 --- src/sys/kern/kern_mutex.c 2007/09/17 21:33:34 1.17 +++ src/sys/kern/kern_mutex.c 2007/12/05 06:50:07 1.25 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_mutex.c,v 1.17 2007/09/17 21:33:34 ad Exp $ */ +/* $NetBSD: kern_mutex.c,v 1.25 2007/12/05 06:50:07 ad Exp $ */ /*- * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. @@ -44,12 +44,12 @@ * Richard McDougall. */ -#include "opt_multiprocessor.h" - #define __MUTEX_PRIVATE #include -__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.17 2007/09/17 21:33:34 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.25 2007/12/05 06:50:07 ad Exp $"); + +#include "opt_multiprocessor.h" #include #include @@ -59,11 +59,11 @@ __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c #include #include #include +#include +#include #include -#include - /* * When not running a debug kernel, spin mutexes are not much * more than an splraiseipl() and splx() pair. @@ -78,13 +78,13 @@ __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c */ #define MUTEX_WANTLOCK(mtx) \ - LOCKDEBUG_WANTLOCK(MUTEX_GETID(mtx), \ + LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ (uintptr_t)__builtin_return_address(0), 0) #define MUTEX_LOCKED(mtx) \ - LOCKDEBUG_LOCKED(MUTEX_GETID(mtx), \ + LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ (uintptr_t)__builtin_return_address(0), 0) #define MUTEX_UNLOCKED(mtx) \ - LOCKDEBUG_UNLOCKED(MUTEX_GETID(mtx), \ + LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ (uintptr_t)__builtin_return_address(0), 0) #define MUTEX_ABORT(mtx, msg) \ mutex_abort(mtx, __func__, msg) @@ -154,28 +154,27 @@ do { \ #define MUTEX_OWNER(owner) \ (owner & MUTEX_THREAD) -#define MUTEX_OWNED(owner) \ - (owner != 0) #define MUTEX_HAS_WAITERS(mtx) \ (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) -#define MUTEX_INITIALIZE_ADAPTIVE(mtx, id) \ +#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \ do { \ - (mtx)->mtx_id = (id); \ + if (dodebug) \ + (mtx)->mtx_owner |= MUTEX_BIT_DEBUG; \ } while (/* CONSTCOND */ 0); -#define MUTEX_INITIALIZE_SPIN(mtx, id, ipl) \ +#define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \ do { \ (mtx)->mtx_owner = MUTEX_BIT_SPIN; \ + if (dodebug) \ + (mtx)->mtx_owner |= MUTEX_BIT_DEBUG; \ (mtx)->mtx_ipl = makeiplcookie((ipl)); \ - (mtx)->mtx_id = (id); \ __cpu_simple_lock_init(&(mtx)->mtx_lock); \ } while (/* CONSTCOND */ 0) #define MUTEX_DESTROY(mtx) \ do { \ (mtx)->mtx_owner = MUTEX_THREAD; \ - (mtx)->mtx_id = -1; \ } while (/* CONSTCOND */ 0); #define MUTEX_SPIN_P(mtx) \ @@ -183,13 +182,25 @@ do { \ #define MUTEX_ADAPTIVE_P(mtx) \ (((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0) -#define MUTEX_GETID(mtx) ((mtx)->mtx_id) +#define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_DEBUG) != 0) +#if defined(LOCKDEBUG) +#define MUTEX_OWNED(owner) (((owner) & ~MUTEX_BIT_DEBUG) != 0) +#define MUTEX_INHERITDEBUG(new, old) (new) |= (old) & MUTEX_BIT_DEBUG +#else /* defined(LOCKDEBUG) */ +#define MUTEX_OWNED(owner) ((owner) != 0) +#define MUTEX_INHERITDEBUG(new, old) /* nothing */ +#endif /* defined(LOCKDEBUG) */ static inline int MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread) { int rv; - rv = MUTEX_CAS(&mtx->mtx_owner, 0UL, curthread); + uintptr_t old = 0; + uintptr_t new = curthread; + + MUTEX_INHERITDEBUG(old, mtx->mtx_owner); + MUTEX_INHERITDEBUG(new, old); + rv = MUTEX_CAS(&mtx->mtx_owner, old, new); MUTEX_RECEIVE(mtx); return rv; } @@ -206,8 +217,12 @@ MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr static inline void MUTEX_RELEASE(kmutex_t *mtx) { + uintptr_t new; + MUTEX_GIVE(mtx); - mtx->mtx_owner = 0; + new = 0; + MUTEX_INHERITDEBUG(new, mtx->mtx_owner); + mtx->mtx_owner = new; } static inline void @@ -291,7 +306,7 @@ void mutex_abort(kmutex_t *mtx, const char *func, const char *msg) { - LOCKDEBUG_ABORT(MUTEX_GETID(mtx), mtx, (MUTEX_SPIN_P(mtx) ? + LOCKDEBUG_ABORT(mtx, (MUTEX_SPIN_P(mtx) ? &mutex_spin_lockops : &mutex_adaptive_lockops), func, msg); /* NOTREACHED */ } @@ -308,17 +323,28 @@ mutex_abort(kmutex_t *mtx, const char *f void mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) { - u_int id; + bool dodebug; memset(mtx, 0, sizeof(*mtx)); switch (type) { case MUTEX_ADAPTIVE: - case MUTEX_DEFAULT: KASSERT(ipl == IPL_NONE); break; + case MUTEX_DEFAULT: case MUTEX_DRIVER: - type = (ipl == IPL_NONE ? MUTEX_ADAPTIVE : MUTEX_SPIN); + switch (ipl) { + case IPL_NONE: + case IPL_SOFTCLOCK: + case IPL_SOFTBIO: + case IPL_SOFTNET: + case IPL_SOFTSERIAL: + type = MUTEX_ADAPTIVE; + break; + default: + type = MUTEX_SPIN; + break; + } break; default: break; @@ -326,17 +352,19 @@ mutex_init(kmutex_t *mtx, kmutex_type_t switch (type) { case MUTEX_NODEBUG: - id = LOCKDEBUG_ALLOC(mtx, NULL); - MUTEX_INITIALIZE_SPIN(mtx, id, ipl); + dodebug = LOCKDEBUG_ALLOC(mtx, NULL, + (uintptr_t)__builtin_return_address(0)); + MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); break; case MUTEX_ADAPTIVE: - case MUTEX_DEFAULT: - id = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops); - MUTEX_INITIALIZE_ADAPTIVE(mtx, id); + dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops, + (uintptr_t)__builtin_return_address(0)); + MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug); break; case MUTEX_SPIN: - id = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops); - MUTEX_INITIALIZE_SPIN(mtx, id, ipl); + dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops, + (uintptr_t)__builtin_return_address(0)); + MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); break; default: panic("mutex_init: impossible type"); @@ -360,7 +388,7 @@ mutex_destroy(kmutex_t *mtx) MUTEX_ASSERT(mtx, !__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)); } - LOCKDEBUG_FREE(mtx, MUTEX_GETID(mtx)); + LOCKDEBUG_FREE(MUTEX_DEBUG_P(mtx), mtx); MUTEX_DESTROY(mtx); } @@ -592,7 +620,7 @@ mutex_vector_enter(kmutex_t *mtx) * or preempted). * * o At any given time, MUTEX_SET_WAITERS() can only ever - * be in progress on one CPU in the system - guarenteed + * be in progress on one CPU in the system - guaranteed * by the turnstile chain lock. * * o No other operations other than MUTEX_SET_WAITERS() @@ -652,8 +680,8 @@ mutex_vector_enter(kmutex_t *mtx) * If the waiters bit is not set it's unsafe to go asleep, * as we might never be awoken. */ - if ((mb_read(), mutex_onproc(owner, &ci)) || - (mb_read(), !MUTEX_HAS_WAITERS(mtx))) { + if ((membar_consumer(), mutex_onproc(owner, &ci)) || + (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) { turnstile_exit(mtx); continue; }