src/sys/kern/kern_mutex.c - diff

Return to kern_mutex.c CVS log

Up to [cvs.NetBSD.org] / src / sys / kern

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/sys/kern/kern_mutex.c between version 1.62 and 1.79

version 1.62, 2015/05/25 21:02:37

version 1.79, 2019/05/09 05:00:31

Line 54 __KERNEL_RCSID(0, "$NetBSD$");

#include <sys/intr.h>

#include <sys/lock.h>

#include <sys/types.h>

#include <sys/cpu.h>

#include <sys/pserialize.h>

#include <dev/lockstat.h>

#include <machine/lock.h>

#define MUTEX_PANIC_SKIP_SPIN 1

#define MUTEX_PANIC_SKIP_ADAPTIVE 1

* When not running a debug kernel, spin mutexes are not much

* more than an splraiseipl() and splx() pair.

Line 75 __KERNEL_RCSID(0, "$NetBSD$");

Line 80 __KERNEL_RCSID(0, "$NetBSD$");

#define MUTEX_WANTLOCK(mtx) \

LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \

(uintptr_t)__builtin_return_address(0), 0)

#define MUTEX_TESTLOCK(mtx) \

LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \

(uintptr_t)__builtin_return_address(0), -1)

#define MUTEX_LOCKED(mtx) \

LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \

(uintptr_t)__builtin_return_address(0), 0)

Line 82 __KERNEL_RCSID(0, "$NetBSD$");

Line 90 __KERNEL_RCSID(0, "$NetBSD$");

LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \

(uintptr_t)__builtin_return_address(0), 0)

#define MUTEX_ABORT(mtx, msg) \

mutex_abort(mtx, __func__, msg)

mutex_abort(__func__, __LINE__, mtx, msg)

#if defined(LOCKDEBUG)

#define MUTEX_DASSERT(mtx, cond) \

do { \

if (!(cond)) \

if (__predict_false(!(cond))) \

MUTEX_ABORT(mtx, "assertion failed: " #cond); \

} while (/* CONSTCOND */ 0);

} while (/* CONSTCOND */ 0)

#else /* LOCKDEBUG */

Line 102 do { \

Line 110 do { \

#define MUTEX_ASSERT(mtx, cond) \

do { \

if (!(cond)) \

if (__predict_false(!(cond))) \

MUTEX_ABORT(mtx, "assertion failed: " #cond); \

} while (/* CONSTCOND */ 0)

Line 174 do { \

Line 182 do { \

(((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0)

#define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \

do { \

if (!dodebug) \

(mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \

do { \

} while (/* CONSTCOND */ 0)

} while (/* CONSTCOND */ 0);

#define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \

do { \

Line 191 do { \

Line 199 do { \

#define MUTEX_DESTROY(mtx) \

do { \

(mtx)->mtx_owner = MUTEX_THREAD; \

} while (/* CONSTCOND */ 0);

} while (/* CONSTCOND */ 0)

#define MUTEX_SPIN_P(mtx) \

(((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0)

Line 261 __strong_alias(mutex_spin_enter,mutex_ve

Line 269 __strong_alias(mutex_spin_enter,mutex_ve

__strong_alias(mutex_spin_exit,mutex_vector_exit);

#endif

static void mutex_abort(kmutex_t *, const char *, const char *);

static void mutex_abort(const char *, size_t, const kmutex_t *,

static void mutex_dump(volatile void *);

const char *);

static void mutex_dump(const volatile void *, lockop_printer_t);

lockops_t mutex_spin_lockops = {

"Mutex",

.lo_name = "Mutex",

LOCKOPS_SPIN,

.lo_type = LOCKOPS_SPIN,

mutex_dump

.lo_dump = mutex_dump,

};

lockops_t mutex_adaptive_lockops = {

"Mutex",

.lo_name = "Mutex",

LOCKOPS_SLEEP,

.lo_type = LOCKOPS_SLEEP,

mutex_dump

.lo_dump = mutex_dump,

};

syncobj_t mutex_syncobj = {

SOBJ_SLEEPQ_SORTED,

.sobj_flag = SOBJ_SLEEPQ_SORTED,

turnstile_unsleep,

.sobj_unsleep = turnstile_unsleep,

turnstile_changepri,

.sobj_changepri = turnstile_changepri,

sleepq_lendpri,

.sobj_lendpri = sleepq_lendpri,

(void *)mutex_owner,

.sobj_owner = (void *)mutex_owner,

};

Line 289 syncobj_t mutex_syncobj = {

Line 298 syncobj_t mutex_syncobj = {

* Dump the contents of a mutex structure.

void

static void

mutex_dump(volatile void *cookie)

mutex_dump(const volatile void *cookie, lockop_printer_t pr)

{

volatile kmutex_t *mtx = cookie;

const volatile kmutex_t *mtx = cookie;

printf_nolog("owner field : %#018lx wait/spin: %16d/%d\n",

pr("owner field : %#018lx wait/spin: %16d/%d\n",

(long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx),

MUTEX_SPIN_P(mtx));

}

Line 306 mutex_dump(volatile void *cookie)

Line 315 mutex_dump(volatile void *cookie)

* generates a lot of machine code in the DIAGNOSTIC case, so

* we ask the compiler to not inline it.

void __noinline

static void __noinline

mutex_abort(kmutex_t *mtx, const char *func, const char *msg)

mutex_abort(const char *func, size_t line, const kmutex_t *mtx, const char *msg)

{

LOCKDEBUG_ABORT(mtx, (MUTEX_SPIN_P(mtx) ?

LOCKDEBUG_ABORT(func, line, mtx, (MUTEX_SPIN_P(mtx) ?

&mutex_spin_lockops : &mutex_adaptive_lockops), func, msg);

&mutex_spin_lockops : &mutex_adaptive_lockops), msg);

}

Line 323 mutex_abort(kmutex_t *mtx, const char *f

Line 332 mutex_abort(kmutex_t *mtx, const char *f

* sleeps - see comments in mutex_vector_enter() about releasing

* mutexes unlocked.

void _mutex_init(kmutex_t *, kmutex_type_t, int, uintptr_t);

void

mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl)

_mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl,

uintptr_t return_address)

{

bool dodebug;

Line 350 mutex_init(kmutex_t *mtx, kmutex_type_t

Line 361 mutex_init(kmutex_t *mtx, kmutex_type_t

switch (type) {

case MUTEX_NODEBUG:

dodebug = LOCKDEBUG_ALLOC(mtx, NULL,

dodebug = LOCKDEBUG_ALLOC(mtx, NULL, return_address);

(uintptr_t)__builtin_return_address(0));

MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl);

break;

case MUTEX_ADAPTIVE:

dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops,

(uintptr_t)__builtin_return_address(0));

return_address);

MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug);

break;

case MUTEX_SPIN:

dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops,

(uintptr_t)__builtin_return_address(0));

return_address);

MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl);

break;

default:

Line 370 mutex_init(kmutex_t *mtx, kmutex_type_t

Line 380 mutex_init(kmutex_t *mtx, kmutex_type_t

}

void

mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl)

{

_mutex_init(mtx, type, ipl, (uintptr_t)__builtin_return_address(0));

}

* mutex_destroy:

Line 476 mutex_vector_enter(kmutex_t *mtx)

Line 493 mutex_vector_enter(kmutex_t *mtx)

* to reduce cache line ping-ponging between CPUs.

do {

#if MUTEX_PANIC_SKIP_SPIN

if (panicstr != NULL)

break;

#endif

while (MUTEX_SPINBIT_LOCKED_P(mtx)) {

SPINLOCK_BACKOFF(count);

#ifdef LOCKDEBUG

if (SPINLOCK_SPINOUT(spins))

MUTEX_ABORT(mtx, "spinout");

Line 503 mutex_vector_enter(kmutex_t *mtx)

Line 522 mutex_vector_enter(kmutex_t *mtx)

MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));

MUTEX_ASSERT(mtx, curthread != 0);

MUTEX_ASSERT(mtx, !cpu_intr_p());

MUTEX_WANTLOCK(mtx);

if (panicstr == NULL) {

KDASSERT(pserialize_not_in_read_section());

LOCKDEBUG_BARRIER(&kernel_lock, 1);

}

Line 533 mutex_vector_enter(kmutex_t *mtx)

Line 554 mutex_vector_enter(kmutex_t *mtx)

owner = mtx->mtx_owner;

continue;

}

#if MUTEX_PANIC_SKIP_ADAPTIVE

if (__predict_false(panicstr != NULL)) {

KPREEMPT_ENABLE(curlwp);

return;

}

#endif

if (__predict_false(MUTEX_OWNER(owner) == curthread)) {

MUTEX_ABORT(mtx, "locking against myself");

}

Line 566 mutex_vector_enter(kmutex_t *mtx)

Line 589 mutex_vector_enter(kmutex_t *mtx)

* Once we have the turnstile chain interlock, mark the

* mutex has having waiters. If that fails, spin again:

* mutex as having waiters. If that fails, spin again:

* chances are that the mutex has been released.

if (!MUTEX_SET_WAITERS(mtx, owner)) {

Line 712 mutex_vector_exit(kmutex_t *mtx)

Line 735 mutex_vector_exit(kmutex_t *mtx)

if (MUTEX_SPIN_P(mtx)) {

#ifdef FULL

if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) {

#if MUTEX_PANIC_SKIP_SPIN

if (panicstr != NULL)

return;

#endif

MUTEX_ABORT(mtx, "exiting unheld spin mutex");

}

MUTEX_UNLOCKED(mtx);

Line 723 mutex_vector_exit(kmutex_t *mtx)

Line 748 mutex_vector_exit(kmutex_t *mtx)

return;

}

#ifdef MUTEX_PANIC_SKIP_ADAPTIVE

if (__predict_false((uintptr_t)panicstr | cold)) {

MUTEX_UNLOCKED(mtx);

MUTEX_RELEASE(mtx);

return;

}

#endif

curthread = (uintptr_t)curlwp;

MUTEX_DASSERT(mtx, curthread != 0);

Line 802 mutex_wakeup(kmutex_t *mtx)

Line 829 mutex_wakeup(kmutex_t *mtx)

* holds the mutex.

int

mutex_owned(kmutex_t *mtx)

mutex_owned(const kmutex_t *mtx)

{

if (mtx == NULL)

Line 823 mutex_owned(kmutex_t *mtx)

Line 850 mutex_owned(kmutex_t *mtx)

* priority inheritance.

lwp_t *

mutex_owner(kmutex_t *mtx)

mutex_owner(const kmutex_t *mtx)

{

MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));

Line 831 mutex_owner(kmutex_t *mtx)

Line 858 mutex_owner(kmutex_t *mtx)

}

* mutex_ownable:

* When compiled with DEBUG and LOCKDEBUG defined, ensure that

* the mutex is available. We cannot use !mutex_owned() since

* that won't work correctly for spin mutexes.

int

mutex_ownable(const kmutex_t *mtx)

{

#ifdef LOCKDEBUG

MUTEX_TESTLOCK(mtx);

#endif

return 1;

}

* mutex_tryenter:

* Try to acquire the mutex; return non-zero if we did.

Line 901 mutex_spin_retry(kmutex_t *mtx)

Line 945 mutex_spin_retry(kmutex_t *mtx)

* to reduce cache line ping-ponging between CPUs.

do {

#if MUTEX_PANIC_SKIP_SPIN

if (panicstr != NULL)

break;

#endif

while (MUTEX_SPINBIT_LOCKED_P(mtx)) {

SPINLOCK_BACKOFF(count);

#ifdef LOCKDEBUG

if (SPINLOCK_SPINOUT(spins))

MUTEX_ABORT(mtx, "spinout");

CVSweb <webmaster@jp.NetBSD.org>