Annotation of src/sys/kern/kern_rwlock.c, Revision 1.19
1.19 ! ad 1: /* $NetBSD: kern_rwlock.c,v 1.18 2008/01/28 19:58:32 ad Exp $ */
1.2 ad 2:
3: /*-
1.16 ad 4: * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
1.2 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Jason R. Thorpe and Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: * 3. All advertising materials mentioning features or use of this software
19: * must display the following acknowledgement:
20: * This product includes software developed by the NetBSD
21: * Foundation, Inc. and its contributors.
22: * 4. Neither the name of The NetBSD Foundation nor the names of its
23: * contributors may be used to endorse or promote products derived
24: * from this software without specific prior written permission.
25: *
26: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36: * POSSIBILITY OF SUCH DAMAGE.
37: */
38:
39: /*
40: * Kernel reader/writer lock implementation, modeled after those
41: * found in Solaris, a description of which can be found in:
42: *
43: * Solaris Internals: Core Kernel Architecture, Jim Mauro and
44: * Richard McDougall.
45: */
46:
1.10 dsl 47: #include <sys/cdefs.h>
1.19 ! ad 48: __KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.18 2008/01/28 19:58:32 ad Exp $");
1.10 dsl 49:
1.2 ad 50: #include "opt_multiprocessor.h"
51:
52: #define __RWLOCK_PRIVATE
53:
54: #include <sys/param.h>
55: #include <sys/proc.h>
56: #include <sys/rwlock.h>
57: #include <sys/sched.h>
58: #include <sys/sleepq.h>
59: #include <sys/systm.h>
60: #include <sys/lockdebug.h>
1.11 ad 61: #include <sys/cpu.h>
1.14 ad 62: #include <sys/atomic.h>
1.15 ad 63: #include <sys/lock.h>
1.2 ad 64:
65: #include <dev/lockstat.h>
66:
67: /*
68: * LOCKDEBUG
69: */
70:
71: #if defined(LOCKDEBUG)
72:
73: #define RW_WANTLOCK(rw, op) \
1.12 yamt 74: LOCKDEBUG_WANTLOCK(RW_DEBUG_P(rw), (rw), \
1.2 ad 75: (uintptr_t)__builtin_return_address(0), op == RW_READER);
76: #define RW_LOCKED(rw, op) \
1.12 yamt 77: LOCKDEBUG_LOCKED(RW_DEBUG_P(rw), (rw), \
1.2 ad 78: (uintptr_t)__builtin_return_address(0), op == RW_READER);
79: #define RW_UNLOCKED(rw, op) \
1.12 yamt 80: LOCKDEBUG_UNLOCKED(RW_DEBUG_P(rw), (rw), \
1.2 ad 81: (uintptr_t)__builtin_return_address(0), op == RW_READER);
82: #define RW_DASSERT(rw, cond) \
83: do { \
84: if (!(cond)) \
1.11 ad 85: rw_abort(rw, __func__, "assertion failed: " #cond); \
1.2 ad 86: } while (/* CONSTCOND */ 0);
87:
88: #else /* LOCKDEBUG */
89:
90: #define RW_WANTLOCK(rw, op) /* nothing */
91: #define RW_LOCKED(rw, op) /* nothing */
92: #define RW_UNLOCKED(rw, op) /* nothing */
93: #define RW_DASSERT(rw, cond) /* nothing */
94:
95: #endif /* LOCKDEBUG */
96:
97: /*
98: * DIAGNOSTIC
99: */
100:
101: #if defined(DIAGNOSTIC)
102:
103: #define RW_ASSERT(rw, cond) \
104: do { \
105: if (!(cond)) \
1.11 ad 106: rw_abort(rw, __func__, "assertion failed: " #cond); \
1.2 ad 107: } while (/* CONSTCOND */ 0)
108:
109: #else
110:
111: #define RW_ASSERT(rw, cond) /* nothing */
112:
113: #endif /* DIAGNOSTIC */
114:
115: /*
116: * For platforms that use 'simple' RW locks.
117: */
118: #ifdef __HAVE_SIMPLE_RW_LOCKS
1.12 yamt 119: #define RW_ACQUIRE(rw, old, new) RW_CAS1(&(rw)->rw_owner, old, new)
120: #define RW_RELEASE(rw, old, new) RW_CAS1(&(rw)->rw_owner, old, new)
121: #define RW_SETDEBUG(rw, on) ((rw)->rw_owner |= (on) ? RW_DEBUG : 0)
122: #define RW_DEBUG_P(rw) (((rw)->rw_owner & RW_DEBUG) != 0)
123: #if defined(LOCKDEBUG)
124: #define RW_INHERITDEBUG(new, old) (new) |= (old) & RW_DEBUG
125: #else /* defined(LOCKDEBUG) */
126: #define RW_INHERITDEBUG(new, old) /* nothing */
127: #endif /* defined(LOCKDEBUG) */
128:
129: static inline int
130: RW_CAS1(volatile uintptr_t *ptr, uintptr_t old, uintptr_t new)
131: {
132:
133: RW_INHERITDEBUG(new, old);
134: return RW_CAS(ptr, old, new);
135: }
1.2 ad 136:
137: static inline int
138: RW_SET_WAITERS(krwlock_t *rw, uintptr_t need, uintptr_t set)
139: {
140: uintptr_t old;
141:
142: if (((old = rw->rw_owner) & need) == 0)
143: return 0;
144: return RW_CAS(&rw->rw_owner, old, old | set);
145: }
146: #endif /* __HAVE_SIMPLE_RW_LOCKS */
147:
148: /*
149: * For platforms that do not provide stubs, or for the LOCKDEBUG case.
150: */
151: #ifdef LOCKDEBUG
152: #undef __HAVE_RW_STUBS
153: #endif
154:
155: #ifndef __HAVE_RW_STUBS
1.6 itohy 156: __strong_alias(rw_enter,rw_vector_enter);
157: __strong_alias(rw_exit,rw_vector_exit);
1.16 ad 158: __strong_alias(rw_tryenter,rw_vector_tryenter);
1.2 ad 159: #endif
160:
1.7 ad 161: static void rw_dump(volatile void *);
162: static lwp_t *rw_owner(wchan_t);
1.19 ! ad 163: extern int mutex_onproc(uintptr_t, struct cpu_info **); /* XXX */
1.2 ad 164:
165: lockops_t rwlock_lockops = {
166: "Reader / writer lock",
167: 1,
168: rw_dump
169: };
170:
1.4 yamt 171: syncobj_t rw_syncobj = {
172: SOBJ_SLEEPQ_SORTED,
173: turnstile_unsleep,
174: turnstile_changepri,
175: sleepq_lendpri,
176: rw_owner,
177: };
178:
1.2 ad 179: /*
180: * rw_dump:
181: *
182: * Dump the contents of a rwlock structure.
183: */
1.11 ad 184: static void
1.2 ad 185: rw_dump(volatile void *cookie)
186: {
187: volatile krwlock_t *rw = cookie;
188:
189: printf_nolog("owner/count : %#018lx flags : %#018x\n",
190: (long)RW_OWNER(rw), (int)RW_FLAGS(rw));
191: }
192:
193: /*
1.11 ad 194: * rw_abort:
195: *
196: * Dump information about an error and panic the system. This
197: * generates a lot of machine code in the DIAGNOSTIC case, so
198: * we ask the compiler to not inline it.
199: */
200: #if __GNUC_PREREQ__(3, 0)
201: __attribute ((noinline))
202: #endif
203: static void
204: rw_abort(krwlock_t *rw, const char *func, const char *msg)
205: {
206:
207: if (panicstr != NULL)
208: return;
209:
1.12 yamt 210: LOCKDEBUG_ABORT(rw, &rwlock_lockops, func, msg);
1.11 ad 211: }
212:
213: /*
1.2 ad 214: * rw_init:
215: *
216: * Initialize a rwlock for use.
217: */
218: void
219: rw_init(krwlock_t *rw)
220: {
1.12 yamt 221: bool dodebug;
1.2 ad 222:
223: memset(rw, 0, sizeof(*rw));
224:
1.12 yamt 225: dodebug = LOCKDEBUG_ALLOC(rw, &rwlock_lockops,
1.11 ad 226: (uintptr_t)__builtin_return_address(0));
1.12 yamt 227: RW_SETDEBUG(rw, dodebug);
1.2 ad 228: }
229:
230: /*
231: * rw_destroy:
232: *
233: * Tear down a rwlock.
234: */
235: void
236: rw_destroy(krwlock_t *rw)
237: {
238:
1.12 yamt 239: RW_ASSERT(rw, (rw->rw_owner & ~RW_DEBUG) == 0);
240: LOCKDEBUG_FREE(RW_DEBUG_P(rw), rw);
1.2 ad 241: }
242:
243: /*
244: * rw_vector_enter:
245: *
246: * Acquire a rwlock.
247: */
248: void
249: rw_vector_enter(krwlock_t *rw, const krw_t op)
250: {
251: uintptr_t owner, incr, need_wait, set_wait, curthread;
1.19 ! ad 252: struct cpu_info *ci;
1.2 ad 253: turnstile_t *ts;
254: int queue;
1.7 ad 255: lwp_t *l;
1.2 ad 256: LOCKSTAT_TIMER(slptime);
1.19 ! ad 257: LOCKSTAT_TIMER(spintime);
! 258: LOCKSTAT_COUNTER(spincnt);
1.2 ad 259: LOCKSTAT_FLAG(lsflag);
260:
261: l = curlwp;
262: curthread = (uintptr_t)l;
263:
1.13 ad 264: RW_ASSERT(rw, !cpu_intr_p());
1.2 ad 265: RW_ASSERT(rw, curthread != 0);
266: RW_WANTLOCK(rw, op);
267:
268: if (panicstr == NULL) {
269: LOCKDEBUG_BARRIER(&kernel_lock, 1);
270: }
271:
272: /*
273: * We play a slight trick here. If we're a reader, we want
274: * increment the read count. If we're a writer, we want to
275: * set the owner field and whe WRITE_LOCKED bit.
276: *
277: * In the latter case, we expect those bits to be zero,
278: * therefore we can use an add operation to set them, which
279: * means an add operation for both cases.
280: */
281: if (__predict_true(op == RW_READER)) {
282: incr = RW_READ_INCR;
283: set_wait = RW_HAS_WAITERS;
284: need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
285: queue = TS_READER_Q;
286: } else {
287: RW_DASSERT(rw, op == RW_WRITER);
288: incr = curthread | RW_WRITE_LOCKED;
289: set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED;
290: need_wait = RW_WRITE_LOCKED | RW_THREAD;
291: queue = TS_WRITER_Q;
292: }
293:
294: LOCKSTAT_ENTER(lsflag);
295:
1.19 ! ad 296: for (ci = NULL;;) {
1.2 ad 297: /*
298: * Read the lock owner field. If the need-to-wait
299: * indicator is clear, then try to acquire the lock.
300: */
301: owner = rw->rw_owner;
302: if ((owner & need_wait) == 0) {
303: if (RW_ACQUIRE(rw, owner, owner + incr)) {
304: /* Got it! */
305: break;
306: }
307:
308: /*
309: * Didn't get it -- spin around again (we'll
310: * probably sleep on the next iteration).
311: */
312: continue;
313: }
314:
315: if (panicstr != NULL)
316: return;
317: if (RW_OWNER(rw) == curthread)
1.11 ad 318: rw_abort(rw, __func__, "locking against myself");
1.2 ad 319:
1.19 ! ad 320: #ifdef MULTIPROCESSOR
! 321: /*
! 322: * If the lock owner is running on another CPU, and
! 323: * there are no existing waiters, then spin.
! 324: */
! 325: if ((owner & (RW_WRITE_LOCKED|RW_HAS_WAITERS)) ==
! 326: RW_WRITE_LOCKED && mutex_onproc(owner, &ci)) {
! 327: LOCKSTAT_START_TIMER(lsflag, spintime);
! 328: u_int count = SPINLOCK_BACKOFF_MIN;
! 329: for (;;) {
! 330: owner = rw->rw_owner;
! 331: if ((owner & (RW_WRITE_LOCKED|RW_HAS_WAITERS))
! 332: != RW_WRITE_LOCKED)
! 333: break;
! 334: if (!mutex_onproc(owner, &ci))
! 335: break;
! 336: SPINLOCK_BACKOFF(count);
! 337: }
! 338: LOCKSTAT_STOP_TIMER(lsflag, spintime);
! 339: LOCKSTAT_COUNT(spincnt, 1);
! 340: if ((owner & need_wait) == 0)
! 341: continue;
! 342: }
! 343: #endif
! 344:
1.2 ad 345: /*
346: * Grab the turnstile chain lock. Once we have that, we
347: * can adjust the waiter bits and sleep queue.
348: */
349: ts = turnstile_lookup(rw);
350:
351: /*
352: * Mark the rwlock as having waiters. If the set fails,
353: * then we may not need to sleep and should spin again.
354: */
355: if (!RW_SET_WAITERS(rw, need_wait, set_wait)) {
356: turnstile_exit(rw);
357: continue;
358: }
359:
360: LOCKSTAT_START_TIMER(lsflag, slptime);
361:
1.4 yamt 362: turnstile_block(ts, queue, rw, &rw_syncobj);
1.2 ad 363:
364: /* If we wake up and arrive here, we've been handed the lock. */
365: RW_RECEIVE(rw);
366:
367: LOCKSTAT_STOP_TIMER(lsflag, slptime);
368: LOCKSTAT_EVENT(lsflag, rw,
369: LB_RWLOCK | (op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2),
370: 1, slptime);
371:
372: break;
373: }
374:
1.19 ! ad 375: LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime);
1.2 ad 376: LOCKSTAT_EXIT(lsflag);
377:
378: RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
379: (op == RW_READER && RW_COUNT(rw) != 0));
380: RW_LOCKED(rw, op);
381: }
382:
383: /*
384: * rw_vector_exit:
385: *
386: * Release a rwlock.
387: */
388: void
389: rw_vector_exit(krwlock_t *rw)
390: {
391: uintptr_t curthread, owner, decr, new;
392: turnstile_t *ts;
393: int rcnt, wcnt;
1.7 ad 394: lwp_t *l;
1.2 ad 395:
396: curthread = (uintptr_t)curlwp;
397: RW_ASSERT(rw, curthread != 0);
398:
1.11 ad 399: if (panicstr != NULL)
1.2 ad 400: return;
401:
402: /*
403: * Again, we use a trick. Since we used an add operation to
404: * set the required lock bits, we can use a subtract to clear
405: * them, which makes the read-release and write-release path
406: * the same.
407: */
408: owner = rw->rw_owner;
409: if (__predict_false((owner & RW_WRITE_LOCKED) != 0)) {
410: RW_UNLOCKED(rw, RW_WRITER);
411: RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0);
412: RW_ASSERT(rw, RW_OWNER(rw) == curthread);
413: decr = curthread | RW_WRITE_LOCKED;
414: } else {
415: RW_UNLOCKED(rw, RW_READER);
416: RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0);
417: RW_ASSERT(rw, RW_COUNT(rw) != 0);
418: decr = RW_READ_INCR;
419: }
420:
421: /*
422: * Compute what we expect the new value of the lock to be. Only
423: * proceed to do direct handoff if there are waiters, and if the
424: * lock would become unowned.
425: */
426: for (;; owner = rw->rw_owner) {
427: new = (owner - decr);
428: if ((new & (RW_THREAD | RW_HAS_WAITERS)) == RW_HAS_WAITERS)
429: break;
430: if (RW_RELEASE(rw, owner, new))
431: return;
432: }
433:
434: for (;;) {
435: /*
436: * Grab the turnstile chain lock. This gets the interlock
437: * on the sleep queue. Once we have that, we can adjust the
438: * waiter bits.
439: */
440: ts = turnstile_lookup(rw);
441: RW_DASSERT(rw, ts != NULL);
1.3 ad 442: RW_DASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0);
1.2 ad 443:
444: owner = rw->rw_owner;
445: wcnt = TS_WAITERS(ts, TS_WRITER_Q);
446: rcnt = TS_WAITERS(ts, TS_READER_Q);
447:
448: /*
449: * Give the lock away.
450: *
451: * If we are releasing a write lock, then wake all
452: * outstanding readers. If we are releasing a read
453: * lock, then wake one writer.
454: */
455: if (rcnt == 0 || (decr == RW_READ_INCR && wcnt != 0)) {
456: RW_DASSERT(rw, wcnt != 0);
457: RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0);
458:
459: /*
460: * Give the lock to the longest waiting
461: * writer.
462: */
463: l = TS_FIRST(ts, TS_WRITER_Q);
464: new = (uintptr_t)l | RW_WRITE_LOCKED;
465:
466: if (wcnt > 1)
467: new |= RW_HAS_WAITERS | RW_WRITE_WANTED;
468: else if (rcnt != 0)
469: new |= RW_HAS_WAITERS;
470:
471: RW_GIVE(rw);
472: if (!RW_RELEASE(rw, owner, new)) {
473: /* Oops, try again. */
474: turnstile_exit(rw);
475: continue;
476: }
477:
478: /* Wake the writer. */
1.7 ad 479: turnstile_wakeup(ts, TS_WRITER_Q, 1, l);
1.2 ad 480: } else {
481: RW_DASSERT(rw, rcnt != 0);
482:
483: /*
1.3 ad 484: * Give the lock to all blocked readers. If there
485: * is a writer waiting, new readers that arrive
486: * after the release will be blocked out.
1.2 ad 487: */
488: new = rcnt << RW_READ_COUNT_SHIFT;
489: if (wcnt != 0)
490: new |= RW_HAS_WAITERS | RW_WRITE_WANTED;
1.12 yamt 491:
1.2 ad 492: RW_GIVE(rw);
493: if (!RW_RELEASE(rw, owner, new)) {
494: /* Oops, try again. */
495: turnstile_exit(rw);
496: continue;
497: }
498:
499: /* Wake up all sleeping readers. */
500: turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL);
501: }
502:
503: break;
504: }
505: }
506:
507: /*
1.16 ad 508: * rw_vector_tryenter:
1.2 ad 509: *
510: * Try to acquire a rwlock.
511: */
512: int
1.16 ad 513: rw_vector_tryenter(krwlock_t *rw, const krw_t op)
1.2 ad 514: {
515: uintptr_t curthread, owner, incr, need_wait;
516:
517: curthread = (uintptr_t)curlwp;
518:
519: RW_ASSERT(rw, curthread != 0);
520:
521: if (op == RW_READER) {
522: incr = RW_READ_INCR;
523: need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
524: } else {
525: RW_DASSERT(rw, op == RW_WRITER);
526: incr = curthread | RW_WRITE_LOCKED;
527: need_wait = RW_WRITE_LOCKED | RW_THREAD;
528: }
529:
530: for (;;) {
531: owner = rw->rw_owner;
532: if ((owner & need_wait) == 0) {
533: if (RW_ACQUIRE(rw, owner, owner + incr)) {
534: /* Got it! */
535: break;
536: }
537: continue;
538: }
539: return 0;
540: }
541:
1.16 ad 542: RW_WANTLOCK(rw, op);
1.2 ad 543: RW_LOCKED(rw, op);
544: RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
545: (op == RW_READER && RW_COUNT(rw) != 0));
1.7 ad 546:
1.2 ad 547: return 1;
548: }
549:
550: /*
551: * rw_downgrade:
552: *
553: * Downgrade a write lock to a read lock.
554: */
555: void
556: rw_downgrade(krwlock_t *rw)
557: {
558: uintptr_t owner, curthread, new;
559: turnstile_t *ts;
560: int rcnt, wcnt;
561:
562: curthread = (uintptr_t)curlwp;
563: RW_ASSERT(rw, curthread != 0);
564: RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0);
565: RW_ASSERT(rw, RW_OWNER(rw) == curthread);
566: RW_UNLOCKED(rw, RW_WRITER);
567:
568: owner = rw->rw_owner;
569: if ((owner & RW_HAS_WAITERS) == 0) {
570: /*
571: * There are no waiters, so we can do this the easy way.
572: * Try swapping us down to one read hold. If it fails, the
573: * lock condition has changed and we most likely now have
574: * waiters.
575: */
576: if (RW_RELEASE(rw, owner, RW_READ_INCR)) {
577: RW_LOCKED(rw, RW_READER);
578: RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0);
579: RW_DASSERT(rw, RW_COUNT(rw) != 0);
580: return;
581: }
582: }
583:
584: /*
585: * Grab the turnstile chain lock. This gets the interlock
586: * on the sleep queue. Once we have that, we can adjust the
587: * waiter bits.
588: */
589: for (;;) {
590: ts = turnstile_lookup(rw);
591: RW_DASSERT(rw, ts != NULL);
592:
593: owner = rw->rw_owner;
594: rcnt = TS_WAITERS(ts, TS_READER_Q);
595: wcnt = TS_WAITERS(ts, TS_WRITER_Q);
596:
597: /*
598: * If there are no readers, just preserve the waiters
599: * bits, swap us down to one read hold and return.
600: */
601: if (rcnt == 0) {
602: RW_DASSERT(rw, wcnt != 0);
603: RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0);
604: RW_DASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0);
605:
606: new = RW_READ_INCR | RW_HAS_WAITERS | RW_WRITE_WANTED;
607: if (!RW_RELEASE(rw, owner, new)) {
608: /* Oops, try again. */
609: turnstile_exit(ts);
610: continue;
611: }
612: break;
613: }
614:
615: /*
616: * Give the lock to all blocked readers. We may
617: * retain one read hold if downgrading. If there
618: * is a writer waiting, new readers will be blocked
619: * out.
620: */
621: new = (rcnt << RW_READ_COUNT_SHIFT) + RW_READ_INCR;
622: if (wcnt != 0)
623: new |= RW_HAS_WAITERS | RW_WRITE_WANTED;
624:
625: RW_GIVE(rw);
626: if (!RW_RELEASE(rw, owner, new)) {
627: /* Oops, try again. */
628: turnstile_exit(rw);
629: continue;
630: }
631:
632: /* Wake up all sleeping readers. */
633: turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL);
634: break;
635: }
636:
637: RW_LOCKED(rw, RW_READER);
638: RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0);
639: RW_DASSERT(rw, RW_COUNT(rw) != 0);
640: }
641:
642: /*
643: * rw_tryupgrade:
644: *
645: * Try to upgrade a read lock to a write lock. We must be the
646: * only reader.
647: */
648: int
649: rw_tryupgrade(krwlock_t *rw)
650: {
651: uintptr_t owner, curthread, new;
652:
653: curthread = (uintptr_t)curlwp;
654: RW_ASSERT(rw, curthread != 0);
655: RW_WANTLOCK(rw, RW_WRITER);
656:
657: for (;;) {
658: owner = rw->rw_owner;
659: RW_ASSERT(rw, (owner & RW_WRITE_LOCKED) == 0);
660: if ((owner & RW_THREAD) != RW_READ_INCR) {
661: RW_ASSERT(rw, (owner & RW_THREAD) != 0);
662: return 0;
663: }
664: new = curthread | RW_WRITE_LOCKED | (owner & ~RW_THREAD);
665: if (RW_ACQUIRE(rw, owner, new))
666: break;
667: }
668:
669: RW_UNLOCKED(rw, RW_READER);
670: RW_LOCKED(rw, RW_WRITER);
671: RW_DASSERT(rw, rw->rw_owner & RW_WRITE_LOCKED);
672: RW_DASSERT(rw, RW_OWNER(rw) == curthread);
673:
674: return 1;
675: }
676:
677: /*
678: * rw_read_held:
679: *
680: * Returns true if the rwlock is held for reading. Must only be
681: * used for diagnostic assertions, and never be used to make
682: * decisions about how to use a rwlock.
683: */
684: int
685: rw_read_held(krwlock_t *rw)
686: {
687: uintptr_t owner;
688:
689: if (panicstr != NULL)
690: return 1;
691:
692: owner = rw->rw_owner;
693: return (owner & RW_WRITE_LOCKED) == 0 && (owner & RW_THREAD) != 0;
694: }
695:
696: /*
697: * rw_write_held:
698: *
699: * Returns true if the rwlock is held for writing. Must only be
700: * used for diagnostic assertions, and never be used to make
701: * decisions about how to use a rwlock.
702: */
703: int
704: rw_write_held(krwlock_t *rw)
705: {
706:
707: if (panicstr != NULL)
708: return 1;
709:
1.17 ad 710: return (rw->rw_owner & (RW_WRITE_LOCKED | RW_THREAD)) ==
1.18 ad 711: (RW_WRITE_LOCKED | (uintptr_t)curlwp);
1.2 ad 712: }
713:
714: /*
715: * rw_lock_held:
716: *
717: * Returns true if the rwlock is held for reading or writing. Must
718: * only be used for diagnostic assertions, and never be used to make
719: * decisions about how to use a rwlock.
720: */
721: int
722: rw_lock_held(krwlock_t *rw)
723: {
724:
725: if (panicstr != NULL)
726: return 1;
727:
728: return (rw->rw_owner & RW_THREAD) != 0;
729: }
1.4 yamt 730:
1.5 ad 731: /*
732: * rw_owner:
733: *
734: * Return the current owner of an RW lock, but only if it is write
735: * held. Used for priority inheritance.
736: */
1.7 ad 737: static lwp_t *
1.4 yamt 738: rw_owner(wchan_t obj)
739: {
740: krwlock_t *rw = (void *)(uintptr_t)obj; /* discard qualifiers */
741: uintptr_t owner = rw->rw_owner;
742:
743: if ((owner & RW_WRITE_LOCKED) == 0)
744: return NULL;
745:
746: return (void *)(owner & RW_THREAD);
747: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to kern_rwlock.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / kern