Annotation of src/sys/arch/i386/i386/lock_stubs.S, Revision 1.13.8.2
1.13.8.2! yamt 1: /* $NetBSD: lock_stubs.S,v 1.13.8.1 2008/05/18 12:32:10 yamt Exp $ */
1.2 ad 2:
3: /*-
1.13 ad 4: * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
1.2 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: /*
33: * The sizes listed against each function are for a kernel compiled
1.7 ad 34: * with options MULTIPROCESSOR && DIAGNOSTIC. Where possible we make
35: * each routine fit into an assumed 64-byte cache line. Please check
36: * alignment with 'objdump -d' after making changes.
1.2 ad 37: */
38:
1.11 lukem 39: #include <machine/asm.h>
1.13.8.2! yamt 40: __KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.13.8.1 2008/05/18 12:32:10 yamt Exp $");
1.11 lukem 41:
1.2 ad 42: #include "opt_multiprocessor.h"
43: #include "opt_lockdebug.h"
44: #include "opt_ddb.h"
45:
46: #include <machine/cputypes.h>
47:
48: #include "assym.h"
49:
50: #if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
51: #define FULL
52: #endif
53:
54: #define ALIGN64 .align 64
55: #define ALIGN32 .align 32
1.12 ad 56: #define LOCK(num) .Lpatch/**/num: lock
1.13.8.1 yamt 57: #define ENDLABEL(name,a) .align a; LABEL(name)
1.2 ad 58:
59: #if !defined(LOCKDEBUG)
60:
61: /*
62: * void mutex_enter(kmutex_t *mtx);
63: *
64: * Acquire a mutex and post a load fence.
65: */
66: ALIGN64
67:
1.7 ad 68: ENTRY(mutex_enter)
69: movl 4(%esp), %edx
1.2 ad 70: movl CPUVAR(CURLWP), %ecx
71: xorl %eax, %eax
1.12 ad 72: LOCK(1)
1.2 ad 73: cmpxchgl %ecx, MTX_OWNER(%edx)
1.13.8.1 yamt 74: jnz 1f
1.2 ad 75: ret
1.13.8.1 yamt 76: 1:
77: jmp _C_LABEL(mutex_vector_enter)
78: END(mutex_enter)
1.2 ad 79:
80: /*
81: * void mutex_exit(kmutex_t *mtx);
82: *
83: * Release a mutex and post a load fence.
84: *
85: * See comments in mutex_vector_enter() about doing this operation unlocked
86: * on multiprocessor systems, and comments in arch/x86/include/lock.h about
87: * memory ordering on Intel x86 systems.
88: */
1.7 ad 89: ENTRY(mutex_exit)
90: movl 4(%esp), %edx
1.2 ad 91: movl CPUVAR(CURLWP), %eax
92: xorl %ecx, %ecx
93: cmpxchgl %ecx, MTX_OWNER(%edx)
1.13.8.1 yamt 94: jnz 1f
1.2 ad 95: ret
1.13.8.1 yamt 96: 1:
97: jmp _C_LABEL(mutex_vector_exit)
98: END(mutex_exit)
1.2 ad 99:
100: /*
101: * void rw_enter(krwlock_t *rwl, krw_t op);
102: *
103: * Acquire one hold on a RW lock.
104: */
1.7 ad 105: ENTRY(rw_enter)
106: movl 4(%esp), %edx
1.2 ad 107: cmpl $RW_READER, 8(%esp)
108: jne 2f
109:
110: /*
111: * Reader
112: */
113: 1: movl RW_OWNER(%edx), %eax
114: testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
115: leal RW_READ_INCR(%eax), %ecx
1.13.8.1 yamt 116: jnz 3f
1.12 ad 117: LOCK(2)
1.2 ad 118: cmpxchgl %ecx, RW_OWNER(%edx)
119: jnz,pn 1b
120: ret
121:
122: /*
123: * Writer
124: */
125: 2: movl CPUVAR(CURLWP), %ecx
126: xorl %eax, %eax
127: orl $RW_WRITE_LOCKED, %ecx
1.12 ad 128: LOCK(3)
1.2 ad 129: cmpxchgl %ecx, RW_OWNER(%edx)
1.13.8.1 yamt 130: jnz 3f
1.2 ad 131: ret
1.13.8.1 yamt 132: 3:
133: jmp _C_LABEL(rw_vector_enter)
134: END(rw_enter)
1.2 ad 135:
136: /*
137: * void rw_exit(krwlock_t *rwl);
138: *
139: * Release one hold on a RW lock.
140: */
1.7 ad 141: ENTRY(rw_exit)
142: movl 4(%esp), %edx
1.2 ad 143: movl RW_OWNER(%edx), %eax
144: testb $RW_WRITE_LOCKED, %al
145: jnz 2f
146:
147: /*
148: * Reader
149: */
150: 1: testb $RW_HAS_WAITERS, %al
1.13.8.1 yamt 151: jnz 3f
1.2 ad 152: cmpl $RW_READ_INCR, %eax
153: leal -RW_READ_INCR(%eax), %ecx
1.13.8.1 yamt 154: jb 3f
1.12 ad 155: LOCK(4)
1.2 ad 156: cmpxchgl %ecx, RW_OWNER(%edx)
157: jnz,pn 1b
158: ret
159:
160: /*
161: * Writer
162: */
163: 2: leal -RW_WRITE_LOCKED(%eax), %ecx
164: subl CPUVAR(CURLWP), %ecx
1.13.8.1 yamt 165: jnz 3f
1.12 ad 166: LOCK(5)
1.2 ad 167: cmpxchgl %ecx, RW_OWNER(%edx)
1.13.8.1 yamt 168: jnz 3f
1.2 ad 169: ret
170:
171: /*
172: * Slow path.
173: */
174: 3: jmp _C_LABEL(rw_vector_exit)
1.13.8.1 yamt 175: END(rw_exit)
1.2 ad 176:
1.13 ad 177: /*
178: * int rw_tryenter(krwlock_t *rwl, krw_t op);
179: *
180: * Try to acquire one hold on a RW lock.
181: */
182: ENTRY(rw_tryenter)
183: movl 4(%esp), %edx
184: cmpl $RW_READER, 8(%esp)
185: jne 2f
186:
187: /*
188: * Reader
189: */
1.13.8.1 yamt 190: movl RW_OWNER(%edx), %eax
191: 1:
1.13 ad 192: leal RW_READ_INCR(%eax), %ecx
1.13.8.1 yamt 193: testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
1.13 ad 194: jnz 3f
195: LOCK(12)
196: cmpxchgl %ecx, RW_OWNER(%edx)
1.13.8.1 yamt 197: jnz 1b
198: movl %edx, %eax /* nonzero */
1.13 ad 199: ret
200:
201: /*
202: * Writer
203: */
1.13.8.1 yamt 204: 2:
205: movl CPUVAR(CURLWP), %ecx
1.13 ad 206: orl $RW_WRITE_LOCKED, %ecx
1.13.8.1 yamt 207: xorl %eax, %eax
1.13 ad 208: LOCK(13)
209: cmpxchgl %ecx, RW_OWNER(%edx)
210: movl $0, %eax
211: setz %al
212: ret
1.13.8.1 yamt 213: 3:
214: xorl %eax, %eax
1.13 ad 215: ret
1.13.8.1 yamt 216: END(rw_tryenter)
1.13 ad 217:
1.2 ad 218: #ifndef __XEN__
219:
220: /*
221: * void mutex_spin_enter(kmutex_t *mtx);
222: *
223: * Acquire a spin mutex and post a load fence.
224: */
1.7 ad 225: ENTRY(mutex_spin_enter)
226: movl 4(%esp), %edx
1.13.8.1 yamt 227: movb CPUVAR(ILEVEL), %cl
228: movb MTX_IPL(%edx), %ch
1.2 ad 229: cmpb %ch, %cl
1.13.8.1 yamt 230: jg 1f
1.7 ad 231: movb %ch, CPUVAR(ILEVEL) /* splraiseipl() */
1.13.8.1 yamt 232: 1:
233: subl $1, CPUVAR(MTX_COUNT) /* decl does not set CF */
234: jnc 2f
235: movb %cl, CPUVAR(MTX_OLDSPL)
1.2 ad 236: 2:
1.4 ad 237: #ifdef FULL
1.2 ad 238: mov $0x0100, %eax /* new + expected value */
1.12 ad 239: LOCK(11)
1.2 ad 240: cmpxchgb %ah, MTX_LOCK(%edx) /* lock it */
1.13.8.1 yamt 241: jnz 3f
1.2 ad 242: #endif
243: ret
1.13.8.1 yamt 244: 3:
245: jmp _C_LABEL(mutex_spin_retry)
1.2 ad 246:
247: ALIGN64
248: LABEL(mutex_spin_enter_end)
1.13.8.1 yamt 249: END(mutex_spin_enter)
1.2 ad 250:
251: /*
252: * Release a spin mutex and post a store fence.
253: */
1.7 ad 254: ENTRY(mutex_spin_exit)
255: movl 4(%esp), %edx
256: movl CPUVAR(MTX_OLDSPL), %ecx
257: incl CPUVAR(MTX_COUNT)
258: movb $0, MTX_LOCK(%edx) /* zero */
1.2 ad 259: jnz 1f
1.7 ad 260: movl %fs:CPU_INFO_IUNMASK(,%ecx,4), %edx
261: cli
262: testl CPUVAR(IPENDING), %edx
1.2 ad 263: movl %ecx, 4(%esp)
264: jnz _C_LABEL(Xspllower) /* does sti */
1.7 ad 265: movl %ecx, CPUVAR(ILEVEL)
1.2 ad 266: sti
267: 1: ret
1.4 ad 268: nop /* XXX round up */
269: .align 32
1.2 ad 270: LABEL(mutex_spin_exit_end)
1.13.8.1 yamt 271: END(mutex_spin_exit)
1.2 ad 272:
273: /*
1.13.8.2! yamt 274: * Patch for i686 CPUs where cli/sti is prohibitively expensive.
1.2 ad 275: * Must be the same size as mutex_spin_exit().
276: */
1.4 ad 277: ENTRY(i686_mutex_spin_exit)
1.2 ad 278: mov 4(%esp),%edx
1.7 ad 279: movl CPUVAR(MTX_OLDSPL), %ecx
280: incl CPUVAR(MTX_COUNT)
281: movb %ch, MTX_LOCK(%edx) /* zero */
282: jnz 1f
1.2 ad 283: pushl %ebx
1.3 yamt 284: 0:
1.7 ad 285: movl CPUVAR(IPENDING), %eax
286: testl %eax, %fs:CPU_INFO_IUNMASK(,%ecx,4)
1.2 ad 287: movl %eax, %ebx
288: /*
289: * On a P4 this jump is cheaper than patching in junk using
290: * cmovnz. Is cmpxchg expensive if it fails?
291: */
1.7 ad 292: jnz 2f
293: cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */
294: jnz 0b
295: popl %ebx
1.2 ad 296: 1:
297: ret
298: 2:
299: popl %ebx
300: movl %ecx,4(%esp)
301: LABEL(i686_mutex_spin_exit_patch)
302: jmp _C_LABEL(Xspllower)
1.4 ad 303: .align 32
1.2 ad 304: LABEL(i686_mutex_spin_exit_end)
1.13.8.1 yamt 305: END(i686_mutex_spin_exit)
1.2 ad 306:
307: #else /* !__XEN__ */
308:
309: /* For now; strong alias not working for some reason. */
1.5 ad 310: ENTRY(mutex_spin_enter)
1.2 ad 311: jmp _C_LABEL(mutex_vector_enter)
312:
1.5 ad 313: ENTRY(mutex_spin_exit)
1.2 ad 314: jmp _C_LABEL(mutex_vector_exit)
315:
316: #endif /* !__XEN__ */
317:
318: #endif /* !LOCKDEBUG */
319:
320: /*
1.12 ad 321: * Spinlocks.
1.2 ad 322: */
1.13.8.2! yamt 323: ENTRY(__cpu_simple_lock_init)
1.12 ad 324: movl 4(%esp), %edx
325: movb $0, (%edx)
1.2 ad 326: ret
1.13.8.1 yamt 327: END(__cpu_simple_lock_init)
328:
1.12 ad 329: NENTRY(__cpu_simple_lock)
330: movl 4(%esp), %edx
331: movl $0x0100, %eax
332: 1:
333: LOCK(6)
334: cmpxchgb %ah, (%edx)
335: jnz 2f
1.2 ad 336: ret
1.12 ad 337: 2:
338: movl $0x0100, %eax
339: pause
340: nop
1.7 ad 341: nop
1.12 ad 342: cmpb $0, (%edx)
343: je 1b
344: jmp 2b
1.13.8.1 yamt 345: END(__cpu_simple_lock)
1.2 ad 346:
1.13.8.2! yamt 347: ENTRY(__cpu_simple_unlock)
1.12 ad 348: movl 4(%esp), %edx
349: movb $0, (%edx)
1.2 ad 350: ret
1.13.8.1 yamt 351: END(__cpu_simple_unlock)
1.2 ad 352:
1.13.8.2! yamt 353: ENTRY(__cpu_simple_lock_try)
1.12 ad 354: movl 4(%esp), %edx
355: movl $0x0100, %eax
356: LOCK(7)
357: cmpxchgb %ah, (%edx)
358: movl $0, %eax
359: setz %al
1.2 ad 360: ret
1.13.8.1 yamt 361: END(__cpu_simple_lock_try)
1.2 ad 362:
363: /*
1.7 ad 364: * Patchpoints to replace with NOP when ncpu == 1.
365: */
1.12 ad 366: .data
367:
1.7 ad 368: #ifndef LOCKDEBUG
369: LABEL(x86_lockpatch)
1.12 ad 370: .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4
1.13 ad 371: .long .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch12
372: .long .Lpatch13
1.8 ad 373: #if defined(FULL) && !defined(__XEN__)
1.12 ad 374: .long .Lpatch11
1.7 ad 375: #endif
376: .long 0
377: #endif
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to lock_stubs.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / i386 / i386