Annotation of src/sys/arch/i386/i386/lock_stubs.S, Revision 1.2.8.7
1.2.8.7 ! ad 1: /* $NetBSD: lock_stubs.S,v 1.2.8.6 2007/12/03 18:36:42 ad Exp $ */
1.2 ad 2:
3: /*-
4: * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: * 3. All advertising materials mentioning features or use of this software
19: * must display the following acknowledgement:
20: * This product includes software developed by the NetBSD
21: * Foundation, Inc. and its contributors.
22: * 4. Neither the name of The NetBSD Foundation nor the names of its
23: * contributors may be used to endorse or promote products derived
24: * from this software without specific prior written permission.
25: *
26: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36: * POSSIBILITY OF SUCH DAMAGE.
37: */
38:
39: /*
40: * The sizes listed against each function are for a kernel compiled
1.2.8.6 ad 41: * with options MULTIPROCESSOR && DIAGNOSTIC. Where possible we make
42: * each routine fit into an assumed 64-byte cache line. Please check
43: * alignment with 'objdump -d' after making changes.
1.2 ad 44: */
45:
46: #include "opt_multiprocessor.h"
47: #include "opt_lockdebug.h"
48: #include "opt_ddb.h"
49:
50: #include <machine/asm.h>
51: #include <machine/cputypes.h>
52:
53: #include "assym.h"
54:
1.2.8.7 ! ad 55: #if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
! 56: #define FULL
! 57: #endif
! 58:
1.2 ad 59: #define ALIGN64 .align 64
60: #define ALIGN32 .align 32
1.2.8.6 ad 61: #define LOCK(name) LABEL(name) lock
1.2 ad 62: #define END(name,a) .align a; LABEL(name)
63:
64: #if !defined(LOCKDEBUG)
65:
66: /*
67: * void mutex_enter(kmutex_t *mtx);
68: *
69: * Acquire a mutex and post a load fence.
70: */
71: ALIGN64
72:
1.2.8.6 ad 73: ENTRY(mutex_enter)
74: movl 4(%esp), %edx
1.2 ad 75: movl CPUVAR(CURLWP), %ecx
76: xorl %eax, %eax
1.2.8.6 ad 77: LOCK(lockpatch1)
1.2 ad 78: cmpxchgl %ecx, MTX_OWNER(%edx)
79: jnz,pn _C_LABEL(mutex_vector_enter)
80: ret
81:
82: /*
83: * void mutex_exit(kmutex_t *mtx);
84: *
85: * Release a mutex and post a load fence.
86: *
87: * See comments in mutex_vector_enter() about doing this operation unlocked
88: * on multiprocessor systems, and comments in arch/x86/include/lock.h about
89: * memory ordering on Intel x86 systems.
90: */
1.2.8.6 ad 91: ENTRY(mutex_exit)
92: movl 4(%esp), %edx
1.2 ad 93: movl CPUVAR(CURLWP), %eax
94: xorl %ecx, %ecx
95: cmpxchgl %ecx, MTX_OWNER(%edx)
96: jnz,pn _C_LABEL(mutex_vector_exit)
97: ret
98:
99: /*
100: * void rw_enter(krwlock_t *rwl, krw_t op);
101: *
102: * Acquire one hold on a RW lock.
103: */
1.2.8.6 ad 104: ENTRY(rw_enter)
105: movl 4(%esp), %edx
1.2 ad 106: cmpl $RW_READER, 8(%esp)
107: jne 2f
108:
109: /*
110: * Reader
111: */
112: 1: movl RW_OWNER(%edx), %eax
113: testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
114: leal RW_READ_INCR(%eax), %ecx
115: jnz,pn _C_LABEL(rw_vector_enter)
1.2.8.6 ad 116: LOCK(lockpatch2)
1.2 ad 117: cmpxchgl %ecx, RW_OWNER(%edx)
118: jnz,pn 1b
119: ret
120:
121: /*
122: * Writer
123: */
124: 2: movl CPUVAR(CURLWP), %ecx
125: xorl %eax, %eax
126: orl $RW_WRITE_LOCKED, %ecx
1.2.8.6 ad 127: LOCK(lockpatch3)
1.2 ad 128: cmpxchgl %ecx, RW_OWNER(%edx)
129: jnz,pn _C_LABEL(rw_vector_enter)
130: ret
131:
132: /*
133: * void rw_exit(krwlock_t *rwl);
134: *
135: * Release one hold on a RW lock.
136: */
1.2.8.6 ad 137: ENTRY(rw_exit)
138: movl 4(%esp), %edx
1.2 ad 139: movl RW_OWNER(%edx), %eax
140: testb $RW_WRITE_LOCKED, %al
141: jnz 2f
142:
143: /*
144: * Reader
145: */
146: 1: testb $RW_HAS_WAITERS, %al
147: jnz,pn 3f
148: cmpl $RW_READ_INCR, %eax
149: leal -RW_READ_INCR(%eax), %ecx
150: jb,pn 3f
1.2.8.6 ad 151: LOCK(lockpatch4)
1.2 ad 152: cmpxchgl %ecx, RW_OWNER(%edx)
153: jnz,pn 1b
154: ret
155:
156: /*
157: * Writer
158: */
159: 2: leal -RW_WRITE_LOCKED(%eax), %ecx
160: subl CPUVAR(CURLWP), %ecx
161: jnz,pn 3f
1.2.8.6 ad 162: LOCK(lockpatch5)
1.2 ad 163: cmpxchgl %ecx, RW_OWNER(%edx)
164: jnz,pn 3f
165: ret
166:
167: /*
168: * Slow path.
169: */
170: 3: jmp _C_LABEL(rw_vector_exit)
171:
172: #ifndef __XEN__
173:
174: /*
175: * void mutex_spin_enter(kmutex_t *mtx);
176: *
177: * Acquire a spin mutex and post a load fence.
178: */
1.2.8.7 ! ad 179: ENTRY(mutex_spin_enter)
! 180: movl 4(%esp), %edx
! 181: movl CPUVAR(ILEVEL), %ecx
! 182: subl $1, CPUVAR(MTX_COUNT) /* decl does not set CF */
1.2 ad 183: jnc 1f
1.2.8.7 ! ad 184: movl %ecx, CPUVAR(MTX_OLDSPL)
1.2 ad 185: 1: movb MTX_IPL(%edx), %ch
186: cmpb %ch, %cl
187: jg,pn 2f
1.2.8.7 ! ad 188: movb %ch, CPUVAR(ILEVEL) /* splraiseipl() */
1.2 ad 189: 2:
1.2.8.7 ! ad 190: #ifdef FULL
1.2 ad 191: mov $0x0100, %eax /* new + expected value */
1.2.8.7 ! ad 192: LOCK(lockpatch11)
1.2 ad 193: cmpxchgb %ah, MTX_LOCK(%edx) /* lock it */
194: jnz,pn _C_LABEL(mutex_spin_retry)
1.2.8.7 ! ad 195: #endif
1.2 ad 196: ret
197:
198: ALIGN64
199: LABEL(mutex_spin_enter_end)
200:
201: /*
202: * Release a spin mutex and post a store fence.
203: */
1.2.8.7 ! ad 204: ENTRY(mutex_spin_exit)
! 205: movl 4(%esp), %edx
! 206: movl CPUVAR(MTX_OLDSPL), %ecx
! 207: incl CPUVAR(MTX_COUNT)
! 208: movb $0, MTX_LOCK(%edx) /* zero */
1.2 ad 209: jnz 1f
1.2.8.7 ! ad 210: movl %fs:CPU_INFO_IUNMASK(,%ecx,4), %edx
1.2 ad 211: cli
1.2.8.7 ! ad 212: testl CPUVAR(IPENDING), %edx
! 213: movl %ecx, 4(%esp)
1.2 ad 214: jnz _C_LABEL(Xspllower) /* does sti */
1.2.8.7 ! ad 215: movl %ecx, CPUVAR(ILEVEL)
1.2 ad 216: sti
217: 1: ret
1.2.8.4 ad 218: nop /* XXX round up */
219: .align 32
1.2 ad 220: LABEL(mutex_spin_exit_end)
221:
222: /*
223: * Patch for i686 CPUs where cli/sti is prohibitavely expensive.
224: * Must be the same size as mutex_spin_exit().
225: */
1.2.8.2 ad 226: ENTRY(i686_mutex_spin_exit)
1.2 ad 227: mov 4(%esp),%edx
1.2.8.6 ad 228: movl CPUVAR(MTX_OLDSPL), %ecx
229: incl CPUVAR(MTX_COUNT)
230: movb %ch, MTX_LOCK(%edx) /* zero */
1.2 ad 231: jnz 1f
1.2.8.6 ad 232: pushl %ebx
1.2.8.1 ad 233: 0:
1.2.8.6 ad 234: movl CPUVAR(IPENDING), %eax
235: testl %eax, %fs:CPU_INFO_IUNMASK(,%ecx,4)
1.2 ad 236: movl %eax, %ebx
237: /*
238: * On a P4 this jump is cheaper than patching in junk using
239: * cmovnz. Is cmpxchg expensive if it fails?
240: */
1.2.8.6 ad 241: jnz 2f
242: cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */
243: jnz 0b
1.2 ad 244: popl %ebx
1.2.8.6 ad 245: 1:
1.2 ad 246: ret
247: 2:
248: popl %ebx
249: movl %ecx,4(%esp)
250: LABEL(i686_mutex_spin_exit_patch)
251: jmp _C_LABEL(Xspllower)
1.2.8.3 ad 252: .align 32
1.2 ad 253: LABEL(i686_mutex_spin_exit_end)
254:
255: #else /* !__XEN__ */
256:
257: /* For now; strong alias not working for some reason. */
1.2.8.5 ad 258: ENTRY(mutex_spin_enter)
1.2 ad 259: jmp _C_LABEL(mutex_vector_enter)
260:
1.2.8.5 ad 261: ENTRY(mutex_spin_exit)
1.2 ad 262: jmp _C_LABEL(mutex_vector_exit)
263:
264: #endif /* !__XEN__ */
265:
266: #endif /* !LOCKDEBUG */
267:
268: /*
269: * int _lock_cas(uintptr_t *val, uintptr_t old, uintptr_t new);
270: *
271: * Perform an atomic compare-and-set operation.
272: */
1.2.8.6 ad 273: ENTRY(_lock_cas)
274: mov 4(%esp),%edx
1.2 ad 275: movl 8(%esp), %eax
276: movl 12(%esp), %ecx
1.2.8.6 ad 277: LOCK(lockpatch6)
1.2 ad 278: cmpxchgl %ecx, (%edx)
279: movl $0, %eax
280: setz %al
281: ret
282:
283: /*
284: * Memory barrier operations, may be patched at runtime.
285: */
286: .align 8
287:
1.2.8.5 ad 288: ENTRY(mb_read)
1.2.8.6 ad 289: LOCK(lockpatch7)
1.2 ad 290: addl $0, 0(%esp)
291: ret
292: END(mb_read_end, 8)
293:
1.2.8.5 ad 294: ENTRY(mb_write)
1.2.8.6 ad 295: nop
1.2 ad 296: ret
297: END(mb_write_end, 8)
298:
1.2.8.5 ad 299: ENTRY(mb_memory)
1.2.8.6 ad 300: LOCK(lockpatch8)
1.2 ad 301: addl $0, 0(%esp)
302: ret
303: END(mb_memory_end, 8)
304:
1.2.8.5 ad 305: ENTRY(sse2_mb_read)
1.2 ad 306: lfence
307: ret
308: END(sse2_mb_read_end, 8)
309:
1.2.8.5 ad 310: ENTRY(sse2_mb_memory)
1.2 ad 311: mfence
312: ret
313: END(sse2_mb_memory_end, 8)
314:
315: /*
1.2.8.6 ad 316: * Patchpoints to replace with NOP when ncpu == 1.
1.2 ad 317: */
1.2.8.6 ad 318: #ifndef LOCKDEBUG
319: LABEL(x86_lockpatch)
320: .long lockpatch1, lockpatch2, lockpatch3, lockpatch4
321: .long lockpatch5, lockpatch6, lockpatch7, lockpatch8
322: #if defined(FULL) && !defined(__XEN__)
323: .long lockpatch11
324: #endif
325: .long 0
326: #endif
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to lock_stubs.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / i386 / i386